[PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[Breno Leitao]

The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two
underlying primitives: the __daif_* path on most systems, and the
__pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The
leaf primitives are already __always_inline, but the wrappers themselves
are plain "static inline".

That is unsafe for noinstr callers: nothing prevents the compiler from
emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an
out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which
breaks the noinstr contract on the entry/idle paths that rely on these
helpers.

x86 hit and fixed exactly this class of bug in commit 7a745be1cc90
("x86/entry: __always_inline irqflags for noinstr").

Force-inline all of the arch_local_irq_*() wrappers so they cannot be
emitted out-of-line:

  - arch_local_irq_enable()
  - arch_local_irq_disable()
  - arch_local_save_flags()
  - arch_irqs_disabled_flags()
  - arch_irqs_disabled()
  - arch_local_irq_save()
  - arch_local_irq_restore()

The primary motivation is noinstr safety. There is a useful side effect
for fleet-wide profiling: when the wrapper is emitted out-of-line,
samples taken inside it during the post-WFI IRQ unmask in
default_idle_call() are attributed to arch_local_irq_enable rather than
default_idle_call(), and the FP-unwinder loses default_idle_call() from
the chain.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
Changes in v2:
- Expand the functions that uses always_inline in arm64
- Link to v1: https://patch.msgid.link/20260420-arm64_always_inline-v1-1-dba919cf46bc@debian.org
---
 arch/arm64/include/asm/irqflags.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h
index d4d7451c2c129..a8cb5a5c93b78 100644
--- a/arch/arm64/include/asm/irqflags.h
+++ b/arch/arm64/include/asm/irqflags.h
@@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)
 	barrier();
 }
 
-static inline void arch_local_irq_enable(void)
+static __always_inline void arch_local_irq_enable(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_enable();
@@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)
 	barrier();
 }
 
-static inline void arch_local_irq_disable(void)
+static __always_inline void arch_local_irq_disable(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_disable();
@@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)
 /*
  * Save the current interrupt enable state.
  */
-static inline unsigned long arch_local_save_flags(void)
+static __always_inline unsigned long arch_local_save_flags(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_local_save_flags();
@@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)
 	return flags != GIC_PRIO_IRQON;
 }
 
-static inline bool arch_irqs_disabled_flags(unsigned long flags)
+static __always_inline bool arch_irqs_disabled_flags(unsigned long flags)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_irqs_disabled_flags(flags);
@@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)
 	return __pmr_irqs_disabled_flags(__pmr_local_save_flags());
 }
 
-static inline bool arch_irqs_disabled(void)
+static __always_inline bool arch_irqs_disabled(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_irqs_disabled();
@@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)
 	return flags;
 }
 
-static inline unsigned long arch_local_irq_save(void)
+static __always_inline unsigned long arch_local_irq_save(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_local_irq_save();
@@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)
 /*
  * restore saved IRQ state
  */
-static inline void arch_local_irq_restore(unsigned long flags)
+static __always_inline void arch_local_irq_restore(unsigned long flags)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_restore(flags);

---
base-commit: bee6ea30c48788e18348309f891ed8afbf7702ac
change-id: 20260420-arm64_always_inline-6bc9dd3c17e6

Best regards,
--  
Breno Leitao <leitao@debian.org>

Re: [PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[Leonardo Bras]

On Tue, Apr 21, 2026 at 08:58:57AM -0700, Breno Leitao wrote:
> The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two
> underlying primitives: the __daif_* path on most systems, and the
> __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The
> leaf primitives are already __always_inline, but the wrappers themselves
> are plain "static inline".
> 
> That is unsafe for noinstr callers: nothing prevents the compiler from
> emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an
> out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which
> breaks the noinstr contract on the entry/idle paths that rely on these
> helpers.
> 
> x86 hit and fixed exactly this class of bug in commit 7a745be1cc90
> ("x86/entry: __always_inline irqflags for noinstr").
> 
> Force-inline all of the arch_local_irq_*() wrappers so they cannot be
> emitted out-of-line:
> 
>   - arch_local_irq_enable()
>   - arch_local_irq_disable()
>   - arch_local_save_flags()
>   - arch_irqs_disabled_flags()
>   - arch_irqs_disabled()
>   - arch_local_irq_save()
>   - arch_local_irq_restore()
> 
> The primary motivation is noinstr safety. There is a useful side effect
> for fleet-wide profiling: when the wrapper is emitted out-of-line,
> samples taken inside it during the post-WFI IRQ unmask in
> default_idle_call() are attributed to arch_local_irq_enable rather than
> default_idle_call(), and the FP-unwinder loses default_idle_call() from
> the chain.
> 
> Signed-off-by: Breno Leitao <leitao@debian.org>
> ---
> Changes in v2:
> - Expand the functions that uses always_inline in arm64
> - Link to v1: https://patch.msgid.link/20260420-arm64_always_inline-v1-1-dba919cf46bc@debian.org
> ---
>  arch/arm64/include/asm/irqflags.h | 14 +++++++-------
>  1 file changed, 7 insertions(+), 7 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h
> index d4d7451c2c129..a8cb5a5c93b78 100644
> --- a/arch/arm64/include/asm/irqflags.h
> +++ b/arch/arm64/include/asm/irqflags.h
> @@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)
>  	barrier();
>  }
>  
> -static inline void arch_local_irq_enable(void)
> +static __always_inline void arch_local_irq_enable(void)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		__pmr_local_irq_enable();
> @@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)
>  	barrier();
>  }
>  
> -static inline void arch_local_irq_disable(void)
> +static __always_inline void arch_local_irq_disable(void)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		__pmr_local_irq_disable();
> @@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)
>  /*
>   * Save the current interrupt enable state.
>   */
> -static inline unsigned long arch_local_save_flags(void)
> +static __always_inline unsigned long arch_local_save_flags(void)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		return __pmr_local_save_flags();
> @@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)
>  	return flags != GIC_PRIO_IRQON;
>  }
>  
> -static inline bool arch_irqs_disabled_flags(unsigned long flags)
> +static __always_inline bool arch_irqs_disabled_flags(unsigned long flags)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		return __pmr_irqs_disabled_flags(flags);
> @@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)
>  	return __pmr_irqs_disabled_flags(__pmr_local_save_flags());
>  }
>  
> -static inline bool arch_irqs_disabled(void)
> +static __always_inline bool arch_irqs_disabled(void)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		return __pmr_irqs_disabled();
> @@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)
>  	return flags;
>  }
>  
> -static inline unsigned long arch_local_irq_save(void)
> +static __always_inline unsigned long arch_local_irq_save(void)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		return __pmr_local_irq_save();
> @@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)
>  /*
>   * restore saved IRQ state
>   */
> -static inline void arch_local_irq_restore(unsigned long flags)
> +static __always_inline void arch_local_irq_restore(unsigned long flags)
>  {
>  	if (system_uses_irq_prio_masking()) {
>  		__pmr_local_irq_restore(flags);
> 
> ---
> base-commit: bee6ea30c48788e18348309f891ed8afbf7702ac
> change-id: 20260420-arm64_always_inline-6bc9dd3c17e6
> 
> Best regards,
> --  
> Breno Leitao <leitao@debian.org>
> 


Looks correct to what was discussed in V1. FWIW:

Reviewed-by: Leonardo Bras <leo.bras@arm.com>

Re: [PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[Breno Leitao]

On Tue, Apr 21, 2026 at 08:58:57AM -0700, Breno Leitao wrote:
> The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two
> underlying primitives: the __daif_* path on most systems, and the
> __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The
> leaf primitives are already __always_inline, but the wrappers themselves
> are plain "static inline".
> 
> That is unsafe for noinstr callers: nothing prevents the compiler from
> emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an
> out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which
> breaks the noinstr contract on the entry/idle paths that rely on these
> helpers.
> 
> x86 hit and fixed exactly this class of bug in commit 7a745be1cc90
> ("x86/entry: __always_inline irqflags for noinstr").
> 
> Force-inline all of the arch_local_irq_*() wrappers so they cannot be
> emitted out-of-line:
> 
>   - arch_local_irq_enable()
>   - arch_local_irq_disable()
>   - arch_local_save_flags()
>   - arch_irqs_disabled_flags()
>   - arch_irqs_disabled()
>   - arch_local_irq_save()
>   - arch_local_irq_restore()
> 
> The primary motivation is noinstr safety. There is a useful side effect
> for fleet-wide profiling: when the wrapper is emitted out-of-line,
> samples taken inside it during the post-WFI IRQ unmask in
> default_idle_call() are attributed to arch_local_irq_enable rather than
> default_idle_call(), and the FP-unwinder loses default_idle_call() from
> the chain.

FWIW I run scripts/bloat-o-meter on the kernel with and without the
patch, and the the code size is mostly the same. here is the result:

	add/remove: 4/12 grow/shrink: 40/0 up/down: 1684/-652 (1032)
	Function                                     old     new   delta
	__schedule                                  8892    9024    +132
	irqentry_exit                                816     892     +76
	lockdep_hardirqs_off                         396     452     +56
	lock_is_held_type                            412     468     +56
	ct_idle_exit                                  76     132     +56
	cpu_idle_poll                                304     360     +56
	arch_stack_walk_reliable                    1152    1196     +44
	arch_stack_walk                             1184    1228     +44
	arch_bpf_stack_walk                          996    1040     +44
	lockdep_hardirqs_on                          464     504     +40
	el0_watchpt                                  576     616     +40
	el0_undef                                    560     600     +40
	el0_sys                                      560     600     +40
	el0_sve_acc                                  560     600     +40
	el0_svc                                      600     640     +40
	el0_sp                                       564     604     +40
	el0_softstp                                  728     768     +40
	el0_sme_acc                                  560     600     +40
	el0_pc                                       740     780     +40
	el0_mops                                     560     600     +40
	el0_inv                                      564     604     +40
	el0_interrupt                                656     696     +40
	el0_ia                                       716     756     +40
	el0_gcs                                      560     600     +40
	el0_fpsimd_exc                               560     600     +40
	el0_fpsimd_acc                               560     600     +40
	el0_fpac                                     560     600     +40
	el0_da                                       568     608     +40
	el0_bti                                      552     592     +40
	el0_brk64                                    560     600     +40
	el0_breakpt                                  720     760     +40
	asm_exit_to_user_mode                        416     456     +40
	__el0_error_handler_common                   592     632     +40
	cpuidle_enter_state                         1220    1248     +28
	check_preemption_disabled                    228     252     +24
	default_idle_call                            252     272     +20
	ct_kernel_enter                              388     404     +16
	ct_idle_enter                                 52      68     +16
	look_up_lock_class                           364     376     +12
	check_flags                                  492     504     +12
	__CortexA53843419_FFFF800081146000             -       8      +8
	__CortexA53843419_FFFF8000809C3004             -       8      +8
	__CortexA53843419_FFFF8000809AE000             -       8      +8
	__CortexA53843419_FFFF800080248004             -       8      +8
	__CortexA53843419_FFFF80008100C000             8       -      -8
	__CortexA53843419_FFFF8000809A9000             8       -      -8
	__CortexA53843419_FFFF8000809A8004             8       -      -8
	__CortexA53843419_FFFF800080448008             8       -      -8
	__CortexA53843419_FFFF8000801EE000             8       -      -8
	arch_local_irq_restore                        48       -     -48
	arch_local_save_flags                         80       -     -80
	arch_local_irq_save                           80       -     -80
	arch_local_irq_enable                         84       -     -84
	arch_local_irq_disable                        96       -     -96
	arch_irqs_disabled_flags                      96       -     -96
	arch_irqs_disabled                           128       -    -128
	Total: Before=163062863, After=163063895, chg +0.00%

Re: [PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[Catalin Marinas]

On Tue, Apr 21, 2026 at 08:58:57AM -0700, Breno Leitao wrote:
> Force-inline all of the arch_local_irq_*() wrappers so they cannot be
> emitted out-of-line:
> 
>   - arch_local_irq_enable()
>   - arch_local_irq_disable()
>   - arch_local_save_flags()
>   - arch_irqs_disabled_flags()
>   - arch_irqs_disabled()
>   - arch_local_irq_save()
>   - arch_local_irq_restore()

I'll queue this, thanks!

I think we should also do local_daif_{mask,restore,inherit} as they seem
to be called from noinstr locations in entry-common.c.

-- 
Catalin

Re: [PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[Mark Rutland]

On Mon, Apr 27, 2026 at 01:26:18PM +0100, Catalin Marinas wrote:
> On Tue, Apr 21, 2026 at 08:58:57AM -0700, Breno Leitao wrote:
> > Force-inline all of the arch_local_irq_*() wrappers so they cannot be
> > emitted out-of-line:
> > 
> >   - arch_local_irq_enable()
> >   - arch_local_irq_disable()
> >   - arch_local_save_flags()
> >   - arch_irqs_disabled_flags()
> >   - arch_irqs_disabled()
> >   - arch_local_irq_save()
> >   - arch_local_irq_restore()
> 
> I'll queue this, thanks!
> 
> I think we should also do local_daif_{mask,restore,inherit} as they seem
> to be called from noinstr locations in entry-common.c.

I agree we probably should mark those as __always_inline, but I beleive
they're safe as-is. For their current usage in entry-common.c they're
only called between between enter_from_*_mode() and exit_to_*_mode(), in
a period where instrumentation is safe.

Mark.

Re: [PATCH v2] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

[Catalin Marinas]

On Tue, 21 Apr 2026 08:58:57 -0700, Breno Leitao wrote:
> The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two
> underlying primitives: the __daif_* path on most systems, and the
> __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The
> leaf primitives are already __always_inline, but the wrappers themselves
> are plain "static inline".
> 
> That is unsafe for noinstr callers: nothing prevents the compiler from
> emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an
> out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which
> breaks the noinstr contract on the entry/idle paths that rely on these
> helpers.
> 
> [...]

Applied to arm64 (for-next/fixes), thanks!

[1/1] arm64/irqflags: __always_inline the arch_local_irq_*() helpers
      https://git.kernel.org/arm64/c/caecde119e34

[PATCH] arm64/daifflags: Make local_daif_*() helpers __always_inline

[Leonardo Bras]

Make sure those helpers are always inlined and instrumentation safe.

Signed-off-by: Leonardo Bras <leo.bras@arm.com>
---
 arch/arm64/include/asm/daifflags.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h
index 5fca48009043..795b35128467 100644
--- a/arch/arm64/include/asm/daifflags.h
+++ b/arch/arm64/include/asm/daifflags.h
@@ -12,66 +12,66 @@
 #include <asm/cpufeature.h>
 #include <asm/ptrace.h>
 
 #define DAIF_PROCCTX		0
 #define DAIF_PROCCTX_NOIRQ	(PSR_I_BIT | PSR_F_BIT)
 #define DAIF_ERRCTX		(PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
 #define DAIF_MASK		(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
 
 
 /* mask/save/unmask/restore all exceptions, including interrupts. */
-static inline void local_daif_mask(void)
+static __always_inline void local_daif_mask(void)
 {
 	WARN_ON(system_has_prio_mask_debugging() &&
 		(read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF |
 						    GIC_PRIO_PSR_I_SET)));
 
 	asm volatile(
 		"msr	daifset, #0xf		// local_daif_mask\n"
 		:
 		:
 		: "memory");
 
 	/* Don't really care for a dsb here, we don't intend to enable IRQs */
 	if (system_uses_irq_prio_masking())
 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 
 	trace_hardirqs_off();
 }
 
-static inline unsigned long local_daif_save_flags(void)
+static __always_inline unsigned long local_daif_save_flags(void)
 {
 	unsigned long flags;
 
 	flags = read_sysreg(daif);
 
 	if (system_uses_irq_prio_masking()) {
 		/* If IRQs are masked with PMR, reflect it in the flags */
 		if (read_sysreg_s(SYS_ICC_PMR_EL1) != GIC_PRIO_IRQON)
 			flags |= PSR_I_BIT | PSR_F_BIT;
 	}
 
 	return flags;
 }
 
-static inline unsigned long local_daif_save(void)
+static __always_inline unsigned long local_daif_save(void)
 {
 	unsigned long flags;
 
 	flags = local_daif_save_flags();
 
 	local_daif_mask();
 
 	return flags;
 }
 
-static inline void local_daif_restore(unsigned long flags)
+static __always_inline void local_daif_restore(unsigned long flags)
 {
 	bool irq_disabled = flags & PSR_I_BIT;
 
 	WARN_ON(system_has_prio_mask_debugging() &&
 		(read_sysreg(daif) & (PSR_I_BIT | PSR_F_BIT)) != (PSR_I_BIT | PSR_F_BIT));
 
 	if (!irq_disabled) {
 		trace_hardirqs_on();
 
 		if (system_uses_irq_prio_masking()) {
@@ -117,21 +117,21 @@ static inline void local_daif_restore(unsigned long flags)
 	write_sysreg(flags, daif);
 
 	if (irq_disabled)
 		trace_hardirqs_off();
 }
 
 /*
  * Called by synchronous exception handlers to restore the DAIF bits that were
  * modified by taking an exception.
  */
-static inline void local_daif_inherit(struct pt_regs *regs)
+static __always_inline void local_daif_inherit(struct pt_regs *regs)
 {
 	unsigned long flags = regs->pstate & DAIF_MASK;
 
 	if (!regs_irqs_disabled(regs))
 		trace_hardirqs_on();
 
 	if (system_uses_irq_prio_masking())
 		gic_write_pmr(regs->pmr);
 
 	/*
-- 
2.54.0