From: Vladimir Murzin <vladimir.murzin@arm.com>
To: linux-arm-kernel@lists.infradead.org
Cc: catalin.marinas@arm.com, will@kernel.org, mark.rutland@arm.com,
maz@kernel.org, lpieralisi@kernel.org, ruanjinjie@huawei.com,
liaochang1@huawei.com
Subject: [PATCH 06/38] arm64: entry: replace DAIF helpers with entry helpers
Date: Mon, 14 Sep 2026 11:20:30 +0100 [thread overview]
Message-ID: <20260914102102.68597-7-vladimir.murzin@arm.com> (raw)
In-Reply-To: <20260914102102.68597-1-vladimir.murzin@arm.com>
From: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Replace all uses of the `local_daif_...` helpers in `entry-common.c`
with the new entry-specific exception masking helpers.
Also replace `local_irq_disable()` with a switch to NOIRQ context
using the new helpers. This provides:
- consistency checks for hardware state
- a unified style for managing exception context
Now that both the irqflags API and entry code handle DAIF and PMR
correctly, remove `GIC_PRIO_PSR_I_SET` from the PMR value set by
`kernel_entry` in `entry.S` and by `init_gic_priority_masking()` in
`kernel/smp.c`.
The `local_daif_...` helpers and other specialized code still use
`GIC_PRIO_PSR_I_SET` for now; this commit does not change their
behavior.
Signed-off-by: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Signed-off-by: Vladimir Murzin <vladimir.murzin@arm.com>
Reviewed-by: Jinjie Ruan <ruanjinjie@huawei.com>
---
arch/arm64/kernel/entry-common.c | 258 ++++++++++++++++++++-----------
arch/arm64/kernel/entry.S | 2 +-
arch/arm64/kernel/smp.c | 2 +-
3 files changed, 173 insertions(+), 89 deletions(-)
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 72c03ccea59f..fd0bfead6e70 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -18,11 +18,11 @@
#include <linux/thread_info.h>
#include <asm/cpufeature.h>
-#include <asm/daifflags.h>
#include <asm/esr.h>
#include <asm/exception.h>
#include <asm/fpsimd.h>
#include <asm/irq_regs.h>
+#include <asm/interrupts/entry.h>
#include <asm/kprobes.h>
#include <asm/mmu.h>
#include <asm/processor.h>
@@ -53,9 +53,10 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg
* instrumentable code, or any code which may trigger an exception.
*/
static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs,
- irqentry_state_t state)
+ irqentry_state_t state,
+ arm64_exc_hwstate_t hwstate)
{
- local_daif_mask();
+ arm64_mask_exc_context(hwstate);
mte_check_tfsr_exit();
irqentry_exit_to_kernel_mode_after_preempt(regs, state);
}
@@ -64,22 +65,24 @@ static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs,
* We are returning from the context which allows involuntary kernel preemption
*/
static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs,
- irqentry_state_t state)
+ irqentry_state_t state,
+ arm64_exc_hwstate_t hwstate)
{
irqentry_exit_to_kernel_mode_preempt(regs, state);
- __arm64_exit_to_kernel_mode(regs, state);
+ __arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
- irqentry_state_t state)
+ irqentry_state_t state,
+ arm64_exc_hwstate_t hwstate)
{
if (!regs_irqs_disabled(regs)) {
- local_irq_disable();
- arm64_exit_to_kernel_mode_preempt(regs, state);
+ hwstate = arm64_lift_exc_context(hwstate, NOIRQ_CONTEXT);
+ arm64_exit_to_kernel_mode_preempt(regs, state, hwstate);
return;
}
- __arm64_exit_to_kernel_mode(regs, state);
+ __arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs)
@@ -102,11 +105,12 @@ static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs)
sme_enter_from_user_mode();
}
-static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs)
+static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs,
+ arm64_exc_hwstate_t hwstate)
{
- local_irq_disable();
+ hwstate = arm64_lift_exc_context(hwstate, NOIRQ_CONTEXT);
syscall_exit_to_user_mode_prepare(regs);
- local_daif_mask();
+ arm64_mask_exc_context(hwstate);
sme_exit_to_user_mode();
mte_check_tfsr_exit();
exit_to_user_mode();
@@ -117,11 +121,12 @@ static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs
* After this function returns it is not safe to call regular kernel code,
* instrumentable code, or any code which may trigger an exception.
*/
-static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
+static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs,
+ arm64_exc_hwstate_t hwstate)
{
- local_irq_disable();
+ hwstate = arm64_lift_exc_context(hwstate, NOIRQ_CONTEXT);
irqentry_exit_to_user_mode_prepare(regs);
- local_daif_mask();
+ arm64_mask_exc_context(hwstate);
sme_exit_to_user_mode();
mte_check_tfsr_exit();
exit_to_user_mode();
@@ -129,7 +134,10 @@ static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
{
- arm64_syscall_exit_to_user_mode(regs);
+ arm64_exc_hwstate_t hwstate = arm64_exc_hwstate_of_context(PROCESS_CONTEXT);
+
+ arm64_syscall_exit_to_user_mode(regs, hwstate);
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
/*
@@ -335,63 +343,69 @@ UNHANDLED(el1t, 64, error)
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_mem_abort(far, esr, regs);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_sp_pc_abort(far, esr, regs);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_el1_undef(regs, esr);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_el1_bti(regs, esr);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_el1_gcs(regs, esr);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_el1_mops(regs, esr);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr)
@@ -451,12 +465,13 @@ static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr)
static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = arm64_enter_from_kernel_mode(regs);
- local_daif_inherit(regs);
+ hwstate = arm64_inherit_exc_context(regs);
do_el1_fpac(regs, esr);
- arm64_exit_to_kernel_mode(regs, state);
+ arm64_exit_to_kernel_mode(regs, state, hwstate);
}
asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
@@ -506,16 +521,20 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
default:
__panic_unhandled(regs, "64-bit el1h sync", esr);
}
+
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
static __always_inline void __el1_pnmi(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
state = irqentry_nmi_enter(regs);
+ hwstate = arm64_unmask_exc_context(NONMI_CONTEXT);
do_interrupt_handler(regs, handler);
- local_daif_mask();
+ arm64_mask_exc_context(hwstate);
irqentry_nmi_exit(regs, state);
}
@@ -526,21 +545,32 @@ static __always_inline void __el1_irq(struct pt_regs *regs,
state = arm64_enter_from_kernel_mode(regs);
+ arm64_unmask_exc_context(NONMI_CONTEXT);
+
irq_enter_rcu();
do_interrupt_handler(regs, handler);
irq_exit_rcu();
- arm64_exit_to_kernel_mode_preempt(regs, state);
+ /*
+ * If pseudo-NMIs are enabled and the interrupted context had
+ * IRQs unmasked, the interrupt handler will have cleared DAIF
+ * and switched to PMR masking in order to handle
+ * NMIs. Otherwise it would keep IF. In both cases on return
+ * we effectivly have NOIRQ_CONTEXT - keep track of it
+ */
+ arm64_debug_exc_context(NOIRQ_CONTEXT);
+ arm64_exit_to_kernel_mode_preempt(regs, state, arm64_exc_hwstate_of_context(NOIRQ_CONTEXT));
}
+
static void noinstr el1_interrupt(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
- write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
-
if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
__el1_pnmi(regs, handler);
else
__el1_irq(regs, handler);
+
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
@@ -556,28 +586,31 @@ asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
- local_daif_restore(DAIF_ERRCTX);
state = irqentry_nmi_enter(regs);
+ hwstate = arm64_unmask_exc_context(ERROR_CONTEXT);
do_serror(regs, esr);
- local_daif_mask();
+ arm64_mask_exc_context(hwstate);
irqentry_nmi_exit(regs, state);
}
static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ arm64_exc_hwstate_t hwstate;
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_mem_abort(far, esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ arm64_exc_hwstate_t hwstate;
/*
* We've taken an instruction abort from userspace and not yet
@@ -588,114 +621,139 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
arm64_apply_bp_hardening();
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_mem_abort(far, esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_fpsimd_acc(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_sve_acc(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_sme_acc(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_fpsimd_exc(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_sys(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ arm64_exc_hwstate_t hwstate;
if (!is_ttbr0_addr(instruction_pointer(regs)))
arm64_apply_bp_hardening();
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_sp_pc_abort(far, esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_sp_pc_abort(regs->sp, esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_undef(regs, esr);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_bti(struct pt_regs *regs)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_bti(regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_mops(regs, esr);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_gcs(regs, esr);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
bad_el0_sync(regs, 0, esr);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
if (!is_ttbr0_addr(regs->pc))
arm64_apply_bp_hardening();
@@ -703,12 +761,13 @@ static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr)
debug_exception_enter(regs);
do_breakpoint(esr, regs);
debug_exception_exit(regs);
- local_daif_restore(DAIF_PROCCTX);
- arm64_exit_to_user_mode(regs);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
bool step_done;
if (!is_ttbr0_addr(regs->pc))
@@ -722,50 +781,56 @@ static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr)
* the single-step is complete.
*/
step_done = try_step_suspended_breakpoints(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
if (!step_done)
do_el0_softstep(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr)
{
/* Watchpoints are the only debug exception to write FAR_EL1 */
unsigned long far = read_sysreg(far_el1);
+ arm64_exc_hwstate_t hwstate;
arm64_enter_from_user_mode(regs);
debug_exception_enter(regs);
do_watchpoint(far, esr, regs);
debug_exception_exit(regs);
- local_daif_restore(DAIF_PROCCTX);
- arm64_exit_to_user_mode(regs);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_brk64(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_svc(struct pt_regs *regs)
{
+ arm64_exc_hwstate_t hwstate;
arm64_syscall_enter_from_user_mode(regs);
cortex_a76_erratum_1463225_svc_handler();
fpsimd_syscall_enter();
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_svc(regs);
- arm64_syscall_exit_to_user_mode(regs);
+ arm64_syscall_exit_to_user_mode(regs, hwstate);
fpsimd_syscall_exit();
}
static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_fpac(regs, esr);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
@@ -834,6 +899,8 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
default:
el0_inv(regs, esr);
}
+
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
static void noinstr el0_interrupt(struct pt_regs *regs,
@@ -841,7 +908,7 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
{
arm64_enter_from_user_mode(regs);
- write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
+ arm64_unmask_exc_context(NONMI_CONTEXT);
if (regs->pc & BIT(55))
arm64_apply_bp_hardening();
@@ -850,7 +917,14 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
do_interrupt_handler(regs, handler);
irq_exit_rcu();
- arm64_exit_to_user_mode(regs);
+ /*
+ * For the same reason as in el1_irq() we effectivly
+ * have NOIRQ_CONTEXT on return from handler - keep
+ * track of it
+ */
+ arm64_debug_exc_context(NOIRQ_CONTEXT);
+ arm64_exit_to_user_mode(regs, arm64_exc_hwstate_of_context(NOIRQ_CONTEXT));
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
@@ -876,15 +950,17 @@ asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
static void noinstr __el0_error_handler_common(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
+ arm64_exc_hwstate_t hwstate;
irqentry_state_t state;
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_ERRCTX);
+ hwstate = arm64_unmask_exc_context(ERROR_CONTEXT);
state = irqentry_nmi_enter(regs);
do_serror(regs, esr);
irqentry_nmi_exit(regs, state);
- local_daif_restore(DAIF_PROCCTX);
- arm64_exit_to_user_mode(regs);
+ hwstate = arm64_drop_exc_context(hwstate, PROCESS_CONTEXT);
+ arm64_exit_to_user_mode(regs, hwstate);
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
@@ -895,27 +971,33 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
#ifdef CONFIG_COMPAT
static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_cp15(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_svc_compat(struct pt_regs *regs)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_syscall_enter_from_user_mode(regs);
cortex_a76_erratum_1463225_svc_handler();
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_el0_svc_compat(regs);
- arm64_syscall_exit_to_user_mode(regs);
+ arm64_syscall_exit_to_user_mode(regs, hwstate);
}
static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr)
{
+ arm64_exc_hwstate_t hwstate;
+
arm64_enter_from_user_mode(regs);
- local_daif_restore(DAIF_PROCCTX);
+ hwstate = arm64_unmask_exc_context(PROCESS_CONTEXT);
do_bkpt32(esr, regs);
- arm64_exit_to_user_mode(regs);
+ arm64_exit_to_user_mode(regs, hwstate);
}
asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
@@ -966,6 +1048,8 @@ asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
default:
el0_inv(regs, esr);
}
+
+ arm64_debug_exc_context(CRITICAL_CONTEXT);
}
asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index f63049ac32dc..cb3be770f2d0 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -316,7 +316,7 @@ alternative_else_nop_endif
mrs_s x20, SYS_ICC_PMR_EL1
str w20, [sp, #S_PMR]
- mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET
+ mov x20, #GIC_PRIO_IRQON
msr_s SYS_ICC_PMR_EL1, x20
.Lskip_pmr_save\@:
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index a61dc3016a11..17e4557404f2 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -187,7 +187,7 @@ static void init_gic_priority_masking(void)
WARN_ON(!(cpuflags & PSR_I_BIT));
WARN_ON(!(cpuflags & PSR_F_BIT));
- gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
+ gic_write_pmr(GIC_PRIO_IRQON);
}
/*
--
2.34.1
next prev parent reply other threads:[~2026-09-14 10:49 UTC|newest]
Thread overview: 61+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-14 10:20 [PATCH 00/38] arm64: Add support for FEAT_NMI Vladimir Murzin
2026-09-14 10:20 ` [PATCH 01/38] arm64: irqflags: Introduce arm64-specific irqflags type Vladimir Murzin
2026-09-14 10:20 ` [PATCH 02/38] arm64: irqflags: save and use both DAIF and PMR Vladimir Murzin
2026-09-14 10:20 ` [PATCH 03/38] arm64: interrupts: Add common exception state helpers Vladimir Murzin
2026-09-17 8:58 ` Jinjie Ruan
2026-09-17 9:10 ` Jinjie Ruan
2026-09-21 14:10 ` Vladimir Murzin
2026-09-21 23:13 ` Ashok Raj
2026-09-14 10:20 ` [PATCH 04/38] arm64: process: Lift check for exception state to generic infrastructure Vladimir Murzin
2026-09-14 10:20 ` [PATCH 05/38] arm64: entry: Introduce entry specific exception masking helpers Vladimir Murzin
2026-09-17 9:53 ` Jinjie Ruan
2026-09-21 14:15 ` Vladimir Murzin
2026-09-14 10:20 ` Vladimir Murzin [this message]
[not found] ` <15bc779f-e6f5-4c2a-a629-cb020f12a41a@huawei.com>
2026-09-21 14:01 ` [PATCH 06/38] arm64: entry: replace DAIF helpers with entry helpers Vladimir Murzin
2026-09-14 10:20 ` [PATCH 07/38] arm64: interrupts: Introduce exception masking save/restore helpers Vladimir Murzin
2026-09-14 10:20 ` [PATCH 08/38] arm64: interrupts: introduce a helper for GIC priority initialization Vladimir Murzin
2026-09-14 10:20 ` [PATCH 09/38] arm64: replace local_daif helpers Vladimir Murzin
2026-09-17 8:14 ` Jinjie Ruan
2026-09-21 14:07 ` Vladimir Murzin
2026-09-14 10:20 ` [PATCH 10/38] arm64: cpuidle: use new helpers to bypass interrupt priority masking Vladimir Murzin
2026-09-14 10:20 ` [PATCH 11/38] arm64: remove daifflags.h Vladimir Murzin
2026-09-14 10:20 ` [PATCH 12/38] arm64: gicv3: remove GIC_PRIO_PSR_I_SET Vladimir Murzin
2026-09-14 10:20 ` [PATCH 13/38] arm64: cpufeature: Remove system_has_prio_mask_debugging() Vladimir Murzin
2026-09-14 10:20 ` [PATCH 14/38] arm64: irqflags: Switch to CONFIG_DEBUG_IRQFLAGS Vladimir Murzin
2026-09-14 10:20 ` [PATCH 15/38] arm64: Kconfig: Remove CONFIG_ARM64_DEBUG_PRIORITY_MASKING Vladimir Murzin
2026-09-14 10:20 ` [PATCH 16/38] efi/runtime-wrappers: Permit architectures to override IRQ flags checks Vladimir Murzin
2026-09-14 10:20 ` [PATCH 17/38] arm64/efi: Implement override for " Vladimir Murzin
2026-09-14 10:20 ` [PATCH 18/38] arm64: booting: Document boot requirements for FEAT_NMI Vladimir Murzin
2026-09-14 10:20 ` [PATCH 19/38] arm64: sysreg: Add definitions for immediate versions of MSR ALLINT Vladimir Murzin
2026-09-14 10:20 ` [PATCH 20/38] arm64: probes: Reject stepping over ALLINT Vladimir Murzin
2026-09-16 8:53 ` Jinjie Ruan
2026-09-14 10:20 ` [PATCH 21/38] arm64: ptrace: Add PSR_ALLINT_BIT Vladimir Murzin
2026-09-14 10:20 ` [PATCH 22/38] arm64: idreg: Add an override for FEAT_NMI Vladimir Murzin
2026-09-14 10:20 ` [PATCH 23/38] arm64: cpufeature: Detect PE support " Vladimir Murzin
2026-09-16 9:22 ` Jinjie Ruan
2026-09-21 13:53 ` Vladimir Murzin
2026-09-14 10:20 ` [PATCH 24/38] arm64: nmi: Manage masking for superpriority interrupts Vladimir Murzin
2026-09-17 11:41 ` Jinjie Ruan
2026-09-21 14:23 ` Vladimir Murzin
2026-09-14 10:20 ` [PATCH 25/38] arm64: irq: Report FEAT_NMI masking local IRQs Vladimir Murzin
2026-09-14 10:20 ` [PATCH 26/38] arm64: nmi: Add handling of superpriority interrupts as NMIs Vladimir Murzin
2026-09-17 11:37 ` Jinjie Ruan
2026-09-21 14:16 ` Vladimir Murzin
2026-09-14 10:20 ` [PATCH 27/38] arm64: suspend: Always initialise PSTATE.ALLINT Vladimir Murzin
2026-09-14 10:20 ` [PATCH 28/38] arm64/efi: Add ALLINT to IRQ flags checks Vladimir Murzin
2026-09-14 10:20 ` [PATCH 29/38] arm64: kprobes: Disable NMIs Vladimir Murzin
2026-09-14 10:20 ` [PATCH 30/38] arm64: smp: Abstract SGI and LPI operations Vladimir Murzin
2026-09-16 9:01 ` Jinjie Ruan
2026-09-21 13:44 ` Vladimir Murzin
2026-09-14 10:20 ` [PATCH 31/38] arm64: smp: Fall back to IRQ when IPI NMI request fails Vladimir Murzin
2026-09-14 10:20 ` [PATCH 32/38] arm64: nmi: Add Kconfig for NMI Vladimir Murzin
2026-09-16 9:06 ` Jinjie Ruan
2026-09-21 13:47 ` Vladimir Murzin
2026-09-14 10:20 ` [PATCH 33/38] irqchip/gic-v3: Rename nmi to pseudo_nmi Vladimir Murzin
2026-09-14 10:20 ` [PATCH 34/38] irqchip/gic-v3: Implement FEAT_GICv3_NMI support Vladimir Murzin
2026-09-14 10:20 ` [PATCH 35/38] arm64: smp: Add NMI support for LPI-backed IPIs Vladimir Murzin
2026-09-14 10:21 ` [PATCH 36/38] genirq: Trust local irqchip when considering IRQ for NMI Vladimir Murzin
2026-09-17 12:06 ` Jinjie Ruan
2026-09-14 10:21 ` [PATCH 37/38] genirq: Do not call add_interrupt_randomness() in NMI context Vladimir Murzin
2026-09-17 11:55 ` Jinjie Ruan
2026-09-14 10:21 ` [PATCH 38/38] irqchip/gic-v5: Add NMI support Vladimir Murzin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260914102102.68597-7-vladimir.murzin@arm.com \
--to=vladimir.murzin@arm.com \
--cc=catalin.marinas@arm.com \
--cc=liaochang1@huawei.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=lpieralisi@kernel.org \
--cc=mark.rutland@arm.com \
--cc=maz@kernel.org \
--cc=ruanjinjie@huawei.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox