[PATCH 26/38] arm64: nmi: Add handling of superpriority interrupts as NMIs

[Vladimir Murzin <vladimir.murzin@arm.com>]

From: Mark Brown <broonie@kernel.org>

Our goal with superpriority interrupts is to use them as NMIs, taking
advantage of the much smaller regions where they are masked to allow
prompt handling of the most time-critical interrupts.

When an interrupt is configured with superpriority, we enter EL1 as we
do for any other interrupt. The presence of a superpriority interrupt
is indicated by a status bit in ISR_EL1. We check this bit before
unmasking interrupts in elX_interrupt(), and if a superpriority
interrupt is pending, we handle it as an NMI. Otherwise, normal
interrupts are handled as usual.

Since superpriority interrupts are always handled as NMIs, the
interrupt controller can rely on in_nmi() to distinguish them from
ordinary interrupts.

Enable IPIs to use superpriority interrupts as NMIs, matching the
existing pseudo-NMI behaviour.

Signed-off-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Signed-off-by: Vladimir Murzin <vladimir.murzin@arm.com>
Reviewed-by: Jinjie Ruan <ruanjinjie@huawei.com>
---
 arch/arm64/include/asm/entry-common.h |  7 +++
 arch/arm64/kernel/entry-common.c      | 83 +++++++++++++++++++++++----
 arch/arm64/kernel/smp.c               |  2 +-
 3 files changed, 79 insertions(+), 13 deletions(-)

diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h
index 73d82a8d8e95..0681ba91ac3b 100644
--- a/arch/arm64/include/asm/entry-common.h
+++ b/arch/arm64/include/asm/entry-common.h
@@ -37,6 +37,13 @@ static inline bool arch_irqentry_exit_need_resched(void)
 	if (system_uses_irq_prio_masking() && read_sysreg(daif))
 		return false;
 
+	/*
+	 * If AllInt is set then we must have handled an NMI, so skip
+	 * preemption
+	 */
+	if (system_uses_nmi() && read_sysreg_s(SYS_ALLINT))
+		return false;
+
 	/*
 	 * Preempting a task from an IRQ means we leave copies of PSTATE
 	 * on the stack. cpufeature's enable calls may modify PSTATE, but
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index fd0bfead6e70..31dae3b48f23 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -31,6 +31,14 @@
 #include <asm/sysreg.h>
 #include <asm/system_misc.h>
 
+static __always_inline bool is_nmi(void)
+{
+	if (!system_uses_nmi())
+		return false;
+
+	return !!(read_sysreg(isr_el1) & ISR_EL1_IS);
+}
+
 /*
  * Handle IRQ/context state management when entering from kernel mode.
  * Before this function is called it is not safe to call regular kernel code,
@@ -525,8 +533,8 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
 	arm64_debug_exc_context(CRITICAL_CONTEXT);
 }
 
-static __always_inline void __el1_pnmi(struct pt_regs *regs,
-				       void (*handler)(struct pt_regs *))
+static __always_inline void __el1_nmi(struct pt_regs *regs,
+				      void (*handler)(struct pt_regs *))
 {
 	arm64_exc_hwstate_t hwstate;
 	irqentry_state_t state;
@@ -545,7 +553,23 @@ static __always_inline void __el1_irq(struct pt_regs *regs,
 
 	state = arm64_enter_from_kernel_mode(regs);
 
-	arm64_unmask_exc_context(NONMI_CONTEXT);
+	/*
+	 * For non-NMI systems, we can switch to NOIRQ context since
+	 * NMIs cannot occur.
+	 *
+	 * For pseudo-NMI, we must keep NONMI context until we can
+	 * determine the interrupt priority, which is done by the IRQ
+	 * handler.
+	 *
+	 * For FEAT_NMI, we already know the interrupt is an IRQ, so we
+	 * can switch to NOIRQ context immediately. NMIs do not
+	 * interfere because they follow a separate handling path and do
+	 * not share the IRQ handling state.
+	 */
+	if (system_uses_irq_prio_masking())
+		arm64_unmask_exc_context(NONMI_CONTEXT);
+	else
+		arm64_unmask_exc_context(NOIRQ_CONTEXT);
 
 	irq_enter_rcu();
 	do_interrupt_handler(regs, handler);
@@ -565,8 +589,9 @@ static __always_inline void __el1_irq(struct pt_regs *regs,
 static void noinstr el1_interrupt(struct pt_regs *regs,
 				  void (*handler)(struct pt_regs *))
 {
-	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
-		__el1_pnmi(regs, handler);
+	/* Is there a NMI to handle? */
+	if (regs_irqs_disabled(regs) || is_nmi())
+		__el1_nmi(regs, handler);
 	else
 		__el1_irq(regs, handler);
 
@@ -580,6 +605,8 @@ asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
 
 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
 {
+	WARN_ON_ONCE(system_uses_nmi() && (read_sysreg(isr_el1) & ISR_EL1_FS));
+
 	el1_interrupt(regs, handle_arch_fiq);
 }
 
@@ -903,20 +930,36 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
 	arm64_debug_exc_context(CRITICAL_CONTEXT);
 }
 
-static void noinstr el0_interrupt(struct pt_regs *regs,
-				  void (*handler)(struct pt_regs *))
+static __always_inline void __el0_nmi(struct pt_regs *regs,
+				      void (*handler)(struct pt_regs *))
 {
-	arm64_enter_from_user_mode(regs);
+	irqentry_state_t state;
+	arm64_exc_hwstate_t hwstate;
 
-	arm64_unmask_exc_context(NONMI_CONTEXT);
+	state = irqentry_nmi_enter(regs);
+	hwstate = arm64_unmask_exc_context(NONMI_CONTEXT);
+	do_interrupt_handler(regs, handler);
+	arm64_mask_exc_context(hwstate);
+	irqentry_nmi_exit(regs, state);
+}
 
-	if (regs->pc & BIT(55))
-		arm64_apply_bp_hardening();
+static __always_inline void __el0_irq(struct pt_regs *regs,
+				      void (*handler)(struct pt_regs *))
+{
+	arm64_enter_from_user_mode(regs);
+	/*
+	 * For the same reason as in el1_irq() keep NONMI
+	 * context for pesudo-NMI, and switch to NOIRQ
+	 * otherwise
+	 */
+	if (system_uses_irq_prio_masking())
+		arm64_unmask_exc_context(NONMI_CONTEXT);
+	else
+		arm64_unmask_exc_context(NOIRQ_CONTEXT);
 
 	irq_enter_rcu();
 	do_interrupt_handler(regs, handler);
 	irq_exit_rcu();
-
 	/*
 	 * For the same reason as in el1_irq() we effectivly
 	 * have NOIRQ_CONTEXT on return from handler - keep
@@ -924,6 +967,20 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
 	 */
 	arm64_debug_exc_context(NOIRQ_CONTEXT);
 	arm64_exit_to_user_mode(regs, arm64_exc_hwstate_of_context(NOIRQ_CONTEXT));
+}
+
+static void noinstr el0_interrupt(struct pt_regs *regs,
+				  void (*handler)(struct pt_regs *))
+{
+	if (regs->pc & BIT(55))
+		arm64_apply_bp_hardening();
+
+	/* Is there a NMI to handle? */
+	if (is_nmi())
+		__el0_nmi(regs, handler);
+	else
+		__el0_irq(regs, handler);
+
 	arm64_debug_exc_context(CRITICAL_CONTEXT);
 }
 
@@ -939,6 +996,8 @@ asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
 
 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
 {
+	WARN_ON_ONCE(system_uses_nmi() && (read_sysreg(isr_el1) & ISR_EL1_FS));
+
 	el0_interrupt(regs, handle_arch_fiq);
 }
 
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index f29094ec3843..3d7060aedd66 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -1061,7 +1061,7 @@ static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
 
 static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
 {
-	if (!system_uses_irq_prio_masking())
+	if (!system_uses_nmi() && !system_uses_irq_prio_masking())
 		return false;
 
 	switch (ipi) {
-- 
2.34.1