[RFC PATCH 29/36] arm64: nmi: Manage masking for superpriority interrupts

[Vladimir Murzin <vladimir.murzin@arm.com>]

From: Ada Couprie Diaz <ada.coupriediaz@arm.com>

Extend logic to handle and debug exception context/state with knowlage
of FEAT_NMI.

Take care to order writes to ALLINT relative to DAIF, as clearing
ALLINT before DAIF could result in taking an NMI while DAIF is fully
masked, as could setting it after DAIF.

Since superpriority interrupts are not masked through DAIF like pseduo
NMIs are, we also need to modify the assembler macros for managing
DAIF to ensure that the masking is done in the assembly code.

Note that save_and_disable_irq/restore_irq and
save_and_disable_daif/restore_irq pairs are used in distinct
contextes:

- former is used in context of SW PAN to quickly disable/enable
  preemption

- latter is used to completely mask all exceptions.

For that reason split save_and_disable_daif/restore_irq into more
generic exception save restore pair and plumb with FEAT_NMI logic.

Co-developed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Ada Couprie Diaz <ada.coupriediaz@arm.com>
---
 arch/arm64/include/asm/assembler.h            | 20 ++++++++-
 .../include/asm/interrupts/common_flags.h     | 41 +++++++++++++++++--
 arch/arm64/include/asm/interrupts/entry.h     | 18 +++++---
 arch/arm64/include/asm/interrupts/masking.h   | 25 ++++++-----
 arch/arm64/include/asm/irqflags.h             |  1 +
 arch/arm64/kernel/entry.S                     | 12 +++---
 6 files changed, 89 insertions(+), 28 deletions(-)

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 0b58b550e8dc..bcdbc308afba 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -37,11 +37,27 @@
 /*
  * Save/restore interrupts.
  */
-	.macro save_and_disable_daif, flags
-	mrs	\flags, daif
+	.macro save_and_disable_exceptions, flags, tmp
+	mrs     \flags, daif		// updates flags[9:6] with DAIF
+#ifdef CONFIG_ARM64_NMI
+alternative_if ARM64_NMI
+	mrs_s	\tmp, SYS_ALLINT	// updates tmp[13] with AllInt
+	msr_s	SYS_ALLINT_SET, xzr
+	orr	\flags, \flags, \tmp	// now flags[13,9:6] carry pair of AllInt,DAIF
+alternative_else_nop_endif
+#endif
 	msr	daifset, #0xf
 	.endm
 
+	.macro  restore_exceptions, flags
+	msr     daif, \flags		// bits other than flags[9:6] are ignored
+#ifdef CONFIG_ARM64_NMI
+alternative_if ARM64_NMI
+	msr_s	SYS_ALLINT, \flags	// bits other than flags[13] are ignored
+alternative_else_nop_endif
+#endif
+	.endm
+
 	.macro	save_and_disable_irq, flags
 	mrs	\flags, daif
 	msr	daifset, #3
diff --git a/arch/arm64/include/asm/interrupts/common_flags.h b/arch/arm64/include/asm/interrupts/common_flags.h
index 72ed6e75d146..315aaec3ea34 100644
--- a/arch/arm64/include/asm/interrupts/common_flags.h
+++ b/arch/arm64/include/asm/interrupts/common_flags.h
@@ -20,6 +20,14 @@
 /*
  * Exception context mapping
  *
+ * FEAT_NMI
+ *
+ * CRITICAL -> DAIF + AllInt (corresponds to the state on exception entry)
+ * ERROR    ->  AIF + AllInt
+ * NONMI    ->   IF + AllInt
+ * NOIRQ    ->   IF
+ * PROCESS  ->    0
+ *
  * pseudo-NMI
  *
  * CRITICAL -> DAIF + IRQON  (corresponds to the state on exception entry)
@@ -76,6 +84,7 @@ arm64_exc_hwstate_t __arm64_exc_hwstate_of_noirq_context(void)
 		return (arm64_exc_hwstate_t){.daif=DAIF_PROCCTX, .pmr=GIC_PRIO_IRQOFF};
 
 	return (arm64_exc_hwstate_t){.daif=DAIF_PROCCTX_NOIRQ};
+
 }
 
 static __always_inline
@@ -84,6 +93,9 @@ arm64_exc_hwstate_t __arm64_exc_hwstate_of_nonmi_context(void)
 	if (system_uses_irq_prio_masking())
 		return (arm64_exc_hwstate_t){.daif=DAIF_PROCCTX_NOIRQ, .pmr=GIC_PRIO_IRQON};
 
+	if (system_uses_nmi())
+               return (arm64_exc_hwstate_t){.daif=DAIF_PROCCTX_NOIRQ, .allint=ALLINT_ALLINT};
+
 	return (arm64_exc_hwstate_t){.daif=DAIF_PROCCTX_NOIRQ};
 }
 
@@ -93,6 +105,9 @@ arm64_exc_hwstate_t __arm64_exc_hwstate_of_error_context(void)
 	if (system_uses_irq_prio_masking())
 		return (arm64_exc_hwstate_t){.daif=DAIF_ERRCTX, .pmr=GIC_PRIO_IRQON};
 
+	if (system_uses_nmi())
+		return (arm64_exc_hwstate_t){.daif=DAIF_ERRCTX, .allint=ALLINT_ALLINT};
+
 	return (arm64_exc_hwstate_t){.daif=DAIF_ERRCTX};
 }
 
@@ -102,6 +117,9 @@ arm64_exc_hwstate_t __arm64_exc_hwstate_of_critical_context(void)
 	if (system_uses_irq_prio_masking())
 		return (arm64_exc_hwstate_t){.daif=DAIF_MASK, .pmr=GIC_PRIO_IRQON};
 
+	if (system_uses_nmi())
+		return (arm64_exc_hwstate_t){.daif=DAIF_MASK, .allint=ALLINT_ALLINT};
+
 	return (arm64_exc_hwstate_t){.daif=DAIF_MASK};
 }
 
@@ -131,6 +149,9 @@ arm64_exc_hwstate_t arm64_inherit_exc_hwstate(struct pt_regs *regs)
 	if (system_uses_irq_prio_masking())
 		state.pmr = regs->pmr;
 
+	if (system_uses_nmi())
+		state.allint = regs->pstate & PSR_ALLINT_BIT;
+
 	return state;
 }
 
@@ -150,6 +171,9 @@ void arm64_debug_exc_hwstate(arm64_exc_hwstate_t expected)
 	if (system_uses_irq_prio_masking()) {
 		WARN_ONCE(1, "Unexpected DAIF+PMR: 0x%x + 0x%x (expected 0x%x + 0x%x)\n",
 			  actual.daif, actual.pmr, expected.daif, expected.pmr);
+	} else if (system_uses_nmi()) {
+		WARN_ONCE(1, "Unexpected DAIF+ALLINT: 0x%x + 0x%x (expected 0x%x + 0x%x)\n",
+			  actual.daif, actual.allint, expected.daif, expected.allint);
 	} else {
 		WARN_ONCE(1, "Unexpected DAIF: 0x%x (expected 0x%x)\n",
 			  actual.daif, expected.daif);
@@ -165,7 +189,7 @@ void arm64_debug_exc_context(arm64_exc_context_t context)
 }
 
 static __always_inline
-void arm64_update_exc_hwstate(arm64_exc_hwstate_t hwstate, bool update_pmr)
+void arm64_update_exc_hwstate(arm64_exc_hwstate_t hwstate, bool update_pmr, bool update_allint)
 {
 	if (system_uses_irq_prio_masking() &&
 	    update_pmr &&
@@ -193,8 +217,19 @@ void arm64_update_exc_hwstate(arm64_exc_hwstate_t hwstate, bool update_pmr)
 		pmr_sync();
 	}
 
+	/*
+	 * Try to order ALLINT writes to be consistent with the DAIF state :
+	 * we don't want to take an NMI with DAIF masked or when it should
+	 * be masked but isn't yet.
+	 */
+	if (system_uses_nmi() && update_allint && hwstate.allint)
+		_allint_set();
+
 	write_sysreg(hwstate.daif, daif);
 
+	if (system_uses_nmi() && update_allint && !hwstate.allint)
+		_allint_clear();
+
 	if (system_uses_irq_prio_masking() &&
 	    update_pmr &&
 	    hwstate.pmr == GIC_PRIO_IRQON) {
@@ -204,10 +239,10 @@ void arm64_update_exc_hwstate(arm64_exc_hwstate_t hwstate, bool update_pmr)
 }
 
 static __always_inline
-void arm64_update_exc_context(arm64_exc_context_t context, bool update_pmr)
+void arm64_update_exc_context(arm64_exc_context_t context, bool update_pmr,  bool update_allint)
 {
 	arm64_exc_hwstate_t hwstate = arm64_exc_hwstate_of_context(context);
 
-	arm64_update_exc_hwstate(hwstate, update_pmr);
+	arm64_update_exc_hwstate(hwstate, update_pmr, update_allint);
 }
 #endif /* __ASM_INTERRUPTS_COMMON_FLAGS_H */
diff --git a/arch/arm64/include/asm/interrupts/entry.h b/arch/arm64/include/asm/interrupts/entry.h
index 3034c490ed66..986536ecd058 100644
--- a/arch/arm64/include/asm/interrupts/entry.h
+++ b/arch/arm64/include/asm/interrupts/entry.h
@@ -10,12 +10,12 @@
 #include <asm/cpufeature.h>
 #include <asm/interrupts/common_flags.h>
 
-
 static __always_inline
 arm64_exc_hwstate_t __arm64_switch_exc_hwstate_to(arm64_exc_hwstate_t prev,
 						  arm64_exc_hwstate_t next)
 {
 	bool update_pmr = system_uses_irq_prio_masking() && prev.pmr != next.pmr;
+	bool update_allint = system_uses_nmi() && prev.allint != next.allint;
 
 	arm64_debug_exc_hwstate(prev);
 
@@ -25,7 +25,7 @@ arm64_exc_hwstate_t __arm64_switch_exc_hwstate_to(arm64_exc_hwstate_t prev,
 	if (!arch_irqs_disabled_flags(next.flags))
 		trace_hardirqs_on();
 
-	arm64_update_exc_hwstate(next, update_pmr);
+	arm64_update_exc_hwstate(next, update_pmr, update_allint);
 
 	if (arch_irqs_disabled_flags(next.flags))
 		trace_hardirqs_off();
@@ -49,15 +49,18 @@ arm64_exc_hwstate_t arm64_drop_exc_context(arm64_exc_hwstate_t prev, arm64_exc_c
 
 	if (IS_ENABLED(CONFIG_DEBUG_IRQFLAGS)) {
 		bool pnmi = system_uses_irq_prio_masking();
+		bool nmi = system_uses_nmi();
 
 		WARN_ON_ONCE(context > ERROR_CONTEXT &&
 			     prev.daif == DAIF_ERRCTX);
 
 		WARN_ON_ONCE(context > NONMI_CONTEXT &&
-			     prev.daif == DAIF_PROCCTX_NOIRQ);
+			     ((nmi && prev.daif == DAIF_PROCCTX_NOIRQ && prev.allint == ALLINT_ALLINT) ||
+			      (!nmi && prev.daif == DAIF_PROCCTX_NOIRQ)));
 
 		WARN_ON_ONCE(context > NOIRQ_CONTEXT &&
-			     pnmi && prev.pmr == GIC_PRIO_IRQOFF);
+			     ((pnmi && prev.pmr == GIC_PRIO_IRQOFF) ||
+			      (nmi && prev.daif == DAIF_PROCCTX_NOIRQ && prev.allint != ALLINT_ALLINT)));
 
 		WARN_ON_ONCE(context > PROCESS_CONTEXT &&
 			     ((pnmi && prev.daif == DAIF_PROCCTX && prev.pmr == GIC_PRIO_IRQON) ||
@@ -74,6 +77,7 @@ arm64_exc_hwstate_t arm64_lift_exc_context(arm64_exc_hwstate_t prev, arm64_exc_c
 
 	if (IS_ENABLED(CONFIG_DEBUG_IRQFLAGS)) {
 		bool pnmi = system_uses_irq_prio_masking();
+		bool nmi = system_uses_nmi();
 
 		WARN_ON_ONCE(context < CRITICAL_CONTEXT &&
 			     prev.daif == DAIF_MASK);
@@ -82,11 +86,13 @@ arm64_exc_hwstate_t arm64_lift_exc_context(arm64_exc_hwstate_t prev, arm64_exc_c
 			     prev.daif == DAIF_ERRCTX);
 
 		WARN_ON_ONCE(context < NONMI_CONTEXT &&
-			     pnmi && prev.daif == DAIF_PROCCTX_NOIRQ);
+			     ((pnmi && prev.daif == DAIF_PROCCTX_NOIRQ) ||
+			      (nmi &&  prev.daif == DAIF_PROCCTX_NOIRQ && prev.allint == ALLINT_ALLINT)));
 
 		WARN_ON_ONCE(context < NOIRQ_CONTEXT &&
 			     ((pnmi && prev.pmr == GIC_PRIO_IRQOFF) ||
-			      (!pnmi && prev.daif == DAIF_PROCCTX_NOIRQ)));
+			      (nmi && prev.daif == DAIF_PROCCTX_NOIRQ && prev.allint != ALLINT_ALLINT) ||
+			      (!pnmi && !nmi && prev.daif == DAIF_PROCCTX_NOIRQ)));
 	}
 
 	return __arm64_switch_exc_hwstate_to(prev, next);
diff --git a/arch/arm64/include/asm/interrupts/masking.h b/arch/arm64/include/asm/interrupts/masking.h
index 9dc37a571094..178e915f43b9 100644
--- a/arch/arm64/include/asm/interrupts/masking.h
+++ b/arch/arm64/include/asm/interrupts/masking.h
@@ -30,6 +30,7 @@ arm64_exc_hwstates_t local_all_irqs_save_mask(arm64_exc_context_t new)
 
 	if (IS_ENABLED(CONFIG_DEBUG_IRQFLAGS)) {
 		bool pnmi = system_uses_irq_prio_masking();
+		bool nmi = system_uses_nmi();
 
 		WARN_ON_ONCE(new < CRITICAL_CONTEXT &&
 			     actual.daif == DAIF_MASK);
@@ -38,14 +39,16 @@ arm64_exc_hwstates_t local_all_irqs_save_mask(arm64_exc_context_t new)
 			     actual.daif == DAIF_ERRCTX);
 
 		WARN_ON_ONCE(new < NONMI_CONTEXT &&
-			     pnmi && actual.daif == DAIF_PROCCTX_NOIRQ);
+			     ((pnmi && actual.daif == DAIF_PROCCTX_NOIRQ) ||
+			      (nmi &&  actual.daif == DAIF_PROCCTX_NOIRQ && actual.allint == ALLINT_ALLINT)));
 
 		WARN_ON_ONCE(new < NOIRQ_CONTEXT &&
 			     ((pnmi && actual.pmr == GIC_PRIO_IRQOFF) ||
-			      (!pnmi && actual.daif == DAIF_PROCCTX_NOIRQ)));
+			      (nmi && actual.daif == DAIF_PROCCTX_NOIRQ && actual.allint != ALLINT_ALLINT) ||
+			      (!pnmi && !nmi && actual.daif == DAIF_PROCCTX_NOIRQ)));
 	}
 
-	arm64_update_exc_hwstate(state, actual.pmr != state.pmr);
+	arm64_update_exc_hwstate(state, actual.pmr != state.pmr, actual.allint != state.allint);
 
 	if (!arch_irqs_disabled_flags(actual.flags))
 		trace_hardirqs_off();
@@ -60,7 +63,7 @@ static inline void local_all_irqs_restore(arm64_exc_hwstates_t states)
 	if (!arch_irqs_disabled_flags(states.saved.flags))
 		trace_hardirqs_on();
 
-	arm64_update_exc_hwstate(states.saved, true);
+	arm64_update_exc_hwstate(states.saved, true, true);
 }
 
 #ifdef CONFIG_DEBUG_IRQFLAGS
@@ -70,7 +73,7 @@ void local_all_irqs_cpu_init_mask(arm64_exc_context_t context)
 	WARN_ON(__this_cpu_read(irqs_masks_cpu_init_done));
 	if (context == PROCESS_CONTEXT)
 		trace_hardirqs_on();
-	arm64_update_exc_context(context, true);
+	arm64_update_exc_context(context, true, true);
 	__this_cpu_write(irqs_masks_cpu_init_done, true);
 	__this_cpu_write(irqs_masks_cpu_final_done, false);
 }
@@ -78,7 +81,7 @@ void local_all_irqs_cpu_init_mask(arm64_exc_context_t context)
 static inline void local_all_irqs_final_mask(void)
 {
 	WARN_ON(__this_cpu_read(irqs_masks_cpu_final_done));
-	arm64_update_exc_context(CRITICAL_CONTEXT, true);
+	arm64_update_exc_context(CRITICAL_CONTEXT, true, true);
 	trace_hardirqs_off();
 	__this_cpu_write(irqs_masks_cpu_final_done, true);
 	__this_cpu_write(irqs_masks_cpu_init_done, false);
@@ -89,12 +92,12 @@ void local_all_irqs_cpu_init_mask(arm64_exc_context_t context)
 {
 	if (context == PROCESS_CONTEXT)
 		trace_hardirqs_on();
-	arm64_update_exc_context(context, true);
+	arm64_update_exc_context(context, true, true);
 }
 
 static inline void local_all_irqs_final_mask(void)
 {
-	arm64_update_exc_context(CRITICAL_CONTEXT, true);
+	arm64_update_exc_context(CRITICAL_CONTEXT, true, true);
 	trace_hardirqs_off();
 }
 #endif /* CONFIG_DEBUG_IRQFLAGS */
@@ -130,7 +133,7 @@ static inline arm64_exc_hwstates_t local_all_irqs_force_daif_save(void)
 		states.expected.daif = states.saved.daif | DAIF_PROCCTX_NOIRQ;
 		states.expected.pmr = GIC_PRIO_IRQON;
 
-		arm64_update_exc_hwstate(states.expected, true);
+		arm64_update_exc_hwstate(states.expected, true, false);
 	}
 
 	return states;
@@ -153,7 +156,7 @@ void local_all_irqs_force_daif_restore(arm64_exc_hwstates_t states)
 
 	if (system_uses_irq_prio_masking()) {
 		arm64_debug_exc_hwstate(states.expected);
-		arm64_update_exc_hwstate(states.saved, true);
+		arm64_update_exc_hwstate(states.saved, true, false);
 	}
 }
 
@@ -167,7 +170,7 @@ static inline void local_interrupt_priority_init(void)
 	WARN_ON(read_sysreg(daif) & PSR_A_BIT);
 	lockdep_assert_irqs_disabled();
 
-	arm64_update_exc_context(NOIRQ_CONTEXT, true);
+	arm64_update_exc_context(NOIRQ_CONTEXT, true, true);
 }
 
 #endif /* __ASM_INTERRUPTS_MASKING_H */
diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h
index 585d3ce15d80..b571d34bf11d 100644
--- a/arch/arm64/include/asm/irqflags.h
+++ b/arch/arm64/include/asm/irqflags.h
@@ -31,6 +31,7 @@
 typedef union arm64_exc_hwstate {
 	struct {
 		u16 daif;
+		u16 allint;
 		u8 pmr;
 	};
 	unsigned long flags;
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index cb3be770f2d0..39ea3fdeb03a 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -815,7 +815,7 @@ SYM_CODE_END(__bp_harden_el1_vectors)
  *
  */
 SYM_FUNC_START(cpu_switch_to)
-	save_and_disable_daif x11
+	save_and_disable_exceptions x11, x12
 	mov	x10, #THREAD_CPU_CONTEXT
 	add	x8, x0, x10
 	mov	x9, sp
@@ -839,7 +839,7 @@ SYM_FUNC_START(cpu_switch_to)
 	ptrauth_keys_install_kernel x1, x8, x9, x10
 	scs_save x0
 	scs_load_current
-	restore_irq x11
+	restore_exceptions x11
 	ret
 SYM_FUNC_END(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
@@ -866,7 +866,7 @@ NOKPROBE(ret_from_fork)
  * Calls func(regs) using this CPU's irq stack and shadow irq stack.
  */
 SYM_FUNC_START(call_on_irq_stack)
-	save_and_disable_daif x9
+	save_and_disable_exceptions x9, x10
 #ifdef CONFIG_SHADOW_CALL_STACK
 	get_current_task x16
 	scs_save x16
@@ -881,10 +881,10 @@ SYM_FUNC_START(call_on_irq_stack)
 
 	/* Move to the new stack and call the function there */
 	add	sp, x16, #IRQ_STACK_SIZE
-	restore_irq x9
+	restore_exceptions x9
 	blr	x1
 
-	save_and_disable_daif x9
+	save_and_disable_exceptions x9, x10
 	/*
 	 * Restore the SP from the FP, and restore the FP and LR from the frame
 	 * record.
@@ -892,7 +892,7 @@ SYM_FUNC_START(call_on_irq_stack)
 	mov	sp, x29
 	ldp	x29, x30, [sp], #16
 	scs_load_current
-	restore_irq x9
+	restore_exceptions x9
 	ret
 SYM_FUNC_END(call_on_irq_stack)
 NOKPROBE(call_on_irq_stack)
-- 
2.34.1