[PATCH 07/38] arm64: interrupts: Introduce exception masking save/restore helpers

[Vladimir Murzin <vladimir.murzin@arm.com>]

From: Ada Couprie Diaz <ada.coupriediaz@arm.com>

Outside the entry code, the local_daif_... helpers need to be replaced
so that both DAIF and PMR are handled correctly and their use can be
controlled more strongly.

Introduce local_exceptions_save_mask() and local_exceptions_restore(),
which must be used as a pair. Save both the original and requested
exception state so that inconsistent changes between the save and
restore can be detected.

There are two cases where paired save and restore operations do not
make sense:

- when initializing a CPU, and
- when preparing to turn it off.

Handle those cases with local_exceptions_cpu_init_mask() and
local_exceptions_final_mask(). When CONFIG_DEBUG_IRQFLAGS is enabled,
enforce that these helpers are not used outside their intended
contexts.

Signed-off-by: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Signed-off-by: Vladimir Murzin <vladimir.murzin@arm.com>
---
 arch/arm64/include/asm/interrupts/masking.h | 92 +++++++++++++++++++++
 arch/arm64/kernel/irq.c                     | 31 +++++++
 2 files changed, 123 insertions(+)
 create mode 100644 arch/arm64/include/asm/interrupts/masking.h

diff --git a/arch/arm64/include/asm/interrupts/masking.h b/arch/arm64/include/asm/interrupts/masking.h
new file mode 100644
index 000000000000..db3439853b64
--- /dev/null
+++ b/arch/arm64/include/asm/interrupts/masking.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Arm Ltd.
+ */
+#ifndef __ASM_INTERRUPTS_MASKING_H
+#define __ASM_INTERRUPTS_MASKING_H
+
+#include <asm/arch_gicv3.h>
+#include <asm/bug.h>
+#include <asm/cpufeature.h>
+#include <asm/interrupts/common_flags.h>
+#include <asm/ptrace.h>
+
+typedef struct arm64_exc_hwstates {
+	arm64_exc_hwstate_t saved;
+	arm64_exc_hwstate_t expected;
+} arm64_exc_hwstates_t;
+
+static inline
+arm64_exc_hwstates_t local_exceptions_save_mask(arm64_exc_context_t new)
+{
+	arm64_exc_hwstate_t actual = {.flags = arch_local_save_flags()};
+	arm64_exc_hwstate_t state = arm64_exc_hwstate_of_context(new);
+	bool irqs_disabled = arch_irqs_disabled_flags(state.flags);
+	bool force;
+
+	/*
+	 * We've just got actual HW state so we can rely on that to
+	 * optimize some unnecessary updates.
+	 */
+	force = system_uses_irq_prio_masking() && actual.pmr != state.pmr;
+
+	if (!irqs_disabled)
+		trace_hardirqs_on();
+
+	__arm64_update_exc_hwstate(state, force);
+
+	if (irqs_disabled)
+		trace_hardirqs_off();
+
+	return (arm64_exc_hwstates_t){.saved = actual, .expected = state};
+}
+
+static inline void local_exceptions_restore(arm64_exc_hwstates_t states)
+{
+	bool irqs_disabled = arch_irqs_disabled_flags(states.saved.flags);
+
+	arm64_debug_exc_hwstate(states.expected);
+
+	if (!irqs_disabled)
+		trace_hardirqs_on();
+
+	arm64_update_exc_hwstate(states.saved);
+
+	if (irqs_disabled)
+		trace_hardirqs_off();
+}
+
+static __always_inline
+void __local_exceptions_cpu_init_mask(arm64_exc_context_t context)
+{
+	if (context == PROCESS_CONTEXT)
+		trace_hardirqs_on();
+
+	arm64_update_exc_context(context);
+
+	if (context != PROCESS_CONTEXT)
+		trace_hardirqs_off();
+}
+
+static __always_inline void __local_exceptions_final_mask(void)
+{
+	arm64_update_exc_context(CRITICAL_CONTEXT);
+	trace_hardirqs_off();
+}
+
+#ifdef CONFIG_DEBUG_IRQFLAGS
+void local_exceptions_cpu_init_mask(arm64_exc_context_t context);
+void local_exceptions_final_mask(void);
+#else
+static inline
+void local_exceptions_cpu_init_mask(arm64_exc_context_t context)
+{
+	__local_exceptions_cpu_init_mask(context);
+}
+
+static inline void local_exceptions_final_mask(void)
+{
+	__local_exceptions_final_mask();
+}
+#endif /* CONFIG_DEBUG_IRQFLAGS */
+#endif /* __ASM_INTERRUPTS_MASKING_H */
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 9fafd826002b..58b778083576 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -22,6 +22,7 @@
 #include <linux/smp.h>
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
+#include <asm/interrupts/masking.h>
 #include <asm/exception.h>
 #include <asm/numa.h>
 #include <asm/softirq_stack.h>
@@ -39,6 +40,12 @@ DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
 DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
 #endif
 
+#ifdef CONFIG_DEBUG_IRQFLAGS
+/* Make sure the CPU init/tear down masking functions are only used once. */
+static cpumask_t irqs_masks_init_done;
+static cpumask_t irqs_masks_final_done;
+#endif
+
 static int __init init_irq_scs(void)
 {
 	int cpu;
@@ -133,3 +140,27 @@ void __init init_IRQ(void)
 		local_daif_restore(DAIF_PROCCTX_NOIRQ);
 	}
 }
+
+#ifdef CONFIG_DEBUG_IRQFLAGS
+void local_exceptions_cpu_init_mask(arm64_exc_context_t context)
+{
+	int cpu = smp_processor_id();
+
+	WARN_ON(cpumask_test_cpu(cpu, &irqs_masks_init_done));
+
+	__local_exceptions_cpu_init_mask(context);
+	cpumask_set_cpu(cpu, &irqs_masks_init_done);
+	cpumask_clear_cpu(cpu, &irqs_masks_final_done);
+}
+
+void local_exceptions_final_mask(void)
+{
+	int cpu = smp_processor_id();
+
+	WARN_ON(cpumask_test_cpu(cpu, &irqs_masks_final_done));
+
+	__local_exceptions_final_mask();
+	cpumask_set_cpu(cpu, &irqs_masks_final_done);
+	cpumask_clear_cpu(cpu, &irqs_masks_init_done);
+}
+#endif
-- 
2.34.1