[PATCH RFC v2 05/29] mm: asi: ASI support in interrupts/exceptions

Brendan Jackman
2025-01-10T18:41:00Z

Add support for potentially switching address spaces from within
interrupts/exceptions/NMIs etc. An interrupt does not automatically
switch to the unrestricted address space. It can switch if needed to
access some memory not available in the restricted address space, using
the normal asi_exit call.

On return from the outermost interrupt, if the target address space was
the restricted address space (e.g. we were in the critical code path
between ASI Enter and VM Enter), the restricted address space will be
automatically restored. Otherwise, execution will continue in the
unrestricted address space until the next explicit ASI Enter.

In order to keep track of when to restore the restricted address space,
an interrupt/exception nesting depth counter is maintained per-task.
An alternative implementation without needing this counter is also
possible, but the counter unlocks an additional nice-to-have benefit by
allowing detection of whether or not we are currently executing inside
an exception context, which would be useful in a later patch.

Note that for KVM on SVM, this is not actually necessary as NMIs are in
fact maskable via CLGI. It's not clear to me if VMX has something
equivalent but we will need this infrastructure in place for userspace
support anyway.

RFC: Once userspace ASI is implemented, this idtentry integration
looks a bit heavy-handed. For example, we don't need this logic
for INT 80 emulation, so having it in DEFINE_IDTENTRY_RAW is
confusing. It could lead to a bug if the order of interrupter counter
modifications and ASI transition logic gets flipped around somehow.

checkpatch.pl SPACING is false positive. AVOID_BUG ignored for RFC.

Checkpatch-args: --ignore=SPACING,AVOID_BUG
Signed-off-by: Junaid Shahid <junaids@google.com>
Signed-off-by: Brendan Jackman <jackmanb@google.com>
---
arch/x86/include/asm/asi.h | 68 ++++++++++++++++++++++++++++++++++++++--
arch/x86/include/asm/idtentry.h | 50 ++++++++++++++++++++++++-----
arch/x86/include/asm/processor.h | 5 +++
arch/x86/kernel/process.c | 2 ++
arch/x86/kernel/traps.c | 22 +++++++++++++
arch/x86/mm/asi.c | 7 ++++-
include/asm-generic/asi.h | 10 ++++++
7 files changed, 153 insertions(+), 11 deletions(-)

diff --git a/arch/x86/include/asm/asi.h b/arch/x86/include/asm/asi.h
index b9671ef2dd3278adceed18507fd260e21954d574..9a9a139518289fc65f26a4d1cd311aa52cc5357f 100644
--- a/arch/x86/include/asm/asi.h
+++ b/arch/x86/include/asm/asi.h
@@ -157,6 +157,11 @@ void asi_relax(void);
/* Immediately exit the restricted address space if in it */
void asi_exit(void);

+static inline void asi_init_thread_state(struct thread_struct *thread)
+{
+ thread->asi_state.intr_nest_depth = 0;
+}
+
/* The target is the domain we'll enter when returning to process context. */
static __always_inline struct asi *asi_get_target(struct task_struct *p)
{
@@ -197,9 +202,10 @@ static __always_inline bool asi_is_relaxed(void)
/*
* Is the current task in the critical section?
*
- * This is just the inverse of !asi_is_relaxed(). We have both functions in order to
- * help write intuitive client code. In particular, asi_is_tense returns false
- * when ASI is disabled, which is judged to make user code more obvious.
+ * This is just the inverse of !asi_is_relaxed(). We have both functions in
+ * order to help write intuitive client code. In particular, asi_is_tense
+ * returns false when ASI is disabled, which is judged to make user code more
+ * obvious.
*/
static __always_inline bool asi_is_tense(void)
{
@@ -211,6 +217,62 @@ static __always_inline pgd_t *asi_pgd(struct asi *asi)
return asi ? asi->pgd : NULL;
}

+static __always_inline void asi_intr_enter(void)
+{
+ if (static_asi_enabled() && asi_is_tense()) {
+ current->thread.asi_state.intr_nest_depth++;
+ barrier();
+ }
+}
+
+void __asi_enter(void);
+
+static __always_inline void asi_intr_exit(void)
+{
+ if (static_asi_enabled() && asi_is_tense()) {
+ /*
+ * If an access to sensitive memory got reordered after the
+ * decrement, the #PF handler for that access would see a value
+ * of 0 for the counter and re-__asi_enter before returning to
+ * the faulting access, triggering an infinite PF loop.
+ */
+ barrier();
+
+ if (--current->thread.asi_state.intr_nest_depth == 0) {
+ /*
+ * If the decrement got reordered after __asi_enter, an
+ * interrupt that came between __asi_enter and the
+ * decrement would always see a nonzero value for the
+ * counter so it wouldn't call __asi_enter again and we
+ * would return to process context in the wrong address
+ * space.
+ */
+ barrier();
+ __asi_enter();
+ }
+ }
+}
+
+/*
+ * Returns the nesting depth of interrupts/exceptions that have interrupted the
+ * ongoing critical section. If the current task is not in a critical section
+ * this is 0.
+ */
+static __always_inline int asi_intr_nest_depth(void)
+{
+ return current->thread.asi_state.intr_nest_depth;
+}
+
+/*
+ * Remember that interrupts/exception don't count as the critical section. If
+ * you want to know if the current task is in the critical section use
+ * asi_is_tense().
+ */
+static __always_inline bool asi_in_critical_section(void)
+{
+ return asi_is_tense() && !asi_intr_nest_depth();
+}
+
#define INIT_MM_ASI(init_mm) \
.asi_init_lock = __MUTEX_INITIALIZER(init_mm.asi_init_lock),

diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index ad5c68f0509d4dfd0834303c0f9dabc93ef73aa4..9e00da0a3b08f83ca5e603dc2abbfd5fa3059811 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -12,6 +12,7 @@
#include <linux/hardirq.h>

#include <asm/irq_stack.h>
+#include <asm/asi.h>

typedef void (*idtentry_t)(struct pt_regs *regs);

@@ -55,12 +56,15 @@ static __always_inline void __##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
- irqentry_state_t state = irqentry_enter(regs); \
+ irqentry_state_t state; \
\
+ asi_intr_enter(); \
+ state = irqentry_enter(regs); \
instrumentation_begin(); \
__##func (regs); \
instrumentation_end(); \
irqentry_exit(regs, state); \
+ asi_intr_exit(); \
} \
\
static __always_inline void __##func(struct pt_regs *regs)
@@ -102,12 +106,15 @@ static __always_inline void __##func(struct pt_regs *regs, \
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code) \
{ \
- irqentry_state_t state = irqentry_enter(regs); \
+ irqentry_state_t state; \
\
+ asi_intr_enter(); \
+ state = irqentry_enter(regs); \
instrumentation_begin(); \
__##func (regs, error_code); \
instrumentation_end(); \
irqentry_exit(regs, state); \
+ asi_intr_exit(); \
} \
\
static __always_inline void __##func(struct pt_regs *regs, \
@@ -139,7 +146,16 @@ static __always_inline void __##func(struct pt_regs *regs, \
* is required before the enter/exit() helpers are invoked.
*/
#define DEFINE_IDTENTRY_RAW(func) \
-__visible noinstr void func(struct pt_regs *regs)
+static __always_inline void __##func(struct pt_regs *regs); \
+ \
+__visible noinstr void func(struct pt_regs *regs) \
+{ \
+ asi_intr_enter(); \
+ __##func (regs); \
+ asi_intr_exit(); \
+} \
+ \
+static __always_inline void __##func(struct pt_regs *regs)

/**
* DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points
@@ -178,7 +194,18 @@ noinstr void fred_##func(struct pt_regs *regs)
* is required before the enter/exit() helpers are invoked.
*/
#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \
-__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
+static __always_inline void __##func(struct pt_regs *regs, \
+ unsigned long error_code); \
+ \
+__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)\
+{ \
+ asi_intr_enter(); \
+ __##func (regs, error_code); \
+ asi_intr_exit(); \
+} \
+ \
+static __always_inline void __##func(struct pt_regs *regs, \
+ unsigned long error_code)

/**
* DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
@@ -209,14 +236,17 @@ static void __##func(struct pt_regs *regs, u32 vector); \
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code) \
{ \
- irqentry_state_t state = irqentry_enter(regs); \
+ irqentry_state_t state; \
u32 vector = (u32)(u8)error_code; \
\
+ asi_intr_enter(); \
+ state = irqentry_enter(regs); \
kvm_set_cpu_l1tf_flush_l1d(); \
instrumentation_begin(); \
run_irq_on_irqstack_cond(__##func, regs, vector); \
instrumentation_end(); \
irqentry_exit(regs, state); \
+ asi_intr_exit(); \
} \
\
static noinline void __##func(struct pt_regs *regs, u32 vector)
@@ -255,13 +285,16 @@ static __always_inline void instr_##func(struct pt_regs *regs) \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
- irqentry_state_t state = irqentry_enter(regs); \
+ irqentry_state_t state; \
\
+ asi_intr_enter(); \
+ state = irqentry_enter(regs); \
kvm_set_cpu_l1tf_flush_l1d(); \
instrumentation_begin(); \
instr_##func (regs); \
instrumentation_end(); \
irqentry_exit(regs, state); \
+ asi_intr_exit(); \
} \
\
void fred_##func(struct pt_regs *regs) \
@@ -294,13 +327,16 @@ static __always_inline void instr_##func(struct pt_regs *regs) \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
- irqentry_state_t state = irqentry_enter(regs); \
+ irqentry_state_t state; \
\
+ asi_intr_enter(); \
+ state = irqentry_enter(regs); \
kvm_set_cpu_l1tf_flush_l1d(); \
instrumentation_begin(); \
instr_##func (regs); \
instrumentation_end(); \
irqentry_exit(regs, state); \
+ asi_intr_exit(); \
} \
\
void fred_##func(struct pt_regs *regs) \
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index f02220e6b4df911d87e2fee4b497eade61a27161..a32a53405f45e4c0473fe081e216029cf5bd0cdd 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -508,6 +508,11 @@ struct thread_struct {
struct {
/* Domain to enter when returning to process context. */
struct asi *target;
+ /*
+ * The depth of interrupt/exceptions interrupting an ASI
+ * critical section
+ */
+ int intr_nest_depth;
} asi_state;
#endif

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index f63f8fd00a91f3d1171f307b92179556ba2d716d..44abc161820153b7f68664b97267658b8e011101 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -96,6 +96,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
#ifdef CONFIG_VM86
dst->thread.vm86 = NULL;
#endif
+ asi_init_thread_state(&dst->thread);
+
/* Drop the copied pointer to current's fpstate */
dst->thread.fpu.fpstate = NULL;

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 2dbadf347b5f4f66625c4f49b76c41b412270d57..beea861da8d3e9a4e2afb3a92ed5f66f11d67bd6 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -65,6 +65,7 @@
#include <asm/umip.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
+#include <asm/asi.h>
#include <asm/vdso.h>
#include <asm/tdx.h>
#include <asm/cfi.h>
@@ -463,6 +464,27 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
}
#endif

+ /*
+ * Do an asi_exit() only here because a #DF usually indicates
+ * the system is in a really bad state, and we don't want to
+ * cause any additional issue that would prevent us from
+ * printing a correct stack trace.
+ *
+ * The additional issues are not related to a possible triple
+ * fault, which can only occurs if a fault is encountered while
+ * invoking this handler, but here we are already executing it.
+ * Instead, an ASI-induced #PF here could potentially end up
+ * getting another #DF. For example, if there was some issue in
+ * invoking the #PF handler. The handler for the second #DF
+ * could then again cause an ASI-induced #PF leading back to the
+ * same recursion.
+ *
+ * This is not needed in the espfix64 case above, since that
+ * code is about turning a #DF into a #GP which is okay to
+ * handle in the restricted domain. That's also why we don't
+ * asi_exit() in the #GP handler.
+ */
+ asi_exit();
irqentry_nmi_enter(regs);
instrumentation_begin();
notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
diff --git a/arch/x86/mm/asi.c b/arch/x86/mm/asi.c
index 5baf563a078f5b3a6cd4b9f5e92baaf81b0774c4..054315d566c082c0925a00ce3a0877624c8b9957 100644
--- a/arch/x86/mm/asi.c
+++ b/arch/x86/mm/asi.c
@@ -235,7 +235,7 @@ static __always_inline void maybe_flush_data(struct asi *next_asi)
this_cpu_and(asi_taints, ~ASI_TAINTS_DATA_MASK);
}

-static noinstr void __asi_enter(void)
+noinstr void __asi_enter(void)
{
u64 asi_cr3;
struct asi *target = asi_get_target(current);
@@ -250,6 +250,7 @@ static noinstr void __asi_enter(void)
* disabling preemption should be fine.
*/
VM_BUG_ON(preemptible());
+ VM_BUG_ON(current->thread.asi_state.intr_nest_depth != 0);

if (!target || target == this_cpu_read(curr_asi))
return;
@@ -290,6 +291,7 @@ noinstr void asi_enter(struct asi *asi)
if (!static_asi_enabled())
return;

+ VM_WARN_ON_ONCE(asi_intr_nest_depth());
VM_WARN_ON_ONCE(!asi);

/* Should not have an asi_enter() without a prior asi_relax(). */
@@ -305,6 +307,7 @@ EXPORT_SYMBOL_GPL(asi_enter);
noinstr void asi_relax(void)
{
if (static_asi_enabled()) {
+ VM_WARN_ON_ONCE(asi_intr_nest_depth());
barrier();
asi_set_target(current, NULL);
}
@@ -326,6 +329,8 @@ noinstr void asi_exit(void)

asi = this_cpu_read(curr_asi);
if (asi) {
+ WARN_ON_ONCE(asi_in_critical_section());
+
maybe_flush_control(NULL);

unrestricted_cr3 =
diff --git a/include/asm-generic/asi.h b/include/asm-generic/asi.h
index eedc961ee916a9e1da631ca489ea4a7bc9e6089f..7f542c59c2b8a2b74432e4edb7199f9171db8a84 100644
--- a/include/asm-generic/asi.h
+++ b/include/asm-generic/asi.h
@@ -52,6 +52,8 @@ static inline bool asi_is_relaxed(void) { return true; }

static inline bool asi_is_tense(void) { return false; }

+static inline bool asi_in_critical_section(void) { return false; }
+
static inline void asi_exit(void) { }

static inline bool asi_is_restricted(void) { return false; }
@@ -65,6 +67,14 @@ static inline pgd_t *asi_pgd(struct asi *asi) { return NULL; }

static inline void asi_handle_switch_mm(void) { }

+static inline void asi_init_thread_state(struct thread_struct *thread) { }
+
+static inline void asi_intr_enter(void) { }
+
+static inline int asi_intr_nest_depth(void) { return 0; }
+
+static inline void asi_intr_exit(void) { }
+
#define static_asi_enabled() false

static inline void asi_check_boottime_disable(void) { }

--
2.47.1.613.gc27f4b7a9f-goog