[patch V3 5/9] MDS basics 5

[Thomas Gleixner <tglx@linutronix.de>]

Subject: [patch V3 5/9] x86/speculation/mds: Clear CPU buffers on exit to user
From: Thomas Gleixner <tglx@linutronix.de>

Add a static key which controls the invocation of the CPU buffer clear
mechanism on exit to user space and add the call into
prepare_exit_to_usermode() and do_nmi() right before actually returning.

Add documentation which kernel to user space transition this covers and
explain why some corner cases are not mitigated.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
V3: Add NMI conditional on user regs and update documentation accordingly.
    Use the static branch scheme suggested by Peter. Fix typos ...
---
 Documentation/x86/mds.rst            |   33 +++++++++++++++++++++++++++++++++
 arch/x86/entry/common.c              |   10 ++++++++++
 arch/x86/include/asm/nospec-branch.h |    2 ++
 arch/x86/kernel/cpu/bugs.c           |    4 +++-
 arch/x86/kernel/nmi.c                |    6 ++++++
 5 files changed, 54 insertions(+), 1 deletion(-)

--- a/Documentation/x86/mds.rst
+++ b/Documentation/x86/mds.rst
@@ -92,3 +92,36 @@ According to current knowledge additiona
 itself are not required because the necessary gadgets to expose the leaked
 data cannot be controlled in a way which allows exploitation from malicious
 user space or VM guests.
+
+Mitigation points
+-----------------
+
+1. Return to user space
+^^^^^^^^^^^^^^^^^^^^^^^
+   When transitioning from kernel to user space the CPU buffers are flushed
+   on affected CPUs:
+
+   - always when the mitigation mode is full. The migitation is enabled
+     through the static key mds_user_clear.
+
+   This covers transitions from kernel to user space through a return to
+   user space from a syscall and from an interrupt or a regular exception.
+
+   There are other kernel to user space transitions which are not covered
+   by this: NMIs and all non maskable exceptions which go through the
+   paranoid exit, which means that they are not invoking the regular
+   prepare_exit_to_usermode() which handles the CPU buffer clearing.
+
+   Access to sensible data like keys, credentials in the NMI context is
+   mostly theoretical: The CPU can do prefetching or execute a
+   misspeculated code path and thereby fetching data which might end up
+   leaking through a buffer.
+
+   But for mounting other attacks the kernel stack address of the task is
+   already valuable information. So in full mitigation mode, the NMI is
+   mitigated on the return from do_nmi() to provide almost complete
+   coverage.
+
+   There is one non maskable exception which returns through paranoid exit
+   and is not mitigated: #DF. If user space is able to trigger a double
+   fault the possible MDS leakage is the least problem to worry about.
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -31,6 +31,7 @@
 #include <asm/vdso.h>
 #include <linux/uaccess.h>
 #include <asm/cpufeature.h>
+#include <asm/nospec-branch.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/syscalls.h>
@@ -180,6 +181,13 @@ static void exit_to_usermode_loop(struct
 	}
 }
 
+static inline void mds_user_clear_cpu_buffers(void)
+{
+	if (!static_branch_likely(&mds_user_clear))
+		return;
+	mds_clear_cpu_buffers();
+}
+
 /* Called with IRQs disabled. */
 __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 {
@@ -212,6 +220,8 @@ static void exit_to_usermode_loop(struct
 #endif
 
 	user_enter_irqoff();
+
+	mds_user_clear_cpu_buffers();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS				\
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -318,6 +318,8 @@ DECLARE_STATIC_KEY_FALSE(switch_to_cond_
 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 
+DECLARE_STATIC_KEY_FALSE(mds_user_clear);
+
 #include <asm/segment.h>
 
 /**
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -63,10 +63,12 @@ DEFINE_STATIC_KEY_FALSE(switch_mm_cond_i
 /* Control unconditional IBPB in switch_mm() */
 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 
+/* Control MDS CPU buffer clear before returning to user space */
+DEFINE_STATIC_KEY_FALSE(mds_user_clear);
+
 void __init check_bugs(void)
 {
 	identify_boot_cpu();
-
 	/*
 	 * identify_boot_cpu() initialized SMT support information, let the
 	 * core code know.
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -34,6 +34,7 @@
 #include <asm/x86_init.h>
 #include <asm/reboot.h>
 #include <asm/cache.h>
+#include <asm/nospec-branch.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/nmi.h>
@@ -533,6 +534,11 @@ do_nmi(struct pt_regs *regs, long error_
 		write_cr2(this_cpu_read(nmi_cr2));
 	if (this_cpu_dec_return(nmi_state))
 		goto nmi_restart;
+
+	if (!static_branch_likely(&mds_user_clear))
+		return;
+	if (user_mode(regs))
+		mds_clear_cpu_buffers();
 }
 NOKPROBE_SYMBOL(do_nmi);