[MODERATED] [PATCH v4 07/28] MDSv4 0

[Andi Kleen <andi@firstfloor.org>]

From: Andi Kleen <ak@linux.intel.com>
Subject:  x86/speculation/mds: Support mds=full for NMIs

NMIs don't go through C code when exiting to user space, so we need
to add an assembler clear cpu for this case. Only used with
mds=full, because otherwise we assume NMIs don't touch
other users or kernel sensitive data.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/entry/entry_64.S       | 12 ++++++++++++
 arch/x86/include/asm/clearcpu.h | 11 +++++++++++
 2 files changed, 23 insertions(+)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 1f0efdb7b629..57f194e3e253 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -39,6 +39,7 @@
 #include <asm/frame.h>
 #include <asm/nospec-branch.h>
 #include <linux/err.h>
+#include <asm/clearcpu.h>
 
 #include "calling.h"
 
@@ -1407,6 +1408,17 @@ ENTRY(nmi)
 	movq	$-1, %rsi
 	call	do_nmi
 
+	/*
+	 * Clear only when force clearing was enabled. Otherwise
+	 * we assume NMI code is not sensitive.
+	 * If you don't have jump labels we always clear too.
+	 */
+#ifdef HAVE_JUMP_LABEL
+	STATIC_BRANCH_JMP l_yes=.Lno_clear_cpu key=force_cpu_clear, branch=1
+#endif
+	CLEAR_CPU
+.Lno_clear_cpu:
+
 	/*
 	 * Return back to user mode.  We must *not* do the normal exit
 	 * work, because we don't want to enable interrupts.
diff --git a/arch/x86/include/asm/clearcpu.h b/arch/x86/include/asm/clearcpu.h
index b83ef1a5268f..67c4e0d38802 100644
--- a/arch/x86/include/asm/clearcpu.h
+++ b/arch/x86/include/asm/clearcpu.h
@@ -2,6 +2,8 @@
 #ifndef _ASM_CLEARCPU_H
 #define _ASM_CLEARCPU_H 1
 
+#ifndef __ASSEMBLY__
+
 #include <linux/jump_label.h>
 #include <linux/sched/smt.h>
 #include <asm/alternative.h>
@@ -41,4 +43,13 @@ static inline void clear_cpu_idle(void)
 
 DECLARE_STATIC_KEY_FALSE(force_cpu_clear);
 
+#else
+
+.macro CLEAR_CPU
+	ALTERNATIVE __stringify(push $__USER_DS ; verw (% _ASM_SP ) ; add $8, % _ASM_SP ),\
+		"", X86_FEATURE_NO_VERW
+.endm
+
+#endif
+
 #endif
-- 
2.17.2