Support interrupt stuffing for Skylake

Support interrupt stuffing for Skylake

[Andi Kleen]

Skylake with retpoline needs additional protection against return
stack underflows. While full coverage is more involved, there
are some low hanging fruits that are relatively easy.  We already
did stuffing on context switch earlier, this adds the next
low hanging fruit: stuffing on interrupts and exception return
to the kernel. 

We only need to stuff when returning to the kernel so there is
no impact on anything interrupting user space directly. Also the
additional stuffing is only enabled on Skylake.

Git tree available here
git://git.kernel.org/pub/scm/linux/kernel/git/ak/linux-misc.git spec/interrupt-stuff-1

v1: Initial post, but based on the earlier skl-rsb code.

[PATCH 1/2] x86/retpoline: Add feature bit for additional return buffer protections

[Andi Kleen]

From: Andi Kleen <ak@linux.intel.com>

On Skylake we want additional protections against spectre_v2
over the normal RETPOLINE against underflowing return buffers.

On return buffer underflow the CPU could fall back to
the poisoned indirect branch predictor.

This patch adds a feature bit, only enabled on Skylake,
to enable additional return buffer underflow protections

This will be used in the next patches

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/include/asm/cpufeatures.h | 1 +
 arch/x86/kernel/cpu/bugs.c         | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 25b9375c1484..85108eb1876b 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 
 #define X86_FEATURE_MBA			( 7*32+18) /* Memory Bandwidth Allocation */
 #define X86_FEATURE_RSB_CTXSW		( 7*32+19) /* Fill RSB on context switches */
+#define X86_FEATURE_RSB_UNDERFLOW	( 7*32+20) /* Avoid return stack underflows */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW		( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 390b3dc3d438..68d449d51673 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -249,6 +249,11 @@ static void __init spectre_v2_select_mitigation(void)
 		setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 		pr_info("Filling RSB on context switch\n");
 	}
+
+	if (is_skylake_era()) {
+		setup_force_cpu_cap(X86_FEATURE_RSB_UNDERFLOW);
+		pr_info("Enabling return buffer underflow protections");
+	}
 }
 
 #undef pr_fmt
-- 
2.14.3

[PATCH 2/2] x86/retpoline: Fill return buffer on interrupt return to kernel

[Andi Kleen]

From: Andi Kleen <ak@linux.intel.com>

Interrupts can have rather deep call chains on top of the original
call chain. Fill the return buffer on Skylake when returning from
an interrupt/NMI to the kernel, to avoid return buffer underflows
later.

This only needs to be done when returning to the kernel,
so interrupts interrupting user space are not impacted.

The patch also do the same for returns from exceptions
to the kernel. This is useful because get_user page faults
can also have deep call chains. It's all unified in the
same code path.

This patch changes the code for 32 and 64bit.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/entry/entry_32.S | 15 ++++++++++++---
 arch/x86/entry/entry_64.S | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 60c4c342316c..77cf739d018e 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -65,7 +65,6 @@
 # define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
 #else
 # define preempt_stop(clobbers)
-# define resume_kernel		restore_all
 #endif
 
 .macro TRACE_IRQS_IRET
@@ -346,8 +345,16 @@ ENTRY(resume_userspace)
 	jmp	restore_all
 END(ret_from_exception)
 
-#ifdef CONFIG_PREEMPT
 ENTRY(resume_kernel)
+	/*
+	 * Interrupts/faults could cause the return buffer of the CPU
+	 * to overflow, which would lead to a underflow later,
+	 * which may lead to a uncontrolled indirect branch.
+	 * Fill the return buffer when returning to the kernel.
+	 */
+	FILL_RETURN_BUFFER %eax, RSB_FILL_LOOPS, X86_FEATURE_RSB_UNDERFLOW
+
+#ifdef CONFIG_PREEMPT
 	DISABLE_INTERRUPTS(CLBR_ANY)
 .Lneed_resched:
 	cmpl	$0, PER_CPU_VAR(__preempt_count)
@@ -356,8 +363,10 @@ ENTRY(resume_kernel)
 	jz	restore_all
 	call	preempt_schedule_irq
 	jmp	.Lneed_resched
-END(resume_kernel)
+#else
+	jmp	restore_all
 #endif
+END(resume_kernel)
 
 GLOBAL(__begin_SYSENTER_singlestep_region)
 /*
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 63f4320602a3..ec36af4b0b36 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -789,6 +789,14 @@ retint_kernel:
 	TRACE_IRQS_IRETQ
 
 GLOBAL(restore_regs_and_return_to_kernel)
+	/*
+	 * Interrupts/faults could cause the return buffer of the CPU
+	 * to overflow, which would lead to a underflow later,
+	 * which may lead to a uncontrolled indirect branch.
+	 * Fill the return buffer when returning to the kernel.
+	 */
+	FILL_RETURN_BUFFER %rax, RSB_FILL_LOOPS, X86_FEATURE_RSB_UNDERFLOW
+
 #ifdef CONFIG_DEBUG_ENTRY
 	/* Assert that pt_regs indicates kernel mode. */
 	testb	$3, CS(%rsp)
@@ -1657,6 +1665,10 @@ nested_nmi:
 nested_nmi_out:
 	popq	%rdx
 
+	/*
+	 * No need to clear return buffer here because the outter NMI will do it,
+	 * and we assume two NMIs will not overflow the return buffer.
+	 */
 	/* We are returning to kernel mode, so this cannot result in a fault. */
 	iretq
 
@@ -1754,6 +1766,14 @@ end_repeat_nmi:
 nmi_swapgs:
 	SWAPGS_UNSAFE_STACK
 nmi_restore:
+	/*
+	 * NMI could cause the return buffer of the CPU
+	 * to overflow, which would lead to a underflow later,
+	 * which may lead to a uncontrolled indirect branch.
+	 * Fill the return buffer when returning to the kernel.
+	 */
+	FILL_RETURN_BUFFER %rax, RSB_FILL_LOOPS, X86_FEATURE_RSB_UNDERFLOW
+
 	POP_EXTRA_REGS
 	POP_C_REGS
 
-- 
2.14.3