[RFC PATCH 1/4] bpf, x64: Implement BPF call retpoline

[Rick Edgecombe <rick.p.edgecombe@intel.com>]

Add x86 call retpoline sequence from the "Intel Retpoline: A Branch Target
Injection Mitigation White Paper" for BPF JIT compiler. Unlike the paper
it uses RBX instead of RAX since RAX is part of the BPF calling
convetions.

Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Alexei Starovoitov <ast@fb.com>
Signed-off-by: Rick Edgecombe <rick.p.edgecombe@intel.com>
---
 arch/x86/include/asm/nospec-branch.h | 45 ++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index dad12b767ba0..70b3f6534134 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -320,6 +320,8 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 
 #endif /* __ASSEMBLY__ */
 
+#ifdef CONFIG_RETPOLINE
+# ifdef CONFIG_X86_64
 /*
  * Below is used in the eBPF JIT compiler and emits the byte sequence
  * for the following assembly:
@@ -341,8 +343,6 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  *    jmp *%rax for x86_64
  *    jmp *%edx for x86_32
  */
-#ifdef CONFIG_RETPOLINE
-# ifdef CONFIG_X86_64
 #  define RETPOLINE_RAX_BPF_JIT_SIZE	17
 #  define RETPOLINE_RAX_BPF_JIT()				\
 do {								\
@@ -355,6 +355,44 @@ do {								\
 	EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */	\
 	EMIT1(0xC3);             /* retq */			\
 } while (0)
+
+/* Modified from "Intel Retpoline: A Branch Target Injection Mitigation White
+ * Paper" to use RBX register in order to not intefere with BPF calling
+ * conventions:
+ *
+ *   jmp label2
+ *
+ * label0:
+ *   call label1
+ *
+ * capture_ret_spec:
+ *   pause
+ *   lfence
+ *   jmp capture_ret_spec
+ *
+ * label1:
+ *   mov %rbx,(%rsp)
+ *   ret
+ *
+ * label2:
+ *   call label0
+ */
+#  define RETPOLINE_RBX_BPF_JIT_CALL_SIZE	24
+#  define RETPOLINE_RBX_BPF_JIT_CALL()				\
+do {								\
+	EMIT2(0xEB, 0x11);	 /* jump label2 */		\
+	/* label2: */						\
+	EMIT1_off32(0xE8, 7);	 /* call label1 */		\
+	/* capture_ret_spec: */					\
+	EMIT2(0xF3, 0x90);	 /* pause */			\
+	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\
+	EMIT2(0xEB, 0xF9);	 /* jmp capture_ret_spec */	\
+	/* label1: */						\
+	EMIT4(0x48, 0x89, 0x1c, 0x24); /* mov %rbx,(%rsp) */	\
+	EMIT1(0xC3);		 /* ret */			\
+	/* label2: */						\
+	EMIT1_off32(0xE8, -22);	 /* call label0 */		\
+} while (0)
 # else /* !CONFIG_X86_64 */
 #  define RETPOLINE_EDX_BPF_JIT()				\
 do {								\
@@ -373,6 +411,9 @@ do {								\
 #  define RETPOLINE_RAX_BPF_JIT_SIZE	2
 #  define RETPOLINE_RAX_BPF_JIT()				\
 	EMIT2(0xFF, 0xE0);       /* jmp *%rax */
+#  define RETPOLINE_RBX_BPF_JIT_CALL_SIZE	2
+#  define RETPOLINE_RBX_BPF_JIT_CALL()				\
+	EMIT2(0xff, 0xD3)        /* call *%rbx */
 # else /* !CONFIG_X86_64 */
 #  define RETPOLINE_EDX_BPF_JIT()				\
 	EMIT2(0xFF, 0xE2)        /* jmp *%edx */
-- 
2.17.1