[PATCH v2 1/3] x86/bhi: Add BHB clearing for CPUs with larger branch history

Pawan Gupta
2025-10-16T01:51:57Z

Add a version of clear_bhb_loop() that works on CPUs with larger branch
history table such as Alder Lake and newer. This could serve as a cheaper
alternative to IBPB mitigation for VMSCAPE.

clear_bhb_loop() and the new clear_bhb_long_loop() only differ in the loop
counter. Convert the asm implementation of clear_bhb_loop() into a macro
that is used by both the variants, passing counter as an argument.

There is no difference in the output of:

$ objdump --disassemble=clear_bhb_loop vmlinux

before and after this commit.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
arch/x86/entry/entry_64.S | 47 ++++++++++++++++++++++++++----------
arch/x86/include/asm/nospec-branch.h | 3 +++
2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed04a968cc7d0095ab0185b2e3b5beffb7680afd..f5f62af080d8ec6fe81e4dbe78ce44d08e62aa59 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1499,11 +1499,6 @@ SYM_CODE_END(rewind_stack_and_make_dead)
* from the branch history tracker in the Branch Predictor, therefore removing
* user influence on subsequent BTB lookups.
*
- * It should be used on parts prior to Alder Lake. Newer parts should use the
- * BHI_DIS_S hardware control instead. If a pre-Alder Lake part is being
- * virtualized on newer hardware the VMM should protect against BHI attacks by
- * setting BHI_DIS_S for the guests.
- *
* CALLs/RETs are necessary to prevent Loop Stream Detector(LSD) from engaging
* and not clearing the branch history. The call tree looks like:
*
@@ -1529,11 +1524,12 @@ SYM_CODE_END(rewind_stack_and_make_dead)
* that all RETs are in the second half of a cacheline to mitigate Indirect
* Target Selection, rather than taking the slowpath via its_return_thunk.
*/
-SYM_FUNC_START(clear_bhb_loop)
+.macro __CLEAR_BHB_LOOP outer_loop_count:req, inner_loop_count:req
ANNOTATE_NOENDBR
push %rbp
mov %rsp, %rbp
- movl $5, %ecx
+
+ movl $\outer_loop_count, %ecx
ANNOTATE_INTRA_FUNCTION_CALL
call 1f
jmp 5f
@@ -1542,29 +1538,54 @@ SYM_FUNC_START(clear_bhb_loop)
* Shift instructions so that the RET is in the upper half of the
* cacheline and don't take the slowpath to its_return_thunk.
*/
- .skip 32 - (.Lret1 - 1f), 0xcc
+ .skip 32 - (.Lret1_\@ - 1f), 0xcc
ANNOTATE_INTRA_FUNCTION_CALL
1: call 2f
-.Lret1: RET
+.Lret1_\@:
+ RET
.align 64, 0xcc
/*
- * As above shift instructions for RET at .Lret2 as well.
+ * As above shift instructions for RET at .Lret2_\@ as well.
*
- * This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc
+ * This should be ideally be: .skip 32 - (.Lret2_\@ - 2f), 0xcc
* but some Clang versions (e.g. 18) don't like this.
*/
.skip 32 - 18, 0xcc
-2: movl $5, %eax
+2: movl $\inner_loop_count, %eax
3: jmp 4f
nop
4: sub $1, %eax
jnz 3b
sub $1, %ecx
jnz 1b
-.Lret2: RET
+.Lret2_\@:
+ RET
5: lfence
+
pop %rbp
RET
+.endm
+
+/*
+ * This should be used on parts prior to Alder Lake. Newer parts should use the
+ * BHI_DIS_S hardware control instead. If a pre-Alder Lake part is being
+ * virtualized on newer hardware the VMM should protect against BHI attacks by
+ * setting BHI_DIS_S for the guests.
+ */
+SYM_FUNC_START(clear_bhb_loop)
+ __CLEAR_BHB_LOOP 5, 5
SYM_FUNC_END(clear_bhb_loop)
EXPORT_SYMBOL_GPL(clear_bhb_loop)
STACK_FRAME_NON_STANDARD(clear_bhb_loop)
+
+/*
+ * A longer version of clear_bhb_loop to ensure that the BHB is cleared on CPUs
+ * with larger branch history tables (i.e. Alder Lake and newer). BHI_DIS_S
+ * protects the kernel, but to mitigate the guest influence on the host
+ * userspace either IBPB or this sequence should be used. See VMSCAPE bug.
+ */
+SYM_FUNC_START(clear_bhb_long_loop)
+ __CLEAR_BHB_LOOP 12, 7
+SYM_FUNC_END(clear_bhb_long_loop)
+EXPORT_SYMBOL_GPL(clear_bhb_long_loop)
+STACK_FRAME_NON_STANDARD(clear_bhb_long_loop)
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 08ed5a2e46a5fd790bcb1b73feb6469518809c06..49707e563bdf71bdd05d3827f10dd2b8ac6bca2c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -388,6 +388,9 @@ extern void write_ibpb(void);

#ifdef CONFIG_X86_64
extern void clear_bhb_loop(void);
+extern void clear_bhb_long_loop(void);
+#else
+static inline void clear_bhb_long_loop(void) {}
#endif

extern void (*x86_return_thunk)(void);

--
2.34.1