[PATCH 2/2] x86/bitops: Fix false output register dependency of TZCNT insn

Uros Bizjak
2025-03-25T17:52:29Z

On Haswell and later Intel processors, the TZCNT instruction appears
to have a false dependency on the destination register. Even though
the instruction only writes to it, the instruction will wait until
destination is ready before executing. This false dependency
was fixed for Skylake (and later) processors.

Fix false dependency by clearing the destination register first.

The x86_64 defconfig object size increases by 4215 bytes:

text data bss dec hex filename
27342396 4642999 814852 32800247 1f47df7 vmlinux-old.o
27346611 4643015 814852 32804478 1f48e7e vmlinux-new.o

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/include/asm/bitops.h | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index bbaf75ea6703..7e3d1cc97c5a 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -248,8 +248,9 @@ arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

static __always_inline unsigned long variable__ffs(unsigned long word)
{
- asm("tzcnt %1,%0"
- : "=r" (word)
+ asm("xor %k0,%k0\n\t" /* avoid false dependency on dest register */
+ "tzcnt %1,%0"
+ : "=&r" (word)
: ASM_INPUT_RM (word));
return word;
}
@@ -267,8 +268,9 @@ static __always_inline unsigned long variable__ffs(unsigned long word)

static __always_inline unsigned long variable_ffz(unsigned long word)
{
- asm("tzcnt %1,%0"
- : "=r" (word)
+ asm("xor %k0,%k0\n\t" /* avoid false dependency on dest register */
+ "tzcnt %1,%0"
+ : "=&r" (word)
: "r" (~word));
return word;
}
--
2.42.0