* [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands
2026-07-06 6:30 [PATCH v2 0/2] Add support for cmpxchg16b emulation Sairaj Kodilkar
@ 2026-07-06 6:30 ` Sairaj Kodilkar
2026-07-06 6:52 ` sashiko-bot
2026-07-06 6:30 ` [PATCH v2 2/2] KVM: x86: Add support for cmpxchg16b emulation Sairaj Kodilkar
1 sibling, 1 reply; 5+ messages in thread
From: Sairaj Kodilkar @ 2026-07-06 6:30 UTC (permalink / raw)
To: H. Peter Anvin, Peter Zijlstra (Intel), Borislav Petkov,
Dave Hansen, Ingo Molnar, Mathieu Desnoyers, Paolo Bonzini,
Sairaj Kodilkar, Sean Christopherson, Thomas Gleixner,
Uros Bizjak, kvm, linux-kernel, x86
Cc: vasant.hegde, suravee.suthikulpanit
Extend the existing user CMPXCHG helpers to support 16-byte operands on
x86-64, using LOCK_PREFIX "cmpxchg16b". This mirrors the existing
__try_cmpxchg64_user_asm() / cmpxchg8b path provided for 32-bit kernels,
where KVM needs an atomic compare-exchange wider than the generic
cmpxchg helper can provide.
On 32-bit kernels, stub the helper to always return failure because cmpxchg16b
requires 64-bit GPRs and is not available.
KVM uses this to atomically emulate guest cmpxchg16b on guest RAM mapped
via userspace addresses.
Signed-off-by: Sairaj Kodilkar <sarunkod@amd.com>
---
arch/x86/include/asm/uaccess.h | 56 +++++++++++++++++++++++++++++++++-
1 file changed, 55 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 367297b188c3..123755d47109 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -407,6 +407,25 @@ do { \
if (unlikely(!success)) \
*_old = __old; \
likely(success); })
+#else // !CONFIG_X86_32
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) ({ \
+ bool success; \
+ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
+ __typeof__(*(_ptr)) __old = *_old; \
+ __typeof__(*(_ptr)) __new = (_new); \
+ asm_goto_output("\n" \
+ "1: " LOCK_PREFIX "cmpxchg16b %[ptr]\n" \
+ _ASM_EXTABLE_UA(1b, %l[label]) \
+ : "=@ccz" (success), \
+ "+A" (__old), \
+ [ptr] "+m" (*_ptr) \
+ : "b" ((u64)__new), \
+ "c" ((u64)((u128)__new >> 64)) \
+ : "memory" \
+ : label); \
+ if (unlikely(!success)) \
+ *_old = __old; \
+ likely(success); })
#endif // CONFIG_X86_32
#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
@@ -463,6 +482,30 @@ do { \
if (unlikely(!__result)) \
*_old = __old; \
likely(__result); })
+#else //!CONFIG_X86_32
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) ({ \
+ int __result; \
+ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
+ __typeof__(*(_ptr)) __old = *_old; \
+ __typeof__(*(_ptr)) __new = (_new); \
+ asm volatile("\n" \
+ "1: " LOCK_PREFIX "cmpxchg16b %[ptr]\n" \
+ "mov $0, %[result]\n\t" \
+ "setz %b[result]\n" \
+ "2:\n" \
+ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \
+ %[result]) \
+ : [result] "=q" (__result), \
+ "+A" (__old), \
+ [ptr] "+m" (*_ptr) \
+ : "b" ((u64)__new), \
+ "c" ((u64)((u128)__new >> 64)) \
+ : "memory", "cc"); \
+ if (unlikely(__result < 0)) \
+ goto label; \
+ if (unlikely(!__result)) \
+ *_old = __old; \
+ likely(__result); })
#endif // CONFIG_X86_32
#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
@@ -551,11 +594,18 @@ do { \
extern void __try_cmpxchg_user_wrong_size(void);
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
+#else
#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \
__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
+
#endif
+
/*
* Force the pointer to u<size> to match the size expected by the asm helper.
* clang/LLVM compiles all cases and only discards the unused paths after
@@ -580,6 +630,10 @@ extern void __try_cmpxchg_user_wrong_size(void);
case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
(_nval), _label); \
break; \
+ case 16: \
+ __ret = __try_cmpxchg128_user_asm((__force u128 *)(_ptr), \
+ (_oldp), (_nval), _label); \
+ break; \
default: __try_cmpxchg_user_wrong_size(); \
} \
__ret; })
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH v2 2/2] KVM: x86: Add support for cmpxchg16b emulation
2026-07-06 6:30 [PATCH v2 0/2] Add support for cmpxchg16b emulation Sairaj Kodilkar
2026-07-06 6:30 ` [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands Sairaj Kodilkar
@ 2026-07-06 6:30 ` Sairaj Kodilkar
1 sibling, 0 replies; 5+ messages in thread
From: Sairaj Kodilkar @ 2026-07-06 6:30 UTC (permalink / raw)
To: H. Peter Anvin, Peter Zijlstra (Intel), Borislav Petkov,
Dave Hansen, Ingo Molnar, Mathieu Desnoyers, Paolo Bonzini,
Sairaj Kodilkar, Sean Christopherson, Thomas Gleixner,
Uros Bizjak, kvm, linux-kernel, x86
Cc: vasant.hegde, suravee.suthikulpanit
AMD and Intel both provides support for 128 bit cmpxchg operands using
cmpxchg8b/cmpxchg16b instructions (opcode 0FC7). However, kvm does not
support emulating cmpxchg16b (i.e when destination memory is 128 bit and
REX.W = 1) which causes emulation failure when QEMU guest performs a
cmpxchg16b on a memory region setup as a IO.
This failure is seen on the AMD IOMMU driver which writes 256-bit device
table entries with two 128-bit cmpxchg operations. For guests using
hardware-accelerated vIOMMU, QEMU traps device table accesses to set up
nested page tables (see [1]). Without 128-bit cmpxchg emulation, KVM
cannot handle these traps and DTE access emulation fails.
Hence extend cmpxchg8b to perform cmpxchg16b when the destination memory
is 128 bit.
[1] https://github.com/AMDESE/qemu-iommu/blob/wip/for_iommufd_hw_queue-v8_amd_viommu_20260106/hw/i386/amd_viommu.c#L517
Signed-off-by: Sairaj Kodilkar <sarunkod@amd.com>
---
arch/x86/kvm/emulate.c | 50 ++++++++++++++++++++++++++------------
arch/x86/kvm/kvm_emulate.h | 6 +++++
arch/x86/kvm/x86.c | 7 +++++-
3 files changed, 46 insertions(+), 17 deletions(-)
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index c8e292e9a24d..9df55b3c5627 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -2188,24 +2188,36 @@ static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
return rc;
}
+#define em_cmpxchg8b_16b(__c, rbits, mbits)\
+do { \
+ u##mbits old = __c->dst.orig_val##mbits; \
+ \
+ BUILD_BUG_ON(rbits * 2 != mbits); \
+ \
+ if (((u##rbits) (old >> 0) != (u##rbits) reg_read(ctxt, VCPU_REGS_RAX)) || \
+ ((u##rbits) (old >> rbits) != (u##rbits) reg_read(ctxt, VCPU_REGS_RDX))) { \
+ *reg_write(ctxt, VCPU_REGS_RAX) = (u##rbits) (old >> 0); \
+ *reg_write(ctxt, VCPU_REGS_RDX) = (u##rbits) (old >> rbits); \
+ ctxt->eflags &= ~X86_EFLAGS_ZF; \
+ } else { \
+ ctxt->dst.val##mbits = ((u##mbits)reg_read(ctxt, VCPU_REGS_RCX) << rbits) | \
+ (u##rbits) reg_read(ctxt, VCPU_REGS_RBX); \
+ \
+ ctxt->eflags |= X86_EFLAGS_ZF; \
+ } \
+} while(0)
+
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
{
- u64 old = ctxt->dst.orig_val64;
-
- if (ctxt->dst.bytes == 16)
+ if (WARN_ON_ONCE(8 + !!(ctxt->rex_bits & REX_W) * 8 != ctxt->dst.bytes))
return X86EMUL_UNHANDLEABLE;
- if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
- ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
- *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
- *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
- ctxt->eflags &= ~X86_EFLAGS_ZF;
- } else {
- ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
- (u32) reg_read(ctxt, VCPU_REGS_RBX);
-
- ctxt->eflags |= X86_EFLAGS_ZF;
- }
+ if (!(ctxt->rex_bits & REX_W))
+ em_cmpxchg8b_16b(ctxt, 32, 64);
+#ifdef CONFIG_X86_64
+ else
+ em_cmpxchg8b_16b(ctxt, 64, 128);
+#endif
return X86EMUL_CONTINUE;
}
@@ -5405,8 +5417,14 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, bool check_intercepts)
goto done;
}
}
- /* Copy full 64-bit value for CMPXCHG8B. */
- ctxt->dst.orig_val64 = ctxt->dst.val64;
+ /* Copy full 64/128-bit value for CMPXCHG8B. */
+
+#ifdef CONFIG_X86_64
+ if (ctxt->dst.bytes == 16)
+ ctxt->dst.orig_val128 = ctxt->dst.val128;
+ else
+#endif
+ ctxt->dst.orig_val64 = ctxt->dst.val64;
special_insn:
diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h
index fb3dab4b5a53..a51677217ada 100644
--- a/arch/x86/kvm/kvm_emulate.h
+++ b/arch/x86/kvm/kvm_emulate.h
@@ -255,6 +255,9 @@ struct operand {
union {
unsigned long orig_val;
u64 orig_val64;
+#ifdef CONFIG_X86_64
+ u128 orig_val128;
+#endif
};
union {
unsigned long *reg;
@@ -268,6 +271,9 @@ struct operand {
union {
unsigned long val;
u64 val64;
+#ifdef CONFIG_X86_64
+ u128 val128;
+#endif
char valptr[sizeof(avx256_t)];
sse128_t vec_val;
avx256_t vec_val2;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3fb64905d190..40c84f8c4912 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8322,7 +8322,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
int r;
/* guests cmpxchg8b have to be emulated atomically */
- if (bytes > 8 || (bytes & (bytes - 1)))
+ if (bytes > 2 * sizeof(unsigned long) || (bytes & (bytes - 1)))
goto emul_write;
gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
@@ -8362,6 +8362,11 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
case 8:
r = emulator_try_cmpxchg_user(u64, hva, old, new);
break;
+#ifdef CONFIG_X86_64
+ case 16:
+ r = emulator_try_cmpxchg_user(u128, hva, old, new);
+ break;
+#endif
default:
BUG();
}
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread