[PATCH v2 0/2] Add support for cmpxchg16b emulation

[PATCH v2 0/2] Add support for cmpxchg16b emulation

[Sairaj Kodilkar]

This series adds 128-bit compare-exchange support needed for KVM to
emulate guest cmpxchg16b instructions.

The AMD IOMMU driver updates 256-bit device table entries with two
128-bit cmpxchg operations.  For hardware-accelerated vIOMMU, QEMU traps
those DTE accesses to install nested page tables [1].  KVM must emulate
guest cmpxchg16b on the trapped MMIO regions; without that, DTE access
emulation fails.

Patch 1: extends the x86 user CMPXCHG helpers with
__try_cmpxchg128_user_asm() (cmpxchg16b on x86-64), wired into
unsafe_try_cmpxchg_user().

Patch 2: extends KVM CMPXCHG8B emulation to 16-byte operands (REX.W=1) and
uses the new helper for atomic 16-byte compare-exchange on guest RAM via
emulator_cmpxchg_emulated().

You can find the KUT to test this series at [2].

[1] https://github.com/AMDESE/qemu-iommu/blob/wip/for_iommufd_hw_queue-v8_amd_viommu_20260106/hw/i386/amd_viommu.c#L517
[2] https://lore.kernel.org/kvm/20260706062153.346-1-sarunkod@amd.com/T/#u

---

Base: 8cd9520d35a6c38db6567e97dd93b1f11f185dc6

---

Sairaj Kodilkar (2):
  x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands
  KVM: x86: Add support for cmpxchg16b emulation

 arch/x86/include/asm/uaccess.h | 56 +++++++++++++++++++++++++++++++++-
 arch/x86/kvm/emulate.c         | 50 ++++++++++++++++++++----------
 arch/x86/kvm/kvm_emulate.h     |  6 ++++
 arch/x86/kvm/x86.c             |  7 ++++-
 4 files changed, 101 insertions(+), 18 deletions(-)

-- 
2.34.1

[PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

[Sairaj Kodilkar]

Extend the existing user CMPXCHG helpers to support 16-byte operands on
x86-64, using LOCK_PREFIX "cmpxchg16b".  This mirrors the existing
__try_cmpxchg64_user_asm() / cmpxchg8b path provided for 32-bit kernels,
where KVM needs an atomic compare-exchange wider than the generic
cmpxchg helper can provide.

On 32-bit kernels, stub the helper to always return failure because cmpxchg16b
requires 64-bit GPRs and is not available.

KVM uses this to atomically emulate guest cmpxchg16b on guest RAM mapped
via userspace addresses.

Signed-off-by: Sairaj Kodilkar <sarunkod@amd.com>
---
 arch/x86/include/asm/uaccess.h | 56 +++++++++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 367297b188c3..123755d47109 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -407,6 +407,25 @@ do {									\
 	if (unlikely(!success))						\
 		*_old = __old;						\
 	likely(success);					})
+#else // !CONFIG_X86_32
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label)	({	\
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_goto_output("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg16b %[ptr]\n"		\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : "=@ccz" (success),				\
+		       "+A" (__old),					\
+		       [ptr] "+m" (*_ptr)				\
+		     : "b" ((u64)__new),				\
+		       "c" ((u64)((u128)__new >> 64))			\
+		     : "memory"						\
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
 #endif // CONFIG_X86_32
 #else  // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 #define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
@@ -463,6 +482,30 @@ do {									\
 	if (unlikely(!__result))					\
 		*_old = __old;						\
 	likely(__result);					})
+#else //!CONFIG_X86_32
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label)	({	\
+	int __result;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm volatile("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg16b %[ptr]\n"		\
+		     "mov $0, %[result]\n\t"				\
+		     "setz %b[result]\n"				\
+		     "2:\n"						\
+		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
+					   %[result])			\
+		     : [result] "=q" (__result),			\
+		       "+A" (__old),					\
+		       [ptr] "+m" (*_ptr)				\
+		     : "b" ((u64)__new),				\
+		       "c" ((u64)((u128)__new >> 64))			\
+		     : "memory", "cc");					\
+	if (unlikely(__result < 0))					\
+		goto label;						\
+	if (unlikely(!__result))					\
+		*_old = __old;						\
+	likely(__result);					})
 #endif // CONFIG_X86_32
 #endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 
@@ -551,11 +594,18 @@ do {										\
 
 extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
+#else
 #define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)		\
 	__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
+
 #endif
 
+
 /*
  * Force the pointer to u<size> to match the size expected by the asm helper.
  * clang/LLVM compiles all cases and only discards the unused paths after
@@ -580,6 +630,10 @@ extern void __try_cmpxchg_user_wrong_size(void);
 	case 8:	__ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
 						 (_nval), _label);		\
 		break;								\
+	case 16:								\
+		__ret = __try_cmpxchg128_user_asm((__force u128 *)(_ptr),	\
+						  (_oldp), (_nval), _label);	\
+		break;								\
 	default: __try_cmpxchg_user_wrong_size();				\
 	}									\
 	__ret;						})
-- 
2.34.1

[PATCH v2 2/2] KVM: x86: Add support for cmpxchg16b emulation

[Sairaj Kodilkar]

AMD and Intel both provides support for 128 bit cmpxchg operands using
cmpxchg8b/cmpxchg16b instructions (opcode 0FC7). However, kvm does not
support emulating cmpxchg16b (i.e when destination memory is 128 bit and
REX.W = 1) which causes emulation failure when QEMU guest performs a
cmpxchg16b on a memory region setup as a IO.

This failure is seen on the AMD IOMMU driver which writes 256-bit device
table entries with two 128-bit cmpxchg operations. For guests using
hardware-accelerated vIOMMU, QEMU traps device table accesses to set up
nested page tables (see [1]).  Without 128-bit cmpxchg emulation, KVM
cannot handle these traps and DTE access emulation fails.

Hence extend cmpxchg8b to perform cmpxchg16b when the destination memory
is 128 bit.

[1] https://github.com/AMDESE/qemu-iommu/blob/wip/for_iommufd_hw_queue-v8_amd_viommu_20260106/hw/i386/amd_viommu.c#L517

Signed-off-by: Sairaj Kodilkar <sarunkod@amd.com>
---
 arch/x86/kvm/emulate.c     | 50 ++++++++++++++++++++++++++------------
 arch/x86/kvm/kvm_emulate.h |  6 +++++
 arch/x86/kvm/x86.c         |  7 +++++-
 3 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index c8e292e9a24d..9df55b3c5627 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -2188,24 +2188,36 @@ static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
 	return rc;
 }
 
+#define em_cmpxchg8b_16b(__c, rbits, mbits)\
+do {												\
+	u##mbits old = __c->dst.orig_val##mbits;						\
+												\
+	BUILD_BUG_ON(rbits * 2 != mbits);							\
+												\
+	if (((u##rbits) (old >> 0) != (u##rbits) reg_read(ctxt, VCPU_REGS_RAX)) ||		\
+	    ((u##rbits) (old >> rbits) != (u##rbits) reg_read(ctxt, VCPU_REGS_RDX))) {		\
+		*reg_write(ctxt, VCPU_REGS_RAX) = (u##rbits) (old >> 0);			\
+		*reg_write(ctxt, VCPU_REGS_RDX) = (u##rbits) (old >> rbits);			\
+		ctxt->eflags &= ~X86_EFLAGS_ZF;							\
+	} else {										\
+		ctxt->dst.val##mbits = ((u##mbits)reg_read(ctxt, VCPU_REGS_RCX) << rbits) |	\
+					(u##rbits) reg_read(ctxt, VCPU_REGS_RBX);		\
+												\
+		ctxt->eflags |= X86_EFLAGS_ZF;							\
+	}											\
+} while(0)
+
 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
 {
-	u64 old = ctxt->dst.orig_val64;
-
-	if (ctxt->dst.bytes == 16)
+	if (WARN_ON_ONCE(8 + !!(ctxt->rex_bits & REX_W) * 8 != ctxt->dst.bytes))
 		return X86EMUL_UNHANDLEABLE;
 
-	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
-	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
-		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
-		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
-		ctxt->eflags &= ~X86_EFLAGS_ZF;
-	} else {
-		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
-			(u32) reg_read(ctxt, VCPU_REGS_RBX);
-
-		ctxt->eflags |= X86_EFLAGS_ZF;
-	}
+	if (!(ctxt->rex_bits & REX_W))
+		em_cmpxchg8b_16b(ctxt, 32, 64);
+#ifdef CONFIG_X86_64
+	else
+		em_cmpxchg8b_16b(ctxt, 64, 128);
+#endif
 	return X86EMUL_CONTINUE;
 }
 
@@ -5405,8 +5417,14 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, bool check_intercepts)
 			goto done;
 		}
 	}
-	/* Copy full 64-bit value for CMPXCHG8B.  */
-	ctxt->dst.orig_val64 = ctxt->dst.val64;
+	/* Copy full 64/128-bit value for CMPXCHG8B.  */
+
+#ifdef CONFIG_X86_64
+	if (ctxt->dst.bytes == 16)
+		ctxt->dst.orig_val128 = ctxt->dst.val128;
+	else
+#endif
+		ctxt->dst.orig_val64 = ctxt->dst.val64;
 
 special_insn:
 
diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h
index fb3dab4b5a53..a51677217ada 100644
--- a/arch/x86/kvm/kvm_emulate.h
+++ b/arch/x86/kvm/kvm_emulate.h
@@ -255,6 +255,9 @@ struct operand {
 	union {
 		unsigned long orig_val;
 		u64 orig_val64;
+#ifdef CONFIG_X86_64
+		u128 orig_val128;
+#endif
 	};
 	union {
 		unsigned long *reg;
@@ -268,6 +271,9 @@ struct operand {
 	union {
 		unsigned long val;
 		u64 val64;
+#ifdef CONFIG_X86_64
+		u128 val128;
+#endif
 		char valptr[sizeof(avx256_t)];
 		sse128_t vec_val;
 		avx256_t vec_val2;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3fb64905d190..40c84f8c4912 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8322,7 +8322,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
 	int r;
 
 	/* guests cmpxchg8b have to be emulated atomically */
-	if (bytes > 8 || (bytes & (bytes - 1)))
+	if (bytes > 2 * sizeof(unsigned long) || (bytes & (bytes - 1)))
 		goto emul_write;
 
 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
@@ -8362,6 +8362,11 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
 	case 8:
 		r = emulator_try_cmpxchg_user(u64, hva, old, new);
 		break;
+#ifdef CONFIG_X86_64
+	case 16:
+		r = emulator_try_cmpxchg_user(u128, hva, old, new);
+		break;
+#endif
 	default:
 		BUG();
 	}
-- 
2.34.1

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] The 32-bit stub for `__try_cmpxchg128_user_asm` incorrectly evaluates to 1 (success) instead of 0 (failure), silently skipping emulation while falsely signaling a successful memory update.
--

commit 11b4c82317b376374b32df0435151f876d6239e5
Author: Sairaj Kodilkar <sarunkod@amd.com>

x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

This commit extends the user CMPXCHG helpers to support 16-byte operands
on x86-64 using the cmpxchg16b instruction. It also adds a 32-bit stub
that is intended to always return failure since cmpxchg16b requires
64-bit GPRs.

> diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
> index 367297b188c39..123755d471099 100644
> --- a/arch/x86/include/asm/uaccess.h
> +++ b/arch/x86/include/asm/uaccess.h

[ ... ]

> @@ -551,11 +594,18 @@ do {										\
>  
>  extern void __try_cmpxchg_user_wrong_size(void);
>  
> -#ifndef CONFIG_X86_32
> +#ifdef CONFIG_X86_32
> +/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
> + * instruction).
> + */
> +#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user
macro treat this as success (true) rather than failure (false)?

This could cause the wrapper to negate the value and return 0 (success) to the
caller, falsely signaling a successful memory exchange without actually
updating memory or the old value. Even though there are no current 32-bit
callers using a 16-byte operand, should this evaluate to 0 instead of 1 to
ensure theoretical future callers don't encounter a silent atomicity break?

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260706063035.1139-1-sarunkod@amd.com?part=1

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

[Sairaj Kodilkar]

On 7/6/2026 12:22 PM, sashiko-bot@kernel.org wrote:
> [You don't often get email from sashiko-bot@kernel.org. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
>
> Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
> - [Medium] The 32-bit stub for `__try_cmpxchg128_user_asm` incorrectly evaluates to 1 (success) instead of 0 (failure), silently skipping emulation while falsely signaling a successful memory update.
> --
>
> commit 11b4c82317b376374b32df0435151f876d6239e5
> Author: Sairaj Kodilkar <sarunkod@amd.com>
>
> x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands
>
> This commit extends the user CMPXCHG helpers to support 16-byte operands
> on x86-64 using the cmpxchg16b instruction. It also adds a 32-bit stub
> that is intended to always return failure since cmpxchg16b requires
> 64-bit GPRs.
>
>> diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
>> index 367297b188c39..123755d471099 100644
>> --- a/arch/x86/include/asm/uaccess.h
>> +++ b/arch/x86/include/asm/uaccess.h
> [ ... ]
>
>> @@ -551,11 +594,18 @@ do {                                                                            \
>>
>>   extern void __try_cmpxchg_user_wrong_size(void);
>>
>> -#ifndef CONFIG_X86_32
>> +#ifdef CONFIG_X86_32
>> +/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
>> + * instruction).
>> + */
>> +#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
> [Severity: Medium]
> Does this 32-bit stub correctly signal failure?
>
> Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user
> macro treat this as success (true) rather than failure (false)?
>
> This could cause the wrapper to negate the value and return 0 (success) to the
> caller, falsely signaling a successful memory exchange without actually
> updating memory or the old value. Even though there are no current 32-bit
> callers using a 16-byte operand, should this evaluate to 0 instead of 1 to
> ensure theoretical future callers don't encounter a silent atomicity break?
>
> --
> Sashiko AI review · https://sashiko.dev/#/patchset/20260706063035.1139-1-sarunkod@amd.com?part=1
Hi,

Right, It should return 0 instead of 1, will rectify this in the next patch.

Thanks
Sairaj

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

[Sean Christopherson]

On Mon, Jul 06, 2026, Sairaj Kodilkar wrote:
> On 7/6/2026 12:22 PM, sashiko-bot@kernel.org wrote:
> > > diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
> > > index 367297b188c39..123755d471099 100644
> > > --- a/arch/x86/include/asm/uaccess.h
> > > +++ b/arch/x86/include/asm/uaccess.h
> > [ ... ]
> > 
> > > @@ -551,11 +594,18 @@ do {                                                                            \
> > > 
> > >   extern void __try_cmpxchg_user_wrong_size(void);
> > > 
> > > -#ifndef CONFIG_X86_32
> > > +#ifdef CONFIG_X86_32
> > > +/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
> > > + * instruction).
> > > + */
> > > +#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
> > [Severity: Medium]
> > Does this 32-bit stub correctly signal failure?
> > 
> > Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user
> > macro treat this as success (true) rather than failure (false)?
> > 
> > This could cause the wrapper to negate the value and return 0 (success) to the
> > caller, falsely signaling a successful memory exchange without actually
> > updating memory or the old value. Even though there are no current 32-bit
> > callers using a 16-byte operand, should this evaluate to 0 instead of 1 to
> > ensure theoretical future callers don't encounter a silent atomicity break?
> > 
> > --
> > Sashiko AI review · https://sashiko.dev/#/patchset/20260706063035.1139-1-sarunkod@amd.com?part=1
> Hi,
> 
> Right, It should return 0 instead of 1, will rectify this in the next patch.

Is it possible to fail the build instead?  Attempting to generate CMPXCHG16B on
32-bit should straigt up fail.