[PATCH] riscv: Add native this_cpu_cmpxchg() support

[PATCH] riscv: Add native this_cpu_cmpxchg() support

[Xie Bo]

RISC-V falls back to the generic this_cpu_cmpxchg() implementation,
which serializes the operation by disabling local interrupts.
Consequently, HAVE_CMPXCHG_LOCAL is unset and users such as
percpu_counter cannot use their cmpxchg-based fast paths.

Implement the 4-byte this_cpu_cmpxchg() operation with cmpxchg_local(),
and provide the 8-byte operation on RV64. Pin execution while resolving
the current CPU pointer so the LR/SC loop operates on one per-CPU
instance, while allowing interrupt-context updates to race through the
atomic operation.

Copy the old and new values to private temporaries before invoking
cmpxchg_local(). This avoids collisions between local variable names in
nested statement-expression macros.

The 1- and 2-byte operations continue to use the generic fallback.

Select HAVE_CMPXCHG_LOCAL and update the architecture feature matrix.

A percpu_counter_add() benchmark using the default batch value reduced
the median time per operation by 77.3% (10 runs of 5,000,000
operations).

In three 60-second stress-ng fork runs, median throughput increased by
4.6%. A 120-second combined fork and VM stress test completed without
rss-counter errors or validation failures.

Signed-off-by: Xie Bo <xb@ultrarisc.com>
---
 .../locking/cmpxchg-local/arch-support.txt    |  2 +-
 arch/riscv/Kconfig                            |  1 +
 arch/riscv/include/asm/percpu.h               | 31 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/percpu.h

diff --git a/Documentation/features/locking/cmpxchg-local/arch-support.txt b/Documentation/features/locking/cmpxchg-local/arch-support.txt
index 2c3a4b9..28d5fa8 100644
--- a/Documentation/features/locking/cmpxchg-local/arch-support.txt
+++ b/Documentation/features/locking/cmpxchg-local/arch-support.txt
@@ -20,7 +20,7 @@
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index f7028ca..62e5794 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -153,6 +153,7 @@ config RISCV
 	select HAVE_ARCH_VMAP_STACK if MMU && 64BIT
 	select HAVE_ASM_MODVERSIONS
 	select HAVE_BUILDTIME_MCOUNT_SORT
+	select HAVE_CMPXCHG_LOCAL
 	select HAVE_CONTEXT_TRACKING_USER
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_CONTIGUOUS if MMU
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
new file mode 100644
index 0000000..d389bb1
--- /dev/null
+++ b/arch/riscv/include/asm/percpu.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_RISCV_PERCPU_H
+#define _ASM_RISCV_PERCPU_H
+
+#include <linux/preempt.h>
+
+#include <asm/cmpxchg.h>
+
+#define _protect_cmpxchg_local(pcp, o, n)				\
+({									\
+	typeof(pcp) *__ptr;						\
+	typeof(pcp) __pcpu_old = (o);					\
+	typeof(pcp) __pcpu_new = (n);					\
+	typeof(*raw_cpu_ptr(&(pcp))) __ret;				\
+	preempt_disable_notrace();					\
+	__ptr = raw_cpu_ptr(&(pcp));					\
+	__ret = cmpxchg_local(__ptr, __pcpu_old, __pcpu_new);		\
+	preempt_enable_notrace();					\
+	__ret;								\
+})
+
+#define this_cpu_cmpxchg_4(pcp, o, n)	_protect_cmpxchg_local(pcp, o, n)
+
+#ifdef CONFIG_64BIT
+#define this_cpu_cmpxchg_8(pcp, o, n)	_protect_cmpxchg_local(pcp, o, n)
+#define this_cpu_cmpxchg64(pcp, o, n)	this_cpu_cmpxchg_8(pcp, o, n)
+#endif
+
+#include <asm-generic/percpu.h>
+
+#endif /* _ASM_RISCV_PERCPU_H */
-- 
2.17.1

Re: [PATCH] riscv: Add native this_cpu_cmpxchg() support

[Paul Walmsley]

Hi, 

On Thu, 30 Jul 2026, Xie Bo wrote:

> RISC-V falls back to the generic this_cpu_cmpxchg() implementation,
> which serializes the operation by disabling local interrupts.
> Consequently, HAVE_CMPXCHG_LOCAL is unset and users such as
> percpu_counter cannot use their cmpxchg-based fast paths.
> 
> Implement the 4-byte this_cpu_cmpxchg() operation with cmpxchg_local(),
> and provide the 8-byte operation on RV64. Pin execution while resolving
> the current CPU pointer so the LR/SC loop operates on one per-CPU
> instance, while allowing interrupt-context updates to race through the
> atomic operation.
> 
> Copy the old and new values to private temporaries before invoking
> cmpxchg_local(). This avoids collisions between local variable names in
> nested statement-expression macros.
> 
> The 1- and 2-byte operations continue to use the generic fallback.
> 
> Select HAVE_CMPXCHG_LOCAL and update the architecture feature matrix.
> 
> A percpu_counter_add() benchmark using the default batch value reduced
> the median time per operation by 77.3% (10 runs of 5,000,000
> operations).
> 
> In three 60-second stress-ng fork runs, median throughput increased by
> 4.6%. A 120-second combined fork and VM stress test completed without
> rss-counter errors or validation failures.
> 
> Signed-off-by: Xie Bo <xb@ultrarisc.com>

This patch seems to be missing 128-bit cmpxchg support.  Care to add it?


thanks,

- Paul

[PATCH v2] riscv: Add native this_cpu_cmpxchg() support

[Xie Bo]

RISC-V falls back to the generic this_cpu_cmpxchg() implementation,
which serializes the operation by disabling local interrupts.
Consequently, HAVE_CMPXCHG_LOCAL is unset and users such as
percpu_counter cannot use their cmpxchg-based fast paths.

Implement the 4-byte this_cpu_cmpxchg() operation with cmpxchg_local(),
and provide the 8-byte operation on RV64. Pin execution while resolving
the current CPU pointer so the LR/SC loop operates on one per-CPU
instance, while allowing interrupt-context updates to race through the
atomic operation.

Provide this_cpu_cmpxchg128() on RV64 when the kernel and toolchain support
the Zacas extension, using cmpxchg128_local().

Changes in v2:
- Add the RV64 Zacas-gated this_cpu_cmpxchg128() implementation.
- Rebase and test against latest upstream.

Copy the old and new values to private temporaries before invoking
cmpxchg_local(). This avoids collisions between local variable names in
nested statement-expression macros.

The 1- and 2-byte operations continue to use the generic fallback.

Select HAVE_CMPXCHG_LOCAL and update the architecture feature matrix.

Tested on latest upstream commit 7f063b2f17ea with GCC 15.2:
RV32 and RV64 builds of arch/riscv/kernel/ and lib/percpu_counter.o
pass, and an RV64 compile probe emits amocas.q for
this_cpu_cmpxchg128().

Signed-off-by: Xie Bo <xb@ultrarisc.com>
---
 .../locking/cmpxchg-local/arch-support.txt    |  2 +-
 arch/riscv/Kconfig                            |  1 +
 arch/riscv/include/asm/percpu.h               | 49 +++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/percpu.h

diff --git a/Documentation/features/locking/cmpxchg-local/arch-support.txt b/Documentation/features/locking/cmpxchg-local/arch-support.txt
index 2c3a4b91f..28d5fa8c3 100644
--- a/Documentation/features/locking/cmpxchg-local/arch-support.txt
+++ b/Documentation/features/locking/cmpxchg-local/arch-support.txt
@@ -20,7 +20,7 @@
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index bf2e81d25..1f655a9a6 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -155,6 +155,7 @@ config RISCV
 	select HAVE_ARCH_VMAP_STACK if MMU && 64BIT
 	select HAVE_ASM_MODVERSIONS
 	select HAVE_BUILDTIME_MCOUNT_SORT
+	select HAVE_CMPXCHG_LOCAL
 	select HAVE_CONTEXT_TRACKING_USER
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_CONTIGUOUS if MMU
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
new file mode 100644
index 000000000..ab048b9c9
--- /dev/null
+++ b/arch/riscv/include/asm/percpu.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_RISCV_PERCPU_H
+#define _ASM_RISCV_PERCPU_H
+
+#include <linux/preempt.h>
+
+#include <asm/cmpxchg.h>
+
+#define _protect_cmpxchg_local(pcp, o, n)				\
+({									\
+	typeof(pcp) *__ptr;						\
+	typeof(pcp) __pcpu_old = (o);					\
+	typeof(pcp) __pcpu_new = (n);					\
+	typeof(*raw_cpu_ptr(&(pcp))) __ret;				\
+	preempt_disable_notrace();					\
+	__ptr = raw_cpu_ptr(&(pcp));					\
+	__ret = cmpxchg_local(__ptr, __pcpu_old, __pcpu_new);		\
+	preempt_enable_notrace();					\
+	__ret;								\
+})
+
+#define this_cpu_cmpxchg_4(pcp, o, n)	_protect_cmpxchg_local(pcp, o, n)
+
+#ifdef CONFIG_64BIT
+#define this_cpu_cmpxchg_8(pcp, o, n)	_protect_cmpxchg_local(pcp, o, n)
+#define this_cpu_cmpxchg64(pcp, o, n)	this_cpu_cmpxchg_8(pcp, o, n)
+
+#if defined(CONFIG_RISCV_ISA_ZACAS) && defined(CONFIG_TOOLCHAIN_HAS_ZACAS)
+#define _protect_cmpxchg128_local(pcp, o, n)			\
+({								\
+	typeof(pcp) *__ptr;						\
+	typeof(pcp) __pcpu_old = (o);					\
+	typeof(pcp) __pcpu_new = (n);					\
+	typeof(pcp) __ret;						\
+	preempt_disable_notrace();					\
+	__ptr = raw_cpu_ptr(&(pcp));					\
+	__ret = cmpxchg128_local(__ptr, __pcpu_old, __pcpu_new);	\
+	preempt_enable_notrace();					\
+	__ret;							\
+})
+
+#define this_cpu_cmpxchg128(pcp, o, n)	\
+	_protect_cmpxchg128_local(pcp, o, n)
+#endif
+#endif
+
+#include <asm-generic/percpu.h>
+
+#endif /* _ASM_RISCV_PERCPU_H */
-- 
2.17.1