Re: [PATCH v4 16/21] arm64: percpu: Implement preemptible void RMW ops

[Vladimir Murzin <vladimir.murzin@arm.com>]

On 9/8/26 16:17, Mark Rutland wrote:
> Use the PCPU GPR infrastructure to implement all of the RMW ops which do
> not return a value.
> 
> Test case:
> 
> | void outline_this_cpu_add_u64(u64 __percpu *p, u64 v)
> | {
> | 	this_cpu_add(*p, v);
> | }
> 
> Generated code before this patch (v7.2-rc4):
> 
> | <outline_this_cpu_add_u64>:
> |        paciasp
> |        stp     x29, x30, [sp, #-16]!
> |        mrs     x2, sp_el0
> |        mov     x29, sp
> |        ldr     w3, [x2, #8]
> |        add     w3, w3, #0x1
> |        str     w3, [x2, #8]
> |        mrs     x3, tpidr_el1
> |        add     x0, x0, x3
> | 1:     ldxr    x5, [x0]
> |        add     x5, x5, x1
> |        stxr    w4, x5, [x0]
> |        cbnz    w4, 1b
> |        ldr     x0, [x2, #8]
> |        sub     x0, x0, #0x1
> |        str     w0, [x2, #8]
> |        cbz     x0, 2f
> |        ldr     x0, [x2, #8]
> |        cbnz    x0, 3f
> | 2:     bl      preempt_schedule_notrace
> | 3:     ldp     x29, x30, [sp], #16
> |        autiasp
> |        ret
> 
> Generated code after this patch:
> 
> | <outline_this_cpu_add_u64>:
> |        mrs     x2, sp_el0
> |        mov     w4, #0xc80
> |        strh    w4, [x2, #20]
> |        mrs     x4, tpidr_el1
> |        add     x3, x0, x4
> | 1:     ldxr    x6, [x3]
> |        add     x6, x6, x1
> |        stxr    w5, x6, [x3]
> |        cbnz    w5, 1b
> |        strh    wzr, [x2, #20]
> |        ret
> 
> Signed-off-by: Mark Rutland <mark.rutland@arm.com>
> Tested-by: Muhammad Usama Anjum <usama.anjum@arm.com>
> Cc: Ada Couprie Diaz <ada.coupriediaz@arm.com>
> Cc: Ard Biesheuvel <ardb@kernel.org>
> Cc: Catalin Marinas <catalin.marinas@arm.com>
> Cc: James Morse <james.morse@arm.com>
> Cc: Jinjie Ruan <ruanjinjie@huawei.com>
> Cc: Marc Zyngier <maz@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Vladimir Murzin <vladimir.murzin@arm.com>
> Cc: Will Deacon <will@kernel.org>
> Cc: Yang Shi <yang@os.amperecomputing.com>
> ---
>  arch/arm64/include/asm/percpu.h | 52 ++++++++++++++++++++-------------
>  1 file changed, 32 insertions(+), 20 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 984f69846cc09..504a7fefa2015 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -168,23 +168,35 @@ static inline void __percpu_write_##sz(void __percpu *pcp, unsigned long val)	\
>  
>  #define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse)		\
>  static inline void							\
> -__percpu_##name##_case_##sz(void *ptr, unsigned long val)		\
> +__percpu_##name##_case_##sz(void __percpu *pcp, unsigned long val)	\
>  {									\
> +	u16 *gprs = &current_thread_info()->pcpu_gprs;			\
> +	unsigned long addr;						\
> +	unsigned long off;						\
>  	unsigned int loop;						\
>  	u##sz tmp;							\
>  									\
> -	asm volatile (ARM64_LSE_ATOMIC_INSN(				\
> +	asm volatile (							\
> +	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")	\
> +	ARM64_LSE_ATOMIC_INSN(						\
>  	/* LL/SC */							\
> -	"1:	ldxr" #sfx "\t%" #w "[tmp], %[ptr]\n"			\
> +	"1:	ldxr" #sfx "\t%" #w "[tmp], [%[addr]]\n"		\
>  		#op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n"	\
> -	"	stxr" #sfx "\t%w[loop], %" #w "[tmp], %[ptr]\n"		\
> +	"	stxr" #sfx "\t%w[loop], %" #w "[tmp], [%[addr]]\n"	\
>  	"	cbnz	%w[loop], 1b",					\
>  	/* LSE atomics */						\
> -		#op_lse #sfx "\t%" #w "[val], %" #w "[tmp], %[ptr]\n"	\
> +		#op_lse #sfx "\t%" #w "[val], %" #w "[tmp], [%[addr]]\n"\
>  		__nops(3))						\
> -	: [loop] "=&r" (loop), [tmp] "=&r" (tmp),			\
> -	  [ptr] "+Q"(*(u##sz *)ptr)					\
> -	: [val] "r" ((u##sz)(val)));					\
> +	__PCPU_GPRS_END("%[gprs]")					\
> +	: [gprs] "=Qo" (*gprs),						\
> +	  [addr] "=&r" (addr),						\
> +	  [off] "=&r" (off),						\
> +	  [loop] "=&r" (loop),						\
> +	  [tmp] "=&r" (tmp)						\
> +	: [pcp] "r" (pcp),						\
> +	  [val] "r" ((u##sz)(val))					\
> +	: "memory"							\
> +	);								\
>  }
>  
>  #define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse)		\
> @@ -301,13 +313,13 @@ PERCPU_RET_OP(add, add, ldadd)
>  	_pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val))
>  
>  #define this_cpu_add_1(pcp, val)	\
> -	_pcp_protect(__percpu_add_case_8, pcp, val)
> +	_pcp_wrap(__percpu_add_case_8, pcp, val)
>  #define this_cpu_add_2(pcp, val)	\
> -	_pcp_protect(__percpu_add_case_16, pcp, val)
> +	_pcp_wrap(__percpu_add_case_16, pcp, val)
>  #define this_cpu_add_4(pcp, val)	\
> -	_pcp_protect(__percpu_add_case_32, pcp, val)
> +	_pcp_wrap(__percpu_add_case_32, pcp, val)
>  #define this_cpu_add_8(pcp, val)	\
> -	_pcp_protect(__percpu_add_case_64, pcp, val)
> +	_pcp_wrap(__percpu_add_case_64, pcp, val)
>  
>  #define this_cpu_add_return_1(pcp, val)	\
>  	_pcp_protect_return(__percpu_add_return_case_8, pcp, val)
> @@ -319,22 +331,22 @@ PERCPU_RET_OP(add, add, ldadd)
>  	_pcp_protect_return(__percpu_add_return_case_64, pcp, val)
>  
>  #define this_cpu_and_1(pcp, val)	\
> -	_pcp_protect(__percpu_andnot_case_8, pcp, ~(u8)(val))
> +	_pcp_wrap(__percpu_andnot_case_8, pcp, ~(u8)(val))
>  #define this_cpu_and_2(pcp, val)	\
> -	_pcp_protect(__percpu_andnot_case_16, pcp, ~(u16)(val))
> +	_pcp_wrap(__percpu_andnot_case_16, pcp, ~(u16)(val))
>  #define this_cpu_and_4(pcp, val)	\
> -	_pcp_protect(__percpu_andnot_case_32, pcp, ~(u32)(val))
> +	_pcp_wrap(__percpu_andnot_case_32, pcp, ~(u32)(val))
>  #define this_cpu_and_8(pcp, val)	\
> -	_pcp_protect(__percpu_andnot_case_64, pcp, ~(u64)(val))
> +	_pcp_wrap(__percpu_andnot_case_64, pcp, ~(u64)(val))
>  
>  #define this_cpu_or_1(pcp, val)		\
> -	_pcp_protect(__percpu_or_case_8, pcp, val)
> +	_pcp_wrap(__percpu_or_case_8, pcp, val)
>  #define this_cpu_or_2(pcp, val)		\
> -	_pcp_protect(__percpu_or_case_16, pcp, val)
> +	_pcp_wrap(__percpu_or_case_16, pcp, val)
>  #define this_cpu_or_4(pcp, val)		\
> -	_pcp_protect(__percpu_or_case_32, pcp, val)
> +	_pcp_wrap(__percpu_or_case_32, pcp, val)
>  #define this_cpu_or_8(pcp, val)		\
> -	_pcp_protect(__percpu_or_case_64, pcp, val)
> +	_pcp_wrap(__percpu_or_case_64, pcp, val)
>  
>  #define this_cpu_xchg_1(pcp, val)	\
>  	_pcp_protect_return(xchg_relaxed, pcp, val)
> -- 2.30.2
> 

FWIW,

Reviewed-by: Vladimir Murzin <vladimir.murzin@arm.com>