* [PATCH] arm64: percpu: Implement this_cpu operations
@ 2014-11-06 11:12 Steve Capper
2014-11-06 11:26 ` Ard Biesheuvel
2014-11-06 12:27 ` [PATCH] " Will Deacon
0 siblings, 2 replies; 12+ messages in thread
From: Steve Capper @ 2014-11-06 11:12 UTC (permalink / raw)
To: linux-arm-kernel
The generic this_cpu operations disable interrupts to ensure that the
requested operation is protected from pre-emption. For arm64, this is
overkill and can hurt throughput and latency.
This patch provides arm64 specific implementations for the this_cpu
operations. Rather than disable interrupts, we use the exclusive
monitor or atomic operations as appropriate.
The following operations are implemented: add, add_return, and, or,
read, write, xchg. We also wire up a cmpxchg implementation from
cmpxchg.h.
Testing was performed using the percpu_test module and hackbench on a
Juno board running 3.18-rc3.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
---
This patch applies on top of one I sent out earlier:
"arm64: xchg: Implement cmpxchg_double"
http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html
My two least favourite things in the kernel are excessive pre-processor
use and assembler, so this patch has been fun to write ;-).
I've tried to make the inline assembler clearer with named parameters,
it really upset checkpatch, if it upsets people then please shout and I
will change it.
Cheers,
--
Steve
---
arch/arm64/include/asm/cmpxchg.h | 6 +-
arch/arm64/include/asm/percpu.h | 231 +++++++++++++++++++++++++++++++++++++++
2 files changed, 235 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 3e02245..3e51f49 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
__ret; \
})
-#define this_cpu_cmpxchg_8(ptr, o, n) \
- cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
+#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 5279e57..e751681 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
#endif /* CONFIG_SMP */
+#define PERCPU_OP(op, asm_op) \
+static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+{ \
+ unsigned long loop, ret; \
+ \
+ switch (size) { \
+ case 1: \
+ do { \
+ asm ("//__per_cpu_" #op "_1\n" \
+ "ldxrb %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrb %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u8 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 2: \
+ do { \
+ asm ("//__per_cpu_" #op "_2\n" \
+ "ldxrh %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrh %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u16 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 4: \
+ do { \
+ asm ("//__per_cpu_" #op "_4\n" \
+ "ldxr %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxr %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u32 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 8: \
+ do { \
+ asm ("//__per_cpu_" #op "_8\n" \
+ "ldxr %[ret], %[ptr]\n" \
+ #asm_op " %[ret], %[ret], %[val]\n" \
+ "stxr %w[loop], %[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u64 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ default: \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret; \
+}
+
+PERCPU_OP(add, add)
+PERCPU_OP(and, and)
+PERCPU_OP(or, orr)
+#undef PERCPU_OP
+
+static inline unsigned long __percpu_read(void *ptr, int size)
+{
+ unsigned long ret;
+
+ switch (size) {
+ case 1:
+ asm ("//__per_cpu_read_1\n"
+ "ldrb %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr));
+ break;
+ case 2:
+ asm ("//__per_cpu_read_2\n"
+ "ldrh %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr));
+ break;
+ case 4:
+ asm ("//__per_cpu_read_4\n"
+ "ldr %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr));
+ break;
+ case 8:
+ asm ("//__per_cpu_read_8\n"
+ "ldr %[ret], %[ptr]\n" :
+ [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr));
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+static inline void __percpu_write(void *ptr, unsigned long val, int size)
+{
+ switch (size) {
+ case 1:
+ asm ("//__per_cpu_write_1\n"
+ "strb %w[val], %[ptr]\n" :
+ [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val));
+ break;
+ case 2:
+ asm ("//__per_cpu_write_2\n"
+ "strh %w[val], %[ptr]\n" :
+ [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val));
+ break;
+ case 4:
+ asm ("//__per_cpu_write_4\n"
+ "str %w[val], %[ptr]\n" :
+ [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val));
+ break;
+ case 8:
+ asm ("//__per_cpu_write_8\n"
+ "str %[val], %[ptr]\n" :
+ [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val));
+ break;
+ default:
+ BUILD_BUG();
+ }
+}
+
+static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
+ int size)
+{
+ unsigned long ret, loop;
+
+ switch (size) {
+ case 1:
+ do {
+ asm ("//__percpu_xchg_1\n"
+ "ldxrb %w[ret], %[ptr]\n"
+ "stxrb %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u8 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 2:
+ do {
+ asm ("//__percpu_xchg_2\n"
+ "ldxrh %w[ret], %[ptr]\n"
+ "stxrh %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u16 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 4:
+ do {
+ asm ("//__percpu_xchg_4\n"
+ "ldxr %w[ret], %[ptr]\n"
+ "stxr %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u32 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 8:
+ do {
+ asm ("//__percpu_xchg_8\n"
+ "ldxr %[ret], %[ptr]\n"
+ "stxr %w[loop], %[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u64 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+#define _percpu_add(pcp, val) \
+ __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
+
+#define _percpu_and(pcp, val) \
+ __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_or(pcp, val) \
+ __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_read(pcp) (typeof(pcp)) \
+ (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
+
+#define _percpu_write(pcp, val) \
+ __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
+
+#define _percpu_xchg(pcp, val) (typeof(pcp)) \
+ (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
+
+#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_read_1(pcp) _percpu_read(pcp)
+#define this_cpu_read_2(pcp) _percpu_read(pcp)
+#define this_cpu_read_4(pcp) _percpu_read(pcp)
+#define this_cpu_read_8(pcp) _percpu_read(pcp)
+
+#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
+
+#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
--
1.9.3
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH] arm64: percpu: Implement this_cpu operations
2014-11-06 11:12 [PATCH] arm64: percpu: Implement this_cpu operations Steve Capper
@ 2014-11-06 11:26 ` Ard Biesheuvel
2014-11-06 11:52 ` Steve Capper
2014-11-06 12:27 ` [PATCH] " Will Deacon
1 sibling, 1 reply; 12+ messages in thread
From: Ard Biesheuvel @ 2014-11-06 11:26 UTC (permalink / raw)
To: linux-arm-kernel
Hi Steve,
On 6 November 2014 12:12, Steve Capper <steve.capper@linaro.org> wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
>
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
>
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
>
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc3.
>
Got any numbers?
> Signed-off-by: Steve Capper <steve.capper@linaro.org>
> ---
>
> This patch applies on top of one I sent out earlier:
> "arm64: xchg: Implement cmpxchg_double"
> http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html
>
> My two least favourite things in the kernel are excessive pre-processor
> use and assembler, so this patch has been fun to write ;-).
>
> I've tried to make the inline assembler clearer with named parameters,
> it really upset checkpatch, if it upsets people then please shout and I
> will change it.
>
> Cheers,
> --
> Steve
>
> ---
> arch/arm64/include/asm/cmpxchg.h | 6 +-
> arch/arm64/include/asm/percpu.h | 231 +++++++++++++++++++++++++++++++++++++++
> 2 files changed, 235 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> index 3e02245..3e51f49 100644
> --- a/arch/arm64/include/asm/cmpxchg.h
> +++ b/arch/arm64/include/asm/cmpxchg.h
> @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
> __ret; \
> })
>
> -#define this_cpu_cmpxchg_8(ptr, o, n) \
> - cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
>
> #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
> cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 5279e57..e751681 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
>
> #endif /* CONFIG_SMP */
>
> +#define PERCPU_OP(op, asm_op) \
> +static inline unsigned long __percpu_##op(void *ptr, \
> + unsigned long val, int size) \
> +{ \
> + unsigned long loop, ret; \
> + \
> + switch (size) { \
> + case 1: \
> + do { \
> + asm ("//__per_cpu_" #op "_1\n" \
> + "ldxrb %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxrb %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u8 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 2: \
> + do { \
> + asm ("//__per_cpu_" #op "_2\n" \
> + "ldxrh %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxrh %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u16 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 4: \
> + do { \
> + asm ("//__per_cpu_" #op "_4\n" \
> + "ldxr %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxr %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u32 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 8: \
> + do { \
> + asm ("//__per_cpu_" #op "_8\n" \
> + "ldxr %[ret], %[ptr]\n" \
> + #asm_op " %[ret], %[ret], %[val]\n" \
> + "stxr %w[loop], %[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u64 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + default: \
> + BUILD_BUG(); \
> + } \
> + \
> + return ret; \
> +}
> +
> +PERCPU_OP(add, add)
> +PERCPU_OP(and, and)
> +PERCPU_OP(or, orr)
> +#undef PERCPU_OP
> +
> +static inline unsigned long __percpu_read(void *ptr, int size)
> +{
> + unsigned long ret;
> +
> + switch (size) {
> + case 1:
> + asm ("//__per_cpu_read_1\n"
> + "ldrb %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr));
> + break;
> + case 2:
> + asm ("//__per_cpu_read_2\n"
> + "ldrh %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr));
> + break;
> + case 4:
> + asm ("//__per_cpu_read_4\n"
> + "ldr %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr));
> + break;
> + case 8:
> + asm ("//__per_cpu_read_8\n"
> + "ldr %[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr));
> + break;
> + default:
> + BUILD_BUG();
> + }
> +
> + return ret;
> +}
> +
Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs?
> +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> +{
> + switch (size) {
> + case 1:
> + asm ("//__per_cpu_write_1\n"
> + "strb %w[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val));
> + break;
> + case 2:
> + asm ("//__per_cpu_write_2\n"
> + "strh %w[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val));
> + break;
> + case 4:
> + asm ("//__per_cpu_write_4\n"
> + "str %w[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val));
> + break;
> + case 8:
> + asm ("//__per_cpu_write_8\n"
> + "str %[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val));
> + break;
> + default:
> + BUILD_BUG();
> + }
> +}
> +
... and similarly, why are these '+Q' and not just '=Q' ?
> +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
> + int size)
> +{
> + unsigned long ret, loop;
> +
> + switch (size) {
> + case 1:
> + do {
> + asm ("//__percpu_xchg_1\n"
> + "ldxrb %w[ret], %[ptr]\n"
> + "stxrb %w[loop], %w[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u8 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + case 2:
> + do {
> + asm ("//__percpu_xchg_2\n"
> + "ldxrh %w[ret], %[ptr]\n"
> + "stxrh %w[loop], %w[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u16 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + case 4:
> + do {
> + asm ("//__percpu_xchg_4\n"
> + "ldxr %w[ret], %[ptr]\n"
> + "stxr %w[loop], %w[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u32 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + case 8:
> + do {
> + asm ("//__percpu_xchg_8\n"
> + "ldxr %[ret], %[ptr]\n"
> + "stxr %w[loop], %[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u64 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + default:
> + BUILD_BUG();
> + }
> +
> + return ret;
> +}
> +
> +#define _percpu_add(pcp, val) \
> + __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
> +
> +#define _percpu_and(pcp, val) \
> + __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_or(pcp, val) \
> + __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_read(pcp) (typeof(pcp)) \
> + (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
> +
> +#define _percpu_write(pcp, val) \
> + __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
> +
> +#define _percpu_xchg(pcp, val) (typeof(pcp)) \
> + (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
> +
> +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
> +
> +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
> +
> +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
> +
> +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
> +
> +#define this_cpu_read_1(pcp) _percpu_read(pcp)
> +#define this_cpu_read_2(pcp) _percpu_read(pcp)
> +#define this_cpu_read_4(pcp) _percpu_read(pcp)
> +#define this_cpu_read_8(pcp) _percpu_read(pcp)
> +
> +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
> +
> +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
> +
> #include <asm-generic/percpu.h>
>
> #endif /* __ASM_PERCPU_H */
> --
> 1.9.3
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH] arm64: percpu: Implement this_cpu operations
2014-11-06 11:26 ` Ard Biesheuvel
@ 2014-11-06 11:52 ` Steve Capper
2014-11-06 12:23 ` [PATCH V2] " Steve Capper
0 siblings, 1 reply; 12+ messages in thread
From: Steve Capper @ 2014-11-06 11:52 UTC (permalink / raw)
To: linux-arm-kernel
On Thu, Nov 06, 2014 at 12:26:46PM +0100, Ard Biesheuvel wrote:
> Hi Steve,
Hey Ard,
>
> On 6 November 2014 12:12, Steve Capper <steve.capper@linaro.org> wrote:
> > The generic this_cpu operations disable interrupts to ensure that the
> > requested operation is protected from pre-emption. For arm64, this is
> > overkill and can hurt throughput and latency.
> >
> > This patch provides arm64 specific implementations for the this_cpu
> > operations. Rather than disable interrupts, we use the exclusive
> > monitor or atomic operations as appropriate.
> >
> > The following operations are implemented: add, add_return, and, or,
> > read, write, xchg. We also wire up a cmpxchg implementation from
> > cmpxchg.h.
> >
> > Testing was performed using the percpu_test module and hackbench on a
> > Juno board running 3.18-rc3.
> >
>
> Got any numbers?
For `./hackbench 100 process 1000' on a Juno system running all cores I get:
w/o patch | with patch
---------------+------------
mean 47.05 | 44.93
stddev 0.38 | 0.68
---------------+------------
(times in seconds, rounded to 2d.p., six runs performed)
So a just under 5% speed boost.
[...]
> > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> > index 5279e57..e751681 100644
> > --- a/arch/arm64/include/asm/percpu.h
> > +++ b/arch/arm64/include/asm/percpu.h
> > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
[...]
> > +static inline unsigned long __percpu_read(void *ptr, int size)
> > +{
> > + unsigned long ret;
> > +
> > + switch (size) {
> > + case 1:
> > + asm ("//__per_cpu_read_1\n"
> > + "ldrb %w[ret], %[ptr]\n" :
> > + [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr));
> > + break;
> > + case 2:
> > + asm ("//__per_cpu_read_2\n"
> > + "ldrh %w[ret], %[ptr]\n" :
> > + [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr));
> > + break;
> > + case 4:
> > + asm ("//__per_cpu_read_4\n"
> > + "ldr %w[ret], %[ptr]\n" :
> > + [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr));
> > + break;
> > + case 8:
> > + asm ("//__per_cpu_read_8\n"
> > + "ldr %[ret], %[ptr]\n" :
> > + [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr));
> > + break;
> > + default:
> > + BUILD_BUG();
> > + }
> > +
> > + return ret;
> > +}
> > +
>
> Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs?
>
Because I fudged the constraint :-).
Thanks, you're quite right it should be 'Q' constraint as we're only reading
from that address.
> > +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> > +{
> > + switch (size) {
> > + case 1:
> > + asm ("//__per_cpu_write_1\n"
> > + "strb %w[val], %[ptr]\n" :
> > + [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val));
> > + break;
> > + case 2:
> > + asm ("//__per_cpu_write_2\n"
> > + "strh %w[val], %[ptr]\n" :
> > + [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val));
> > + break;
> > + case 4:
> > + asm ("//__per_cpu_write_4\n"
> > + "str %w[val], %[ptr]\n" :
> > + [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val));
> > + break;
> > + case 8:
> > + asm ("//__per_cpu_write_8\n"
> > + "str %[val], %[ptr]\n" :
> > + [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val));
> > + break;
> > + default:
> > + BUILD_BUG();
> > + }
> > +}
> > +
>
> ... and similarly, why are these '+Q' and not just '=Q' ?
Another mistake on my part, it should be "=Q" as we're only writing to that
address.
Thanks for going through this carefully Ard, I'll send out a corrected V2 once
I've sanity checked it.
Cheers,
--
Steve
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH V2] arm64: percpu: Implement this_cpu operations
2014-11-06 11:52 ` Steve Capper
@ 2014-11-06 12:23 ` Steve Capper
0 siblings, 0 replies; 12+ messages in thread
From: Steve Capper @ 2014-11-06 12:23 UTC (permalink / raw)
To: linux-arm-kernel
The generic this_cpu operations disable interrupts to ensure that the
requested operation is protected from pre-emption. For arm64, this is
overkill and can hurt throughput and latency.
This patch provides arm64 specific implementations for the this_cpu
operations. Rather than disable interrupts, we use the exclusive
monitor or atomic operations as appropriate.
The following operations are implemented: add, add_return, and, or,
read, write, xchg. We also wire up a cmpxchg implementation from
cmpxchg.h.
Testing was performed using the percpu_test module and hackbench on a
Juno board running 3.18-rc3.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
---
Changed in V2: corrected address constraint for __percpu_read and
__percpu_write.
---
arch/arm64/include/asm/cmpxchg.h | 6 +-
arch/arm64/include/asm/percpu.h | 231 +++++++++++++++++++++++++++++++++++++++
2 files changed, 235 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 3e02245..3e51f49 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
__ret; \
})
-#define this_cpu_cmpxchg_8(ptr, o, n) \
- cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
+#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 5279e57..30b84f2 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
#endif /* CONFIG_SMP */
+#define PERCPU_OP(op, asm_op) \
+static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+{ \
+ unsigned long loop, ret; \
+ \
+ switch (size) { \
+ case 1: \
+ do { \
+ asm ("//__per_cpu_" #op "_1\n" \
+ "ldxrb %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrb %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u8 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 2: \
+ do { \
+ asm ("//__per_cpu_" #op "_2\n" \
+ "ldxrh %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrh %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u16 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 4: \
+ do { \
+ asm ("//__per_cpu_" #op "_4\n" \
+ "ldxr %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxr %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u32 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 8: \
+ do { \
+ asm ("//__per_cpu_" #op "_8\n" \
+ "ldxr %[ret], %[ptr]\n" \
+ #asm_op " %[ret], %[ret], %[val]\n" \
+ "stxr %w[loop], %[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u64 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ default: \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret; \
+}
+
+PERCPU_OP(add, add)
+PERCPU_OP(and, and)
+PERCPU_OP(or, orr)
+#undef PERCPU_OP
+
+static inline unsigned long __percpu_read(void *ptr, int size)
+{
+ unsigned long ret;
+
+ switch (size) {
+ case 1:
+ asm ("//__per_cpu_read_1\n"
+ "ldrb %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u8 *)ptr));
+ break;
+ case 2:
+ asm ("//__per_cpu_read_2\n"
+ "ldrh %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u16 *)ptr));
+ break;
+ case 4:
+ asm ("//__per_cpu_read_4\n"
+ "ldr %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u32 *)ptr));
+ break;
+ case 8:
+ asm ("//__per_cpu_read_8\n"
+ "ldr %[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u64 *)ptr));
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+static inline void __percpu_write(void *ptr, unsigned long val, int size)
+{
+ switch (size) {
+ case 1:
+ asm ("//__per_cpu_write_1\n"
+ "strb %w[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u8 *)ptr) : [val] "r"(val));
+ break;
+ case 2:
+ asm ("//__per_cpu_write_2\n"
+ "strh %w[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u16 *)ptr) : [val] "r"(val));
+ break;
+ case 4:
+ asm ("//__per_cpu_write_4\n"
+ "str %w[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u32 *)ptr) : [val] "r"(val));
+ break;
+ case 8:
+ asm ("//__per_cpu_write_8\n"
+ "str %[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u64 *)ptr) : [val] "r"(val));
+ break;
+ default:
+ BUILD_BUG();
+ }
+}
+
+static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
+ int size)
+{
+ unsigned long ret, loop;
+
+ switch (size) {
+ case 1:
+ do {
+ asm ("//__percpu_xchg_1\n"
+ "ldxrb %w[ret], %[ptr]\n"
+ "stxrb %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u8 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 2:
+ do {
+ asm ("//__percpu_xchg_2\n"
+ "ldxrh %w[ret], %[ptr]\n"
+ "stxrh %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u16 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 4:
+ do {
+ asm ("//__percpu_xchg_4\n"
+ "ldxr %w[ret], %[ptr]\n"
+ "stxr %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u32 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 8:
+ do {
+ asm ("//__percpu_xchg_8\n"
+ "ldxr %[ret], %[ptr]\n"
+ "stxr %w[loop], %[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u64 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+#define _percpu_add(pcp, val) \
+ __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
+
+#define _percpu_and(pcp, val) \
+ __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_or(pcp, val) \
+ __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_read(pcp) (typeof(pcp)) \
+ (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
+
+#define _percpu_write(pcp, val) \
+ __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
+
+#define _percpu_xchg(pcp, val) (typeof(pcp)) \
+ (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
+
+#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_read_1(pcp) _percpu_read(pcp)
+#define this_cpu_read_2(pcp) _percpu_read(pcp)
+#define this_cpu_read_4(pcp) _percpu_read(pcp)
+#define this_cpu_read_8(pcp) _percpu_read(pcp)
+
+#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
+
+#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
--
1.9.3
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH] arm64: percpu: Implement this_cpu operations
2014-11-06 11:12 [PATCH] arm64: percpu: Implement this_cpu operations Steve Capper
2014-11-06 11:26 ` Ard Biesheuvel
@ 2014-11-06 12:27 ` Will Deacon
2014-11-07 13:52 ` Steve Capper
1 sibling, 1 reply; 12+ messages in thread
From: Will Deacon @ 2014-11-06 12:27 UTC (permalink / raw)
To: linux-arm-kernel
Hi Steve,
Thanks for looking at this!
On Thu, Nov 06, 2014 at 11:12:57AM +0000, Steve Capper wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
>
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
>
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
>
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc3.
[...]
> diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> index 3e02245..3e51f49 100644
> --- a/arch/arm64/include/asm/cmpxchg.h
> +++ b/arch/arm64/include/asm/cmpxchg.h
> @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
> __ret; \
> })
>
> -#define this_cpu_cmpxchg_8(ptr, o, n) \
> - cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
You can use cmpxchg_local here, as we don't require barrier semantics.
> #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
> cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 5279e57..e751681 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
>
> #endif /* CONFIG_SMP */
>
> +#define PERCPU_OP(op, asm_op) \
> +static inline unsigned long __percpu_##op(void *ptr, \
> + unsigned long val, int size) \
> +{ \
> + unsigned long loop, ret; \
> + \
> + switch (size) { \
> + case 1: \
> + do { \
> + asm ("//__per_cpu_" #op "_1\n" \
> + "ldxrb %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxrb %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u8 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
Curious, but do you see any difference in code generation over an explicit
cbnz, like we use in the ATOMIC_OP macro?
> + case 2: \
> + do { \
> + asm ("//__per_cpu_" #op "_2\n" \
> + "ldxrh %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxrh %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u16 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 4: \
> + do { \
> + asm ("//__per_cpu_" #op "_4\n" \
> + "ldxr %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxr %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u32 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 8: \
> + do { \
> + asm ("//__per_cpu_" #op "_8\n" \
> + "ldxr %[ret], %[ptr]\n" \
> + #asm_op " %[ret], %[ret], %[val]\n" \
> + "stxr %w[loop], %[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u64 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + default: \
> + BUILD_BUG(); \
> + } \
> + \
> + return ret; \
> +}
> +
> +PERCPU_OP(add, add)
> +PERCPU_OP(and, and)
> +PERCPU_OP(or, orr)
Can you use these to generate local_t versions too?
Will
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH] arm64: percpu: Implement this_cpu operations
2014-11-06 12:27 ` [PATCH] " Will Deacon
@ 2014-11-07 13:52 ` Steve Capper
2014-11-14 11:37 ` [PATCH V3] " Steve Capper
0 siblings, 1 reply; 12+ messages in thread
From: Steve Capper @ 2014-11-07 13:52 UTC (permalink / raw)
To: linux-arm-kernel
On Thu, Nov 06, 2014 at 12:27:53PM +0000, Will Deacon wrote:
> Hi Steve,
>
> Thanks for looking at this!
Hey Will,
No problem, it's quite beneficial for performance.
>
> On Thu, Nov 06, 2014 at 11:12:57AM +0000, Steve Capper wrote:
> > The generic this_cpu operations disable interrupts to ensure that the
> > requested operation is protected from pre-emption. For arm64, this is
> > overkill and can hurt throughput and latency.
> >
> > This patch provides arm64 specific implementations for the this_cpu
> > operations. Rather than disable interrupts, we use the exclusive
> > monitor or atomic operations as appropriate.
> >
> > The following operations are implemented: add, add_return, and, or,
> > read, write, xchg. We also wire up a cmpxchg implementation from
> > cmpxchg.h.
> >
> > Testing was performed using the percpu_test module and hackbench on a
> > Juno board running 3.18-rc3.
>
> [...]
>
> > diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> > index 3e02245..3e51f49 100644
> > --- a/arch/arm64/include/asm/cmpxchg.h
> > +++ b/arch/arm64/include/asm/cmpxchg.h
> > @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
> > __ret; \
> > })
> >
> > -#define this_cpu_cmpxchg_8(ptr, o, n) \
> > - cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> > +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> > +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> > +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> > +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
>
> You can use cmpxchg_local here, as we don't require barrier semantics.
Agreed, thanks, I will update that.
>
> > #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
> > cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> > index 5279e57..e751681 100644
> > --- a/arch/arm64/include/asm/percpu.h
> > +++ b/arch/arm64/include/asm/percpu.h
> > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
> >
> > #endif /* CONFIG_SMP */
> >
> > +#define PERCPU_OP(op, asm_op) \
> > +static inline unsigned long __percpu_##op(void *ptr, \
> > + unsigned long val, int size) \
> > +{ \
> > + unsigned long loop, ret; \
> > + \
> > + switch (size) { \
> > + case 1: \
> > + do { \
> > + asm ("//__per_cpu_" #op "_1\n" \
> > + "ldxrb %w[ret], %[ptr]\n" \
> > + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> > + "stxrb %w[loop], %w[ret], %[ptr]\n" \
> > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> > + [ptr] "+Q"(*(u8 *)ptr) \
> > + : [val] "Ir" (val)); \
> > + } while (loop); \
> > + break; \
>
> Curious, but do you see any difference in code generation over an explicit
> cbnz, like we use in the ATOMIC_OP macro?
I've not noticed any substancial difference in the code paths I've
inspected. Theoretically, a compiler that is extremely averse to
branches could do some unrolling with this.
I decided to give the compiler as much control as possible, so elected
to put the minimum amount of assembler in.
>
> > + case 2: \
> > + do { \
> > + asm ("//__per_cpu_" #op "_2\n" \
> > + "ldxrh %w[ret], %[ptr]\n" \
> > + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> > + "stxrh %w[loop], %w[ret], %[ptr]\n" \
> > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> > + [ptr] "+Q"(*(u16 *)ptr) \
> > + : [val] "Ir" (val)); \
> > + } while (loop); \
> > + break; \
> > + case 4: \
> > + do { \
> > + asm ("//__per_cpu_" #op "_4\n" \
> > + "ldxr %w[ret], %[ptr]\n" \
> > + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> > + "stxr %w[loop], %w[ret], %[ptr]\n" \
> > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> > + [ptr] "+Q"(*(u32 *)ptr) \
> > + : [val] "Ir" (val)); \
> > + } while (loop); \
> > + break; \
> > + case 8: \
> > + do { \
> > + asm ("//__per_cpu_" #op "_8\n" \
> > + "ldxr %[ret], %[ptr]\n" \
> > + #asm_op " %[ret], %[ret], %[val]\n" \
> > + "stxr %w[loop], %[ret], %[ptr]\n" \
> > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> > + [ptr] "+Q"(*(u64 *)ptr) \
> > + : [val] "Ir" (val)); \
> > + } while (loop); \
> > + break; \
> > + default: \
> > + BUILD_BUG(); \
> > + } \
> > + \
> > + return ret; \
> > +}
> > +
> > +PERCPU_OP(add, add)
> > +PERCPU_OP(and, and)
> > +PERCPU_OP(or, orr)
>
> Can you use these to generate local_t versions too?
Sure, I forgot about them, I will add them to V3.
Cheers,
--
Steve
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH V3] arm64: percpu: Implement this_cpu operations
2014-11-07 13:52 ` Steve Capper
@ 2014-11-14 11:37 ` Steve Capper
2014-11-14 13:46 ` Will Deacon
0 siblings, 1 reply; 12+ messages in thread
From: Steve Capper @ 2014-11-14 11:37 UTC (permalink / raw)
To: linux-arm-kernel
The generic this_cpu operations disable interrupts to ensure that the
requested operation is protected from pre-emption. For arm64, this is
overkill and can hurt throughput and latency.
This patch provides arm64 specific implementations for the this_cpu
operations. Rather than disable interrupts, we use the exclusive
monitor or atomic operations as appropriate.
The following operations are implemented: add, add_return, and, or,
read, write, xchg. We also wire up a cmpxchg implementation from
cmpxchg.h.
Testing was performed using the percpu_test module and hackbench on a
Juno board running 3.18-rc4.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
---
Having thought more about it, I will send out a separate patch to
implement local.h for arm64 as it's logically separate.
Changed in V3: use cmpxchg_local rather than cmpxchg for
this_cpu_cmpxchg.
Changed in V2: corrected address constraint for __percpu_read and
__percpu_write.
---
arch/arm64/include/asm/cmpxchg.h | 6 +-
arch/arm64/include/asm/percpu.h | 231 +++++++++++++++++++++++++++++++++++++++
2 files changed, 235 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 3e02245..9248066 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
__ret; \
})
-#define this_cpu_cmpxchg_8(ptr, o, n) \
- cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
+#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 5279e57..30b84f2 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
#endif /* CONFIG_SMP */
+#define PERCPU_OP(op, asm_op) \
+static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+{ \
+ unsigned long loop, ret; \
+ \
+ switch (size) { \
+ case 1: \
+ do { \
+ asm ("//__per_cpu_" #op "_1\n" \
+ "ldxrb %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrb %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u8 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 2: \
+ do { \
+ asm ("//__per_cpu_" #op "_2\n" \
+ "ldxrh %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrh %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u16 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 4: \
+ do { \
+ asm ("//__per_cpu_" #op "_4\n" \
+ "ldxr %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxr %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u32 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 8: \
+ do { \
+ asm ("//__per_cpu_" #op "_8\n" \
+ "ldxr %[ret], %[ptr]\n" \
+ #asm_op " %[ret], %[ret], %[val]\n" \
+ "stxr %w[loop], %[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u64 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ default: \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret; \
+}
+
+PERCPU_OP(add, add)
+PERCPU_OP(and, and)
+PERCPU_OP(or, orr)
+#undef PERCPU_OP
+
+static inline unsigned long __percpu_read(void *ptr, int size)
+{
+ unsigned long ret;
+
+ switch (size) {
+ case 1:
+ asm ("//__per_cpu_read_1\n"
+ "ldrb %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u8 *)ptr));
+ break;
+ case 2:
+ asm ("//__per_cpu_read_2\n"
+ "ldrh %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u16 *)ptr));
+ break;
+ case 4:
+ asm ("//__per_cpu_read_4\n"
+ "ldr %w[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u32 *)ptr));
+ break;
+ case 8:
+ asm ("//__per_cpu_read_8\n"
+ "ldr %[ret], %[ptr]\n" :
+ [ret] "=&r"(ret) : [ptr] "Q"(*(u64 *)ptr));
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+static inline void __percpu_write(void *ptr, unsigned long val, int size)
+{
+ switch (size) {
+ case 1:
+ asm ("//__per_cpu_write_1\n"
+ "strb %w[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u8 *)ptr) : [val] "r"(val));
+ break;
+ case 2:
+ asm ("//__per_cpu_write_2\n"
+ "strh %w[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u16 *)ptr) : [val] "r"(val));
+ break;
+ case 4:
+ asm ("//__per_cpu_write_4\n"
+ "str %w[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u32 *)ptr) : [val] "r"(val));
+ break;
+ case 8:
+ asm ("//__per_cpu_write_8\n"
+ "str %[val], %[ptr]\n" :
+ [ptr] "=Q"(*(u64 *)ptr) : [val] "r"(val));
+ break;
+ default:
+ BUILD_BUG();
+ }
+}
+
+static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
+ int size)
+{
+ unsigned long ret, loop;
+
+ switch (size) {
+ case 1:
+ do {
+ asm ("//__percpu_xchg_1\n"
+ "ldxrb %w[ret], %[ptr]\n"
+ "stxrb %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u8 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 2:
+ do {
+ asm ("//__percpu_xchg_2\n"
+ "ldxrh %w[ret], %[ptr]\n"
+ "stxrh %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u16 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 4:
+ do {
+ asm ("//__percpu_xchg_4\n"
+ "ldxr %w[ret], %[ptr]\n"
+ "stxr %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u32 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 8:
+ do {
+ asm ("//__percpu_xchg_8\n"
+ "ldxr %[ret], %[ptr]\n"
+ "stxr %w[loop], %[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u64 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+#define _percpu_add(pcp, val) \
+ __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
+
+#define _percpu_and(pcp, val) \
+ __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_or(pcp, val) \
+ __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_read(pcp) (typeof(pcp)) \
+ (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
+
+#define _percpu_write(pcp, val) \
+ __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
+
+#define _percpu_xchg(pcp, val) (typeof(pcp)) \
+ (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
+
+#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_read_1(pcp) _percpu_read(pcp)
+#define this_cpu_read_2(pcp) _percpu_read(pcp)
+#define this_cpu_read_4(pcp) _percpu_read(pcp)
+#define this_cpu_read_8(pcp) _percpu_read(pcp)
+
+#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
+
+#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
--
1.9.3
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH V3] arm64: percpu: Implement this_cpu operations
2014-11-14 11:37 ` [PATCH V3] " Steve Capper
@ 2014-11-14 13:46 ` Will Deacon
2014-11-14 15:03 ` [PATCH V4] " Steve Capper
0 siblings, 1 reply; 12+ messages in thread
From: Will Deacon @ 2014-11-14 13:46 UTC (permalink / raw)
To: linux-arm-kernel
Hi Steve,
On Fri, Nov 14, 2014 at 11:37:57AM +0000, Steve Capper wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
>
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
>
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
>
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc4.
What does this patch apply against? I'm struggling to apply it to our
for-next branch (perhaps it conflicts with your other cmpxchg patch?)
Anyway, one comment below.
> +static inline unsigned long __percpu_read(void *ptr, int size)
> +{
> + unsigned long ret;
> +
> + switch (size) {
> + case 1:
> + asm ("//__per_cpu_read_1\n"
> + "ldrb %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret) : [ptr] "Q"(*(u8 *)ptr));
> + break;
> + case 2:
> + asm ("//__per_cpu_read_2\n"
> + "ldrh %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret) : [ptr] "Q"(*(u16 *)ptr));
> + break;
> + case 4:
> + asm ("//__per_cpu_read_4\n"
> + "ldr %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret) : [ptr] "Q"(*(u32 *)ptr));
> + break;
> + case 8:
> + asm ("//__per_cpu_read_8\n"
> + "ldr %[ret], %[ptr]\n" :
> + [ret] "=&r"(ret) : [ptr] "Q"(*(u64 *)ptr));
> + break;
> + default:
> + BUILD_BUG();
> + }
> +
> + return ret;
> +}
> +
> +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> +{
> + switch (size) {
> + case 1:
> + asm ("//__per_cpu_write_1\n"
> + "strb %w[val], %[ptr]\n" :
> + [ptr] "=Q"(*(u8 *)ptr) : [val] "r"(val));
> + break;
> + case 2:
> + asm ("//__per_cpu_write_2\n"
> + "strh %w[val], %[ptr]\n" :
> + [ptr] "=Q"(*(u16 *)ptr) : [val] "r"(val));
> + break;
> + case 4:
> + asm ("//__per_cpu_write_4\n"
> + "str %w[val], %[ptr]\n" :
> + [ptr] "=Q"(*(u32 *)ptr) : [val] "r"(val));
> + break;
> + case 8:
> + asm ("//__per_cpu_write_8\n"
> + "str %[val], %[ptr]\n" :
> + [ptr] "=Q"(*(u64 *)ptr) : [val] "r"(val));
> + break;
> + default:
> + BUILD_BUG();
> + }
> +}
Can you implement the read/write accessors with ACCESS_ONCE instead?
I think we're just after a single-copy atomic access without barrier
semantics, so that should work if you get your types right.
Will
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH V4] arm64: percpu: Implement this_cpu operations
2014-11-14 13:46 ` Will Deacon
@ 2014-11-14 15:03 ` Steve Capper
2014-11-17 10:40 ` Will Deacon
0 siblings, 1 reply; 12+ messages in thread
From: Steve Capper @ 2014-11-14 15:03 UTC (permalink / raw)
To: linux-arm-kernel
The generic this_cpu operations disable interrupts to ensure that the
requested operation is protected from pre-emption. For arm64, this is
overkill and can hurt throughput and latency.
This patch provides arm64 specific implementations for the this_cpu
operations. Rather than disable interrupts, we use the exclusive
monitor or atomic operations as appropriate.
The following operations are implemented: add, add_return, and, or,
read, write, xchg. We also wire up a cmpxchg implementation from
cmpxchg.h.
Testing was performed using the percpu_test module and hackbench on a
Juno board running 3.18-rc4.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
---
Changed in V4: rebased to allow for merge in Will's for-next branch.
(I had erroneously changed my cmpxchg_double patch).
Changed the read/write accessors to use ACCESS_ONCE rather than asm.
Changed in V3: use cmpxchg_local rather than cmpxchg for
this_cpu_cmpxchg.
Changed in V2: corrected address constraint for __percpu_read and
__percpu_write.
---
arch/arm64/include/asm/cmpxchg.h | 6 +-
arch/arm64/include/asm/percpu.h | 215 +++++++++++++++++++++++++++++++++++++++
2 files changed, 219 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 89e397b..cb95930 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -246,8 +246,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
__ret; \
})
-#define this_cpu_cmpxchg_8(ptr, o, n) \
- cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
cmpxchg_double_local(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 5279e57..b8b2c8a 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -44,6 +44,221 @@ static inline unsigned long __my_cpu_offset(void)
#endif /* CONFIG_SMP */
+#define PERCPU_OP(op, asm_op) \
+static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+{ \
+ unsigned long loop, ret; \
+ \
+ switch (size) { \
+ case 1: \
+ do { \
+ asm ("//__per_cpu_" #op "_1\n" \
+ "ldxrb %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrb %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u8 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 2: \
+ do { \
+ asm ("//__per_cpu_" #op "_2\n" \
+ "ldxrh %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrh %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u16 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 4: \
+ do { \
+ asm ("//__per_cpu_" #op "_4\n" \
+ "ldxr %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxr %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u32 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 8: \
+ do { \
+ asm ("//__per_cpu_" #op "_8\n" \
+ "ldxr %[ret], %[ptr]\n" \
+ #asm_op " %[ret], %[ret], %[val]\n" \
+ "stxr %w[loop], %[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u64 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ default: \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret; \
+}
+
+PERCPU_OP(add, add)
+PERCPU_OP(and, and)
+PERCPU_OP(or, orr)
+#undef PERCPU_OP
+
+static inline unsigned long __percpu_read(void *ptr, int size)
+{
+ unsigned long ret;
+
+ switch (size) {
+ case 1:
+ ret = ACCESS_ONCE(*(u8 *)ptr);
+ break;
+ case 2:
+ ret = ACCESS_ONCE(*(u16 *)ptr);
+ break;
+ case 4:
+ ret = ACCESS_ONCE(*(u32 *)ptr);
+ break;
+ case 8:
+ ret = ACCESS_ONCE(*(u64 *)ptr);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+static inline void __percpu_write(void *ptr, unsigned long val, int size)
+{
+ switch (size) {
+ case 1:
+ ACCESS_ONCE(*((u8 *)ptr)) = (u8) val;
+ break;
+ case 2:
+ ACCESS_ONCE(*((u16 *)ptr)) = (u16) val;
+ break;
+ case 4:
+ ACCESS_ONCE(*((u32 *)ptr)) = (u32) val;
+ break;
+ case 8:
+ ACCESS_ONCE(*((u64 *)ptr)) = (u64) val;
+ break;
+ default:
+ BUILD_BUG();
+ }
+}
+
+static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
+ int size)
+{
+ unsigned long ret, loop;
+
+ switch (size) {
+ case 1:
+ do {
+ asm ("//__percpu_xchg_1\n"
+ "ldxrb %w[ret], %[ptr]\n"
+ "stxrb %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u8 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 2:
+ do {
+ asm ("//__percpu_xchg_2\n"
+ "ldxrh %w[ret], %[ptr]\n"
+ "stxrh %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u16 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 4:
+ do {
+ asm ("//__percpu_xchg_4\n"
+ "ldxr %w[ret], %[ptr]\n"
+ "stxr %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u32 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 8:
+ do {
+ asm ("//__percpu_xchg_8\n"
+ "ldxr %[ret], %[ptr]\n"
+ "stxr %w[loop], %[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u64 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+#define _percpu_add(pcp, val) \
+ __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
+
+#define _percpu_and(pcp, val) \
+ __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_or(pcp, val) \
+ __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_read(pcp) (typeof(pcp)) \
+ (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
+
+#define _percpu_write(pcp, val) \
+ __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
+
+#define _percpu_xchg(pcp, val) (typeof(pcp)) \
+ (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
+
+#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_read_1(pcp) _percpu_read(pcp)
+#define this_cpu_read_2(pcp) _percpu_read(pcp)
+#define this_cpu_read_4(pcp) _percpu_read(pcp)
+#define this_cpu_read_8(pcp) _percpu_read(pcp)
+
+#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
+
+#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
--
1.9.3
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH V4] arm64: percpu: Implement this_cpu operations
2014-11-14 15:03 ` [PATCH V4] " Steve Capper
@ 2014-11-17 10:40 ` Will Deacon
2014-11-19 15:49 ` Steve Capper
0 siblings, 1 reply; 12+ messages in thread
From: Will Deacon @ 2014-11-17 10:40 UTC (permalink / raw)
To: linux-arm-kernel
Hi Steve,
On Fri, Nov 14, 2014 at 03:03:33PM +0000, Steve Capper wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
>
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
>
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
>
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc4.
Looks good. I notice that this change drops the compiler barriers too,
which we used to get via local_irq_{enable/disable}. I *think* that's fine
(at least, I couldn't find a place that breaks due to that) but it would be
nice to know that it was deliberate :)
> +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> +{
> + switch (size) {
> + case 1:
> + ACCESS_ONCE(*((u8 *)ptr)) = (u8) val;
> + break;
> + case 2:
> + ACCESS_ONCE(*((u16 *)ptr)) = (u16) val;
> + break;
> + case 4:
> + ACCESS_ONCE(*((u32 *)ptr)) = (u32) val;
> + break;
> + case 8:
> + ACCESS_ONCE(*((u64 *)ptr)) = (u64) val;
> + break;
I think you've gone a bit overboard with brackets and spacing here. Can't
you just do something like:
ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
Anyway:
Reviewed-by: Will Deacon <will.deacon@arm.com>
Will
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH V4] arm64: percpu: Implement this_cpu operations
2014-11-17 10:40 ` Will Deacon
@ 2014-11-19 15:49 ` Steve Capper
2014-11-19 16:53 ` [PATCH V5] " Steve Capper
0 siblings, 1 reply; 12+ messages in thread
From: Steve Capper @ 2014-11-19 15:49 UTC (permalink / raw)
To: linux-arm-kernel
On Mon, Nov 17, 2014 at 10:40:11AM +0000, Will Deacon wrote:
> Hi Steve,
Hey Will,
[sorry for the late reply, was recovering from plague]
>
> On Fri, Nov 14, 2014 at 03:03:33PM +0000, Steve Capper wrote:
> > The generic this_cpu operations disable interrupts to ensure that the
> > requested operation is protected from pre-emption. For arm64, this is
> > overkill and can hurt throughput and latency.
> >
> > This patch provides arm64 specific implementations for the this_cpu
> > operations. Rather than disable interrupts, we use the exclusive
> > monitor or atomic operations as appropriate.
> >
> > The following operations are implemented: add, add_return, and, or,
> > read, write, xchg. We also wire up a cmpxchg implementation from
> > cmpxchg.h.
> >
> > Testing was performed using the percpu_test module and hackbench on a
> > Juno board running 3.18-rc4.
>
> Looks good. I notice that this change drops the compiler barriers too,
> which we used to get via local_irq_{enable/disable}. I *think* that's fine
> (at least, I couldn't find a place that breaks due to that) but it would be
> nice to know that it was deliberate :)
Yes it was deliberate. :-)
My understanding of the this_cpu... set of accessors is that they are
there to solely protect the access with respect to pre-emption, not other
CPUs, thus we shouldn't need barriers?
>
> > +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> > +{
> > + switch (size) {
> > + case 1:
> > + ACCESS_ONCE(*((u8 *)ptr)) = (u8) val;
> > + break;
> > + case 2:
> > + ACCESS_ONCE(*((u16 *)ptr)) = (u16) val;
> > + break;
> > + case 4:
> > + ACCESS_ONCE(*((u32 *)ptr)) = (u32) val;
> > + break;
> > + case 8:
> > + ACCESS_ONCE(*((u64 *)ptr)) = (u64) val;
> > + break;
>
> I think you've gone a bit overboard with brackets and spacing here. Can't
> you just do something like:
>
> ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
Yeah, I went a little paranthesis crazy there, I'll correct this lest
it gets confused with LISP.
>
> Anyway:
>
> Reviewed-by: Will Deacon <will.deacon@arm.com>
Ta Will,
I'll send out a V5 with the parantheses sanetised.
Cheers,
--
Steve
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH V5] arm64: percpu: Implement this_cpu operations
2014-11-19 15:49 ` Steve Capper
@ 2014-11-19 16:53 ` Steve Capper
0 siblings, 0 replies; 12+ messages in thread
From: Steve Capper @ 2014-11-19 16:53 UTC (permalink / raw)
To: linux-arm-kernel
The generic this_cpu operations disable interrupts to ensure that the
requested operation is protected from pre-emption. For arm64, this is
overkill and can hurt throughput and latency.
This patch provides arm64 specific implementations for the this_cpu
operations. Rather than disable interrupts, we use the exclusive
monitor or atomic operations as appropriate.
The following operations are implemented: add, add_return, and, or,
read, write, xchg. We also wire up a cmpxchg implementation from
cmpxchg.h.
Testing was performed using the percpu_test module and hackbench on a
Juno board running 3.18-rc4.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
Reviewed-by: Will Deacon <will.deacon@arm.com>
---
Changed in V5: spacing and parantheses sanetised for read/write
accessors.
Changed in V4: rebased to allow for merge in Will's for-next branch.
(I had erroneously changed my cmpxchg_double patch).
Changed the read/write accessors to use ACCESS_ONCE rather than asm.
Changed in V3: use cmpxchg_local rather than cmpxchg for
this_cpu_cmpxchg.
Changed in V2: corrected address constraint for __percpu_read and
__percpu_write.
---
arch/arm64/include/asm/cmpxchg.h | 6 +-
arch/arm64/include/asm/percpu.h | 215 +++++++++++++++++++++++++++++++++++++++
2 files changed, 219 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 89e397b..cb95930 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -246,8 +246,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
__ret; \
})
-#define this_cpu_cmpxchg_8(ptr, o, n) \
- cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n)
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
cmpxchg_double_local(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 5279e57..09da25b 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -44,6 +44,221 @@ static inline unsigned long __my_cpu_offset(void)
#endif /* CONFIG_SMP */
+#define PERCPU_OP(op, asm_op) \
+static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+{ \
+ unsigned long loop, ret; \
+ \
+ switch (size) { \
+ case 1: \
+ do { \
+ asm ("//__per_cpu_" #op "_1\n" \
+ "ldxrb %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrb %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u8 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 2: \
+ do { \
+ asm ("//__per_cpu_" #op "_2\n" \
+ "ldxrh %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxrh %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u16 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 4: \
+ do { \
+ asm ("//__per_cpu_" #op "_4\n" \
+ "ldxr %w[ret], %[ptr]\n" \
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
+ "stxr %w[loop], %w[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u32 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ case 8: \
+ do { \
+ asm ("//__per_cpu_" #op "_8\n" \
+ "ldxr %[ret], %[ptr]\n" \
+ #asm_op " %[ret], %[ret], %[val]\n" \
+ "stxr %w[loop], %[ret], %[ptr]\n" \
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
+ [ptr] "+Q"(*(u64 *)ptr) \
+ : [val] "Ir" (val)); \
+ } while (loop); \
+ break; \
+ default: \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret; \
+}
+
+PERCPU_OP(add, add)
+PERCPU_OP(and, and)
+PERCPU_OP(or, orr)
+#undef PERCPU_OP
+
+static inline unsigned long __percpu_read(void *ptr, int size)
+{
+ unsigned long ret;
+
+ switch (size) {
+ case 1:
+ ret = ACCESS_ONCE(*(u8 *)ptr);
+ break;
+ case 2:
+ ret = ACCESS_ONCE(*(u16 *)ptr);
+ break;
+ case 4:
+ ret = ACCESS_ONCE(*(u32 *)ptr);
+ break;
+ case 8:
+ ret = ACCESS_ONCE(*(u64 *)ptr);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+static inline void __percpu_write(void *ptr, unsigned long val, int size)
+{
+ switch (size) {
+ case 1:
+ ACCESS_ONCE(*(u8 *)ptr) = (u8)val;
+ break;
+ case 2:
+ ACCESS_ONCE(*(u16 *)ptr) = (u16)val;
+ break;
+ case 4:
+ ACCESS_ONCE(*(u32 *)ptr) = (u32)val;
+ break;
+ case 8:
+ ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
+ break;
+ default:
+ BUILD_BUG();
+ }
+}
+
+static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
+ int size)
+{
+ unsigned long ret, loop;
+
+ switch (size) {
+ case 1:
+ do {
+ asm ("//__percpu_xchg_1\n"
+ "ldxrb %w[ret], %[ptr]\n"
+ "stxrb %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u8 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 2:
+ do {
+ asm ("//__percpu_xchg_2\n"
+ "ldxrh %w[ret], %[ptr]\n"
+ "stxrh %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u16 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 4:
+ do {
+ asm ("//__percpu_xchg_4\n"
+ "ldxr %w[ret], %[ptr]\n"
+ "stxr %w[loop], %w[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u32 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ case 8:
+ do {
+ asm ("//__percpu_xchg_8\n"
+ "ldxr %[ret], %[ptr]\n"
+ "stxr %w[loop], %[val], %[ptr]\n"
+ : [loop] "=&r"(loop), [ret] "=&r"(ret),
+ [ptr] "+Q"(*(u64 *)ptr)
+ : [val] "r" (val));
+ } while (loop);
+ break;
+ default:
+ BUILD_BUG();
+ }
+
+ return ret;
+}
+
+#define _percpu_add(pcp, val) \
+ __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
+
+#define _percpu_and(pcp, val) \
+ __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_or(pcp, val) \
+ __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
+
+#define _percpu_read(pcp) (typeof(pcp)) \
+ (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
+
+#define _percpu_write(pcp, val) \
+ __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
+
+#define _percpu_xchg(pcp, val) (typeof(pcp)) \
+ (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
+
+#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_read_1(pcp) _percpu_read(pcp)
+#define this_cpu_read_2(pcp) _percpu_read(pcp)
+#define this_cpu_read_4(pcp) _percpu_read(pcp)
+#define this_cpu_read_8(pcp) _percpu_read(pcp)
+
+#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
+#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
+
+#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
--
1.9.3
^ permalink raw reply related [flat|nested] 12+ messages in thread
end of thread, other threads:[~2014-11-19 16:53 UTC | newest]
Thread overview: 12+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2014-11-06 11:12 [PATCH] arm64: percpu: Implement this_cpu operations Steve Capper
2014-11-06 11:26 ` Ard Biesheuvel
2014-11-06 11:52 ` Steve Capper
2014-11-06 12:23 ` [PATCH V2] " Steve Capper
2014-11-06 12:27 ` [PATCH] " Will Deacon
2014-11-07 13:52 ` Steve Capper
2014-11-14 11:37 ` [PATCH V3] " Steve Capper
2014-11-14 13:46 ` Will Deacon
2014-11-14 15:03 ` [PATCH V4] " Steve Capper
2014-11-17 10:40 ` Will Deacon
2014-11-19 15:49 ` Steve Capper
2014-11-19 16:53 ` [PATCH V5] " Steve Capper
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).