* [PATCH] csky: cmpxchg: Optimize with acquire & release
@ 2022-04-06 12:40 guoren
2022-04-11 9:37 ` Mark Rutland
0 siblings, 1 reply; 2+ messages in thread
From: guoren @ 2022-04-06 12:40 UTC (permalink / raw)
To: arnd, mark.rutland, peterz
Cc: linux-csky, linux-arch, linux-kernel, Guo Ren, Guo Ren
From: Guo Ren <guoren@linux.alibaba.com>
Optimize arch_xchg|cmpxchg|cmpxchg_local with ASM acquire|release
instructions instead of previous C based.
Signed-off-by: Guo Ren <guoren@linux.alibaba.com>
Signed-off-by: Guo Ren <guoren@kernel.org>
---
arch/csky/include/asm/barrier.h | 8 +-
arch/csky/include/asm/cmpxchg.h | 173 ++++++++++++++++++++++++++++++--
2 files changed, 172 insertions(+), 9 deletions(-)
diff --git a/arch/csky/include/asm/barrier.h b/arch/csky/include/asm/barrier.h
index f4045dd53e17..a075f17d02dd 100644
--- a/arch/csky/include/asm/barrier.h
+++ b/arch/csky/include/asm/barrier.h
@@ -37,6 +37,9 @@
* bar.brar
* bar.bwaw
*/
+#define ACQUIRE_FENCE ".long 0x8427c000\n"
+#define RELEASE_FENCE ".long 0x842ec000\n"
+
#define __bar_brw() asm volatile (".long 0x842cc000\n":::"memory")
#define __bar_br() asm volatile (".long 0x8424c000\n":::"memory")
#define __bar_bw() asm volatile (".long 0x8428c000\n":::"memory")
@@ -44,10 +47,10 @@
#define __bar_ar() asm volatile (".long 0x8421c000\n":::"memory")
#define __bar_aw() asm volatile (".long 0x8422c000\n":::"memory")
#define __bar_brwarw() asm volatile (".long 0x842fc000\n":::"memory")
-#define __bar_brarw() asm volatile (".long 0x8427c000\n":::"memory")
+#define __bar_brarw() asm volatile (ACQUIRE_FENCE:::"memory")
#define __bar_bwarw() asm volatile (".long 0x842bc000\n":::"memory")
#define __bar_brwar() asm volatile (".long 0x842dc000\n":::"memory")
-#define __bar_brwaw() asm volatile (".long 0x842ec000\n":::"memory")
+#define __bar_brwaw() asm volatile (RELEASE_FENCE:::"memory")
#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory")
#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory")
#define __bar_bwaw() asm volatile (".long 0x842ac000\n":::"memory")
@@ -56,7 +59,6 @@
#define __smp_rmb() __bar_brar()
#define __smp_wmb() __bar_bwaw()
-#define ACQUIRE_FENCE ".long 0x8427c000\n"
#define __smp_acquire_fence() __bar_brarw()
#define __smp_release_fence() __bar_brwaw()
diff --git a/arch/csky/include/asm/cmpxchg.h b/arch/csky/include/asm/cmpxchg.h
index d1bef11f8dc9..7baf9a706b5b 100644
--- a/arch/csky/include/asm/cmpxchg.h
+++ b/arch/csky/include/asm/cmpxchg.h
@@ -30,10 +30,88 @@ extern void __bad_xchg(void);
} \
__ret; \
})
-
#define arch_xchg_relaxed(ptr, x) \
(__xchg_relaxed((x), (ptr), sizeof(*(ptr))))
+#define __xchg_acquire(new, ptr, size) \
+({ \
+ __typeof__(ptr) __ptr = (ptr); \
+ __typeof__(new) __new = (new); \
+ __typeof__(*(ptr)) __ret; \
+ unsigned long tmp; \
+ switch (size) { \
+ case 4: \
+ asm volatile ( \
+ "1: ldex.w %0, (%3) \n" \
+ ACQUIRE_FENCE \
+ " mov %1, %2 \n" \
+ " stex.w %1, (%3) \n" \
+ " bez %1, 1b \n" \
+ : "=&r" (__ret), "=&r" (tmp) \
+ : "r" (__new), "r"(__ptr) \
+ :); \
+ break; \
+ default: \
+ __bad_xchg(); \
+ } \
+ __ret; \
+})
+#define arch_xchg_acquire(ptr, x) \
+ (__xchg_acquire((x), (ptr), sizeof(*(ptr))))
+
+#define __xchg_release(new, ptr, size) \
+({ \
+ __typeof__(ptr) __ptr = (ptr); \
+ __typeof__(new) __new = (new); \
+ __typeof__(*(ptr)) __ret; \
+ unsigned long tmp; \
+ switch (size) { \
+ case 4: \
+ asm volatile ( \
+ "1: ldex.w %0, (%3) \n" \
+ " mov %1, %2 \n" \
+ RELEASE_FENCE \
+ " stex.w %1, (%3) \n" \
+ " bez %1, 1b \n" \
+ : "=&r" (__ret), "=&r" (tmp) \
+ : "r" (__new), "r"(__ptr) \
+ :); \
+ break; \
+ default: \
+ __bad_xchg(); \
+ } \
+ __ret; \
+})
+#define arch_xchg_release(ptr, x) \
+ (__xchg_release((x), (ptr), sizeof(*(ptr))))
+
+#define __xchg(new, ptr, size) \
+({ \
+ __typeof__(ptr) __ptr = (ptr); \
+ __typeof__(new) __new = (new); \
+ __typeof__(*(ptr)) __ret; \
+ unsigned long tmp; \
+ switch (size) { \
+ case 4: \
+ asm volatile ( \
+ "1: ldex.w %0, (%3) \n" \
+ ACQUIRE_FENCE \
+ " mov %1, %2 \n" \
+ RELEASE_FENCE \
+ " stex.w %1, (%3) \n" \
+ " bez %1, 1b \n" \
+ : "=&r" (__ret), "=&r" (tmp) \
+ : "r" (__new), "r"(__ptr) \
+ :); \
+ break; \
+ default: \
+ __bad_xchg(); \
+ } \
+ __ret; \
+})
+#define arch_xchg(ptr, x) \
+ (__xchg((x), (ptr), sizeof(*(ptr))))
+
#define __cmpxchg_relaxed(ptr, old, new, size) \
({ \
__typeof__(ptr) __ptr = (ptr); \
@@ -60,19 +138,102 @@ extern void __bad_xchg(void);
} \
__ret; \
})
-
#define arch_cmpxchg_relaxed(ptr, o, n) \
(__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
-#define arch_cmpxchg(ptr, o, n) \
+#define __cmpxchg_acquire(ptr, old, new, size) \
+({ \
+ __typeof__(ptr) __ptr = (ptr); \
+ __typeof__(new) __new = (new); \
+ __typeof__(new) __tmp; \
+ __typeof__(old) __old = (old); \
+ __typeof__(*(ptr)) __ret; \
+ switch (size) { \
+ case 4: \
+ asm volatile ( \
+ "1: ldex.w %0, (%3) \n" \
+ ACQUIRE_FENCE \
+ " cmpne %0, %4 \n" \
+ " bt 2f \n" \
+ " mov %1, %2 \n" \
+ " stex.w %1, (%3) \n" \
+ " bez %1, 1b \n" \
+ "2: \n" \
+ : "=&r" (__ret), "=&r" (__tmp) \
+ : "r" (__new), "r"(__ptr), "r"(__old) \
+ :); \
+ break; \
+ default: \
+ __bad_xchg(); \
+ } \
+ __ret; \
+})
+#define arch_cmpxchg_acquire(ptr, o, n) \
+ (__cmpxchg_acquire((ptr), (o), (n), sizeof(*(ptr))))
+
+#define __cmpxchg_release(ptr, old, new, size) \
({ \
+ __typeof__(ptr) __ptr = (ptr); \
+ __typeof__(new) __new = (new); \
+ __typeof__(new) __tmp; \
+ __typeof__(old) __old = (old); \
__typeof__(*(ptr)) __ret; \
- __smp_release_fence(); \
- __ret = arch_cmpxchg_relaxed(ptr, o, n); \
- __smp_acquire_fence(); \
+ switch (size) { \
+ case 4: \
+ asm volatile ( \
+ "1: ldex.w %0, (%3) \n" \
+ " cmpne %0, %4 \n" \
+ " bt 2f \n" \
+ " mov %1, %2 \n" \
+ RELEASE_FENCE \
+ " stex.w %1, (%3) \n" \
+ " bez %1, 1b \n" \
+ "2: \n" \
+ : "=&r" (__ret), "=&r" (__tmp) \
+ : "r" (__new), "r"(__ptr), "r"(__old) \
+ :); \
+ break; \
+ default: \
+ __bad_xchg(); \
+ } \
__ret; \
})
+#define arch_cmpxchg_release(ptr, o, n) \
+ (__cmpxchg_release((ptr), (o), (n), sizeof(*(ptr))))
+#define __cmpxchg(ptr, old, new, size) \
+({ \
+ __typeof__(ptr) __ptr = (ptr); \
+ __typeof__(new) __new = (new); \
+ __typeof__(new) __tmp; \
+ __typeof__(old) __old = (old); \
+ __typeof__(*(ptr)) __ret; \
+ switch (size) { \
+ case 4: \
+ asm volatile ( \
+ "1: ldex.w %0, (%3) \n" \
+ ACQUIRE_FENCE \
+ " cmpne %0, %4 \n" \
+ " bt 2f \n" \
+ " mov %1, %2 \n" \
+ RELEASE_FENCE \
+ " stex.w %1, (%3) \n" \
+ " bez %1, 1b \n" \
+ "2: \n" \
+ : "=&r" (__ret), "=&r" (__tmp) \
+ : "r" (__new), "r"(__ptr), "r"(__old) \
+ :); \
+ break; \
+ default: \
+ __bad_xchg(); \
+ } \
+ __ret; \
+})
+#define arch_cmpxchg(ptr, o, n) \
+ (__cmpxchg((ptr), (o), (n), sizeof(*(ptr))))
+
+#define arch_cmpxchg_local(ptr, o, n) \
+ (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
#else
#include <asm-generic/cmpxchg.h>
#endif
--
2.25.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH] csky: cmpxchg: Optimize with acquire & release
2022-04-06 12:40 [PATCH] csky: cmpxchg: Optimize with acquire & release guoren
@ 2022-04-11 9:37 ` Mark Rutland
0 siblings, 0 replies; 2+ messages in thread
From: Mark Rutland @ 2022-04-11 9:37 UTC (permalink / raw)
To: guoren; +Cc: arnd, peterz, linux-csky, linux-arch, linux-kernel, Guo Ren
Hi Guo,
On Wed, Apr 06, 2022 at 08:40:56PM +0800, guoren@kernel.org wrote:
> From: Guo Ren <guoren@linux.alibaba.com>
>
> Optimize arch_xchg|cmpxchg|cmpxchg_local with ASM acquire|release
> instructions instead of previous C based.
>
> Signed-off-by: Guo Ren <guoren@linux.alibaba.com>
> Signed-off-by: Guo Ren <guoren@kernel.org>
> ---
> arch/csky/include/asm/barrier.h | 8 +-
> arch/csky/include/asm/cmpxchg.h | 173 ++++++++++++++++++++++++++++++--
> 2 files changed, 172 insertions(+), 9 deletions(-)
>
> diff --git a/arch/csky/include/asm/barrier.h b/arch/csky/include/asm/barrier.h
> index f4045dd53e17..a075f17d02dd 100644
> --- a/arch/csky/include/asm/barrier.h
> +++ b/arch/csky/include/asm/barrier.h
> @@ -37,6 +37,9 @@
> * bar.brar
> * bar.bwaw
> */
> +#define ACQUIRE_FENCE ".long 0x8427c000\n"
> +#define RELEASE_FENCE ".long 0x842ec000\n"
[...]
> +#define __xchg(new, ptr, size) \
> +({ \
> + __typeof__(ptr) __ptr = (ptr); \
> + __typeof__(new) __new = (new); \
> + __typeof__(*(ptr)) __ret; \
> + unsigned long tmp; \
> + switch (size) { \
> + case 4: \
> + asm volatile ( \
> + "1: ldex.w %0, (%3) \n" \
> + ACQUIRE_FENCE \
> + " mov %1, %2 \n" \
> + RELEASE_FENCE \
> + " stex.w %1, (%3) \n" \
> + " bez %1, 1b \n" \
> + : "=&r" (__ret), "=&r" (tmp) \
> + : "r" (__new), "r"(__ptr) \
> + :); \
As with the riscv case, I believe this suffers the problem described in:
8e86f0b409a44193 ("arm64: atomics: fix use of acquire + release for full barrier semantics")
... and does not provide FULL ordering semantics.
[...]
> +#define __cmpxchg(ptr, old, new, size) \
> +({ \
> + __typeof__(ptr) __ptr = (ptr); \
> + __typeof__(new) __new = (new); \
> + __typeof__(new) __tmp; \
> + __typeof__(old) __old = (old); \
> + __typeof__(*(ptr)) __ret; \
> + switch (size) { \
> + case 4: \
> + asm volatile ( \
> + "1: ldex.w %0, (%3) \n" \
> + ACQUIRE_FENCE \
> + " cmpne %0, %4 \n" \
> + " bt 2f \n" \
> + " mov %1, %2 \n" \
> + RELEASE_FENCE \
> + " stex.w %1, (%3) \n" \
> + " bez %1, 1b \n" \
> + "2: \n" \
> + : "=&r" (__ret), "=&r" (__tmp) \
> + : "r" (__new), "r"(__ptr), "r"(__old) \
> + :); \
Likewise.
Thanks,
Mark.
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2022-04-11 9:38 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-04-06 12:40 [PATCH] csky: cmpxchg: Optimize with acquire & release guoren
2022-04-11 9:37 ` Mark Rutland
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox