Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Peter Zijlstra <peterz@infradead.org>]

On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> From: wangrui <wangrui@loongson.cn>
> 
> This patch introduce a new atomic primitive 'and_or', It may be have three
> types of implemeations:
> 
>  * The generic implementation is based on arch_cmpxchg.
>  * The hardware supports atomic 'and_or' of single instruction.
>  * The hardware supports LL/SC style atomic operations:
> 
>    1:  ll  v1, mem
>        and t1, v1, arg1
>        or  t1, t1, arg2
>        sc  t1, mem
>        beq t1, 0, 1b
> 
> Now that all the architectures have implemented it.
> 
> Signed-by-off: Rui Wang <wangrui@loongson.cn>
> Signed-by-off: hev <r@hev.cc>
> ---

>  include/asm-generic/atomic-instrumented.h | 28 ++++++++++++
>  include/asm-generic/atomic.h              | 29 +++++++++++++
>  include/linux/atomic-arch-fallback.h      | 42 ++++++++++++++++++
>  22 files changed, 588 insertions(+)

> diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h
> index bc45af52c93b..231a8386ac80 100644
> --- a/include/asm-generic/atomic-instrumented.h
> +++ b/include/asm-generic/atomic-instrumented.h
> @@ -441,6 +441,34 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v)
>  	return arch_atomic_fetch_xor_relaxed(i, v);
>  }
>  
> +static __always_inline int
> +atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_acquire(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_release(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_relaxed(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +
>  static __always_inline int
>  atomic_xchg(atomic_t *v, int i)
>  {
> diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
> index 04b8be9f1a77..474e8cd8e58d 100644
> --- a/include/asm-generic/atomic.h
> +++ b/include/asm-generic/atomic.h
> @@ -50,6 +50,18 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return c;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	int c, old;							\
> +									\
> +	c = v->counter;							\
> +	while ((old = arch_cmpxchg(&v->counter, c, (c c_op1 i) c_op2 j)) != c)	\
> +		c = old;						\
> +									\
> +	return c;							\
> +}
> +
>  #else
>  
>  #include <linux/irqflags.h>
> @@ -91,6 +103,20 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return ret;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned long flags;						\
> +	int ret;							\
> +									\
> +	raw_local_irq_save(flags);					\
> +	ret = v->counter;						\
> +	v->counter = (v->counter c_op1 i) c_op2 j;			\
> +	raw_local_irq_restore(flags);					\
> +									\
> +	return ret;							\
> +}
> +
>  #endif /* CONFIG_SMP */
>  
>  ATOMIC_OP_RETURN(add, +)
> @@ -101,6 +127,7 @@ ATOMIC_FETCH_OP(sub, -)
>  ATOMIC_FETCH_OP(and, &)
>  ATOMIC_FETCH_OP(or, |)
>  ATOMIC_FETCH_OP(xor, ^)
> +ATOMIC_FETCH_OP2(and_or, &, |)
>  
>  ATOMIC_OP(add, +)
>  ATOMIC_OP(sub, -)
> @@ -109,6 +136,7 @@ ATOMIC_OP(or, |)
>  ATOMIC_OP(xor, ^)
>  
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> @@ -120,6 +148,7 @@ ATOMIC_OP(xor, ^)
>  #define arch_atomic_fetch_and			generic_atomic_fetch_and
>  #define arch_atomic_fetch_or			generic_atomic_fetch_or
>  #define arch_atomic_fetch_xor			generic_atomic_fetch_xor
> +#define arch_atomic_fetch_and_or		generic_atomic_fetch_and_or
>  
>  #define arch_atomic_add				generic_atomic_add
>  #define arch_atomic_sub				generic_atomic_sub
> diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h
> index a3dba31df01e..92043a8d5b79 100644
> --- a/include/linux/atomic-arch-fallback.h
> +++ b/include/linux/atomic-arch-fallback.h
> @@ -891,6 +891,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v)
>  
>  #endif /* arch_atomic_fetch_xor_relaxed */
>  
> +#ifndef arch_atomic_fetch_and_or_relaxed
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or
> +#else /* arch_atomic_fetch_and_or_relaxed */
> +
> +#ifndef arch_atomic_fetch_and_or_acquire
> +static __always_inline int
> +arch_atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> +	int ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> +	__atomic_acquire_fence();
> +	return ret;
> +}
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or_acquire
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or_release
> +static __always_inline int
> +arch_atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> +	__atomic_release_fence();
> +	return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or_release
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or
> +static __always_inline int
> +arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	int ret;
> +	__atomic_pre_full_fence();
> +	ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> +	__atomic_post_full_fence();
> +	return ret;
> +}
> +#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
> +#endif
> +
> +#endif /* arch_atomic_fetch_and_or_relaxed */
> +
>  #ifndef arch_atomic_xchg_relaxed
>  #define arch_atomic_xchg_acquire arch_atomic_xchg
>  #define arch_atomic_xchg_release arch_atomic_xchg

Urgh.. please start from something like the below and then run:

  scripts/atomic/gen-atomics.sh

The below isn't quite right, because it'll use try_cmpxchg() for
atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
not entirely sure how to make that happen in a hurry.

---

diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
index fbee2f6190d9..3aaa0caa6b2d 100755
--- a/scripts/atomic/atomics.tbl
+++ b/scripts/atomic/atomics.tbl
@@ -39,3 +39,4 @@ inc_not_zero		b	v
 inc_unless_negative	b	v
 dec_unless_positive	b	v
 dec_if_positive		i	v
+andnot_or		vF	v	i:m	i:o
diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
new file mode 100644
index 000000000000..f50e78d6c53a
--- /dev/null
+++ b/scripts/atomic/fallbacks/andnot_or
@@ -0,0 +1,15 @@
+cat <<EOF
+static __always_inline ${ret}
+arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
+{
+	${retstmt}({
+		${int} N, O = atomic_read(v);
+		do {
+			N = O;
+			N &= ~m;
+			N |= o;
+		} while (!arch_${atomic}_try_cmpxchg${order}(v, &O, N));
+		O;
+	});
+}
+EOF
diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
deleted file mode 100644
index e69de29bb2d1..000000000000