From: Peter Zijlstra <peterz@infradead.org>
To: torvalds@linux-foundation.org, mingo@kernel.org,
tglx@linutronix.de, will.deacon@arm.com,
paulmck@linux.vnet.ibm.com, boqun.feng@gmail.com,
waiman.long@hpe.com, fweisbec@gmail.com
Cc: linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org,
rth@twiddle.net, vgupta@synopsys.com, linux@arm.linux.org.uk,
egtvedt@samfundet.no, realmz6@gmail.com,
ysato@users.sourceforge.jp, rkuo@codeaurora.org,
tony.luck@intel.com, geert@linux-m68k.org,
james.hogan@imgtec.com, ralf@linux-mips.org, dhowells@redhat.com,
jejb@parisc-linux.org, mpe@ellerman.id.au,
schwidefsky@de.ibm.com, dalias@libc.org, davem@davemloft.net,
cmetcalf@mellanox.com, jcmvbkbc@gmail.com, arnd@arndb.de,
peterz@infradead.org, dbueso@suse.de, fengguang.wu@intel.com
Subject: [RFC][PATCH 21/31] locking,sparc: Implement atomic{,64}_fetch_{add,sub,and,or,xor}()
Date: Fri, 22 Apr 2016 11:04:34 +0200 [thread overview]
Message-ID: <20160422093924.408974831@infradead.org> (raw)
In-Reply-To: 20160422090413.393652501@infradead.org
[-- Attachment #1: peterz-atomic-fetch-sparc.patch --]
[-- Type: text/plain, Size: 7872 bytes --]
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/sparc/include/asm/atomic.h | 1
arch/sparc/include/asm/atomic_32.h | 15 +++++++--
arch/sparc/include/asm/atomic_64.h | 16 +++++++--
arch/sparc/lib/atomic32.c | 29 ++++++++++-------
arch/sparc/lib/atomic_64.S | 61 ++++++++++++++++++++++++++++++-------
arch/sparc/lib/ksyms.c | 17 +++++++---
6 files changed, 105 insertions(+), 34 deletions(-)
--- a/arch/sparc/include/asm/atomic.h
+++ b/arch/sparc/include/asm/atomic.h
@@ -5,4 +5,5 @@
#else
#include <asm/atomic_32.h>
#endif
+#define atomic_fetch_or atomic_fetch_or
#endif
--- a/arch/sparc/include/asm/atomic_32.h
+++ b/arch/sparc/include/asm/atomic_32.h
@@ -20,9 +20,10 @@
#define ATOMIC_INIT(i) { (i) }
int atomic_add_return(int, atomic_t *);
-void atomic_and(int, atomic_t *);
-void atomic_or(int, atomic_t *);
-void atomic_xor(int, atomic_t *);
+int atomic_fetch_add(int, atomic_t *);
+int atomic_fetch_and(int, atomic_t *);
+int atomic_fetch_or(int, atomic_t *);
+int atomic_fetch_xor(int, atomic_t *);
int atomic_cmpxchg(atomic_t *, int, int);
int atomic_xchg(atomic_t *, int);
int __atomic_add_unless(atomic_t *, int, int);
@@ -35,7 +36,15 @@ void atomic_set(atomic_t *, int);
#define atomic_inc(v) ((void)atomic_add_return( 1, (v)))
#define atomic_dec(v) ((void)atomic_add_return( -1, (v)))
+#define atomic_fetch_or atomic_fetch_or
+
+#define atomic_and(i, v) ((void)atomic_fetch_and((i), (v)))
+#define atomic_or(i, v) ((void)atomic_fetch_or((i), (v)))
+#define atomic_xor(i, v) ((void)atomic_fetch_xor((i), (v)))
+
#define atomic_sub_return(i, v) (atomic_add_return(-(int)(i), (v)))
+#define atomic_fetch_sub(i, v) (atomic_fetch_add (-(int)(i), (v)))
+
#define atomic_inc_return(v) (atomic_add_return( 1, (v)))
#define atomic_dec_return(v) (atomic_add_return( -1, (v)))
--- a/arch/sparc/include/asm/atomic_64.h
+++ b/arch/sparc/include/asm/atomic_64.h
@@ -28,16 +28,24 @@ void atomic64_##op(long, atomic64_t *);
int atomic_##op##_return(int, atomic_t *); \
long atomic64_##op##_return(long, atomic64_t *);
-#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
+#define ATOMIC_FETCH_OP(op) \
+int atomic_fetch_##op(int, atomic_t *); \
+long atomic64_fetch_##op(long, atomic64_t *);
+
+#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
ATOMIC_OPS(add)
ATOMIC_OPS(sub)
-ATOMIC_OP(and)
-ATOMIC_OP(or)
-ATOMIC_OP(xor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
+
+ATOMIC_OPS(and)
+ATOMIC_OPS(or)
+ATOMIC_OPS(xor)
#undef ATOMIC_OPS
+#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
--- a/arch/sparc/lib/atomic32.c
+++ b/arch/sparc/lib/atomic32.c
@@ -27,39 +27,44 @@ static DEFINE_SPINLOCK(dummy);
#endif /* SMP */
-#define ATOMIC_OP_RETURN(op, c_op) \
-int atomic_##op##_return(int i, atomic_t *v) \
+#define ATOMIC_FETCH_OP(op, c_op) \
+int atomic_fetch_##op(int i, atomic_t *v) \
{ \
int ret; \
unsigned long flags; \
spin_lock_irqsave(ATOMIC_HASH(v), flags); \
\
- ret = (v->counter c_op i); \
+ ret = v->counter; \
+ v->counter c_op i; \
\
spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
return ret; \
} \
-EXPORT_SYMBOL(atomic_##op##_return);
+EXPORT_SYMBOL(atomic_fetch_##op);
-#define ATOMIC_OP(op, c_op) \
-void atomic_##op(int i, atomic_t *v) \
+#define ATOMIC_OP_RETURN(op, c_op) \
+int atomic_##op##_return(int i, atomic_t *v) \
{ \
+ int ret; \
unsigned long flags; \
spin_lock_irqsave(ATOMIC_HASH(v), flags); \
\
- v->counter c_op i; \
+ ret = (v->counter c_op i); \
\
spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
+ return ret; \
} \
-EXPORT_SYMBOL(atomic_##op);
+EXPORT_SYMBOL(atomic_##op##_return);
ATOMIC_OP_RETURN(add, +=)
-ATOMIC_OP(and, &=)
-ATOMIC_OP(or, |=)
-ATOMIC_OP(xor, ^=)
+ATOMIC_FETCH_OP(add, +=)
+ATOMIC_FETCH_OP(and, &=)
+ATOMIC_FETCH_OP(or, |=)
+ATOMIC_FETCH_OP(xor, ^=)
+
+#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
-#undef ATOMIC_OP
int atomic_xchg(atomic_t *v, int new)
{
--- a/arch/sparc/lib/atomic_64.S
+++ b/arch/sparc/lib/atomic_64.S
@@ -9,10 +9,11 @@
.text
- /* Two versions of the atomic routines, one that
+ /* Three versions of the atomic routines, one that
* does not return a value and does not perform
- * memory barriers, and a second which returns
- * a value and does the barriers.
+ * memory barriers, and a two which return
+ * a value, the new and old value resp. and does the
+ * barriers.
*/
#define ATOMIC_OP(op) \
@@ -43,15 +44,34 @@ ENTRY(atomic_##op##_return) /* %o0 = inc
2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic_##op##_return);
-#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
+#define ATOMIC_FETCH_OP(op) \
+ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
+ BACKOFF_SETUP(%o2); \
+1: lduw [%o1], %g1; \
+ op %g1, %o0, %g7; \
+ cas [%o1], %g1, %g7; \
+ cmp %g1, %g7; \
+ bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
+ nop; \
+ retl; \
+ sra %g1, 0, %o0; \
+2: BACKOFF_SPIN(%o2, %o3, 1b); \
+ENDPROC(atomic_fetch_##op);
+
+#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
ATOMIC_OPS(add)
ATOMIC_OPS(sub)
-ATOMIC_OP(and)
-ATOMIC_OP(or)
-ATOMIC_OP(xor)
#undef ATOMIC_OPS
+#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
+
+ATOMIC_OPS(and)
+ATOMIC_OPS(or)
+ATOMIC_OPS(xor)
+
+#undef ATOMIC_OPS
+#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
@@ -83,15 +103,34 @@ ENTRY(atomic64_##op##_return) /* %o0 = i
2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic64_##op##_return);
-#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op)
+#define ATOMIC64_FETCH_OP(op) \
+ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
+ BACKOFF_SETUP(%o2); \
+1: ldx [%o1], %g1; \
+ op %g1, %o0, %g7; \
+ casx [%o1], %g1, %g7; \
+ cmp %g1, %g7; \
+ bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
+ nop; \
+ retl; \
+ mov %g1, %o0; \
+2: BACKOFF_SPIN(%o2, %o3, 1b); \
+ENDPROC(atomic64_fetch_##op);
+
+#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op)
ATOMIC64_OPS(add)
ATOMIC64_OPS(sub)
-ATOMIC64_OP(and)
-ATOMIC64_OP(or)
-ATOMIC64_OP(xor)
#undef ATOMIC64_OPS
+#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op)
+
+ATOMIC64_OPS(and)
+ATOMIC64_OPS(or)
+ATOMIC64_OPS(xor)
+
+#undef ATOMIC64_OPS
+#undef ATOMIC64_FETCH_OP
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
--- a/arch/sparc/lib/ksyms.c
+++ b/arch/sparc/lib/ksyms.c
@@ -107,15 +107,24 @@ EXPORT_SYMBOL(atomic64_##op);
EXPORT_SYMBOL(atomic_##op##_return); \
EXPORT_SYMBOL(atomic64_##op##_return);
-#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
+#define ATOMIC_FETCH_OP(op) \
+EXPORT_SYMBOL(atomic_fetch_##op); \
+EXPORT_SYMBOL(atomic64_fetch_##op);
+
+#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
ATOMIC_OPS(add)
ATOMIC_OPS(sub)
-ATOMIC_OP(and)
-ATOMIC_OP(or)
-ATOMIC_OP(xor)
#undef ATOMIC_OPS
+#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
+
+ATOMIC_OPS(and)
+ATOMIC_OPS(or)
+ATOMIC_OPS(xor)
+
+#undef ATOMIC_OPS
+#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
next prev parent reply other threads:[~2016-04-22 9:42 UTC|newest]
Thread overview: 106+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-04-22 9:04 [RFC][PATCH 00/31] implement atomic_fetch_$op Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 01/31] locking: Flip arguments to atomic_fetch_or Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 10:54 ` Will Deacon
2016-04-22 11:09 ` Geert Uytterhoeven
2016-04-22 11:09 ` Geert Uytterhoeven
2016-04-22 14:18 ` Peter Zijlstra
2016-04-22 14:18 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 02/31] locking,alpha: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 16:57 ` Richard Henderson
2016-04-23 1:55 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 03/31] locking,arc: Implement atomic_fetch_{add,sub,and,andnot,or,xor}() Peter Zijlstra
2016-04-22 10:50 ` Vineet Gupta
2016-04-22 10:50 ` Vineet Gupta
2016-04-22 14:16 ` Peter Zijlstra
2016-04-22 14:16 ` Peter Zijlstra
2016-04-25 4:26 ` Vineet Gupta
2016-04-25 4:26 ` Vineet Gupta
2016-04-22 14:26 ` Peter Zijlstra
2016-04-22 14:26 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 04/31] locking,arm: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 11:35 ` Will Deacon
2016-04-22 11:35 ` Will Deacon
2016-04-22 9:04 ` [RFC][PATCH 05/31] locking,arm64: " Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 11:08 ` Will Deacon
2016-04-22 14:23 ` Will Deacon
2016-04-22 14:23 ` Will Deacon
2016-04-22 9:04 ` [RFC][PATCH 06/31] locking,avr32: Implement atomic_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 11:58 ` Hans-Christian Noren Egtvedt
2016-04-22 11:58 ` Hans-Christian Noren Egtvedt
2016-04-22 9:04 ` [RFC][PATCH 07/31] locking,blackfin: " Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 08/31] locking,frv: Implement atomic{,64}_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 09/31] locking,h8300: Implement atomic_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 10/31] locking,hexagon: " Peter Zijlstra
2016-04-23 2:16 ` Peter Zijlstra
2016-04-26 0:39 ` Richard Kuo
2016-04-22 9:04 ` [RFC][PATCH 11/31] locking,ia64: Implement atomic{,64}_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 12/31] locking,m32r: Implement atomic_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 13/31] locking,m68k: " Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 14/31] locking,metag: " Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-30 0:20 ` James Hogan
2016-05-02 8:15 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 15/31] locking,mips: Implement atomic{,64}_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 16/31] locking,mn10300: Implement atomic_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 17/31] locking,parisc: Implement atomic{,64}_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 18/31] locking,powerpc: Implement atomic{,64}_fetch_{add,sub,and,or,xor}{,_relaxed,_acquire,_release}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 16:41 ` Boqun Feng
2016-04-23 2:31 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 19/31] locking,s390: Implement atomic{,64}_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-25 8:06 ` Martin Schwidefsky
2016-04-25 8:06 ` Martin Schwidefsky
2016-04-25 8:26 ` Peter Zijlstra
2016-04-25 8:26 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 20/31] locking,sh: Implement atomic_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra [this message]
2016-04-22 9:04 ` [RFC][PATCH 22/31] locking,tile: Implement atomic{,64}_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-25 21:10 ` Chris Metcalf
[not found] ` <571E840A.8090703@mellanox.com>
2016-04-26 15:28 ` Peter Zijlstra
2016-04-26 15:32 ` Chris Metcalf
2016-04-22 9:04 ` [RFC][PATCH 23/31] locking,x86: " Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 24/31] locking,xtensa: Implement atomic_fetch_{add,sub,and,or,xor}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 25/31] locking: Fix atomic64_relaxed bits Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 26/31] locking: Implement atomic{,64,_long}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 27/31] locking: Remove linux/atomic.h:atomic_fetch_or Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 13:02 ` Will Deacon
2016-04-22 14:21 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 28/31] locking: Remove the deprecated atomic_{set,clear}_mask() functions Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 29/31] locking,alpha: Convert to _relaxed atomics Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 30/31] locking,mips: " Peter Zijlstra
2016-04-22 9:04 ` Peter Zijlstra
2016-04-22 9:04 ` [RFC][PATCH 31/31] locking,qrwlock: Employ atomic_fetch_add_acquire() Peter Zijlstra
2016-04-22 14:25 ` Waiman Long
2016-04-22 9:44 ` [RFC][PATCH 00/31] implement atomic_fetch_$op Peter Zijlstra
2016-04-22 9:44 ` Peter Zijlstra
2016-04-22 12:56 ` Fengguang Wu
2016-04-22 13:03 ` Will Deacon
2016-04-22 13:03 ` Will Deacon
2016-04-22 14:23 ` Peter Zijlstra
2016-04-23 1:59 ` Fengguang Wu
2016-04-22 18:35 ` Kalle Valo
2016-04-23 3:23 ` Fengguang Wu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20160422093924.408974831@infradead.org \
--to=peterz@infradead.org \
--cc=arnd@arndb.de \
--cc=boqun.feng@gmail.com \
--cc=cmetcalf@mellanox.com \
--cc=dalias@libc.org \
--cc=davem@davemloft.net \
--cc=dbueso@suse.de \
--cc=dhowells@redhat.com \
--cc=egtvedt@samfundet.no \
--cc=fengguang.wu@intel.com \
--cc=fweisbec@gmail.com \
--cc=geert@linux-m68k.org \
--cc=james.hogan@imgtec.com \
--cc=jcmvbkbc@gmail.com \
--cc=jejb@parisc-linux.org \
--cc=linux-arch@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux@arm.linux.org.uk \
--cc=mingo@kernel.org \
--cc=mpe@ellerman.id.au \
--cc=paulmck@linux.vnet.ibm.com \
--cc=ralf@linux-mips.org \
--cc=realmz6@gmail.com \
--cc=rkuo@codeaurora.org \
--cc=rth@twiddle.net \
--cc=schwidefsky@de.ibm.com \
--cc=tglx@linutronix.de \
--cc=tony.luck@intel.com \
--cc=torvalds@linux-foundation.org \
--cc=vgupta@synopsys.com \
--cc=waiman.long@hpe.com \
--cc=will.deacon@arm.com \
--cc=ysato@users.sourceforge.jp \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).