From: Christoph Lameter <cl@linux.com>
To: akpm@linux-foundation.org
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: linux-kernel@vger.kernel.org
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Tejun Heo <tj@kernel.org>
Subject: [thisops uV2 08/10] percpu: generic this_cpu_cmpxchg() and this_cpu_cmpxchg_double support
Date: Fri, 26 Nov 2010 15:09:45 -0600 [thread overview]
Message-ID: <20101126210953.591278825@linux.com> (raw)
In-Reply-To: 20101126210937.383047168@linux.com
[-- Attachment #1: this_cpu_cmpxchg --]
[-- Type: text/plain, Size: 8222 bytes --]
Provide arch code to create the (local atomic) instructions.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/percpu.h | 179 ++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 178 insertions(+), 1 deletion(-)
Index: linux-2.6/include/linux/percpu.h
===================================================================
--- linux-2.6.orig/include/linux/percpu.h 2010-11-24 10:37:57.000000000 -0600
+++ linux-2.6/include/linux/percpu.h 2010-11-24 10:38:11.000000000 -0600
@@ -254,6 +254,21 @@ extern void __bad_size_call_parameter(vo
pscr_ret__; \
})
+/* Special handling for cmpxchg_double */
+#define __pcpu_size_call_return_int(stem, pcp, ...) \
+({ int pscr_ret__; \
+ __verify_pcpu_ptr(pcp); \
+ switch(sizeof(*pcp)) { \
+ case 1: pscr_ret__ = stem##1(pcp, __VA_ARGS__);break; \
+ case 2: pscr_ret__ = stem##2(pcp, __VA_ARGS__);break; \
+ case 4: pscr_ret__ = stem##4(pcp, __VA_ARGS__);break; \
+ case 8: pscr_ret__ = stem##8(pcp, __VA_ARGS__);break; \
+ default: \
+ __bad_size_call_parameter();break; \
+ } \
+ pscr_ret__; \
+})
+
#define __pcpu_size_call(stem, variable, ...) \
do { \
__verify_pcpu_ptr(&(variable)); \
@@ -317,6 +332,134 @@ do { \
# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
#endif
+#define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
+({ typeof(pcp) ret__; \
+ preempt_disable(); \
+ ret__ = __this_cpu_read(pcp); \
+ if (ret__ == (oval)) \
+ __this_cpu_write(pcp, nval); \
+ preempt_enable(); \
+ ret__; \
+})
+
+#ifndef this_cpu_cmpxchg
+# ifndef this_cpu_cmpxchg_1
+# define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# ifndef this_cpu_cmpxchg_2
+# define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# ifndef this_cpu_cmpxchg_4
+# define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# ifndef this_cpu_cmpxchg_8
+# define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# define this_cpu_cmpxchg(pcp, oval, nval) __pcpu_size_call_return2(this_cpu_cmpxchg_, (pcp), oval, nval)
+#endif
+
+#define __this_cpu_generic_cmpxchg(pcp, oval, nval) \
+({ typeof(pcp) ret__; \
+ ret__ = __this_cpu_read(pcp); \
+ if (ret__ == (oval)) \
+ __this_cpu_write(pcp, nval); \
+ ret__; \
+})
+
+#ifndef __this_cpu_cmpxchg
+# ifndef __this_cpu_cmpxchg_1
+# define __this_cpu_cmpxchg_1(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# ifndef __this_cpu_cmpxchg_2
+# define __this_cpu_cmpxchg_2(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# ifndef __this_cpu_cmpxchg_4
+# define __this_cpu_cmpxchg_4(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# ifndef __this_cpu_cmpxchg_8
+# define __this_cpu_cmpxchg_8(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
+# endif
+# define __this_cpu_cmpxchg(pcp, oval, nval) __pcpu_size_call_return2(\
+ __this_cpu_cmpxchg_, (pcp), oval, nval)
+#endif
+
+/*
+ * cmpxchg_double replaces two adjacent scalars at once. The first parameter
+ * passed is a percpu pointer, not a scalar like the other this_cpu
+ * operations. This is so because the function operates on two scalars
+ * (must be of same size). A truth value is returned to indicate success or
+ * failure (since a double register result is difficult to handle).
+ * There is very limited hardware support for these operations. So only certain
+ * sizes may work.
+ */
+#define __this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2) \
+({ typeof(oval2) * __percpu pcp2 = (typeof(oval2) *)((pcp) + 1); \
+ int __ret = 0; \
+ if (__this_cpu_read(*pcp) == (oval1) && \
+ __this_cpu_read(*pcp2) == (oval2)) { \
+ __this_cpu_write(*pcp, (nval1)); \
+ __this_cpu_write(*pcp2, (nval2)); \
+ __ret = 1; \
+ } \
+ (__ret); \
+})
+
+#ifndef __this_cpu_cmpxchg_double
+# ifndef __this_cpu_cmpxchg_double_1
+# define __this_cpu_cmpxchg_double_1(pcp, oval1, oval2, nval1, nval2) \
+ __this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef __this_cpu_cmpxchg_double_2
+# define __this_cpu_cmpxchg_double_2(pcp, oval1, oval2, nval1, nval2) \
+ __this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef __this_cpu_cmpxchg_double_4
+# define __this_cpu_cmpxchg_double_4(pcp, oval1, oval2, nval1, nval2) \
+ __this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef __this_cpu_cmpxchg_double_8
+# define __this_cpu_cmpxchg_double_8(pcp, oval1, oval2, nval1, nval2) \
+ __this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# define __this_cpu_cmpxchg_double(pcp, oval1, oval2, nval1, nval2) \
+ __pcpu_size_call_return_int(__this_cpu_cmpxchg_double_, (pcp), \
+ oval1, oval2, nval1, nval2)
+#endif
+
+#define _this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2) \
+({ int ret__; \
+ preempt_disable(); \
+ ret__ = __this_cpu_generic_cmpxchg_double(pcp, \
+ oval1, oval2, nval1, nval2); \
+ preempt_enable(); \
+ ret__; \
+})
+
+#ifndef this_cpu_cmpxchg_double
+# ifndef this_cpu_cmpxchg_double_1
+# define this_cpu_cmpxchg_double_1(pcp, oval1, oval2, nval1, nval2) \
+ _this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef this_cpu_cmpxchg_double_2
+# define this_cpu_cmpxchg_double_2(pcp, oval1, oval2, nval1, nval2) \
+ _this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef this_cpu_cmpxchg_double_4
+# define this_cpu_cmpxchg_double_4(pcp, oval1, oval2, nval1, nval2) \
+ _this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef this_cpu_cmpxchg_double_8
+# define this_cpu_cmpxchg_double_8(pcp, oval1, oval2, nval1, nval2) \
+ _this_cpu_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# define this_cpu_cmpxchg_double(pcp, oval1, oval2, nval1, nval2) \
+ __pcpu_size_call_return_int(this_cpu_cmpxchg_double_, (pcp), \
+ oval1, oval2, nval1, nval2)
+#endif
+
+
+
+
#define _this_cpu_generic_to_op(pcp, val, op) \
do { \
preempt_disable(); \
@@ -603,7 +746,7 @@ do { \
* IRQ safe versions of the per cpu RMW operations. Note that these operations
* are *not* safe against modification of the same variable from another
* processors (which one gets when using regular atomic operations)
- . They are guaranteed to be atomic vs. local interrupts and
+ * They are guaranteed to be atomic vs. local interrupts and
* preemption only.
*/
#define irqsafe_cpu_generic_to_op(pcp, val, op) \
@@ -690,4 +833,38 @@ do { \
# define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))
#endif
+#define irqsafe_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2) \
+({ int ret__; \
+ unsigned long flags; \
+ local_irq_save(flags); \
+ ret__ = __this_cpu_generic_cmpxchg_double(pcp, \
+ oval1, oval2, nval1, nval2); \
+ local_irq_restore(flags); \
+ ret__; \
+})
+
+#ifndef irqsafe_cmpxchg_double
+# ifndef irqsafe_cmpxchg_double_1
+# define irqsafe_cmpxchg_double_1(pcp, oval1, oval2, nval1, nval2) \
+ irqsafe_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef irqsafe_cmpxchg_double_2
+# define irqsafe_cmpxchg_double_2(pcp, oval1, oval2, nval1, nval2) \
+ irqsafe_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef irqsafe_cmpxchg_double_4
+# define irqsafe_cmpxchg_double_4(pcp, oval1, oval2, nval1, nval2) \
+ irqsafe_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# ifndef irqsafe_cmpxchg_double_8
+# define irqsafe_cmpxchg_double_8(pcp, oval1, oval2, nval1, nval2) \
+ irqsafe_generic_cmpxchg_double(pcp, oval1, oval2, nval1, nval2)
+# endif
+# define irqsafe_cmpxchg_double(pcp, oval1, oval2, nval1, nval2) \
+ __pcpu_size_call_return_int(irqsafe_cmpxchg_double_, (pcp), \
+ oval1, oval2, nval1, nval2)
+#endif
+
+
+
#endif /* __LINUX_PERCPU_H */
next prev parent reply other threads:[~2010-11-26 21:11 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-11-26 21:09 [thisops uV2 00/10] Upgrade of this_cpu_ops V2 Christoph Lameter
2010-11-26 21:09 ` [thisops uV2 01/10] percpucounter: Optimize __percpu_counter_add a bit through the use of this_cpu() options Christoph Lameter
2010-11-27 14:42 ` Mathieu Desnoyers
2010-11-26 21:09 ` [thisops uV2 02/10] vmstat: Optimize zone counter modifications through the use of this cpu operations Christoph Lameter
2010-11-27 8:00 ` Pekka Enberg
2010-11-27 14:49 ` Mathieu Desnoyers
2010-11-29 16:16 ` Christoph Lameter
2010-11-29 17:13 ` Christoph Lameter
2010-11-29 19:28 ` Mathieu Desnoyers
2010-11-29 20:07 ` Christoph Lameter
2010-11-29 20:59 ` Christoph Lameter
2010-11-26 21:09 ` [thisops uV2 03/10] percpu: Generic support for this_cpu_add,sub,dec,inc_return Christoph Lameter
2010-11-27 14:58 ` Mathieu Desnoyers
2010-11-29 16:08 ` Christoph Lameter
2010-11-26 21:09 ` [thisops uV2 04/10] x86: Support " Christoph Lameter
2010-11-27 8:06 ` Pekka Enberg
2010-11-29 16:03 ` Christoph Lameter
2010-11-27 15:00 ` Mathieu Desnoyers
2010-11-29 16:31 ` Christoph Lameter
2010-11-29 18:33 ` Mathieu Desnoyers
2010-11-29 18:54 ` Christoph Lameter
2010-11-29 19:22 ` Mathieu Desnoyers
2010-11-29 20:09 ` Christoph Lameter
2010-11-26 21:09 ` [thisops uV2 05/10] x86: Use this_cpu_inc_return for nmi counter Christoph Lameter
2010-11-27 8:07 ` Pekka Enberg
2010-11-27 15:00 ` Mathieu Desnoyers
2010-11-26 21:09 ` [thisops uV2 06/10] vmstat: Use this_cpu_inc_return for vm statistics Christoph Lameter
2010-11-27 8:09 ` Pekka Enberg
2010-11-29 16:04 ` Christoph Lameter
2010-11-26 21:09 ` [thisops uV2 07/10] highmem: Use this_cpu_xx_return() operations Christoph Lameter
2010-11-26 21:09 ` Christoph Lameter [this message]
2010-11-26 21:09 ` [thisops uV2 09/10] x86: this_cpu_cmpxchg and this_cpu_cmpxchg_double operations Christoph Lameter
2010-11-27 6:30 ` Eric Dumazet
2010-11-27 15:20 ` Mathieu Desnoyers
2010-11-29 16:11 ` Christoph Lameter
2010-11-26 21:09 ` [thisops uV2 10/10] slub: Lockless fastpaths Christoph Lameter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20101126210953.591278825@linux.com \
--to=cl@linux.com \
--cc=akpm@linux-foundation.org \
--cc=penberg@cs.helsinki.fi \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.