From: Christoph Lameter <cl@linux.com>
To: Tejun Heo <tj@kernel.org>
Cc: akpm@linux-foundation.org
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: linux-kernel@vger.kernel.org
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Subject: [cpuops cmpxchg V2 2/5] x86: this_cpu_cmpxchg and this_cpu_xchg operations
Date: Tue, 14 Dec 2010 10:28:44 -0600 [thread overview]
Message-ID: <20101214162853.636123666@linux.com> (raw)
In-Reply-To: 20101214162842.542421046@linux.com
[-- Attachment #1: cpuops_cmpxchg_x86 --]
[-- Type: text/plain, Size: 6324 bytes --]
Provide support as far as the hardware capabilities of the x86 cpus
allow.
Define CONFIG_CMPXCHG_LOCAL in Kconfig.cpu to allow core code to test for
fast cpuops implementations.
V1->V2:
- Take out the definition for this_cpu_cmpxchg_8 and move it into
a separate patch.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
arch/x86/Kconfig.cpu | 3 +
arch/x86/include/asm/percpu.h | 109 +++++++++++++++++++++++++++++++++++++++++-
2 files changed, 111 insertions(+), 1 deletion(-)
Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/percpu.h 2010-12-10 12:18:46.000000000 -0600
+++ linux-2.6/arch/x86/include/asm/percpu.h 2010-12-10 12:42:18.000000000 -0600
@@ -212,6 +212,83 @@ do { \
ret__; \
})
+/*
+ * Beware: xchg on x86 has an implied lock prefix. There will be the cost of
+ * full lock semantics even though they are not needed.
+ */
+#define percpu_xchg_op(var, nval) \
+({ \
+ typeof(var) __ret; \
+ typeof(var) __new = (nval); \
+ switch (sizeof(var)) { \
+ case 1: \
+ asm("xchgb %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "q" (__new) \
+ : "memory"); \
+ break; \
+ case 2: \
+ asm("xchgw %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "r" (__new) \
+ : "memory"); \
+ break; \
+ case 4: \
+ asm("xchgl %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "r" (__new) \
+ : "memory"); \
+ break; \
+ case 8: \
+ asm("xchgq %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "r" (__new) \
+ : "memory"); \
+ break; \
+ default: __bad_percpu_size(); \
+ } \
+ __ret; \
+})
+
+/*
+ * cmpxchg has no such implied lock semantics as a result it is much
+ * more efficient for cpu local operations.
+ */
+#define percpu_cmpxchg_op(var, oval, nval) \
+({ \
+ typeof(var) __ret; \
+ typeof(var) __old = (oval); \
+ typeof(var) __new = (nval); \
+ switch (sizeof(var)) { \
+ case 1: \
+ asm("cmpxchgb %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "q" (__new), "0" (__old) \
+ : "memory"); \
+ break; \
+ case 2: \
+ asm("cmpxchgw %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "r" (__new), "0" (__old) \
+ : "memory"); \
+ break; \
+ case 4: \
+ asm("cmpxchgl %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "r" (__new), "0" (__old) \
+ : "memory"); \
+ break; \
+ case 8: \
+ asm("cmpxchgq %2, "__percpu_arg(1) \
+ : "=a" (__ret), "+m" (var) \
+ : "r" (__new), "0" (__old) \
+ : "memory"); \
+ break; \
+ default: __bad_percpu_size(); \
+ } \
+ __ret; \
+})
+
#define percpu_from_op(op, var, constraint) \
({ \
typeof(var) pfo_ret__; \
@@ -335,6 +412,17 @@ do { \
#define irqsafe_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
#define irqsafe_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
+/*
+ * Generic fallback operations for __this_cpu_xchg are okay and much faster
+ * than an xchg with forced lock semantics.
+ */
+#define this_cpu_xchg_1(pcp, nval) percpu_xchg_op(pcp, nval)
+#define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval)
+#define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval)
+#define irqsafe_cpu_xchg_1(pcp, nval) percpu_xchg_op(pcp, nval)
+#define irqsafe_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval)
+#define irqsafe_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval)
+
#ifndef CONFIG_M386
#define __this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
#define __this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
@@ -342,7 +430,18 @@ do { \
#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val)
-#endif
+
+#define __this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define __this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define __this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define irqsafe_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define irqsafe_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define irqsafe_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#endif /* !CONFIG_M386 */
+
/*
* Per cpu atomic 64 bit operations are only available under 64 bit.
* 32 bit must fall back to generic operations.
@@ -370,6 +469,14 @@ do { \
#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
+#define __this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
+#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
+#define irqsafe_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
+
+#define __this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+#define irqsafe_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
+
#endif
/* This is not atomic against other CPUs -- CPU preemption needs to be off */
Index: linux-2.6/arch/x86/Kconfig.cpu
===================================================================
--- linux-2.6.orig/arch/x86/Kconfig.cpu 2010-12-10 12:18:46.000000000 -0600
+++ linux-2.6/arch/x86/Kconfig.cpu 2010-12-10 12:18:49.000000000 -0600
@@ -310,6 +310,9 @@ config X86_INTERNODE_CACHE_SHIFT
config X86_CMPXCHG
def_bool X86_64 || (X86_32 && !M386)
+config CMPXCHG_LOCAL
+ def_bool X86_64 || (X86_32 && !M386)
+
config X86_L1_CACHE_SHIFT
int
default "7" if MPENTIUM4 || MPSC
next prev parent reply other threads:[~2010-12-14 16:29 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-12-14 16:28 [cpuops cmpxchg V2 0/5] Cmpxchg and xchg operations Christoph Lameter
2010-12-14 16:28 ` [cpuops cmpxchg V2 1/5] percpu: Generic this_cpu_cmpxchg() and this_cpu_xchg support Christoph Lameter
2010-12-17 14:55 ` Tejun Heo
2010-12-14 16:28 ` Christoph Lameter [this message]
2010-12-17 15:22 ` [cpuops cmpxchg V2 2/5] x86: this_cpu_cmpxchg and this_cpu_xchg operations Tejun Heo
2010-12-14 16:28 ` [cpuops cmpxchg V2 3/5] irq_work: Use per cpu atomics instead of regular atomics Christoph Lameter
2010-12-15 16:32 ` Tejun Heo
2010-12-15 16:34 ` H. Peter Anvin
2010-12-15 16:50 ` Peter Zijlstra
2010-12-15 17:04 ` Christoph Lameter
2010-12-15 17:18 ` Peter Zijlstra
2010-12-15 17:31 ` H. Peter Anvin
2010-12-15 17:32 ` Christoph Lameter
2010-12-18 15:32 ` Tejun Heo
2010-12-14 16:28 ` [cpuops cmpxchg V2 4/5] vmstat: User per cpu atomics to avoid interrupt disable / enable Christoph Lameter
2010-12-15 16:45 ` Tejun Heo
2010-12-15 17:01 ` Christoph Lameter
2010-12-14 16:28 ` [cpuops cmpxchg V2 5/5] cpuops: Use cmpxchg for xchg to avoid lock semantics Christoph Lameter
2010-12-14 16:35 ` Mathieu Desnoyers
2010-12-14 16:44 ` Eric Dumazet
2010-12-14 16:55 ` Christoph Lameter
2010-12-14 17:00 ` H. Peter Anvin
2010-12-14 17:19 ` Christoph Lameter
2010-12-14 17:22 ` H. Peter Anvin
2010-12-14 17:29 ` Tejun Heo
2010-12-14 17:35 ` Christoph Lameter
2010-12-15 1:06 ` H. Peter Anvin
2010-12-15 16:29 ` Tejun Heo
2010-12-15 16:35 ` H. Peter Anvin
2010-12-15 16:39 ` Tejun Heo
2010-12-16 16:14 ` Tejun Heo
2010-12-16 18:13 ` x86: Use this_cpu_has for thermal_interrupt Christoph Lameter
2010-12-18 15:35 ` Tejun Heo
2010-12-21 0:56 ` H. Peter Anvin
2010-12-30 11:29 ` Tejun Heo
2010-12-30 18:19 ` H. Peter Anvin
2010-12-31 12:43 ` Tejun Heo
2010-12-16 18:14 ` x86: udelay: Use this_cpu_read to avoid address calculation Christoph Lameter
2010-12-16 18:15 ` gameport: use this_cpu_read instead of lookup Christoph Lameter
2010-12-18 15:34 ` Tejun Heo
2010-12-16 18:16 ` acpi throttling: Use this_cpu_has and simplify code Christoph Lameter
2010-12-18 15:50 ` Tejun Heo
2010-12-21 1:52 ` ykzhao
2010-12-21 22:43 ` Christoph Lameter
2010-12-21 4:28 ` Len Brown
2010-12-16 18:19 ` [cpuops cmpxchg V2 5/5] cpuops: Use cmpxchg for xchg to avoid lock semantics H. Peter Anvin
2010-12-16 18:55 ` Tejun Heo
2010-12-16 20:42 ` H. Peter Anvin
2010-12-15 16:47 ` Tejun Heo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20101214162853.636123666@linux.com \
--to=cl@linux.com \
--cc=akpm@linux-foundation.org \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox