[patch 00/23] Atomic operations updates: add cmpxchg_local

[patch 00/23] Atomic operations updates: add cmpxchg_local

[Mathieu Desnoyers]

Hi Andrew,

I have been asked for a cmpxchg_local implementation by Christoph Lameter so he
can use it to accelerate the slub allocator.

On some architectures, where cmpxchg is nothing more that a simple save/restore
interrupts surrounding the cmpxchg non atomic ops, I remove the code and make
them use the new "generic" version.

cmpxchg_local and cmpxchg64_local are made available on all architectures in
these patches. They use the best primitive known, either an atomic operation or
save/restore interrupts.

Mathieu

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and 80486

[Mathieu Desnoyers]

[-- Attachment #1: i386-cmpxchg64-80386-80486-fallback.patch --]
[-- Type: text/plain, Size: 5748 bytes --]

Actually, on 386, cmpxchg and cmpxchg_local fall back on
cmpxchg_386_u8/16/32: it disables interruptions around non atomic
updates to mimic the cmpxchg behavior.

The comment:
/* Poor man's cmpxchg for 386. Unsuitable for SMP */

already present in cmpxchg_386_u32 tells much about how this cmpxchg
implementation should not be used in a SMP context. However, the cmpxchg_local
can perfectly use this fallback, since it only needs to be atomic wrt the local
cpu.

This patch adds a cmpxchg_486_u64 and uses it as a fallback for cmpxchg64
and cmpxchg64_local on 80386 and 80486.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: mingo@redhat.com
---
 arch/i386/kernel/cpu/intel.c |   17 +++++++
 include/asm-i386/cmpxchg.h   |  100 +++++++++++++++++++++++++++++--------------
 2 files changed, 85 insertions(+), 32 deletions(-)

Index: linux-2.6-lttng/arch/i386/kernel/cpu/intel.c
===================================================================
--- linux-2.6-lttng.orig/arch/i386/kernel/cpu/intel.c	2007-08-07 11:00:43.000000000 -0400
+++ linux-2.6-lttng/arch/i386/kernel/cpu/intel.c	2007-08-07 11:11:07.000000000 -0400
@@ -329,5 +329,22 @@ unsigned long cmpxchg_386_u32(volatile v
 EXPORT_SYMBOL(cmpxchg_386_u32);
 #endif
 
+#ifndef CONFIG_X86_CMPXCHG64
+unsigned long long cmpxchg_486_u64(volatile void *ptr, u64 old, u64 new)
+{
+	u64 prev;
+	unsigned long flags;
+
+	/* Poor man's cmpxchg8b for 386 and 486. Unsuitable for SMP */
+	local_irq_save(flags);
+	prev = *(u64 *)ptr;
+	if (prev == old)
+		*(u64 *)ptr = new;
+	local_irq_restore(flags);
+	return prev;
+}
+EXPORT_SYMBOL(cmpxchg_486_u64);
+#endif
+
 // arch_initcall(intel_cpu_init);
 
Index: linux-2.6-lttng/include/asm-i386/cmpxchg.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-i386/cmpxchg.h	2007-08-07 11:01:17.000000000 -0400
+++ linux-2.6-lttng/include/asm-i386/cmpxchg.h	2007-08-07 11:16:42.000000000 -0400
@@ -116,6 +116,15 @@ static inline unsigned long __xchg(unsig
 					(unsigned long)(n),sizeof(*(ptr))))
 #endif
 
+#ifdef CONFIG_X86_CMPXCHG64
+#define cmpxchg64(ptr,o,n)\
+	((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
+					(unsigned long long)(n)))
+#define cmpxchg64_local(ptr,o,n)\
+	((__typeof__(*(ptr)))__cmpxchg64_local((ptr),(unsigned long long)(o),\
+					(unsigned long long)(n)))
+#endif
+
 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 				      unsigned long new, int size)
 {
@@ -203,6 +212,34 @@ static inline unsigned long __cmpxchg_lo
 	return old;
 }
 
+static inline unsigned long long __cmpxchg64(volatile void *ptr,
+			unsigned long long old, unsigned long long new)
+{
+	unsigned long long prev;
+	__asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
+			     : "=A"(prev)
+			     : "b"((unsigned long)new),
+			       "c"((unsigned long)(new >> 32)),
+			       "m"(*__xg(ptr)),
+			       "0"(old)
+			     : "memory");
+	return prev;
+}
+
+static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
+			unsigned long long old, unsigned long long new)
+{
+	unsigned long long prev;
+	__asm__ __volatile__("cmpxchg8b %3"
+			     : "=A"(prev)
+			     : "b"((unsigned long)new),
+			       "c"((unsigned long)(new >> 32)),
+			       "m"(*__xg(ptr)),
+			       "0"(old)
+			     : "memory");
+	return prev;
+}
+
 #ifndef CONFIG_X86_CMPXCHG
 /*
  * Building a kernel capable running on 80386. It may be necessary to
@@ -252,38 +289,37 @@ static inline unsigned long cmpxchg_386(
 })
 #endif
 
-static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
-				      unsigned long long new)
-{
-	unsigned long long prev;
-	__asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
-			     : "=A"(prev)
-			     : "b"((unsigned long)new),
-			       "c"((unsigned long)(new >> 32)),
-			       "m"(*__xg(ptr)),
-			       "0"(old)
-			     : "memory");
-	return prev;
-}
+#ifndef CONFIG_X86_CMPXCHG64
+/*
+ * Building a kernel capable running on 80386 and 80486. It may be necessary
+ * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
+ */
+
+extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
+
+#define cmpxchg64(ptr,o,n)						\
+({									\
+	__typeof__(*(ptr)) __ret;					\
+	if (likely(boot_cpu_data.x86 > 4))				\
+		__ret = __cmpxchg64((ptr), (unsigned long long)(o),	\
+				(unsigned long long)(n));		\
+	else								\
+		__ret = cmpxchg_486_u64((ptr), (unsigned long long)(o),	\
+				(unsigned long long)(n));		\
+	__ret;								\
+})
+#define cmpxchg64_local(ptr,o,n)					\
+({									\
+	__typeof__(*(ptr)) __ret;					\
+	if (likely(boot_cpu_data.x86 > 4))				\
+		__ret = __cmpxchg64_local((ptr), (unsigned long long)(o), \
+				(unsigned long long)(n));		\
+	else								\
+		__ret = cmpxchg_486_u64((ptr), (unsigned long long)(o),	\
+				(unsigned long long)(n));		\
+	__ret;								\
+})
 
-static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
-			unsigned long long old, unsigned long long new)
-{
-	unsigned long long prev;
-	__asm__ __volatile__("cmpxchg8b %3"
-			     : "=A"(prev)
-			     : "b"((unsigned long)new),
-			       "c"((unsigned long)(new >> 32)),
-			       "m"(*__xg(ptr)),
-			       "0"(old)
-			     : "memory");
-	return prev;
-}
+#endif
 
-#define cmpxchg64(ptr,o,n)\
-	((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
-					(unsigned long long)(n)))
-#define cmpxchg64_local(ptr,o,n)\
-	((__typeof__(*(ptr)))__cmpxchg64_local((ptr),(unsigned long long)(o),\
-					(unsigned long long)(n)))
 #endif

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 02/23] Add cmpxchg_local to asm-generic for per cpu atomic operations

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-generic-for-up.patch --]
[-- Type: text/plain, Size: 3537 bytes --]

Emulates the cmpxchg_local by disabling interrupts around variable modification.
This is not reentrant wrt NMIs and MCEs. It is only protected against normal
interrupts, but this is enough for architectures without such interrupt sources
or if used in a context where the data is not shared with such handlers.

It can be used as a fallback for architectures lacking a real cmpxchg
instruction.

For architectures that have a real cmpxchg but does not have NMIs or MCE,
testing which of the generic vs architecture specific cmpxchg is the fastest
should be done.

asm-generic/cmpxchg.h defines a cmpxchg that uses cmpxchg_local. It is meant to
be used as a cmpxchg fallback for architectures that do not support SMP.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
---
 include/asm-generic/cmpxchg-local.h |   60 ++++++++++++++++++++++++++++++++++++
 include/asm-generic/cmpxchg.h       |   21 ++++++++++++
 2 files changed, 81 insertions(+)

Index: linux-2.6-lttng/include/asm-generic/cmpxchg.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6-lttng/include/asm-generic/cmpxchg.h	2007-08-12 09:33:41.000000000 -0400
@@ -0,0 +1,21 @@
+#ifndef __ASM_GENERIC_CMPXCHG_H
+#define __ASM_GENERIC_CMPXCHG_H
+
+/*
+ * Generic cmpxchg
+ *
+ * Uses the local cmpxchg. Does not support SMP.
+ */
+#ifdef CONFIG_SMP
+#error "Cannot use generic cmpxchg on SMP"
+#endif
+
+/*
+ * Atomic compare and exchange.
+ */
+#define __HAVE_ARCH_CMPXCHG	1
+
+#define cmpxchg(ptr,o,n)	cmpxchg_local((ptr), (o), (n))
+#define cmpxchg64(ptr,o,n)	cmpxchg64_local((ptr), (o), (n))
+
+#endif
Index: linux-2.6-lttng/include/asm-generic/cmpxchg-local.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6-lttng/include/asm-generic/cmpxchg-local.h	2007-08-12 09:19:22.000000000 -0400
@@ -0,0 +1,60 @@
+#ifndef __ASM_GENERIC_CMPXCHG_LOCAL_H
+#define __ASM_GENERIC_CMPXCHG_LOCAL_H
+
+#include <linux/types.h>
+
+extern unsigned long wrong_size_cmpxchg(volatile void *ptr);
+
+/*
+ * Generic version of __cmpxchg_local (disables interrupts). Takes an unsigned
+ * long parameter, supporting various types of architectures.
+ */
+static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
+				    unsigned long old,
+				    unsigned long new, int size)
+{
+	unsigned long flags, prev;
+
+	/*
+	 * Sanity checking, compile-time.
+	 */
+	if (size == 8 && sizeof(unsigned long) != 8)
+		wrong_size_cmpxchg(ptr);
+
+	local_irq_save(flags);
+	switch (size) {
+	case 1: if ((prev = *(u8*)ptr) == old)
+			*(u8*)ptr = (u8)new;
+		break;
+	case 2: if ((prev = *(u16*)ptr) == old)
+			*(u16*)ptr = (u16)new;
+		break;
+	case 4: if ((prev = *(u32*)ptr) == old)
+			*(u32*)ptr = (u32)new;
+		break;
+	case 8: if ((prev = *(u64*)ptr) == old)
+			*(u64*)ptr = (u64)new;
+		break;
+	default:
+		wrong_size_cmpxchg(ptr);
+	}
+	local_irq_restore(flags);
+	return prev;
+}
+
+/*
+ * Generic version of __cmpxchg64_local. Takes an u64 parameter.
+ */
+static inline u64 __cmpxchg64_local_generic(volatile void *ptr, u64 old, u64 new)
+{
+	u64 prev;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if ((prev = *(u64*)ptr) == old)
+		*(u64*)ptr = new;
+	local_irq_restore(flags);
+	return prev;
+}
+
+#endif

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 03/23] Add cmpxchg_local to arm

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-arm.patch --]
[-- Type: text/plain, Size: 2207 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: rmk@arm.linux.org.uk
---
 include/asm-arm/system.h |   43 +++++++++++--------------------------------
 1 file changed, 11 insertions(+), 32 deletions(-)

Index: linux-2.6-lttng/include/asm-arm/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-arm/system.h	2007-08-07 11:03:37.000000000 -0400
+++ linux-2.6-lttng/include/asm-arm/system.h	2007-08-07 11:19:40.000000000 -0400
@@ -350,40 +350,19 @@ static inline unsigned long __xchg(unsig
 extern void disable_hlt(void);
 extern void enable_hlt(void);
 
-#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg-local.h>
+
 /*
- * Atomic compare and exchange.
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * then available.
  */
-#define __HAVE_ARCH_CMPXCHG	1
-
-extern unsigned long wrong_size_cmpxchg(volatile void *ptr);
-
-static inline unsigned long __cmpxchg(volatile void *ptr,
-				    unsigned long old,
-				    unsigned long new, int size)
-{
-	unsigned long flags, prev;
-	volatile unsigned long *p = ptr;
-
-	if (size == 4) {
-		local_irq_save(flags);
-		prev = *p;
-		if (prev == old)
-			*p = new;
-		local_irq_restore(flags);
-		return prev;
-	} else
-		return wrong_size_cmpxchg(ptr);
-}
-
-#define cmpxchg(ptr, o, n)					  	\
-({									\
-     __typeof__(*(ptr)) _o_ = (o);					\
-     __typeof__(*(ptr)) _n_ = (n);					\
-     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		\
-		(unsigned long)_n_, sizeof(*(ptr)));	\
-})
-
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
 #endif
 
 #endif /* __ASSEMBLY__ */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 04/23] Add cmpxchg_local to avr32

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-avr32.patch --]
[-- Type: text/plain, Size: 1808 bytes --]

Use the new generic cmpxchg_local (disables interrupt) for 8, 16 and 64 bits
cmpxchg_local. Use the __cmpxchg_u32 primitive for 32 bits cmpxchg_local.

Note that cmpxchg only uses the __cmpxchg_u32 or __cmpxchg_u64 and will cause
a linker error if called with 8 or 16 bits argument.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: hskinnemoen@atmel.com
---
 include/asm-avr32/system.h |   23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

Index: linux-2.6-lttng/include/asm-avr32/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-avr32/system.h	2007-07-20 19:10:57.000000000 -0400
+++ linux-2.6-lttng/include/asm-avr32/system.h	2007-07-20 19:32:36.000000000 -0400
@@ -140,6 +140,29 @@ static inline unsigned long __cmpxchg(vo
 				   (unsigned long)(new),	\
 				   sizeof(*(ptr))))
 
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+		return __cmpxchg_u32(ptr, old, new);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+#define cmpxchg_local(ptr, old, new)					\
+	((typeof(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(old),	\
+				   (unsigned long)(new),		\
+				   sizeof(*(ptr))))
+
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
 struct pt_regs;
 void NORET_TYPE die(const char *str, struct pt_regs *regs, long err);
 void _exception(long signr, struct pt_regs *regs, int code,

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 05/23] Add cmpxchg_local to blackfin, replace __cmpxchg by generic cmpxchg

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-blackfin.patch --]
[-- Type: text/plain, Size: 2856 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set since nobody seems to know why __cmpxchg
has been implemented in assembly in the first place thather than in plain C.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: michael.frysinger@analog.com
---
 include/asm-blackfin/system.h |   59 ++++++++----------------------------------
 1 file changed, 12 insertions(+), 47 deletions(-)

Index: linux-2.6-lttng/include/asm-blackfin/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-blackfin/system.h	2007-08-07 14:31:52.000000000 -0400
+++ linux-2.6-lttng/include/asm-blackfin/system.h	2007-08-07 14:47:31.000000000 -0400
@@ -176,55 +176,20 @@ static inline unsigned long __xchg(unsig
 	return tmp;
 }
 
+#include <asm-generic/cmpxchg-local.h>
+
 /*
- * Atomic compare and exchange.  Compare OLD with MEM, if identical,
- * store NEW in MEM.  Return the initial value in MEM.  Success is
- * indicated by comparing RETURN with OLD.
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
  */
-static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
-				      unsigned long new, int size)
-{
-	unsigned long tmp = 0;
-	unsigned long flags = 0;
-
-	local_irq_save(flags);
-
-	switch (size) {
-	case 1:
-		__asm__ __volatile__
-			("%0 = b%3 (z);\n\t"
-			 "CC = %1 == %0;\n\t"
-			 "IF !CC JUMP 1f;\n\t"
-			 "b%3 = %2;\n\t"
-			 "1:\n\t"
-			 : "=&d" (tmp) : "d" (old), "d" (new), "m" (*__xg(ptr)) : "memory");
-		break;
-	case 2:
-		__asm__ __volatile__
-			("%0 = w%3 (z);\n\t"
-			 "CC = %1 == %0;\n\t"
-			 "IF !CC JUMP 1f;\n\t"
-			 "w%3 = %2;\n\t"
-			 "1:\n\t"
-			 : "=&d" (tmp) : "d" (old), "d" (new), "m" (*__xg(ptr)) : "memory");
-		break;
-	case 4:
-		__asm__ __volatile__
-			("%0 = %3;\n\t"
-			 "CC = %1 == %0;\n\t"
-			 "IF !CC JUMP 1f;\n\t"
-			 "%3 = %2;\n\t"
-			 "1:\n\t"
-			 : "=&d" (tmp) : "d" (old), "d" (new), "m" (*__xg(ptr)) : "memory");
-		break;
-	}
-	local_irq_restore(flags);
-	return tmp;
-}
-
-#define cmpxchg(ptr,o,n)\
-        ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
-                                        (unsigned long)(n),sizeof(*(ptr))))
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
 
 #define prepare_to_switch()     do { } while(0)
 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 06/23] Add cmpxchg_local to cris

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-cris.patch --]
[-- Type: text/plain, Size: 1365 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: starvik@axis.com
---
 include/asm-cris/system.h |   15 +++++++++++++++
 1 file changed, 15 insertions(+)

Index: linux-2.6-lttng/include/asm-cris/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-cris/system.h	2007-08-07 14:31:52.000000000 -0400
+++ linux-2.6-lttng/include/asm-cris/system.h	2007-08-07 14:48:53.000000000 -0400
@@ -66,6 +66,21 @@ static inline unsigned long __xchg(unsig
   return x;
 }
 
+#include <asm-generic/cmpxchg-local.h>
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
+
 #define arch_align_stack(x) (x)
 
 void default_idle(void);

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 07/23] Add cmpxchg_local to frv

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-frv.patch --]
[-- Type: text/plain, Size: 1577 bytes --]

Use the new generic cmpxchg_local (disables interrupt) for 8, 16 and 64 bits
arguments. Use the 32 bits cmpxchg available on the architecture for 32 bits
arguments.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
---
 include/asm-frv/system.h |   24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

Index: linux-2.6-lttng/include/asm-frv/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-frv/system.h	2007-08-07 14:51:22.000000000 -0400
+++ linux-2.6-lttng/include/asm-frv/system.h	2007-08-07 14:51:39.000000000 -0400
@@ -265,5 +265,29 @@ extern uint32_t __cmpxchg_32(uint32_t *v
 
 #endif
 
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+		return cmpxchg(ptr, old, new);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	\
+     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
 
 #endif /* _ASM_SYSTEM_H */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 08/23] Add cmpxchg_local to h8300

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-h8300.patch --]
[-- Type: text/plain, Size: 1371 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
---
 include/asm-h8300/system.h |   15 +++++++++++++++
 1 file changed, 15 insertions(+)

Index: linux-2.6-lttng/include/asm-h8300/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-h8300/system.h	2007-07-20 18:36:09.000000000 -0400
+++ linux-2.6-lttng/include/asm-h8300/system.h	2007-07-20 19:27:23.000000000 -0400
@@ -139,6 +139,21 @@ static inline unsigned long __xchg(unsig
         asm("jmp @@0");			\
 })
 
+#include <asm-generic/cmpxchg-local.h>
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
+
 #define arch_align_stack(x) (x)
 
 #endif /* _H8300_SYSTEM_H */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 09/23] Add cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-ia64.patch --]
[-- Type: text/plain, Size: 1239 bytes --]

Add the primitives cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64. They
use cmpxchg_acq as underlying macro, just like the already existing ia64
cmpxchg().

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: tony.luck@intel.com
---
 include/asm-ia64/intrinsics.h |    4 ++++
 1 file changed, 4 insertions(+)

Index: linux-2.6-lttng/include/asm-ia64/intrinsics.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-ia64/intrinsics.h	2007-07-20 18:36:09.000000000 -0400
+++ linux-2.6-lttng/include/asm-ia64/intrinsics.h	2007-07-20 19:29:17.000000000 -0400
@@ -158,6 +158,10 @@ extern long ia64_cmpxchg_called_with_bad
 
 /* for compatibility with other platforms: */
 #define cmpxchg(ptr,o,n)	cmpxchg_acq(ptr,o,n)
+#define cmpxchg_local(ptr,o,n)	cmpxchg_acq(ptr,o,n)
+
+#define cmpxchg64(ptr,o,n)		cmpxchg_acq(ptr,o,n)
+#define cmpxchg64_local(ptr,o,n)	cmpxchg_acq(ptr,o,n)
 
 #ifdef CONFIG_IA64_DEBUG_CMPXCHG
 # define CMPXCHG_BUGCHECK_DECL	int _cmpxchg_bugcheck_count = 128;

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 10/23] New cmpxchg_local (optimized for UP case) for m32r

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-m32r.patch --]
[-- Type: text/plain, Size: 4343 bytes --]

Add __xchg_local, xchg_local (define), __cmpxchg_local_u32, __cmpxchg_local,
cmpxchg_local(macro).

cmpxchg_local and cmpxchg64_local will use the architecture specific
__cmpxchg_local_u32 for 32 bits arguments, and use the generic
__cmpxchg_local_generic for 8, 16 and 64 bits arguments.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: takata@linux-m32r.org
CC: linux-m32r@ml.linux-m32r.org
---
 include/asm-m32r/system.h |  103 +++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 97 insertions(+), 6 deletions(-)

Index: linux-2.6-lttng/include/asm-m32r/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-m32r/system.h	2007-08-07 14:31:52.000000000 -0400
+++ linux-2.6-lttng/include/asm-m32r/system.h	2007-08-07 14:55:02.000000000 -0400
@@ -123,6 +123,9 @@ static inline void local_irq_disable(voi
 
 #define xchg(ptr,x) \
 	((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
+#define xchg_local(ptr,x) \
+	((__typeof__(*(ptr)))__xchg_local((unsigned long)(x),(ptr), \
+			sizeof(*(ptr))))
 
 #ifdef CONFIG_SMP
 extern void  __xchg_called_with_bad_pointer(void);
@@ -196,6 +199,42 @@ __xchg(unsigned long x, volatile void * 
 	return (tmp);
 }
 
+static __always_inline unsigned long
+__xchg_local(unsigned long x, volatile void * ptr, int size)
+{
+	unsigned long flags;
+	unsigned long tmp = 0;
+
+	local_irq_save(flags);
+
+	switch (size) {
+	case 1:
+		__asm__ __volatile__ (
+			"ldb	%0, @%2 \n\t"
+			"stb	%1, @%2 \n\t"
+			: "=&r" (tmp) : "r" (x), "r" (ptr) : "memory");
+		break;
+	case 2:
+		__asm__ __volatile__ (
+			"ldh	%0, @%2 \n\t"
+			"sth	%1, @%2 \n\t"
+			: "=&r" (tmp) : "r" (x), "r" (ptr) : "memory");
+		break;
+	case 4:
+		__asm__ __volatile__ (
+			"ld	%0, @%2 \n\t"
+			"st	%1, @%2 \n\t"
+			: "=&r" (tmp) : "r" (x), "r" (ptr) : "memory");
+		break;
+	default:
+		__xchg_called_with_bad_pointer();
+	}
+
+	local_irq_restore(flags);
+
+	return (tmp);
+}
+
 #define __HAVE_ARCH_CMPXCHG	1
 
 static inline unsigned long
@@ -228,6 +267,37 @@ __cmpxchg_u32(volatile unsigned int *p, 
 	return retval;
 }
 
+static inline unsigned long
+__cmpxchg_local_u32(volatile unsigned int *p, unsigned int old,
+			unsigned int new)
+{
+	unsigned long flags;
+	unsigned int retval;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+			DCACHE_CLEAR("%0", "r4", "%1")
+			"ld %0, @%1;		\n"
+		"	bne	%0, %2, 1f;	\n"
+			"st %3, @%1;		\n"
+		"	bra	2f;		\n"
+                "       .fillinsn		\n"
+		"1:"
+			"st %0, @%1;		\n"
+                "       .fillinsn		\n"
+		"2:"
+			: "=&r" (retval)
+			: "r" (p), "r" (old), "r" (new)
+			: "cbit", "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+			, "r4"
+#endif  /* CONFIG_CHIP_M32700_TS1 */
+		);
+	local_irq_restore(flags);
+
+	return retval;
+}
+
 /* This function doesn't exist, so you'll get a linker error
    if something tries to do an invalid cmpxchg().  */
 extern void __cmpxchg_called_with_bad_pointer(void);
@@ -248,12 +318,33 @@ __cmpxchg(volatile void *ptr, unsigned l
 }
 
 #define cmpxchg(ptr,o,n)						 \
-  ({									 \
-     __typeof__(*(ptr)) _o_ = (o);					 \
-     __typeof__(*(ptr)) _n_ = (n);					 \
-     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		 \
-				    (unsigned long)_n_, sizeof(*(ptr))); \
-  })
+     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)(o),		 \
+				    (unsigned long)(n), sizeof(*(ptr)))
+
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+		return __cmpxchg_local_u32(ptr, old, new);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	    \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
 
 #endif  /* __KERNEL__ */
 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 11/23] Fix m32r __xchg

[Mathieu Desnoyers]

[-- Attachment #1: fix-m32r-__xchg.patch --]
[-- Type: text/plain, Size: 1032 bytes --]

the #endif  /* CONFIG_SMP */ should cover the default condition, or it may cause
bad parameter to be silently missed.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: takata@linux-m32r.org
CC: linux-m32r@ml.linux-m32r.org
---
 include/asm-m32r/system.h |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux-2.6-lttng/include/asm-m32r/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-m32r/system.h	2007-08-07 14:55:02.000000000 -0400
+++ linux-2.6-lttng/include/asm-m32r/system.h	2007-08-07 14:57:57.000000000 -0400
@@ -189,9 +189,9 @@ __xchg(unsigned long x, volatile void * 
 #endif	/* CONFIG_CHIP_M32700_TS1 */
 		);
 		break;
+#endif  /* CONFIG_SMP */
 	default:
 		__xchg_called_with_bad_pointer();
-#endif  /* CONFIG_SMP */
 	}
 
 	local_irq_restore(flags);

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 12/23] local_t m32r use architecture specific cmpxchg_local

[Mathieu Desnoyers]

[-- Attachment #1: local_t_m32r_optimized.patch --]
[-- Type: text/plain, Size: 10239 bytes --]

On m32r, use the new cmpxchg_local as primitive for the local_cmpxchg
operation.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: takata@linux-m32r.org
CC: linux-m32r@ml.linux-m32r.org
---
 include/asm-m32r/local.h |  362 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 361 insertions(+), 1 deletion(-)

Index: linux-2.6-lttng/include/asm-m32r/local.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-m32r/local.h	2007-07-20 17:29:08.000000000 -0400
+++ linux-2.6-lttng/include/asm-m32r/local.h	2007-07-20 17:43:28.000000000 -0400
@@ -1,6 +1,366 @@
 #ifndef __M32R_LOCAL_H
 #define __M32R_LOCAL_H
 
-#include <asm-generic/local.h>
+/*
+ *  linux/include/asm-m32r/local.h
+ *
+ *  M32R version:
+ *    Copyright (C) 2001, 2002  Hitoshi Yamamoto
+ *    Copyright (C) 2004  Hirokazu Takata <takata at linux-m32r.org>
+ *    Copyright (C) 2007  Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
+ */
+
+#include <linux/percpu.h>
+#include <asm/assembler.h>
+#include <asm/system.h>
+#include <asm/local.h>
+
+/*
+ * Atomic operations that C can't guarantee us.  Useful for
+ * resource counting etc..
+ */
+
+/*
+ * Make sure gcc doesn't try to be clever and move things around
+ * on us. We need to use _exactly_ the address the user gave us,
+ * not some alias that contains the same information.
+ */
+typedef struct { volatile int counter; } local_t;
+
+#define LOCAL_INIT(i)	{ (i) }
+
+/**
+ * local_read - read local variable
+ * @l: pointer of type local_t
+ *
+ * Atomically reads the value of @l.
+ */
+#define local_read(l)	((l)->counter)
+
+/**
+ * local_set - set local variable
+ * @l: pointer of type local_t
+ * @i: required value
+ *
+ * Atomically sets the value of @l to @i.
+ */
+#define local_set(l,i)	(((l)->counter) = (i))
+
+/**
+ * local_add_return - add long to local variable and return it
+ * @i: long value to add
+ * @l: pointer of type local_t
+ *
+ * Atomically adds @i to @l and return (@i + @l).
+ */
+static __inline__ long local_add_return(long i, local_t *l)
+{
+	unsigned long flags;
+	long result;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+		"# local_add_return		\n\t"
+		DCACHE_CLEAR("%0", "r4", "%1")
+		"ld %0, @%1;			\n\t"
+		"add	%0, %2;			\n\t"
+		"st %0, @%1;			\n\t"
+		: "=&r" (result)
+		: "r" (&l->counter), "r" (i)
+		: "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+		, "r4"
+#endif	/* CONFIG_CHIP_M32700_TS1 */
+	);
+	local_irq_restore(flags);
+
+	return result;
+}
+
+/**
+ * local_sub_return - subtract long from local variable and return it
+ * @i: long value to subtract
+ * @l: pointer of type local_t
+ *
+ * Atomically subtracts @i from @l and return (@l - @i).
+ */
+static __inline__ long local_sub_return(long i, local_t *l)
+{
+	unsigned long flags;
+	long result;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+		"# local_sub_return		\n\t"
+		DCACHE_CLEAR("%0", "r4", "%1")
+		"ld %0, @%1;			\n\t"
+		"sub	%0, %2;			\n\t"
+		"st %0, @%1;			\n\t"
+		: "=&r" (result)
+		: "r" (&l->counter), "r" (i)
+		: "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+		, "r4"
+#endif	/* CONFIG_CHIP_M32700_TS1 */
+	);
+	local_irq_restore(flags);
+
+	return result;
+}
+
+/**
+ * local_add - add long to local variable
+ * @i: long value to add
+ * @l: pointer of type local_t
+ *
+ * Atomically adds @i to @l.
+ */
+#define local_add(i,l) ((void) local_add_return((i), (l)))
+
+/**
+ * local_sub - subtract the local variable
+ * @i: long value to subtract
+ * @l: pointer of type local_t
+ *
+ * Atomically subtracts @i from @l.
+ */
+#define local_sub(i,l) ((void) local_sub_return((i), (l)))
+
+/**
+ * local_sub_and_test - subtract value from variable and test result
+ * @i: integer value to subtract
+ * @l: pointer of type local_t
+ *
+ * Atomically subtracts @i from @l and returns
+ * true if the result is zero, or false for all
+ * other cases.
+ */
+#define local_sub_and_test(i,l) (local_sub_return((i), (l)) == 0)
+
+/**
+ * local_inc_return - increment local variable and return it
+ * @l: pointer of type local_t
+ *
+ * Atomically increments @l by 1 and returns the result.
+ */
+static __inline__ long local_inc_return(local_t *l)
+{
+	unsigned long flags;
+	long result;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+		"# local_inc_return		\n\t"
+		DCACHE_CLEAR("%0", "r4", "%1")
+		"ld %0, @%1;			\n\t"
+		"addi	%0, #1;			\n\t"
+		"st %0, @%1;			\n\t"
+		: "=&r" (result)
+		: "r" (&l->counter)
+		: "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+		, "r4"
+#endif	/* CONFIG_CHIP_M32700_TS1 */
+	);
+	local_irq_restore(flags);
+
+	return result;
+}
+
+/**
+ * local_dec_return - decrement local variable and return it
+ * @l: pointer of type local_t
+ *
+ * Atomically decrements @l by 1 and returns the result.
+ */
+static __inline__ long local_dec_return(local_t *l)
+{
+	unsigned long flags;
+	long result;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+		"# local_dec_return		\n\t"
+		DCACHE_CLEAR("%0", "r4", "%1")
+		"ld %0, @%1;			\n\t"
+		"addi	%0, #-1;		\n\t"
+		"st %0, @%1;			\n\t"
+		: "=&r" (result)
+		: "r" (&l->counter)
+		: "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+		, "r4"
+#endif	/* CONFIG_CHIP_M32700_TS1 */
+	);
+	local_irq_restore(flags);
+
+	return result;
+}
+
+/**
+ * local_inc - increment local variable
+ * @l: pointer of type local_t
+ *
+ * Atomically increments @l by 1.
+ */
+#define local_inc(l) ((void)local_inc_return(l))
+
+/**
+ * local_dec - decrement local variable
+ * @l: pointer of type local_t
+ *
+ * Atomically decrements @l by 1.
+ */
+#define local_dec(l) ((void)local_dec_return(l))
+
+/**
+ * local_inc_and_test - increment and test
+ * @l: pointer of type local_t
+ *
+ * Atomically increments @l by 1
+ * and returns true if the result is zero, or false for all
+ * other cases.
+ */
+#define local_inc_and_test(l) (local_inc_return(l) == 0)
+
+/**
+ * local_dec_and_test - decrement and test
+ * @l: pointer of type local_t
+ *
+ * Atomically decrements @l by 1 and
+ * returns true if the result is 0, or false for all
+ * other cases.
+ */
+#define local_dec_and_test(l) (local_dec_return(l) == 0)
+
+/**
+ * local_add_negative - add and test if negative
+ * @l: pointer of type local_t
+ * @i: integer value to add
+ *
+ * Atomically adds @i to @l and returns true
+ * if the result is negative, or false when
+ * result is greater than or equal to zero.
+ */
+#define local_add_negative(i,l) (local_add_return((i), (l)) < 0)
+
+#define local_cmpxchg(l, o, n) (cmpxchg_local(&((l)->counter), (o), (n)))
+#define local_xchg(v, new) (xchg_local(&((l)->counter), new))
+
+/**
+ * local_add_unless - add unless the number is a given value
+ * @l: pointer of type local_t
+ * @a: the amount to add to l...
+ * @u: ...unless l is equal to u.
+ *
+ * Atomically adds @a to @l, so long as it was not @u.
+ * Returns non-zero if @l was not @u, and zero otherwise.
+ */
+static __inline__ int local_add_unless(local_t *l, long a, long u)
+{
+	long c, old;
+	c = local_read(l);
+	for (;;) {
+		if (unlikely(c == (u)))
+			break;
+		old = local_cmpxchg((l), c, c + (a));
+		if (likely(old == c))
+			break;
+		c = old;
+	}
+	return c != (u);
+}
+
+#define local_inc_not_zero(l) local_add_unless((l), 1, 0)
+
+static __inline__ void local_clear_mask(unsigned long  mask, local_t *addr)
+{
+	unsigned long flags;
+	unsigned long tmp;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+		"# local_clear_mask		\n\t"
+		DCACHE_CLEAR("%0", "r5", "%1")
+		"ld %0, @%1;			\n\t"
+		"and	%0, %2;			\n\t"
+		"st %0, @%1;			\n\t"
+		: "=&r" (tmp)
+		: "r" (addr), "r" (~mask)
+		: "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+		, "r5"
+#endif	/* CONFIG_CHIP_M32700_TS1 */
+	);
+	local_irq_restore(flags);
+}
+
+static __inline__ void local_set_mask(unsigned long  mask, local_t *addr)
+{
+	unsigned long flags;
+	unsigned long tmp;
+
+	local_irq_save(flags);
+	__asm__ __volatile__ (
+		"# local_set_mask		\n\t"
+		DCACHE_CLEAR("%0", "r5", "%1")
+		"ld %0, @%1;			\n\t"
+		"or	%0, %2;			\n\t"
+		"st %0, @%1;			\n\t"
+		: "=&r" (tmp)
+		: "r" (addr), "r" (mask)
+		: "memory"
+#ifdef CONFIG_CHIP_M32700_TS1
+		, "r5"
+#endif	/* CONFIG_CHIP_M32700_TS1 */
+	);
+	local_irq_restore(flags);
+}
+
+/* Atomic operations are already serializing on m32r */
+#define smp_mb__before_local_dec()	barrier()
+#define smp_mb__after_local_dec()	barrier()
+#define smp_mb__before_local_inc()	barrier()
+#define smp_mb__after_local_inc()	barrier()
+
+/* Use these for per-cpu local_t variables: on some archs they are
+ * much more efficient than these naive implementations.  Note they take
+ * a variable, not an address.
+ */
+
+#define __local_inc(l)		((l)->a.counter++)
+#define __local_dec(l)		((l)->a.counter++)
+#define __local_add(i,l)	((l)->a.counter+=(i))
+#define __local_sub(i,l)	((l)->a.counter-=(i))
+
+/* Use these for per-cpu local_t variables: on some archs they are
+ * much more efficient than these naive implementations.  Note they take
+ * a variable, not an address.
+ */
+
+/* Need to disable preemption for the cpu local counters otherwise we could
+   still access a variable of a previous CPU in a non local way. */
+#define cpu_local_wrap_v(l)	 	\
+	({ local_t res__;		\
+	   preempt_disable(); 		\
+	   res__ = (l);			\
+	   preempt_enable();		\
+	   res__; })
+#define cpu_local_wrap(l)		\
+	({ preempt_disable();		\
+	   l;				\
+	   preempt_enable(); })		\
+
+#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
+#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
+#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
+#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
+#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
+#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
+
+#define __cpu_local_inc(l)	cpu_local_inc(l)
+#define __cpu_local_dec(l)	cpu_local_dec(l)
+#define __cpu_local_add(i, l)	cpu_local_add((i), (l))
+#define __cpu_local_sub(i, l)	cpu_local_sub((i), (l))
 
 #endif /* __M32R_LOCAL_H */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 13/23] Add cmpxchg_local to m86k

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-m68k.patch --]
[-- Type: text/plain, Size: 1870 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: zippel@linux-m68k.org
CC: linux-m68k@lists.linux-m68k.org
---
 include/asm-m68k/system.h |   21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

Index: linux-2.6-lttng/include/asm-m68k/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-m68k/system.h	2007-08-07 14:31:51.000000000 -0400
+++ linux-2.6-lttng/include/asm-m68k/system.h	2007-08-07 15:01:41.000000000 -0400
@@ -154,6 +154,10 @@ static inline unsigned long __xchg(unsig
 }
 #endif
 
+#include <asm-generic/cmpxchg-local.h>
+
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
 /*
  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
  * store NEW in MEM.  Return the initial value in MEM.  Success is
@@ -188,6 +192,23 @@ static inline unsigned long __cmpxchg(vo
 #define cmpxchg(ptr,o,n)\
 	((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
 					(unsigned long)(n),sizeof(*(ptr))))
+#define cmpxchg_local(ptr,o,n)\
+	((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
+					(unsigned long)(n),sizeof(*(ptr))))
+#else
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
+
 #endif
 
 #define arch_align_stack(x) (x)

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 14/23] Add cmpxchg_local to m68knommu

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-m68knommu.patch --]
[-- Type: text/plain, Size: 1849 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: gerg@uclinux.org
---
 include/asm-m68knommu/system.h |   28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

Index: linux-2.6-lttng/include/asm-m68knommu/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-m68knommu/system.h	2007-07-20 18:36:08.000000000 -0400
+++ linux-2.6-lttng/include/asm-m68knommu/system.h	2007-07-20 19:37:02.000000000 -0400
@@ -187,26 +187,20 @@ static inline unsigned long __xchg(unsig
 }
 #endif
 
+#include <asm-generic/cmpxchg-local.h>
+
 /*
- * Atomic compare and exchange.  Compare OLD with MEM, if identical,
- * store NEW in MEM.  Return the initial value in MEM.  Success is
- * indicated by comparing RETURN with OLD.
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
  */
-#define __HAVE_ARCH_CMPXCHG	1
-
-static __inline__ unsigned long
-cmpxchg(volatile int *p, int old, int new)
-{
-	unsigned long flags;
-	int prev;
-
-	local_irq_save(flags);
-	if ((prev = *p) == old)
-		*p = new;
-	local_irq_restore(flags);
-	return(prev);
-}
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
 
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
 
 #ifdef CONFIG_M68332
 #define HARD_RESET_NOW() ({		\

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 15/23] Add cmpxchg_local to parisc

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-parisc.patch --]
[-- Type: text/plain, Size: 1917 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: parisc-linux@parisc-linux.org
---
 include/asm-parisc/atomic.h |   29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

Index: linux-2.6-lttng/include/asm-parisc/atomic.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-parisc/atomic.h	2007-07-20 19:44:40.000000000 -0400
+++ linux-2.6-lttng/include/asm-parisc/atomic.h	2007-07-20 19:44:47.000000000 -0400
@@ -122,6 +122,35 @@ __cmpxchg(volatile void *ptr, unsigned l
 				    (unsigned long)_n_, sizeof(*(ptr))); \
   })
 
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new_, int size)
+{
+	switch (size) {
+#ifdef CONFIG_64BIT
+	case 8:	return __cmpxchg_u64((unsigned long *)ptr, old, new_);
+#endif
+	case 4:	return __cmpxchg_u32(ptr, old, new_);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new_, size);
+	}
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	\
+     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#ifdef CONFIG_64BIT
+#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr), (o), (n))
+#else
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+#endif
+
 /* Note that we need not lock read accesses - aligned word writes/reads
  * are atomic, so a reader never sees unconsistent values.
  *

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 16/23] Add cmpxchg_local to ppc

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-ppc.patch --]
[-- Type: text/plain, Size: 2796 bytes --]

Add a local processor version of cmpxchg for ppc.

Implements __cmpxchg_u32_local and uses it for 32 bits cmpxchg_local.
It uses the non NMI safe cmpxchg_local_generic for 1, 2 and 8 bytes
cmpxchg_local.

From: Gunnar Larisch <gl@denx.de>
From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Gunnar Larisch <gl@denx.de>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: benh@kernel.crashing.org
---
 include/asm-ppc/system.h |   49 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

Index: linux-2.6-lttng/include/asm-ppc/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-ppc/system.h	2007-08-07 14:31:51.000000000 -0400
+++ linux-2.6-lttng/include/asm-ppc/system.h	2007-08-07 15:03:51.000000000 -0400
@@ -208,12 +208,34 @@ __cmpxchg_u32(volatile unsigned int *p, 
 	return prev;
 }
 
+static __inline__ unsigned long
+__cmpxchg_u32_local(volatile unsigned int *p, unsigned int old,
+	unsigned int new)
+{
+	unsigned int prev;
+
+	__asm__ __volatile__ ("\n\
+1:	lwarx	%0,0,%2 \n\
+	cmpw	0,%0,%3 \n\
+	bne	2f \n"
+	PPC405_ERR77(0,%2)
+"	stwcx.	%4,0,%2 \n\
+	bne-	1b\n"
+"2:"
+	: "=&r" (prev), "=m" (*p)
+	: "r" (p), "r" (old), "r" (new), "m" (*p)
+	: "cc", "memory");
+
+	return prev;
+}
+
 /* This function doesn't exist, so you'll get a linker error
    if something tries to do an invalid cmpxchg().  */
 extern void __cmpxchg_called_with_bad_pointer(void);
 
 static __inline__ unsigned long
-__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
+__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
+	unsigned int size)
 {
 	switch (size) {
 	case 4:
@@ -235,6 +257,31 @@ __cmpxchg(volatile void *ptr, unsigned l
 				    (unsigned long)_n_, sizeof(*(ptr))); \
   })
 
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+		return __cmpxchg_u32_local(ptr, old, new);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	\
+     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
 #define arch_align_stack(x) (x)
 
 #endif /* __KERNEL__ */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 17/23] Add cmpxchg_local to s390

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-s390.patch --]
[-- Type: text/plain, Size: 1873 bytes --]

Use the standard __cmpxchg for every type that can be updated atomically.
Use the new generic cmpxchg_local (disables interrupt) for other types.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: schwidefsky@de.ibm.com
CC: linux390@de.ibm.com
---
 include/asm-s390/system.h |   34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

Index: linux-2.6-lttng/include/asm-s390/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-s390/system.h	2007-08-10 16:16:21.000000000 -0400
+++ linux-2.6-lttng/include/asm-s390/system.h	2007-08-10 19:35:46.000000000 -0400
@@ -353,6 +353,40 @@ __cmpxchg(volatile void *ptr, unsigned l
 
 #include <linux/irqflags.h>
 
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 1:
+	case 2:
+	case 4:
+#ifdef __s390x__
+	case 8:
+#endif
+		return __cmpxchg(ptr, old, new, size);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)						\
+     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#ifdef __s390x__
+#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
+#else
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+#endif
+
 /*
  * Use to set psw mask except for the first byte which
  * won't be changed by this function.

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 18/23] Add cmpxchg_local to sh, use generic cmpxchg() instead of cmpxchg_u32

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-sh.patch --]
[-- Type: text/plain, Size: 2658 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Since cmpxchg_u32 is _exactly_ the cmpxchg_local generic implementation, remove
it and use the generic version instead.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: lethal@linux-sh.org
---
 include/asm-sh/system.h |   48 ++++++++++++------------------------------------
 1 file changed, 12 insertions(+), 36 deletions(-)

Index: linux-2.6-lttng/include/asm-sh/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-sh/system.h	2007-08-07 14:31:51.000000000 -0400
+++ linux-2.6-lttng/include/asm-sh/system.h	2007-08-07 15:08:10.000000000 -0400
@@ -201,44 +201,20 @@ extern void __xchg_called_with_bad_point
 #define xchg(ptr,x)	\
 	((__typeof__(*(ptr)))__xchg((ptr),(unsigned long)(x), sizeof(*(ptr))))
 
-static inline unsigned long __cmpxchg_u32(volatile int * m, unsigned long old,
-	unsigned long new)
-{
-	__u32 retval;
-	unsigned long flags;
-
-	local_irq_save(flags);
-	retval = *m;
-	if (retval == old)
-		*m = new;
-	local_irq_restore(flags);       /* implies memory barrier  */
-	return retval;
-}
-
-/* This function doesn't exist, so you'll get a linker error
- * if something tries to do an invalid cmpxchg(). */
-extern void __cmpxchg_called_with_bad_pointer(void);
-
-#define __HAVE_ARCH_CMPXCHG 1
+#include <asm-generic/cmpxchg-local.h>
 
-static inline unsigned long __cmpxchg(volatile void * ptr, unsigned long old,
-		unsigned long new, int size)
-{
-	switch (size) {
-	case 4:
-		return __cmpxchg_u32(ptr, old, new);
-	}
-	__cmpxchg_called_with_bad_pointer();
-	return old;
-}
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
 
-#define cmpxchg(ptr,o,n)						 \
-  ({									 \
-     __typeof__(*(ptr)) _o_ = (o);					 \
-     __typeof__(*(ptr)) _n_ = (n);					 \
-     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		 \
-				    (unsigned long)_n_, sizeof(*(ptr))); \
-  })
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
 
 extern void die(const char *str, struct pt_regs *regs, long err) __attribute__ ((noreturn));
 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 19/23] Add cmpxchg_local to sh64

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-sh64.patch --]
[-- Type: text/plain, Size: 1376 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: lethal@linux-sh.org
---
 include/asm-sh64/system.h |   15 +++++++++++++++
 1 file changed, 15 insertions(+)

Index: linux-2.6-lttng/include/asm-sh64/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-sh64/system.h	2007-07-23 11:01:16.000000000 -0400
+++ linux-2.6-lttng/include/asm-sh64/system.h	2007-07-23 11:01:20.000000000 -0400
@@ -164,6 +164,21 @@ static __inline__ unsigned long __xchg(u
 	return x;
 }
 
+#include <asm-generic/cmpxchg-local.h>
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
+
 /* XXX
  * disable hlt during certain critical i/o operations
  */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 20/23] Add cmpxchg_local to sparc, move __cmpxchg to system.h

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-sparc.patch --]
[-- Type: text/plain, Size: 4204 bytes --]

Move cmpxchg and add cmpxchg_local to system.h.
Use the new generic cmpxchg_local (disables interrupt).

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: wli@holomorphy.com
CC: sparclinux@vger.kernel.org
---
 include/asm-sparc/atomic.h |   36 ----------------------------------
 include/asm-sparc/system.h |   47 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 36 deletions(-)

Index: linux-2.6-lttng/include/asm-sparc/atomic.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-sparc/atomic.h	2007-08-10 19:36:01.000000000 -0400
+++ linux-2.6-lttng/include/asm-sparc/atomic.h	2007-08-10 19:42:47.000000000 -0400
@@ -17,42 +17,6 @@ typedef struct { volatile int counter; }
 
 #ifdef __KERNEL__
 
-/* Emulate cmpxchg() the same way we emulate atomics,
- * by hashing the object address and indexing into an array
- * of spinlocks to get a bit of performance...
- *
- * See arch/sparc/lib/atomic32.c for implementation.
- *
- * Cribbed from <asm-parisc/atomic.h>
- */
-#define __HAVE_ARCH_CMPXCHG	1
-
-/* bug catcher for when unsupported size is used - won't link */
-extern void __cmpxchg_called_with_bad_pointer(void);
-/* we only need to support cmpxchg of a u32 on sparc */
-extern unsigned long __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_);
-
-/* don't worry...optimizer will get rid of most of this */
-static __inline__ unsigned long
-__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size)
-{
-	switch(size) {
-	case 4:
-		return __cmpxchg_u32((u32 *)ptr, (u32)old, (u32)new_);
-	default:
-		__cmpxchg_called_with_bad_pointer();
-		break;
-	}
-	return old;
-}
-
-#define cmpxchg(ptr,o,n) ({						\
-	__typeof__(*(ptr)) _o_ = (o);					\
-	__typeof__(*(ptr)) _n_ = (n);					\
-	(__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,	\
-			(unsigned long)_n_, sizeof(*(ptr)));		\
-})
-
 #define ATOMIC_INIT(i)  { (i) }
 
 extern int __atomic_add_return(int, atomic_t *);
Index: linux-2.6-lttng/include/asm-sparc/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-sparc/system.h	2007-08-10 19:43:08.000000000 -0400
+++ linux-2.6-lttng/include/asm-sparc/system.h	2007-08-10 19:43:42.000000000 -0400
@@ -245,6 +245,53 @@ static __inline__ unsigned long __xchg(u
 	return x;
 }
 
+/* Emulate cmpxchg() the same way we emulate atomics,
+ * by hashing the object address and indexing into an array
+ * of spinlocks to get a bit of performance...
+ *
+ * See arch/sparc/lib/atomic32.c for implementation.
+ *
+ * Cribbed from <asm-parisc/atomic.h>
+ */
+#define __HAVE_ARCH_CMPXCHG	1
+
+/* bug catcher for when unsupported size is used - won't link */
+extern void __cmpxchg_called_with_bad_pointer(void);
+/* we only need to support cmpxchg of a u32 on sparc */
+extern unsigned long __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_);
+
+/* don't worry...optimizer will get rid of most of this */
+static __inline__ unsigned long
+__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size)
+{
+	switch(size) {
+	case 4:
+		return __cmpxchg_u32((u32 *)ptr, (u32)old, (u32)new_);
+	default:
+		__cmpxchg_called_with_bad_pointer();
+		break;
+	}
+	return old;
+}
+
+#define cmpxchg(ptr,o,n) ({						\
+	__typeof__(*(ptr)) _o_ = (o);					\
+	__typeof__(*(ptr)) _n_ = (n);					\
+	(__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,	\
+			(unsigned long)_n_, sizeof(*(ptr)));		\
+})
+
+#include <asm-generic/cmpxchg-local.h>
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
 extern void die_if_kernel(char *str, struct pt_regs *regs) __attribute__ ((noreturn));
 
 #endif /* __KERNEL__ */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 21/23] Add cmpxchg_local to sparc64

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-sparc64.patch --]
[-- Type: text/plain, Size: 1715 bytes --]

Use cmpxchg_u32 and cmpxchg_u64 for cmpxchg_local and cmpxchg64_local. For other
type sizes, use the new generic cmpxchg_local (disables interrupt).

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: sparclinux@vger.kernel.org
CC: wli@holomorphy.com
---
 include/asm-sparc64/system.h |   26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

Index: linux-2.6-lttng/include/asm-sparc64/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-sparc64/system.h	2007-08-07 14:31:51.000000000 -0400
+++ linux-2.6-lttng/include/asm-sparc64/system.h	2007-08-07 15:13:05.000000000 -0400
@@ -314,6 +314,32 @@ __cmpxchg(volatile void *ptr, unsigned l
 				    (unsigned long)_n_, sizeof(*(ptr))); \
   })
 
+#include <asm-generic/cmpxchg-local.h>
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+	case 8:	return __cmpxchg(ptr, old, new, size);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+#define cmpxchg_local(ptr,o,n)					  	\
+	(__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr), (o), (n))
+
 #endif /* !(__ASSEMBLY__) */
 
 #define arch_align_stack(x) (x)

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 22/23] Add cmpxchg_local to v850

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-v850.patch --]
[-- Type: text/plain, Size: 1353 bytes --]

Use the new generic cmpxchg_local (disables interrupt). Also use the generic
cmpxchg as fallback if SMP is not set.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
---
 include/asm-v850/system.h |   15 +++++++++++++++
 1 file changed, 15 insertions(+)

Index: linux-2.6-lttng/include/asm-v850/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-v850/system.h	2007-07-23 11:28:45.000000000 -0400
+++ linux-2.6-lttng/include/asm-v850/system.h	2007-07-23 11:28:57.000000000 -0400
@@ -104,6 +104,21 @@ static inline unsigned long __xchg (unsi
 	return tmp;
 }
 
+#include <asm-generic/cmpxchg-local.h>
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
+#ifndef CONFIG_SMP
+#include <asm-generic/cmpxchg.h>
+#endif
+
 #define arch_align_stack(x) (x)
 
 #endif /* __V850_SYSTEM_H__ */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 23/23] Add cmpxchg_local to xtensa

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-xtensa.patch --]
[-- Type: text/plain, Size: 1631 bytes --]

Use the architecture specific __cmpxchg_u32 for 32 bits cmpxchg)_local. Else,
use the new generic cmpxchg_local (disables interrupt).

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
chris@zankel.net
---
 include/asm-xtensa/system.h |   22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

Index: linux-2.6-lttng/include/asm-xtensa/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-xtensa/system.h	2007-07-23 11:29:18.000000000 -0400
+++ linux-2.6-lttng/include/asm-xtensa/system.h	2007-07-23 11:32:32.000000000 -0400
@@ -156,8 +156,30 @@ __cmpxchg(volatile void *ptr, unsigned l
 	   			        (unsigned long)_n_, sizeof (*(ptr))); \
 	})
 
+#include <asm-generic/cmpxchg-local.h>
 
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+		return __cmpxchg_u32(ptr, old, new);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
 
+	return old;
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	    \
+     (__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o), \
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
 
 /*
  * xchg_u32

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [patch 00/23] Atomic operations updates: add cmpxchg_local

[Mathieu Desnoyers]

* Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca) wrote:
> Hi Andrew,
> 
> I have been asked for a cmpxchg_local implementation by Christoph Lameter so he
> can use it to accelerate the slub allocator.
> 
> On some architectures, where cmpxchg is nothing more that a simple save/restore
> interrupts surrounding the cmpxchg non atomic ops, I remove the code and make
> them use the new "generic" version.
> 
> cmpxchg_local and cmpxchg64_local are made available on all architectures in
> these patches. They use the best primitive known, either an atomic operation or
> save/restore interrupts.
> 

Forgot to mention: it applies on top of 2.6.23-rc2-mm2

series file:

#
# cmpxchg_local and cmpxchg64_local standardization
#
i386-cmpxchg64-80386-80486-fallback.patch
#
add-cmpxchg-local-to-generic-for-up.patch
add-cmpxchg-local-to-arm.patch
add-cmpxchg-local-to-avr32.patch
add-cmpxchg-local-to-blackfin.patch
add-cmpxchg-local-to-cris.patch
add-cmpxchg-local-to-frv.patch
add-cmpxchg-local-to-h8300.patch
add-cmpxchg-local-to-ia64.patch
add-cmpxchg-local-to-m32r.patch
fix-m32r-__xchg.patch
local_t_m32r_optimized.patch
add-cmpxchg-local-to-m68k.patch
add-cmpxchg-local-to-m68knommu.patch
add-cmpxchg-local-to-parisc.patch
add-cmpxchg-local-to-ppc.patch
add-cmpxchg-local-to-s390.patch
add-cmpxchg-local-to-sh.patch
add-cmpxchg-local-to-sh64.patch
add-cmpxchg-local-to-sparc.patch
add-cmpxchg-local-to-sparc64.patch
add-cmpxchg-local-to-v850.patch
add-cmpxchg-local-to-xtensa.patch

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and 80486

[Jan Engelhardt]

On Aug 12 2007 10:54, Mathieu Desnoyers wrote:

>Date: Sun, 12 Aug 2007 10:54:35 -0400
>From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
>To:  <akpm@linux-foundation.org>,  <linux-kernel@vger.kernel.org>
>Cc: Christoph Lameter <clameter@sgi.com>,
>    Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>,  <mingo@redhat.com>
>Subject: [patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and
>     80486
>
>Actually, on 386, cmpxchg and cmpxchg_local fall back on
>cmpxchg_386_u8/16/32: it disables interruptions around non atomic
>updates to mimic the cmpxchg behavior.
>
>The comment:
>/* Poor man's cmpxchg for 386. Unsuitable for SMP */
>
>already present in cmpxchg_386_u32 tells much about how this cmpxchg
>implementation should not be used in a SMP context. However, the cmpxchg_local
>can perfectly use this fallback, since it only needs to be atomic wrt the local
>cpu.
>
>This patch adds a cmpxchg_486_u64 and uses it as a fallback for cmpxchg64
>and cmpxchg64_local on 80386 and 80486.

hm, but why is it called cmpxchg_486 when the other functions are called
cmpxchg_386?


	Jan
-- 

Re: [patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and 80486

[Mathieu Desnoyers]

* Jan Engelhardt (jengelh@computergmbh.de) wrote:
> 
> On Aug 12 2007 10:54, Mathieu Desnoyers wrote:
> 
> >Date: Sun, 12 Aug 2007 10:54:35 -0400
> >From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> >To:  <akpm@linux-foundation.org>,  <linux-kernel@vger.kernel.org>
> >Cc: Christoph Lameter <clameter@sgi.com>,
> >    Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>,  <mingo@redhat.com>
> >Subject: [patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and
> >     80486
> >
> >Actually, on 386, cmpxchg and cmpxchg_local fall back on
> >cmpxchg_386_u8/16/32: it disables interruptions around non atomic
> >updates to mimic the cmpxchg behavior.
> >
> >The comment:
> >/* Poor man's cmpxchg for 386. Unsuitable for SMP */
> >
> >already present in cmpxchg_386_u32 tells much about how this cmpxchg
> >implementation should not be used in a SMP context. However, the cmpxchg_local
> >can perfectly use this fallback, since it only needs to be atomic wrt the local
> >cpu.
> >
> >This patch adds a cmpxchg_486_u64 and uses it as a fallback for cmpxchg64
> >and cmpxchg64_local on 80386 and 80486.
> 
> hm, but why is it called cmpxchg_486 when the other functions are called
> cmpxchg_386?
> 

Because the standard cmpxchg is missing only on  386, but cmpxchg8b is
missing both on 386 and 486.

Citing Intel's Instruction set reference:

cmpxchg:
This instruction is not supported on Intel processors earlier than the
Intel486 processors.

cmpxchg8b:
This instruction encoding is not supported on Intel processors earlier
than the Pentium processors.

Mathieu

> 
> 	Jan
> -- 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [patch 09/23] Add cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64

[Keith Owens]

Mathieu Desnoyers (on Sun, 12 Aug 2007 10:54:43 -0400) wrote:
>Add the primitives cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64. They
>use cmpxchg_acq as underlying macro, just like the already existing ia64
>cmpxchg().
>
>Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
>CC: clameter@sgi.com
>CC: tony.luck@intel.com
>---
> include/asm-ia64/intrinsics.h |    4 ++++
> 1 file changed, 4 insertions(+)
>
>Index: linux-2.6-lttng/include/asm-ia64/intrinsics.h
>===================================================================
>--- linux-2.6-lttng.orig/include/asm-ia64/intrinsics.h	2007-07-20 18:36:09.000000000 -0400
>+++ linux-2.6-lttng/include/asm-ia64/intrinsics.h	2007-07-20 19:29:17.000000000 -0400
>@@ -158,6 +158,10 @@ extern long ia64_cmpxchg_called_with_bad
> 
> /* for compatibility with other platforms: */
> #define cmpxchg(ptr,o,n)	cmpxchg_acq(ptr,o,n)
>+#define cmpxchg_local(ptr,o,n)	cmpxchg_acq(ptr,o,n)
>+
>+#define cmpxchg64(ptr,o,n)		cmpxchg_acq(ptr,o,n)
>+#define cmpxchg64_local(ptr,o,n)	cmpxchg_acq(ptr,o,n)

As a matter of coding style, I prefer

#define cmpxchg_local	cmpxchg
#define cmpxchg64_local	cmpxchg64

Which makes it absolutely clear that they are the same code.  With your
patch, humans have to do a string compare of two defines to see if they
are the same.

[patch 09/23] Add cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64 (revised)

[Mathieu Desnoyers]

Add cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64 (revised)

Add the primitives cmpxchg_local, cmpxchg64 and cmpxchg64_local to ia64. They
use cmpxchg_acq as underlying macro, just like the already existing ia64
cmpxchg().

Changelog:

ia64 cmpxchg_local coding style fix
Quoting Keith Owens:

As a matter of coding style, I prefer

#define cmpxchg_local   cmpxchg
#define cmpxchg64_local cmpxchg64

Which makes it absolutely clear that they are the same code.  With your
patch, humans have to do a string compare of two defines to see if they
are the same.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: tony.luck@intel.com
CC: Keith Owens <kaos@ocs.com.au>
---
 include/asm-ia64/intrinsics.h |    4 ++++
 1 file changed, 4 insertions(+)

Index: linux-2.6-lttng/include/asm-ia64/intrinsics.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-ia64/intrinsics.h	2007-08-12 09:33:57.000000000 -0400
+++ linux-2.6-lttng/include/asm-ia64/intrinsics.h	2007-08-12 15:09:49.000000000 -0400
@@ -158,6 +158,10 @@ extern long ia64_cmpxchg_called_with_bad
 
 /* for compatibility with other platforms: */
 #define cmpxchg(ptr,o,n)	cmpxchg_acq(ptr,o,n)
+#define cmpxchg64(ptr,o,n)	cmpxchg_acq(ptr,o,n)
+
+#define cmpxchg_local		cmpxchg
+#define cmpxchg64_local		cmpxchg64
 
 #ifdef CONFIG_IA64_DEBUG_CMPXCHG
 # define CMPXCHG_BUGCHECK_DECL	int _cmpxchg_bugcheck_count = 128;
-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [patch 17/23] Add cmpxchg_local to s390

[Heiko Carstens]

On Sun, Aug 12, 2007 at 10:54:51AM -0400, Mathieu Desnoyers wrote:
> Use the standard __cmpxchg for every type that can be updated atomically.
> Use the new generic cmpxchg_local (disables interrupt) for other types.
> 
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> CC: clameter@sgi.com
> CC: schwidefsky@de.ibm.com
> CC: linux390@de.ibm.com
> ---
>  include/asm-s390/system.h |   34 ++++++++++++++++++++++++++++++++++
>  1 file changed, 34 insertions(+)
> 
> Index: linux-2.6-lttng/include/asm-s390/system.h
> ===================================================================
> --- linux-2.6-lttng.orig/include/asm-s390/system.h	2007-08-10 16:16:21.000000000 -0400
> +++ linux-2.6-lttng/include/asm-s390/system.h	2007-08-10 19:35:46.000000000 -0400
> @@ -353,6 +353,40 @@ __cmpxchg(volatile void *ptr, unsigned l
> 
>  #include <linux/irqflags.h>
> 
> +#include <asm-generic/cmpxchg-local.h>
> +
> +static inline unsigned long __cmpxchg_local(volatile void *ptr,
> +				      unsigned long old,
> +				      unsigned long new, int size)
> +{
> +	switch (size) {
> +	case 1:
> +	case 2:
> +	case 4:
> +#ifdef __s390x__
> +	case 8:
> +#endif
> +		return __cmpxchg(ptr, old, new, size);
> +	default:
> +		return __cmpxchg_local_generic(ptr, old, new, size);
> +	}
> +
> +	return old;
> +}
> +
> +/*
> + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
> + * them available.
> + */
> +#define cmpxchg_local(ptr,o,n)						\
> +     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
> +			   	 (unsigned long)(n), sizeof(*(ptr)))
> +#ifdef __s390x__
> +#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
> +#else
> +#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
> +#endif
> +

What's the reason to have cmpxchg64_local on 32 bit architectures?
Without that need all this would just be a few simple defines.

Re: [patch 17/23] Add cmpxchg_local to s390

[Mathieu Desnoyers]

* Heiko Carstens (heiko.carstens@de.ibm.com) wrote:
> On Sun, Aug 12, 2007 at 10:54:51AM -0400, Mathieu Desnoyers wrote:
> > Use the standard __cmpxchg for every type that can be updated atomically.
> > Use the new generic cmpxchg_local (disables interrupt) for other types.
> > 
> > Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> > CC: clameter@sgi.com
> > CC: schwidefsky@de.ibm.com
> > CC: linux390@de.ibm.com
> > ---
> >  include/asm-s390/system.h |   34 ++++++++++++++++++++++++++++++++++
> >  1 file changed, 34 insertions(+)
> > 
> > Index: linux-2.6-lttng/include/asm-s390/system.h
> > ===================================================================
> > --- linux-2.6-lttng.orig/include/asm-s390/system.h	2007-08-10 16:16:21.000000000 -0400
> > +++ linux-2.6-lttng/include/asm-s390/system.h	2007-08-10 19:35:46.000000000 -0400
> > @@ -353,6 +353,40 @@ __cmpxchg(volatile void *ptr, unsigned l
> > 
> >  #include <linux/irqflags.h>
> > 
> > +#include <asm-generic/cmpxchg-local.h>
> > +
> > +static inline unsigned long __cmpxchg_local(volatile void *ptr,
> > +				      unsigned long old,
> > +				      unsigned long new, int size)
> > +{
> > +	switch (size) {
> > +	case 1:
> > +	case 2:
> > +	case 4:
> > +#ifdef __s390x__
> > +	case 8:
> > +#endif
> > +		return __cmpxchg(ptr, old, new, size);
> > +	default:
> > +		return __cmpxchg_local_generic(ptr, old, new, size);
> > +	}
> > +
> > +	return old;
> > +}
> > +
> > +/*
> > + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
> > + * them available.
> > + */
> > +#define cmpxchg_local(ptr,o,n)						\
> > +     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
> > +			   	 (unsigned long)(n), sizeof(*(ptr)))
> > +#ifdef __s390x__
> > +#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
> > +#else
> > +#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
> > +#endif
> > +
> 
> What's the reason to have cmpxchg64_local on 32 bit architectures?
> Without that need all this would just be a few simple defines.

cmpxchg64_local on 32 bits architectures takes unsigned long long
parameters, but cmpxchg_local only takes longs. Since we have cmpxchg8b
to execute a 8 byte cmpxchg atomically on pentium and +, it makes sense
to provide a flavor of cmpxchg and cmpxchg_local using this instruction.

Also, for 32 bits architectures lacking the 64 bits atomic cmpxchg, it
makes sense _not_ to define cmpxchg64 while cmpxchg could still be
available.

Moreover, the fallback for cmpxchg8b on i386 for 386 and 486 is a
different case than cmpxchg (which is only required for 386). Using
different code makes this easier.

However, cmpxchg64_local will be emulated by disabling interrupts on all
architectures where it is not supported atomically.

Therefore, we *could* turn cmpxchg64_local into a cmpxchg_local, but it
would make the 386/486 fallbacks ugly, make its design different from
cmpxchg/cmpxchg64 (which really depends on atomic operations and cannot
be emulated) and require the __cmpxchg_local to be expressed as a macro
rather than an inline function so the parameters would not be fixed to
unsigned long long in every case.

So I think cmpxchg64_local makes sense there, but I am open to
suggestions.

Mathieu

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [patch 04/23] Add cmpxchg_local to avr32

[Haavard Skinnemoen]

On Sun, 12 Aug 2007 10:54:38 -0400
Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> wrote:

> Use the new generic cmpxchg_local (disables interrupt) for 8, 16 and
> 64 bits cmpxchg_local. Use the __cmpxchg_u32 primitive for 32 bits
> cmpxchg_local.
> 
> Note that cmpxchg only uses the __cmpxchg_u32 or __cmpxchg_u64 and
> will cause a linker error if called with 8 or 16 bits argument.

Actually, you'll get a linker error if it's called with a 64-bit
argument too. But your patch doesn't use __cmpxchg_u64 so it's fine.

> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

Looks good to me. I suppose we _could_ implement 8- and 16-bit cmpxchg
using stcond, but I really doubt it's worth it.

Acked-by: Haavard Skinnemoen <hskinnemoen@atmel.com>

Re: [patch 00/23] Atomic operations updates: add cmpxchg_local

[Christoph Lameter]

On Sun, 12 Aug 2007, Mathieu Desnoyers wrote:

> I have been asked for a cmpxchg_local implementation by Christoph Lameter so he
> can use it to accelerate the slub allocator.

Hmmm... I have run my own and tested a variety of approaches on cmpxchg a 
couple of weeks ago on an Athlong64. There was no significant performance 
gain (even on a test to just allocate from the same slab cache) and there 
were actual losses in the free path since the logic got a bit complex.

The main performance issue are still the cacheline that are 
touched and cmpxchg will not change that.

The exact code that was used to run these tests can be found in the 
Performance Counter Patchset that I posted 2 weeks ago.

cmpxchg_local

SLUB Performance testing
========================
1. Kmalloc: Repeatedly allocate then free test
10000 times kmalloc(8) = 2.3ms kfree() = 1.1ms
10000 times kmalloc(16) = 2.4ms kfree() = 1.2ms
10000 times kmalloc(32) = 2.4ms kfree() = 1.2ms
10000 times kmalloc(64) = 2.6ms kfree() = 1.4ms
10000 times kmalloc(128) = 2.8ms kfree() = 1.9ms
10000 times kmalloc(256) = 3ms kfree() = 2ms
10000 times kmalloc(512) = 3.4ms kfree() = 2.1ms
10000 times kmalloc(1024) = 3.9ms kfree() = 2.4ms
10000 times kmalloc(2048) = 4.1ms kfree() = 2.6ms
10000 times kmalloc(4096) = 4.8ms kfree() = 3.5ms
2. Kmalloc: alloc/free test
10000 times kmalloc(8)/kfree = 3ms
10000 times kmalloc(16)/kfree = 3.1ms
10000 times kmalloc(32)/kfree = 3.1ms
10000 times kmalloc(64)/kfree = 3.1ms
10000 times kmalloc(128)/kfree = 3.2ms
10000 times kmalloc(256)/kfree = 3.3ms
10000 times kmalloc(512)/kfree = 3.3ms
10000 times kmalloc(1024)/kfree = 3.4ms
10000 times kmalloc(2048)/kfree = 3.4ms
10000 times kmalloc(4096)/kfree = 3.5ms

base line

1. Repeatedly allocate then free test
10000 times kmalloc(8) = 2.4ms kfree() = 1.1ms
10000 times kmalloc(16) = 2.5ms kfree() = 1.1ms
10000 times kmalloc(32) = 2.5ms kfree() = 1.1ms
10000 times kmalloc(64) = 2.7ms kfree() = 1.4ms
10000 times kmalloc(128) = 2.9ms kfree() = 1.9ms
10000 times kmalloc(256) = 3.1ms kfree() = 2ms
10000 times kmalloc(512) = 3.4ms kfree() = 2.1ms
10000 times kmalloc(1024) = 3.9ms kfree() = 2.4ms
10000 times kmalloc(2048) = 4.2ms kfree() = 2.5ms
10000 times kmalloc(4096) = 4.9ms kfree() = 3.2ms
2. Alloc/Free test
10000 times kmalloc(8)/kfree = 3ms
10000 times kmalloc(16)/kfree = 3.1ms
10000 times kmalloc(32)/kfree = 3.1ms
10000 times kmalloc(64)/kfree = 3.1ms
10000 times kmalloc(128)/kfree = 3.2ms
10000 times kmalloc(256)/kfree = 3.2ms
10000 times kmalloc(512)/kfree = 3.2ms
10000 times kmalloc(1024)/kfree = 3.3ms
10000 times kmalloc(2048)/kfree = 3.3ms
10000 times kmalloc(4096)/kfree = 3.3ms

Re: [patch 11/23] Fix m32r __xchg

[Adrian Bunk]

On Sun, Aug 12, 2007 at 10:54:45AM -0400, Mathieu Desnoyers wrote:
> the #endif  /* CONFIG_SMP */ should cover the default condition, or it may cause
> bad parameter to be silently missed.
> 
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> CC: takata@linux-m32r.org
> CC: linux-m32r@ml.linux-m32r.org
> ---
>  include/asm-m32r/system.h |    2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> Index: linux-2.6-lttng/include/asm-m32r/system.h
> ===================================================================
> --- linux-2.6-lttng.orig/include/asm-m32r/system.h	2007-08-07 14:55:02.000000000 -0400
> +++ linux-2.6-lttng/include/asm-m32r/system.h	2007-08-07 14:57:57.000000000 -0400
> @@ -189,9 +189,9 @@ __xchg(unsigned long x, volatile void * 
>  #endif	/* CONFIG_CHIP_M32700_TS1 */
>  		);
>  		break;
> +#endif  /* CONFIG_SMP */
>  	default:
>  		__xchg_called_with_bad_pointer();
> -#endif  /* CONFIG_SMP */
>  	}
>  
>  	local_irq_restore(flags);


It seems you never checked whether your patch compiles:

<--  snip  -->

...
  CC      init/main.o
In file included from include2/asm/bitops.h:16,
                 from /home/bunk/linux/kernel-2.6/linux-2.6.23-rc2-mm2/include/linux/bitops.h:9,
                 from /home/bunk/linux/kernel-2.6/linux-2.6.23-rc2-mm2/include/linux/kernel.h:15,
                 from include2/asm/processor.h:16,
                 from /home/bunk/linux/kernel-2.6/linux-2.6.23-rc2-mm2/include/linux/prefetch.h:14,
                 from /home/bunk/linux/kernel-2.6/linux-2.6.23-rc2-mm2/include/linux/list.h:8,
                 from /home/bunk/linux/kernel-2.6/linux-2.6.23-rc2-mm2/include/linux/module.h:9,
                 from /home/bunk/linux/kernel-2.6/linux-2.6.23-rc2-mm2/init/main.c:13:
include2/asm/system.h: In function '__xchg':
include2/asm/system.h:191: error: implicit declaration of function '__xchg_called_with_bad_pointer'
make[2]: *** [init/main.o] Error 1

<--  snip  -->


cu
Adrian

-- 

       "Is there not promise of rain?" Ling Tan asked suddenly out
        of the darkness. There had been need of rain for many days.
       "Only a promise," Lao Er said.
                                       Pearl S. Buck - Dragon Seed

[PATCH] Fix m32r __xchg (revised)

[Mathieu Desnoyers]

Fix m32r __xchg (revised)

the #endif  /* CONFIG_SMP */ should cover the default condition, or it may cause
bad parameter to be silently missed.

To make it work correctly, we have to remove the ifdef CONFIG SMP surrounding 
__xchg_called_with_bad_pointer declaration. Thanks to Adrian Bunk for
detecting this.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: takata@linux-m32r.org
CC: linux-m32r@ml.linux-m32r.org
CC: Adrian Bunk <bunk@kernel.org>
---
 include/asm-m32r/system.h |    4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

Index: linux-2.6-lttng/include/asm-m32r/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-m32r/system.h	2007-08-13 18:21:02.000000000 -0400
+++ linux-2.6-lttng/include/asm-m32r/system.h	2007-08-19 07:08:26.000000000 -0400
@@ -127,9 +127,7 @@ static inline void local_irq_disable(voi
 	((__typeof__(*(ptr)))__xchg_local((unsigned long)(x),(ptr), \
 			sizeof(*(ptr))))
 
-#ifdef CONFIG_SMP
 extern void  __xchg_called_with_bad_pointer(void);
-#endif
 
 #ifdef CONFIG_CHIP_M32700_TS1
 #define DCACHE_CLEAR(reg0, reg1, addr)				\
@@ -189,9 +187,9 @@ __xchg(unsigned long x, volatile void * 
 #endif	/* CONFIG_CHIP_M32700_TS1 */
 		);
 		break;
+#endif  /* CONFIG_SMP */
 	default:
 		__xchg_called_with_bad_pointer();
-#endif  /* CONFIG_SMP */
 	}
 
 	local_irq_restore(flags);
-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 16/23] Add cmpxchg_local to ppc

[Mathieu Desnoyers]

[-- Attachment #1: add-cmpxchg-local-to-ppc.patch --]
[-- Type: text/plain, Size: 2796 bytes --]

Add a local processor version of cmpxchg for ppc.

Implements __cmpxchg_u32_local and uses it for 32 bits cmpxchg_local.
It uses the non NMI safe cmpxchg_local_generic for 1, 2 and 8 bytes
cmpxchg_local.

From: Gunnar Larisch <gl@denx.de>
From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Gunnar Larisch <gl@denx.de>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: clameter@sgi.com
CC: benh@kernel.crashing.org
---
 include/asm-ppc/system.h |   49 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

Index: linux-2.6-lttng/include/asm-ppc/system.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-ppc/system.h	2007-08-07 14:31:51.000000000 -0400
+++ linux-2.6-lttng/include/asm-ppc/system.h	2007-08-07 15:03:51.000000000 -0400
@@ -208,12 +208,34 @@ __cmpxchg_u32(volatile unsigned int *p, 
 	return prev;
 }
 
+static __inline__ unsigned long
+__cmpxchg_u32_local(volatile unsigned int *p, unsigned int old,
+	unsigned int new)
+{
+	unsigned int prev;
+
+	__asm__ __volatile__ ("\n\
+1:	lwarx	%0,0,%2 \n\
+	cmpw	0,%0,%3 \n\
+	bne	2f \n"
+	PPC405_ERR77(0,%2)
+"	stwcx.	%4,0,%2 \n\
+	bne-	1b\n"
+"2:"
+	: "=&r" (prev), "=m" (*p)
+	: "r" (p), "r" (old), "r" (new), "m" (*p)
+	: "cc", "memory");
+
+	return prev;
+}
+
 /* This function doesn't exist, so you'll get a linker error
    if something tries to do an invalid cmpxchg().  */
 extern void __cmpxchg_called_with_bad_pointer(void);
 
 static __inline__ unsigned long
-__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
+__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
+	unsigned int size)
 {
 	switch (size) {
 	case 4:
@@ -235,6 +257,31 @@ __cmpxchg(volatile void *ptr, unsigned l
 				    (unsigned long)_n_, sizeof(*(ptr))); \
   })
 
+#include <asm-generic/cmpxchg-local.h>
+
+static inline unsigned long __cmpxchg_local(volatile void *ptr,
+				      unsigned long old,
+				      unsigned long new, int size)
+{
+	switch (size) {
+	case 4:
+		return __cmpxchg_u32_local(ptr, old, new);
+	default:
+		return __cmpxchg_local_generic(ptr, old, new, size);
+	}
+
+	return old;
+}
+
+/*
+ * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
+ * them available.
+ */
+#define cmpxchg_local(ptr,o,n)					  	\
+     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
+			   	 (unsigned long)(n), sizeof(*(ptr)))
+#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
+
 #define arch_align_stack(x) (x)
 
 #endif /* __KERNEL__ */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Grant Grundler]

On Sun, Aug 12, 2007 at 10:54:49AM -0400, Mathieu Desnoyers wrote:
> Use the new generic cmpxchg_local (disables interrupt). Also use the generic
> cmpxchg as fallback if SMP is not set.

Mathieu,
thanks for adding __cmpxchg_local to parisc.... but why do we need it?

By definition, atomic operators are, well, atomic.

I searched for __cmpxchg_local and found this reference:
    http://www.ussg.iu.edu/hypermail/linux/kernel/0612.2/1337.html

but the "root" of that thread (Dec 20, 2006):
    http://www.ussg.iu.edu/hypermail/linux/kernel/0612.2/1334.html

Doesn't explain the difference between "local" and "non-local" either.
Per CPU data should only need memory barriers (in some cases) and
protection against interrupts (in probably more cases). So I'm not
understanding why a new set of APIs is needed.

Can you add a description to Documentation/atomic_ops.txt ?
*sigh* sorry for being "late to the party" on this one...

cheers,
grant

> 
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> CC: clameter@sgi.com
> CC: parisc-linux@parisc-linux.org
> ---
>  include/asm-parisc/atomic.h |   29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
> 
> Index: linux-2.6-lttng/include/asm-parisc/atomic.h
> ===================================================================
> --- linux-2.6-lttng.orig/include/asm-parisc/atomic.h	2007-07-20 19:44:40.000000000 -0400
> +++ linux-2.6-lttng/include/asm-parisc/atomic.h	2007-07-20 19:44:47.000000000 -0400
> @@ -122,6 +122,35 @@ __cmpxchg(volatile void *ptr, unsigned l
>  				    (unsigned long)_n_, sizeof(*(ptr))); \
>    })
>  
> +#include <asm-generic/cmpxchg-local.h>
> +
> +static inline unsigned long __cmpxchg_local(volatile void *ptr,
> +				      unsigned long old,
> +				      unsigned long new_, int size)
> +{
> +	switch (size) {
> +#ifdef CONFIG_64BIT
> +	case 8:	return __cmpxchg_u64((unsigned long *)ptr, old, new_);
> +#endif
> +	case 4:	return __cmpxchg_u32(ptr, old, new_);
> +	default:
> +		return __cmpxchg_local_generic(ptr, old, new_, size);
> +	}
> +}
> +
> +/*
> + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
> + * them available.
> + */
> +#define cmpxchg_local(ptr,o,n)					  	\
> +     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
> +			   	 (unsigned long)(n), sizeof(*(ptr)))
> +#ifdef CONFIG_64BIT
> +#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr), (o), (n))
> +#else
> +#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
> +#endif
> +
>  /* Note that we need not lock read accesses - aligned word writes/reads
>   * are atomic, so a reader never sees unconsistent values.
>   *
> 
> -- 
> Mathieu Desnoyers
> Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
> OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
> _______________________________________________
> parisc-linux mailing list
> parisc-linux@lists.parisc-linux.org
> http://lists.parisc-linux.org/mailman/listinfo/parisc-linux

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Mathieu Desnoyers]

* Grant Grundler (grundler@parisc-linux.org) wrote:
> On Sun, Aug 12, 2007 at 10:54:49AM -0400, Mathieu Desnoyers wrote:
> > Use the new generic cmpxchg_local (disables interrupt). Also use the generic
> > cmpxchg as fallback if SMP is not set.
> 
> Mathieu,
> thanks for adding __cmpxchg_local to parisc.... but why do we need it?
> 
> By definition, atomic operators are, well, atomic.
> 
> I searched for __cmpxchg_local and found this reference:
>     http://www.ussg.iu.edu/hypermail/linux/kernel/0612.2/1337.html
> 
> but the "root" of that thread (Dec 20, 2006):
>     http://www.ussg.iu.edu/hypermail/linux/kernel/0612.2/1334.html
> 
> Doesn't explain the difference between "local" and "non-local" either.
> Per CPU data should only need memory barriers (in some cases) and
> protection against interrupts (in probably more cases). So I'm not
> understanding why a new set of APIs is needed.
> 
> Can you add a description to Documentation/atomic_ops.txt ?
> *sigh* sorry for being "late to the party" on this one...
> 

Does Documentation/local_ops.txt answer your questions ? If not, please
tell me and I'll gladly explain more.

Mathieu

> cheers,
> grant
> 
> > 
> > Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> > CC: clameter@sgi.com
> > CC: parisc-linux@parisc-linux.org
> > ---
> >  include/asm-parisc/atomic.h |   29 +++++++++++++++++++++++++++++
> >  1 file changed, 29 insertions(+)
> > 
> > Index: linux-2.6-lttng/include/asm-parisc/atomic.h
> > ===================================================================
> > --- linux-2.6-lttng.orig/include/asm-parisc/atomic.h	2007-07-20 19:44:40.000000000 -0400
> > +++ linux-2.6-lttng/include/asm-parisc/atomic.h	2007-07-20 19:44:47.000000000 -0400
> > @@ -122,6 +122,35 @@ __cmpxchg(volatile void *ptr, unsigned l
> >  				    (unsigned long)_n_, sizeof(*(ptr))); \
> >    })
> >  
> > +#include <asm-generic/cmpxchg-local.h>
> > +
> > +static inline unsigned long __cmpxchg_local(volatile void *ptr,
> > +				      unsigned long old,
> > +				      unsigned long new_, int size)
> > +{
> > +	switch (size) {
> > +#ifdef CONFIG_64BIT
> > +	case 8:	return __cmpxchg_u64((unsigned long *)ptr, old, new_);
> > +#endif
> > +	case 4:	return __cmpxchg_u32(ptr, old, new_);
> > +	default:
> > +		return __cmpxchg_local_generic(ptr, old, new_, size);
> > +	}
> > +}
> > +
> > +/*
> > + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
> > + * them available.
> > + */
> > +#define cmpxchg_local(ptr,o,n)					  	\
> > +     (__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o),	\
> > +			   	 (unsigned long)(n), sizeof(*(ptr)))
> > +#ifdef CONFIG_64BIT
> > +#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr), (o), (n))
> > +#else
> > +#define cmpxchg64_local(ptr,o,n) __cmpxchg64_local_generic((ptr), (o), (n))
> > +#endif
> > +
> >  /* Note that we need not lock read accesses - aligned word writes/reads
> >   * are atomic, so a reader never sees unconsistent values.
> >   *
> > 
> > -- 
> > Mathieu Desnoyers
> > Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
> > OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
> > _______________________________________________
> > parisc-linux mailing list
> > parisc-linux@lists.parisc-linux.org
> > http://lists.parisc-linux.org/mailman/listinfo/parisc-linux

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[David Miller]

From: Grant Grundler <grundler@parisc-linux.org>
Date: Mon, 27 Aug 2007 15:04:32 -0600

> Doesn't explain the difference between "local" and "non-local" either.
> Per CPU data should only need memory barriers (in some cases) and
> protection against interrupts (in probably more cases). So I'm not
> understanding why a new set of APIs is needed.
> 
> Can you add a description to Documentation/atomic_ops.txt ?

My understanding is that the local versions can run without
memory barriers.  Only local cpu execution and local interrupt
threads of control can access the variable, and interrupts
force all stores to become visible to the local cpu so an
interrupt can't see the datum from before the atomic update
if one was in progress when the interrupt triggered, and
stuff like that.

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Grant Grundler]

On Mon, Aug 27, 2007 at 05:11:40PM -0400, Mathieu Desnoyers wrote:
...
> > Can you add a description to Documentation/atomic_ops.txt ?
> > *sigh* sorry for being "late to the party" on this one...
> 
> Does Documentation/local_ops.txt answer your questions ? If not, please
> tell me and I'll gladly explain more.

Yes, it does mostly - thanks. 

A few questions/nits:
o Did you attempt quantify how many places in the kernel could use this?
  I'm just trying to get a feel for how useful this really is vs just 
  using existing mechanisms (that people understand) to implement a 
  non-SMP-safe counter that protects updates (writes) against interrupts.
  If you did, adding some referencs to local_ops.txt would be helpful
  so folks could look for examples of "correct usage".

o Wording in local_ops.txt: "on the 
	"... it will then appear to be written out of order wrt
	 other memory writes on the owner CPU."

  I'd like to suggest "by the owner CPU".

o How can a local_t counter protect updates (writes) against interrupts 
  but not preemption?
  I always thought preemption required some sort of interrupt or trap.
  Maybe the local_ops.txt explains that and I just missed it.

  DaveM explained updates "in flight" would not be visible to interrupts
  and I suspect that's the answer to my question....but then I don't "feel
  good" the local_ops are safe to update in interrupts _and_ the process
  context kernel.  Maybe the relationship between local_ops, preemption,
  and interrupts could be explained more carefully in local_ops.txt.

o OK to add a reference for local_ops.txt to atomic_ops.txt?
  They are obviously related and anyone "discovering" one of the docs
  should be made aware of the other.
  Patch+log entry appended below. Please sign-off if that's ok with you.


thanks,
grant

Diff+Commit entry against 2.6.22.5:

local_t is a variant of atomic_t and has related ops to match.
Add reference for local_t documentation to atomic_ops.txt. 

Signed-off-by: Grant Grundler <grundler@parisc-linux.org>


--- 2.6.22.5-ORIG/Documentation/atomic_ops.txt	2007-08-27 22:50:27.000000000 -0700
+++ 2.6.22.5-ggg/Documentation/atomic_ops.txt	2007-08-27 22:54:44.000000000 -0700
@@ -14,6 +14,10 @@
 
 	typedef struct { volatile int counter; } atomic_t;
 
+local_t is very similar to atomic_t. If the counter is per CPU and only
+updated by one CPU, local_t is probably more appropriate. Please see
+Documentation/local_ops.txt for the semantics of local_t.
+
 	The first operations to implement for atomic_t's are the
 initializers and plain reads.
 

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Mathieu Desnoyers]

* Grant Grundler (grundler@parisc-linux.org) wrote:
> On Mon, Aug 27, 2007 at 05:11:40PM -0400, Mathieu Desnoyers wrote:
> ...
> > > Can you add a description to Documentation/atomic_ops.txt ?
> > > *sigh* sorry for being "late to the party" on this one...
> > 
> > Does Documentation/local_ops.txt answer your questions ? If not, please
> > tell me and I'll gladly explain more.
> 
> Yes, it does mostly - thanks. 
> 
> A few questions/nits:
> o Did you attempt quantify how many places in the kernel could use this?
>   I'm just trying to get a feel for how useful this really is vs just 
>   using existing mechanisms (that people understand) to implement a 
>   non-SMP-safe counter that protects updates (writes) against interrupts.
>   If you did, adding some referencs to local_ops.txt would be helpful
>   so folks could look for examples of "correct usage".
> 

Good question. Since it is useful to implement fast, interrupt
reentrant, counters of any kind without disabling interrupts, I think it
could be vastely used in the kernel. I also use it in my LTTng kernel
tracer implementation to provide very fast buffer management. It is used
in LTTng, but could be used for most kind of buffering management too;
meaning that we could manage buffers without disabling interrupts.

So I don't expect to come with an "upper bound" about where it can be
used...

> o Wording in local_ops.txt: "on the 
> 	"... it will then appear to be written out of order wrt
> 	 other memory writes on the owner CPU."
> 
>   I'd like to suggest "by the owner CPU".
> 

Ok, fixing.. thanks!

> o How can a local_t counter protect updates (writes) against interrupts 
>   but not preemption?
>   I always thought preemption required some sort of interrupt or trap.
>   Maybe the local_ops.txt explains that and I just missed it.
> 

"Local atomic operations only guarantee variable modification atomicity
wrt the CPU which owns the data. Therefore, care must taken to make sure
that only one CPU writes to the local_t data. This is done by using per
cpu data and making sure that we modify it from within a preemption safe
context." -> therefore, preemption must be disabled around local ops
usage. This is required to be pinned to one CPU anyway.

>   DaveM explained updates "in flight" would not be visible to interrupts
>   and I suspect that's the answer to my question....but then I don't "feel
>   good" the local_ops are safe to update in interrupts _and_ the process
>   context kernel.  Maybe the relationship between local_ops, preemption,
>   and interrupts could be explained more carefully in local_ops.txt.
> 

Does the paragraph above explain it enough or should I add some more
explanation ?

> o OK to add a reference for local_ops.txt to atomic_ops.txt?
>   They are obviously related and anyone "discovering" one of the docs
>   should be made aware of the other.
>   Patch+log entry appended below. Please sign-off if that's ok with you.
> 

I'm perfectly ok with the idea, but suggest a small modification. See
below.

> 
> thanks,
> grant
> 
> Diff+Commit entry against 2.6.22.5:
> 
> local_t is a variant of atomic_t and has related ops to match.
> Add reference for local_t documentation to atomic_ops.txt. 
> 
> Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
> 
> 
> --- 2.6.22.5-ORIG/Documentation/atomic_ops.txt	2007-08-27 22:50:27.000000000 -0700
> +++ 2.6.22.5-ggg/Documentation/atomic_ops.txt	2007-08-27 22:54:44.000000000 -0700
> @@ -14,6 +14,10 @@
>  
>  	typedef struct { volatile int counter; } atomic_t;
>  
> +local_t is very similar to atomic_t. If the counter is per CPU and only
> +updated by one CPU, local_t is probably more appropriate. Please see
> +Documentation/local_ops.txt for the semantics of local_t.
> +
>  	The first operations to implement for atomic_t's are the
>  initializers and plain reads.
>  

The text snippet is good, but I am not sure it belongs between the
description of atomic_t type and its initializers. What if we do
something like: (with context, I tried to explain the distinction
between atomic_t and local_t some more)


                Semantics and Behavior of Atomic and
                         Bitmask Operations

                          David S. Miller

        This document is intended to serve as a guide to Linux port
maintainers on how to implement atomic counter, bitops, and spinlock
interfaces properly.

atomic_t should be used to provide a type with update primitives
executed atomically from any CPU.  If the counter is per CPU and only
updated by one CPU, local_t is probably more appropriate.  Please see
Documentation/local_ops.txt for the semantics of local_t.

        The atomic_t type should be defined as a signed integer.
Also, it should be made opaque such that any kind of cast to a normal
C integer type will fail.  Something like the following should
suffice: 


Mathieu

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Grant Grundler]

On Tue, Aug 28, 2007 at 07:50:18AM -0400, Mathieu Desnoyers wrote:
...
> > A few questions/nits:
> > o Did you attempt quantify how many places in the kernel could use this?
> >   I'm just trying to get a feel for how useful this really is vs just 
> >   using existing mechanisms (that people understand) to implement a 
> >   non-SMP-safe counter that protects updates (writes) against interrupts.
> >   If you did, adding some referencs to local_ops.txt would be helpful
> >   so folks could look for examples of "correct usage".
> > 
> 
> Good question. Since it is useful to implement fast, interrupt
> reentrant, counters of any kind without disabling interrupts, I think it
> could be vastely used in the kernel. I also use it in my LTTng kernel
> tracer implementation to provide very fast buffer management. It is used
> in LTTng, but could be used for most kind of buffering management too;
> meaning that we could manage buffers without disabling interrupts.
> 
> So I don't expect to come with an "upper bound" about where it can be
> used...

Ok...so I'll try to find one in 2.6.22.5:
grundler <1855>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep atomic_t
./arch/s390/kernel/time.c:static DEFINE_PER_CPU(atomic_t, etr_sync_word);
grundler <1856>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep local_t
./arch/x86_64/kernel/nmi.c:static DEFINE_PER_CPU(local_t, alert_counter);

uhm, I was expecting more than that.  Maybe there is some other systemic
problem with how PER_CPU stuff is used/declared?

In any case, some references to LTT usage would be quite helpful.
E.g. a list of file and variable names at the end of local_ops.txt file.


> > o How can a local_t counter protect updates (writes) against interrupts 
> >   but not preemption?
> >   I always thought preemption required some sort of interrupt or trap.
> >   Maybe the local_ops.txt explains that and I just missed it.
> > 
> 
> "Local atomic operations only guarantee variable modification atomicity
> wrt the CPU which owns the data. Therefore, care must taken to make sure
> that only one CPU writes to the local_t data. This is done by using per
> cpu data and making sure that we modify it from within a preemption safe
> context." -> therefore, preemption must be disabled around local ops
> usage. This is required to be pinned to one CPU anyway.

Sorry...the quoted text doesn't answer my question. It's a definition
of semantics, not an explanation of the "mechanics".

I want to know what happens when (if?) an interrupt occurs in the
middle of a read/modify/write sequence that isn't prefixed with LOCK
(or something similar for other arches like "store locked conditional" ops).

Stating the semantics is a good thing - but not a substitution for
describing how it works for a given architecture. Either in the code
or in local_ops.txt. Otherwise people like me won't use it because
we don't believe that (or understand how) it really works.

> >   DaveM explained updates "in flight" would not be visible to interrupts
> >   and I suspect that's the answer to my question....but then I don't "feel
> >   good" the local_ops are safe to update in interrupts _and_ the process
> >   context kernel.  Maybe the relationship between local_ops, preemption,
> >   and interrupts could be explained more carefully in local_ops.txt.
> > 
> 
> Does the paragraph above explain it enough or should I add some more
> explanation ?

Please add a bit more detail. If DaveM is correct (he normally is), then
there must be limits on how the local_t can be used in the kernel process
and interrupt contexts. I'd like those rules spelled out very clearly
since it's easy to get wrong and tracking down such a bug is quite painful.

Note: I already missed the one critical sentence about only the "owning"
CPU can write the value....there seem to be other limitations as well
with respect to interrupts.

> > o OK to add a reference for local_ops.txt to atomic_ops.txt?
> >   They are obviously related and anyone "discovering" one of the docs
> >   should be made aware of the other.
> >   Patch+log entry appended below. Please sign-off if that's ok with you.
> > 
> 
> I'm perfectly ok with the idea, but suggest a small modification. See
> below.

Looks fine to me. Add your "Signed-off-by" and submit to DaveM
since he seems to be the maintainer of atomic_ops.txt.

cheers,
grant

> 
> > 
> > thanks,
> > grant
> > 
> > Diff+Commit entry against 2.6.22.5:
> > 
> > local_t is a variant of atomic_t and has related ops to match.
> > Add reference for local_t documentation to atomic_ops.txt. 
> > 
> > Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
> > 
> > 
> > --- 2.6.22.5-ORIG/Documentation/atomic_ops.txt	2007-08-27 22:50:27.000000000 -0700
> > +++ 2.6.22.5-ggg/Documentation/atomic_ops.txt	2007-08-27 22:54:44.000000000 -0700
> > @@ -14,6 +14,10 @@
> >  
> >  	typedef struct { volatile int counter; } atomic_t;
> >  
> > +local_t is very similar to atomic_t. If the counter is per CPU and only
> > +updated by one CPU, local_t is probably more appropriate. Please see
> > +Documentation/local_ops.txt for the semantics of local_t.
> > +
> >  	The first operations to implement for atomic_t's are the
> >  initializers and plain reads.
> >  
> 
> The text snippet is good, but I am not sure it belongs between the
> description of atomic_t type and its initializers. What if we do
> something like: (with context, I tried to explain the distinction
> between atomic_t and local_t some more)
> 
> 
>                 Semantics and Behavior of Atomic and
>                          Bitmask Operations
> 
>                           David S. Miller
> 
>         This document is intended to serve as a guide to Linux port
> maintainers on how to implement atomic counter, bitops, and spinlock
> interfaces properly.
> 
> atomic_t should be used to provide a type with update primitives
> executed atomically from any CPU.  If the counter is per CPU and only
> updated by one CPU, local_t is probably more appropriate.  Please see
> Documentation/local_ops.txt for the semantics of local_t.
> 
>         The atomic_t type should be defined as a signed integer.
> Also, it should be made opaque such that any kind of cast to a normal
> C integer type will fail.  Something like the following should
> suffice: 
> 
> 
> Mathieu
> 
> -- 
> Mathieu Desnoyers
> Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
> OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Mathieu Desnoyers]

* Grant Grundler (grundler@parisc-linux.org) wrote:
> On Tue, Aug 28, 2007 at 07:50:18AM -0400, Mathieu Desnoyers wrote:
> ...
> > > A few questions/nits:
> > > o Did you attempt quantify how many places in the kernel could use this?
> > >   I'm just trying to get a feel for how useful this really is vs just 
> > >   using existing mechanisms (that people understand) to implement a 
> > >   non-SMP-safe counter that protects updates (writes) against interrupts.
> > >   If you did, adding some referencs to local_ops.txt would be helpful
> > >   so folks could look for examples of "correct usage".
> > > 
> > 
> > Good question. Since it is useful to implement fast, interrupt
> > reentrant, counters of any kind without disabling interrupts, I think it
> > could be vastely used in the kernel. I also use it in my LTTng kernel
> > tracer implementation to provide very fast buffer management. It is used
> > in LTTng, but could be used for most kind of buffering management too;
> > meaning that we could manage buffers without disabling interrupts.
> > 
> > So I don't expect to come with an "upper bound" about where it can be
> > used...
> 
> Ok...so I'll try to find one in 2.6.22.5:
> grundler <1855>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep atomic_t
> ./arch/s390/kernel/time.c:static DEFINE_PER_CPU(atomic_t, etr_sync_word);
> grundler <1856>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep local_t
> ./arch/x86_64/kernel/nmi.c:static DEFINE_PER_CPU(local_t, alert_counter);
> 
> uhm, I was expecting more than that.  Maybe there is some other systemic
> problem with how PER_CPU stuff is used/declared?
> 

the local ops has just been standardized in 2.6.22 though a patchset I
did. I would not expect the code to start using them this quickly. Or
maybe is it just that I am doing a terrible marketing job ;)

> In any case, some references to LTT usage would be quite helpful.
> E.g. a list of file and variable names at the end of local_ops.txt file.
> 

LTT is not mainlined (yet!) ;)

> 
> > > o How can a local_t counter protect updates (writes) against interrupts 
> > >   but not preemption?
> > >   I always thought preemption required some sort of interrupt or trap.
> > >   Maybe the local_ops.txt explains that and I just missed it.
> > > 
> > 
> > "Local atomic operations only guarantee variable modification atomicity
> > wrt the CPU which owns the data. Therefore, care must taken to make sure
> > that only one CPU writes to the local_t data. This is done by using per
> > cpu data and making sure that we modify it from within a preemption safe
> > context." -> therefore, preemption must be disabled around local ops
> > usage. This is required to be pinned to one CPU anyway.
> 
> Sorry...the quoted text doesn't answer my question. It's a definition
> of semantics, not an explanation of the "mechanics".
> 
> I want to know what happens when (if?) an interrupt occurs in the
> middle of a read/modify/write sequence that isn't prefixed with LOCK
> (or something similar for other arches like "store locked conditional" ops).
> 
> Stating the semantics is a good thing - but not a substitution for
> describing how it works for a given architecture. Either in the code
> or in local_ops.txt. Otherwise people like me won't use it because
> we don't believe that (or understand how) it really works.
> 

Quoting Intel 64 and IA-32 Architectures Software Developer's Manual

3.2 Instructions
LOCK - Assert LOCK# Signal Prefix

Causes the processor's LOCK# signal to be asserted during execution of
the accompanying instruction (turns the instruction into an atomic
instruction). In a multiprocessor environment, the LOCK# signal insures
that the processor has exclusive use of any shared memory while the
signal is asserted.

And if we take a look at some of the atomic primitives which are used in
i386 local.h:

add (for inc/dec/add/sub)
xadd
cmpxchg

All these instructions, just like any other, can be interrupted by an
external interrupt or cause a trap, exception, or fault. Interrupt
handler are executing between instructions and traps/exceptions/faults
will either execute instead of the faulty instruction or after is has
been executed. In all these cases, each instruction can be seen as
executing atomically wrt the local CPU. This is exactly what permits
asm-i386/local.h to define out the LOCK prefix for UP kernels.

I use the same trick UP kernel are using, but I deploy it in SMP
context, but I require the CPU to be the only one to access the memory
locations written to by the local ops.

Basically, since the memory location is _not_ shared across CPUs for
writing, we can safely write to it without holding the LOCK signal.


> > >   DaveM explained updates "in flight" would not be visible to interrupts
> > >   and I suspect that's the answer to my question....but then I don't "feel
> > >   good" the local_ops are safe to update in interrupts _and_ the process
> > >   context kernel.  Maybe the relationship between local_ops, preemption,
> > >   and interrupts could be explained more carefully in local_ops.txt.
> > > 
> > 
> > Does the paragraph above explain it enough or should I add some more
> > explanation ?
> 
> Please add a bit more detail. If DaveM is correct (he normally is), then
> there must be limits on how the local_t can be used in the kernel process
> and interrupt contexts. I'd like those rules spelled out very clearly
> since it's easy to get wrong and tracking down such a bug is quite painful.
> 
> Note: I already missed the one critical sentence about only the "owning"
> CPU can write the value....there seem to be other limitations as well
> with respect to interrupts.
> 

Ok, let's give a try at a clear statement:

- Variables touched by local ops must be per cpu variables.
- _Only_ the CPU owner of these variables must write to them.
- This CPU can use local ops from any context (process, irq, softirq, nmi, ...)
  to update its local_t variables.
- Preemption (or interrupts) must be disabled when using local ops in
  process context to   make sure the process won't be migrated to a
  different CPU between getting the per-cpu variable and doing the
  actual local op.
- When using local ops in interrupt context, no special care must be
  taken on a mainline kernel, since they will run on the local CPU with
  preemption already disabled. I suggest, however, to explicitly
  disable preemption anyway to make sure it will still work correctly on
  -rt kernels.
- Reading the local cpu variable will provide the current copy of the
  variable.
- Reads of these variables can be done from any CPU, because updates to
  "long", aligned, variables are always atomic. Since no memory
  synchronization is done by the writer CPU, an outdated copy of the
  variable can be read when reading some _other_ cpu's variables.


> > > o OK to add a reference for local_ops.txt to atomic_ops.txt?
> > >   They are obviously related and anyone "discovering" one of the docs
> > >   should be made aware of the other.
> > >   Patch+log entry appended below. Please sign-off if that's ok with you.
> > > 
> > 
> > I'm perfectly ok with the idea, but suggest a small modification. See
> > below.
> 
> Looks fine to me. Add your "Signed-off-by" and submit to DaveM
> since he seems to be the maintainer of atomic_ops.txt.
> 

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

Thanks,

Mathieu

> cheers,
> grant
> 
> > 
> > > 
> > > thanks,
> > > grant
> > > 
> > > Diff+Commit entry against 2.6.22.5:
> > > 
> > > local_t is a variant of atomic_t and has related ops to match.
> > > Add reference for local_t documentation to atomic_ops.txt. 
> > > 
> > > Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
> > > 
> > > 
> > > --- 2.6.22.5-ORIG/Documentation/atomic_ops.txt	2007-08-27 22:50:27.000000000 -0700
> > > +++ 2.6.22.5-ggg/Documentation/atomic_ops.txt	2007-08-27 22:54:44.000000000 -0700
> > > @@ -14,6 +14,10 @@
> > >  
> > >  	typedef struct { volatile int counter; } atomic_t;
> > >  
> > > +local_t is very similar to atomic_t. If the counter is per CPU and only
> > > +updated by one CPU, local_t is probably more appropriate. Please see
> > > +Documentation/local_ops.txt for the semantics of local_t.
> > > +
> > >  	The first operations to implement for atomic_t's are the
> > >  initializers and plain reads.
> > >  
> > 
> > The text snippet is good, but I am not sure it belongs between the
> > description of atomic_t type and its initializers. What if we do
> > something like: (with context, I tried to explain the distinction
> > between atomic_t and local_t some more)
> > 
> > 
> >                 Semantics and Behavior of Atomic and
> >                          Bitmask Operations
> > 
> >                           David S. Miller
> > 
> >         This document is intended to serve as a guide to Linux port
> > maintainers on how to implement atomic counter, bitops, and spinlock
> > interfaces properly.
> > 
> > atomic_t should be used to provide a type with update primitives
> > executed atomically from any CPU.  If the counter is per CPU and only
> > updated by one CPU, local_t is probably more appropriate.  Please see
> > Documentation/local_ops.txt for the semantics of local_t.
> > 
> >         The atomic_t type should be defined as a signed integer.
> > Also, it should be made opaque such that any kind of cast to a normal
> > C integer type will fail.  Something like the following should
> > suffice: 
> > 
> > 
> > Mathieu
> > 
> > -- 
> > Mathieu Desnoyers
> > Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
> > OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Grant Grundler]

[davem: patch for you at the bottom to Documentation/atomic_ops.txt ]

On Tue, Aug 28, 2007 at 02:38:35PM -0400, Mathieu Desnoyers wrote:
> * Grant Grundler (grundler@parisc-linux.org) wrote:
> > On Tue, Aug 28, 2007 at 07:50:18AM -0400, Mathieu Desnoyers wrote:
...
> > > So I don't expect to come with an "upper bound" about where it can be
> > > used...
> > 
> > Ok...so I'll try to find one in 2.6.22.5:
> > grundler <1855>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep atomic_t
> > ./arch/s390/kernel/time.c:static DEFINE_PER_CPU(atomic_t, etr_sync_word);
> > grundler <1856>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep local_t
> > ./arch/x86_64/kernel/nmi.c:static DEFINE_PER_CPU(local_t, alert_counter);
> > 
> > uhm, I was expecting more than that.  Maybe there is some other systemic
> > problem with how PER_CPU stuff is used/declared?
> 
> the local ops has just been standardized in 2.6.22 though a patchset I
> did. I would not expect the code to start using them this quickly. Or
> maybe is it just that I am doing a terrible marketing job ;)

Yeah, I didn't expect many users of local_t.

The search for atomic_t usage in DEFINE_PER_CPU was an attempt to
find other potential candidates which could be local_t.
Is there any other programmatic way we could look for code which
could (or should) be using local_t?

> > In any case, some references to LTT usage would be quite helpful.
> > E.g. a list of file and variable names at the end of local_ops.txt file.
> > 
> 
> LTT is not mainlined (yet!) ;)

Ok...probably not such a good example then. :)

...
> > Sorry...the quoted text doesn't answer my question. It's a definition
> > of semantics, not an explanation of the "mechanics".
> > 
> > I want to know what happens when (if?) an interrupt occurs in the
> > middle of a read/modify/write sequence that isn't prefixed with LOCK
> > (or something similar for other arches like "store locked conditional" ops).
> > 
> > Stating the semantics is a good thing - but not a substitution for
> > describing how it works for a given architecture. Either in the code
> > or in local_ops.txt. Otherwise people like me won't use it because
> > we don't believe that (or understand how) it really works.
> > 
> 
> Quoting Intel 64 and IA-32 Architectures Software Developer's Manual
> 
> 3.2 Instructions
> LOCK - Assert LOCK# Signal Prefix
...

I've read this before and understand how LOCK works. This isn't helpful
since I want a description of the behavior without LOCK.

> And if we take a look at some of the atomic primitives which are used in
> i386 local.h:
> 
> add (for inc/dec/add/sub)
> xadd
> cmpxchg
> 
> All these instructions, just like any other, can be interrupted by an
> external interrupt or cause a trap, exception, or fault. Interrupt
> handler are executing between instructions and traps/exceptions/faults
> will either execute instead of the faulty instruction or after is has
> been executed. In all these cases, each instruction can be seen as
> executing atomically wrt the local CPU. This is exactly what permits
> asm-i386/local.h to define out the LOCK prefix for UP kernels.

I think what I'm looking for but don't know if it's true:
    The cmpxchg (for example) at the kernel process context will not
    clobber or be clobbered by a cmpxchg done to the same local_t
    performed at the kernel interrupt context by the same CPU.

If that's not true, then it would be good to add that as another
restriction to usage.

> I use the same trick UP kernel are using, but I deploy it in SMP
> context, but I require the CPU to be the only one to access the memory
> locations written to by the local ops.
> 
> Basically, since the memory location is _not_ shared across CPUs for
> writing, we can safely write to it without holding the LOCK signal.

ok.

...
> > Note: I already missed the one critical sentence about only the "owning"
> > CPU can write the value....there seem to be other limitations as well
> > with respect to interrupts.
> > 
> 
> Ok, let's give a try at a clear statement:
> 
> - Variables touched by local ops must be per cpu variables.
> - _Only_ the CPU owner of these variables must write to them.
> - This CPU can use local ops from any context (process, irq, softirq, nmi, ...)
>   to update its local_t variables.
> - Preemption (or interrupts) must be disabled when using local ops in
>   process context to   make sure the process won't be migrated to a
>   different CPU between getting the per-cpu variable and doing the
>   actual local op.
> - When using local ops in interrupt context, no special care must be
>   taken on a mainline kernel, since they will run on the local CPU with
>   preemption already disabled. I suggest, however, to explicitly
>   disable preemption anyway to make sure it will still work correctly on
>   -rt kernels.
> - Reading the local cpu variable will provide the current copy of the
>   variable.
> - Reads of these variables can be done from any CPU, because updates to
>   "long", aligned, variables are always atomic. Since no memory
>   synchronization is done by the writer CPU, an outdated copy of the
>   variable can be read when reading some _other_ cpu's variables.

Perfect! Ship it! ;)
Can you submit a patch to add the above to Documentation/local_ops.txt ?

...
> > Looks fine to me. Add your "Signed-off-by" and submit to DaveM
> > since he seems to be the maintainer of atomic_ops.txt.
> 
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

heh...I think DaveM will want it in git or universal patch form. :)
Patch appended below.

thanks!
grant

Add document reference and simple use example of local_t.

Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

--- linux-2.6.22.5-ORIG/Documentation/atomic_ops.txt	2007-08-22 16:23:54.000000000 -0700
+++ linux-2.6.22.5-ggg/Documentation/atomic_ops.txt	2007-08-28 22:57:26.000000000 -0700
@@ -7,6 +7,11 @@
 maintainers on how to implement atomic counter, bitops, and spinlock
 interfaces properly.
 
+	atomic_t should be used to provide a type with update primitives
+executed atomically from any CPU.  If the counter is per CPU and only
+updated by one CPU, local_t is probably more appropriate.  Please see
+Documentation/local_ops.txt for the semantics of local_t.
+
 	The atomic_t type should be defined as a signed integer.
 Also, it should be made opaque such that any kind of cast to a normal
 C integer type will fail.  Something like the following should

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[David Miller]

From: Grant Grundler <grundler@parisc-linux.org>
Date: Tue, 28 Aug 2007 23:59:55 -0600

> [davem: patch for you at the bottom to Documentation/atomic_ops.txt ]

Looks fine to me.

Re: [parisc-linux] [patch 15/23] Add cmpxchg_local to parisc

[Mathieu Desnoyers]

* Grant Grundler (grundler@parisc-linux.org) wrote:
> [davem: patch for you at the bottom to Documentation/atomic_ops.txt ]
> 
> On Tue, Aug 28, 2007 at 02:38:35PM -0400, Mathieu Desnoyers wrote:
> > * Grant Grundler (grundler@parisc-linux.org) wrote:
> > > On Tue, Aug 28, 2007 at 07:50:18AM -0400, Mathieu Desnoyers wrote:
> ...
> > > > So I don't expect to come with an "upper bound" about where it can be
> > > > used...
> > > 
> > > Ok...so I'll try to find one in 2.6.22.5:
> > > grundler <1855>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep atomic_t
> > > ./arch/s390/kernel/time.c:static DEFINE_PER_CPU(atomic_t, etr_sync_word);
> > > grundler <1856>find -name \*.c | xargs fgrep DEFINE_PER_CPU | fgrep local_t
> > > ./arch/x86_64/kernel/nmi.c:static DEFINE_PER_CPU(local_t, alert_counter);
> > > 
> > > uhm, I was expecting more than that.  Maybe there is some other systemic
> > > problem with how PER_CPU stuff is used/declared?
> > 
> > the local ops has just been standardized in 2.6.22 though a patchset I
> > did. I would not expect the code to start using them this quickly. Or
> > maybe is it just that I am doing a terrible marketing job ;)
> 
> Yeah, I didn't expect many users of local_t.
> 
> The search for atomic_t usage in DEFINE_PER_CPU was an attempt to
> find other potential candidates which could be local_t.
> Is there any other programmatic way we could look for code which
> could (or should) be using local_t?
> 

Yep.. atomic_t static variables used as counters are a perfect match:
they are updated very often and read rarely. Therefore, it is
preferable to have per cpu updates with very cheap locking and to
iterate on all of them to sum them when they should be printed. That's
just an example, but with it comes vmstat and all sorts of counters.

> > > In any case, some references to LTT usage would be quite helpful.
> > > E.g. a list of file and variable names at the end of local_ops.txt file.
> > > 
> > 
> > LTT is not mainlined (yet!) ;)
> 
> Ok...probably not such a good example then. :)
> 
> ...
> > > Sorry...the quoted text doesn't answer my question. It's a definition
> > > of semantics, not an explanation of the "mechanics".
> > > 
> > > I want to know what happens when (if?) an interrupt occurs in the
> > > middle of a read/modify/write sequence that isn't prefixed with LOCK
> > > (or something similar for other arches like "store locked conditional" ops).
> > > 
> > > Stating the semantics is a good thing - but not a substitution for
> > > describing how it works for a given architecture. Either in the code
> > > or in local_ops.txt. Otherwise people like me won't use it because
> > > we don't believe that (or understand how) it really works.
> > > 
> > 
> > Quoting Intel 64 and IA-32 Architectures Software Developer's Manual
> > 
> > 3.2 Instructions
> > LOCK - Assert LOCK# Signal Prefix
> ...
> 
> I've read this before and understand how LOCK works. This isn't helpful
> since I want a description of the behavior without LOCK.
> 
> > And if we take a look at some of the atomic primitives which are used in
> > i386 local.h:
> > 
> > add (for inc/dec/add/sub)
> > xadd
> > cmpxchg
> > 
> > All these instructions, just like any other, can be interrupted by an
> > external interrupt or cause a trap, exception, or fault. Interrupt
> > handler are executing between instructions and traps/exceptions/faults
> > will either execute instead of the faulty instruction or after is has
> > been executed. In all these cases, each instruction can be seen as
> > executing atomically wrt the local CPU. This is exactly what permits
> > asm-i386/local.h to define out the LOCK prefix for UP kernels.
> 
> I think what I'm looking for but don't know if it's true:
>     The cmpxchg (for example) at the kernel process context will not
>     clobber or be clobbered by a cmpxchg done to the same local_t
>     performed at the kernel interrupt context by the same CPU.
> 
> If that's not true, then it would be good to add that as another
> restriction to usage.
> 

Each of these cmpxchg will execute atomically wrt the local CPU, so
there is no limitation there.

> > I use the same trick UP kernel are using, but I deploy it in SMP
> > context, but I require the CPU to be the only one to access the memory
> > locations written to by the local ops.
> > 
> > Basically, since the memory location is _not_ shared across CPUs for
> > writing, we can safely write to it without holding the LOCK signal.
> 
> ok.
> 
> ...
> > > Note: I already missed the one critical sentence about only the "owning"
> > > CPU can write the value....there seem to be other limitations as well
> > > with respect to interrupts.
> > > 
> > 
> > Ok, let's give a try at a clear statement:
> > 
> > - Variables touched by local ops must be per cpu variables.
> > - _Only_ the CPU owner of these variables must write to them.
> > - This CPU can use local ops from any context (process, irq, softirq, nmi, ...)
> >   to update its local_t variables.
> > - Preemption (or interrupts) must be disabled when using local ops in
> >   process context to   make sure the process won't be migrated to a
> >   different CPU between getting the per-cpu variable and doing the
> >   actual local op.
> > - When using local ops in interrupt context, no special care must be
> >   taken on a mainline kernel, since they will run on the local CPU with
> >   preemption already disabled. I suggest, however, to explicitly
> >   disable preemption anyway to make sure it will still work correctly on
> >   -rt kernels.
> > - Reading the local cpu variable will provide the current copy of the
> >   variable.
> > - Reads of these variables can be done from any CPU, because updates to
> >   "long", aligned, variables are always atomic. Since no memory
> >   synchronization is done by the writer CPU, an outdated copy of the
> >   variable can be read when reading some _other_ cpu's variables.
> 
> Perfect! Ship it! ;)
> Can you submit a patch to add the above to Documentation/local_ops.txt ?
> 

Hehe sure :) I'll prepare one.

> ...
> > > Looks fine to me. Add your "Signed-off-by" and submit to DaveM
> > > since he seems to be the maintainer of atomic_ops.txt.
> > 
> > Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> 
> heh...I think DaveM will want it in git or universal patch form. :)
> Patch appended below.
> 
> thanks!
> grant
> 
> Add document reference and simple use example of local_t.
> 
> Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> 
> --- linux-2.6.22.5-ORIG/Documentation/atomic_ops.txt	2007-08-22 16:23:54.000000000 -0700
> +++ linux-2.6.22.5-ggg/Documentation/atomic_ops.txt	2007-08-28 22:57:26.000000000 -0700
> @@ -7,6 +7,11 @@
>  maintainers on how to implement atomic counter, bitops, and spinlock
>  interfaces properly.
>  
> +	atomic_t should be used to provide a type with update primitives
> +executed atomically from any CPU.  If the counter is per CPU and only
> +updated by one CPU, local_t is probably more appropriate.  Please see
> +Documentation/local_ops.txt for the semantics of local_t.
> +
>  	The atomic_t type should be defined as a signed integer.
>  Also, it should be made opaque such that any kind of cast to a normal
>  C integer type will fail.  Something like the following should

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68