[cpuops inc_return V1 0/9] Implement this_cpu_inc_return (and friends) and use it in various places

[cpuops inc_return V1 0/9] Implement this_cpu_inc_return (and friends) and use it in various places

[Christoph Lameter]

Extend the this_cpu_ops infrastructure to support this_cpu_inc_return and
then use the new functionality in various subsystems. Provide an optimize x86
implementation for these operations.

All these patches depends on the initial patch being applied first.

Patches have been reviewed a couple of times and I would suggest that
they go through Tejun's percpu tree for merging.

[cpuops inc_return V1 1/9] percpu: Generic support for this_cpu_add,sub,dec,inc_return

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_generic --]
[-- Type: text/plain, Size: 3797 bytes --]

Introduce generic support for this_cpu_add_return etc.

The fallback is to realize these operations with simpler __this_cpu_ops.

Reviewed-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 include/linux/percpu.h |   77 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

Index: linux-2.6/include/linux/percpu.h
===================================================================
--- linux-2.6.orig/include/linux/percpu.h	2010-11-29 10:19:28.000000000 -0600
+++ linux-2.6/include/linux/percpu.h	2010-11-29 10:24:57.000000000 -0600
@@ -240,6 +240,25 @@ extern void __bad_size_call_parameter(vo
 	pscr_ret__;							\
 })
 
+#define __pcpu_size_call_return2(stem, pcp, ...)			\
+({									\
+	typeof(pcp) ret__;						\
+	__verify_pcpu_ptr(&(pcp));					\
+	switch(sizeof(pcp)) {						\
+	case 1: ret__ = stem##1(pcp, __VA_ARGS__);			\
+		break;							\
+	case 2: ret__ = stem##2(pcp, __VA_ARGS__);			\
+		break;							\
+	case 4: ret__ = stem##4(pcp, __VA_ARGS__);			\
+		break;							\
+	case 8: ret__ = stem##8(pcp, __VA_ARGS__);			\
+		break;							\
+	default:							\
+		__bad_size_call_parameter();break;			\
+	}								\
+	ret__;								\
+})
+
 #define __pcpu_size_call(stem, variable, ...)				\
 do {									\
 	__verify_pcpu_ptr(&(variable));					\
@@ -529,6 +548,64 @@ do {									\
 # define __this_cpu_xor(pcp, val)	__pcpu_size_call(__this_cpu_xor_, (pcp), (val))
 #endif
 
+#define _this_cpu_generic_add_return(pcp, val)				\
+({									\
+	typeof(pcp) ret__;						\
+	preempt_disable();						\
+	__this_cpu_add(pcp, val);					\
+	ret__ = __this_cpu_read(pcp);					\
+	preempt_enable();						\
+	ret__;								\
+})
+
+#ifndef this_cpu_add_return
+# ifndef this_cpu_add_return_1
+#  define this_cpu_add_return_1(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_2
+#  define this_cpu_add_return_2(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_4
+#  define this_cpu_add_return_4(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_8
+#  define this_cpu_add_return_8(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
+#endif
+
+#define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(val))
+#define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
+#define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)
+
+#define __this_cpu_generic_add_return(pcp, val)				\
+({									\
+	typeof(pcp) ret__;						\
+	__this_cpu_add(pcp, val);					\
+	ret__ = __this_cpu_read(pcp);					\
+	ret__;								\
+})
+
+#ifndef __this_cpu_add_return
+# ifndef __this_cpu_add_return_1
+#  define __this_cpu_add_return_1(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef __this_cpu_add_return_2
+#  define __this_cpu_add_return_2(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef __this_cpu_add_return_4
+#  define __this_cpu_add_return_4(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef __this_cpu_add_return_8
+#  define __this_cpu_add_return_8(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# define __this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
+#endif
+
+#define __this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(val))
+#define __this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
+#define __this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)
+
 /*
  * IRQ safe versions of the per cpu RMW operations. Note that these operations
  * are *not* safe against modification of the same variable from another

[cpuops UPDATED 1/9] percpu: Generic support for this_cpu_add,sub,dec,inc_return

[Tejun Heo]

From: Christoph Lameter <cl@linux.com>
Subject: percpu: Generic support for this_cpu_add, sub, dec, inc_return

Introduce generic support for this_cpu_add_return etc.

The fallback is to realize these operations with simpler __this_cpu_ops.

tj: - Reformatted __cpu_size_call_return2() to make it more consistent
      with its neighbors.

    - Dropped unnecessary temp variable ret__ from
      __this_cpu_generic_add_return().

Reviewed-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
---
I updated it a bit.  If you don't like it, please scream.

Thanks.

 include/linux/percpu.h |   71 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

Index: percpu/include/linux/percpu.h
===================================================================
--- percpu.orig/include/linux/percpu.h
+++ percpu/include/linux/percpu.h
@@ -240,6 +240,21 @@ extern void __bad_size_call_parameter(vo
 	pscr_ret__;							\
 })

+#define __pcpu_size_call_return2(stem, variable, ...)			\
+({									\
+	typeof(variable) pscr2_ret__;					\
+	__verify_pcpu_ptr(&(variable));					\
+	switch(sizeof(variable)) {					\
+	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break;	\
+	case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break;	\
+	case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break;	\
+	case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break;	\
+	default:							\
+		__bad_size_call_parameter(); break;			\
+	}								\
+	pscr2_ret__;							\
+})
+
 #define __pcpu_size_call(stem, variable, ...)				\
 do {									\
 	__verify_pcpu_ptr(&(variable));					\
@@ -529,6 +544,62 @@ do {									\
 # define __this_cpu_xor(pcp, val)	__pcpu_size_call(__this_cpu_xor_, (pcp), (val))
 #endif

+#define _this_cpu_generic_add_return(pcp, val)				\
+({									\
+	typeof(pcp) ret__;						\
+	preempt_disable();						\
+	__this_cpu_add(pcp, val);					\
+	ret__ = __this_cpu_read(pcp);					\
+	preempt_enable();						\
+	ret__;								\
+})
+
+#ifndef this_cpu_add_return
+# ifndef this_cpu_add_return_1
+#  define this_cpu_add_return_1(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_2
+#  define this_cpu_add_return_2(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_4
+#  define this_cpu_add_return_4(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_8
+#  define this_cpu_add_return_8(pcp, val)	_this_cpu_generic_add_return(pcp, val)
+# endif
+# define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
+#endif
+
+#define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(val))
+#define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
+#define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)
+
+#define __this_cpu_generic_add_return(pcp, val)				\
+({									\
+	__this_cpu_add(pcp, val);					\
+	__this_cpu_read(pcp);						\
+})
+
+#ifndef __this_cpu_add_return
+# ifndef __this_cpu_add_return_1
+#  define __this_cpu_add_return_1(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef __this_cpu_add_return_2
+#  define __this_cpu_add_return_2(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef __this_cpu_add_return_4
+#  define __this_cpu_add_return_4(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef __this_cpu_add_return_8
+#  define __this_cpu_add_return_8(pcp, val)	__this_cpu_generic_add_return(pcp, val)
+# endif
+# define __this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
+#endif
+
+#define __this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(val))
+#define __this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
+#define __this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)
+
 /*
  * IRQ safe versions of the per cpu RMW operations. Note that these operations
  * are *not* safe against modification of the same variable from another

[cpuops inc_return V1 2/9] x86: Support for this_cpu_add,sub,dec,inc_return

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_x86 --]
[-- Type: text/plain, Size: 2811 bytes --]

Supply an implementation for x86 in order to generate more efficient code.

V2->V3:
	- Cleanup
	- Remove strange type checking from percpu_add_return_op.

Signed-off-by: Christoph Lameter <cl@linux.com>

---
 arch/x86/include/asm/percpu.h |   46 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/percpu.h	2010-11-29 14:29:13.000000000 -0600
+++ linux-2.6/arch/x86/include/asm/percpu.h	2010-11-30 08:42:02.000000000 -0600
@@ -177,6 +177,41 @@ do {									\
 	}								\
 } while (0)
 
+
+/*
+ * Add return operation
+ */
+#define percpu_add_return_op(var, val)					\
+({									\
+	typedef typeof(var) pao_T__;					\
+	typeof(var) ret__ = val;					\
+	switch (sizeof(var)) {						\
+	case 1:								\
+		asm("xaddb %0, "__percpu_arg(1)				\
+			    : "+q" (ret__), "+m" (var)			\
+			    : : "memory");				\
+		break;							\
+	case 2:								\
+		asm("xaddw %0, "__percpu_arg(1)				\
+			    : "+r" (ret__), "+m" (var)			\
+			    : : "memory");				\
+		break;							\
+	case 4:								\
+		asm("xaddl %0, "__percpu_arg(1)				\
+			    : "+r"(ret__), "+m" (var)			\
+			    : : "memory");				\
+		break;							\
+	case 8:								\
+		asm("xaddq %0, "__percpu_arg(1)				\
+			    : "+re" (ret__),  "+m" (var)		\
+			    : : "memory");				\
+		break;							\
+	default: __bad_percpu_size();					\
+	}								\
+	ret__ += val;							\
+	ret__;								\
+})
+
 #define percpu_from_op(op, var, constraint)		\
 ({							\
 	typeof(var) pfo_ret__;				\
@@ -300,6 +335,14 @@ do {									\
 #define irqsafe_cpu_xor_2(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define irqsafe_cpu_xor_4(pcp, val)	percpu_to_op("xor", (pcp), val)
 
+#ifndef CONFIG_M386
+#define __this_cpu_add_return_1(pcp, val)	percpu_add_return_op(pcp, val)
+#define __this_cpu_add_return_2(pcp, val)	percpu_add_return_op(pcp, val)
+#define __this_cpu_add_return_4(pcp, val)	percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_1(pcp, val)		percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_2(pcp, val)		percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_4(pcp, val)		percpu_add_return_op(pcp, val)
+#endif
 /*
  * Per cpu atomic 64 bit operations are only available under 64 bit.
  * 32 bit must fall back to generic operations.
@@ -324,6 +367,9 @@ do {									\
 #define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
 
+#define __this_cpu_add_return_8(pcp, val)	percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_8(pcp, val)	percpu_add_return_op(pcp, val)
+
 #endif
 
 /* This is not atomic against other CPUs -- CPU preemption needs to be off */

[cpuops UPDATED 2/9] x86: Support for this_cpu_add, sub, dec, inc_return

[Tejun Heo]

From: Christoph Lameter <cl@linux.com>

Supply an implementation for x86 in order to generate more efficient code.

V2->V3:
	- Cleanup
	- Remove strange type checking from percpu_add_return_op.

tj: - Dropped unused typedef from percpu_add_return_op().
    - Renamed ret__ to paro_ret__ in percpu_add_return_op().
    - Minor indentation adjustments.

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
---
I got a bit fascistic with this one too.  Here's the updated version.

Thank you.

 arch/x86/include/asm/percpu.h |   43 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

Index: percpu/arch/x86/include/asm/percpu.h
===================================================================
--- percpu.orig/arch/x86/include/asm/percpu.h
+++ percpu/arch/x86/include/asm/percpu.h
@@ -177,6 +177,39 @@ do {									\
 	}								\
 } while (0)

+/*
+ * Add return operation
+ */
+#define percpu_add_return_op(var, val)					\
+({									\
+	typeof(var) paro_ret__ = val;					\
+	switch (sizeof(var)) {						\
+	case 1:								\
+		asm("xaddb %0, "__percpu_arg(1)				\
+			    : "+q" (paro_ret__), "+m" (var)		\
+			    : : "memory");				\
+		break;							\
+	case 2:								\
+		asm("xaddw %0, "__percpu_arg(1)				\
+			    : "+r" (paro_ret__), "+m" (var)		\
+			    : : "memory");				\
+		break;							\
+	case 4:								\
+		asm("xaddl %0, "__percpu_arg(1)				\
+			    : "+r" (paro_ret__), "+m" (var)		\
+			    : : "memory");				\
+		break;							\
+	case 8:								\
+		asm("xaddq %0, "__percpu_arg(1)				\
+			    : "+re" (paro_ret__), "+m" (var)		\
+			    : : "memory");				\
+		break;							\
+	default: __bad_percpu_size();					\
+	}								\
+	paro_ret__ += val;						\
+	paro_ret__;							\
+})
+
 #define percpu_from_op(op, var, constraint)		\
 ({							\
 	typeof(var) pfo_ret__;				\
@@ -300,6 +333,14 @@ do {									\
 #define irqsafe_cpu_xor_2(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define irqsafe_cpu_xor_4(pcp, val)	percpu_to_op("xor", (pcp), val)

+#ifndef CONFIG_M386
+#define __this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
+#define __this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
+#define __this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_1(pcp, val)	percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_2(pcp, val)	percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_4(pcp, val)	percpu_add_return_op(pcp, val)
+#endif
 /*
  * Per cpu atomic 64 bit operations are only available under 64 bit.
  * 32 bit must fall back to generic operations.
@@ -324,6 +365,8 @@ do {									\
 #define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)

+#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
+#define this_cpu_add_return_8(pcp, val)	percpu_add_return_op(pcp, val)
 #endif

 /* This is not atomic against other CPUs -- CPU preemption needs to be off */

[cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_nmi --]
[-- Type: text/plain, Size: 1009 bytes --]

this_cpu_inc_return() saves us a memory access there.

Reviewed-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Pekka Enberg <penberg@kernel.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 arch/x86/kernel/apic/nmi.c |    3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

Index: linux-2.6/arch/x86/kernel/apic/nmi.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/apic/nmi.c	2010-11-23 16:35:19.000000000 -0600
+++ linux-2.6/arch/x86/kernel/apic/nmi.c	2010-11-23 16:38:29.000000000 -0600
@@ -432,8 +432,7 @@ nmi_watchdog_tick(struct pt_regs *regs,
 		 * Ayiee, looks like this CPU is stuck ...
 		 * wait a few IRQs (5 seconds) before doing the oops ...
 		 */
-		__this_cpu_inc(alert_counter);
-		if (__this_cpu_read(alert_counter) == 5 * nmi_hz)
+		if (__this_cpu_inc_return(alert_counter) == 5 * nmi_hz)
 			/*
 			 * die_nmi will return ONLY if NOTIFY_STOP happens..
 			 */

Re: [cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter

[Tejun Heo]

On 12/06/2010 06:40 PM, Christoph Lameter wrote:
> this_cpu_inc_return() saves us a memory access there.
> 
> Reviewed-by: Tejun Heo <tj@kernel.org>
> Reviewed-by: Pekka Enberg <penberg@kernel.org>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> Signed-off-by: Christoph Lameter <cl@linux.com>

03-05, the updated 06, 07-09 and the connector patch applied and
pushed out to percpu#for-next.

Thanks.

-- 
tejun

Re: [cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter

[Christoph Lameter]

On Wed, 8 Dec 2010, Tejun Heo wrote:

> On 12/06/2010 06:40 PM, Christoph Lameter wrote:
> > this_cpu_inc_return() saves us a memory access there.
> >
> > Reviewed-by: Tejun Heo <tj@kernel.org>
> > Reviewed-by: Pekka Enberg <penberg@kernel.org>
> > Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> > Signed-off-by: Christoph Lameter <cl@linux.com>
>
> 03-05, the updated 06, 07-09 and the connector patch applied and
> pushed out to percpu#for-next.

Great. Then you are ready for the next set with the this_cpu_cmpxchg
stuff?

Re: [cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter

[Tejun Heo]

Hello, Christoph.

On 12/08/2010 06:31 PM, Christoph Lameter wrote:
> On Wed, 8 Dec 2010, Tejun Heo wrote:
> 
>> On 12/06/2010 06:40 PM, Christoph Lameter wrote:
>>> this_cpu_inc_return() saves us a memory access there.
>>>
>>> Reviewed-by: Tejun Heo <tj@kernel.org>
>>> Reviewed-by: Pekka Enberg <penberg@kernel.org>
>>> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
>>> Signed-off-by: Christoph Lameter <cl@linux.com>
>>
>> 03-05, the updated 06, 07-09 and the connector patch applied and
>> pushed out to percpu#for-next.
> 
> Great. Then you are ready for the next set with the this_cpu_cmpxchg
> stuff?

Heh, sure, shoot.  :-)

-- 
tejun

[cpuops inc_return V1 4/9] vmstat: Use this_cpu_inc_return for vm statistics

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_vmstat --]
[-- Type: text/plain, Size: 1236 bytes --]

this_cpu_inc_return() saves us a memory access there. Code
size does not change.

V1->V2:
	- Fixed the location of the __per_cpu pointer attributes
	- Sparse checked
V2->V3:
	- Move fixes to __percpu attribute usage to earlier patch

Reviewed-by: Pekka Enberg <penberg@kernel.org>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 mm/vmstat.c |    8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

Index: linux-2.6/mm/vmstat.c
===================================================================
--- linux-2.6.orig/mm/vmstat.c	2010-11-29 10:36:16.000000000 -0600
+++ linux-2.6/mm/vmstat.c	2010-11-29 10:38:50.000000000 -0600
@@ -227,9 +227,7 @@ void __inc_zone_state(struct zone *zone,
 	s8 __percpu *p = pcp->vm_stat_diff + item;
 	s8 v, t;
 
-	__this_cpu_inc(*p);
-
-	v = __this_cpu_read(*p);
+	v = __this_cpu_inc_return(*p);
 	t = __this_cpu_read(pcp->stat_threshold);
 	if (unlikely(v > t)) {
 		s8 overstep = t >> 1;
@@ -251,9 +249,7 @@ void __dec_zone_state(struct zone *zone,
 	s8 __percpu *p = pcp->vm_stat_diff + item;
 	s8 v, t;
 
-	__this_cpu_dec(*p);
-
-	v = __this_cpu_read(*p);
+	v = __this_cpu_dec_return(*p);
 	t = __this_cpu_read(pcp->stat_threshold);
 	if (unlikely(v < - t)) {
 		s8 overstep = t >> 1;

[cpuops inc_return V1 5/9] highmem: Use this_cpu_xx_return() operations

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_highmem --]
[-- Type: text/plain, Size: 1614 bytes --]

Use this_cpu operations to optimize access primitives for highmem.

The main effect is the avoidance of address calculations through the
use of a segment prefix.

V3->V4
	- kmap_atomic_idx: Do not return a value.
	- Use __this_cpu_dec without HIGHMEM_DEBUG

Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 include/linux/highmem.h |   13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

Index: linux-2.6/include/linux/highmem.h
===================================================================
--- linux-2.6.orig/include/linux/highmem.h	2010-12-01 09:47:58.000000000 -0600
+++ linux-2.6/include/linux/highmem.h	2010-12-01 09:48:16.000000000 -0600
@@ -81,7 +81,8 @@ DECLARE_PER_CPU(int, __kmap_atomic_idx);
 
 static inline int kmap_atomic_idx_push(void)
 {
-	int idx = __get_cpu_var(__kmap_atomic_idx)++;
+	int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1;
+
 #ifdef CONFIG_DEBUG_HIGHMEM
 	WARN_ON_ONCE(in_irq() && !irqs_disabled());
 	BUG_ON(idx > KM_TYPE_NR);
@@ -91,16 +92,18 @@ static inline int kmap_atomic_idx_push(v
 
 static inline int kmap_atomic_idx(void)
 {
-	return __get_cpu_var(__kmap_atomic_idx) - 1;
+	return __this_cpu_read(__kmap_atomic_idx) - 1;
 }
 
-static inline int kmap_atomic_idx_pop(void)
+static inline void kmap_atomic_idx_pop(void)
 {
-	int idx = --__get_cpu_var(__kmap_atomic_idx);
 #ifdef CONFIG_DEBUG_HIGHMEM
+	int idx = __this_cpu_dec_return(__kmap_atomic_idx);
+
 	BUG_ON(idx < 0);
+#else
+	__this_cpu_dec(__kmap_atomic_idx);
 #endif
-	return idx;
 }
 
 #endif

[cpuops inc_return V1 6/9] Taskstats: Use this_cpu_ops

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_taskstats --]
[-- Type: text/plain, Size: 1072 bytes --]

Use this_cpu_inc_return in one place and avoid ugly __raw_get_cpu in another.

V3->V4:
	- Fix off by one.

Cc: Michael Holzheu <holzheu@linux.vnet.ibm.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 kernel/taskstats.c |    5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

Index: linux-2.6/kernel/taskstats.c
===================================================================
--- linux-2.6.orig/kernel/taskstats.c	2010-11-30 10:06:35.000000000 -0600
+++ linux-2.6/kernel/taskstats.c	2010-11-30 10:10:14.000000000 -0600
@@ -89,8 +89,7 @@ static int prepare_reply(struct genl_inf
 		return -ENOMEM;
 
 	if (!info) {
-		int seq = get_cpu_var(taskstats_seqnum)++;
-		put_cpu_var(taskstats_seqnum);
+		int seq = this_cpu_inc_return(taskstats_seqnum) - 1;
 
 		reply = genlmsg_put(skb, 0, seq, &family, 0, cmd);
 	} else
@@ -581,7 +580,7 @@ void taskstats_exit(struct task_struct *
 		fill_tgid_exit(tsk);
 	}
 
-	listeners = &__raw_get_cpu_var(listener_array);
+	listeners = __this_cpu_ptr(listener_array);
 	if (list_empty(&listeners->list))
 		return;
 

Re: [cpuops inc_return V1 6/9] Taskstats: Use this_cpu_ops

[Michael Holzheu]

Hello Christoph,

On Mon, 2010-12-06 at 11:40 -0600, Christoph Lameter wrote:
> plain text document attachment (cpuops_inc_return_taskstats)
> Use this_cpu_inc_return in one place and avoid ugly __raw_get_cpu in another.
> 
> V3->V4:
> 	- Fix off by one.
> 
> Cc: Michael Holzheu <holzheu@linux.vnet.ibm.com>
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 
> ---
>  kernel/taskstats.c |    5 ++---
>  1 file changed, 2 insertions(+), 3 deletions(-)
> 
> Index: linux-2.6/kernel/taskstats.c
> ===================================================================
> --- linux-2.6.orig/kernel/taskstats.c	2010-11-30 10:06:35.000000000 -0600
> +++ linux-2.6/kernel/taskstats.c	2010-11-30 10:10:14.000000000 -0600
> @@ -89,8 +89,7 @@ static int prepare_reply(struct genl_inf
>  		return -ENOMEM;
> 
>  	if (!info) {
> -		int seq = get_cpu_var(taskstats_seqnum)++;
> -		put_cpu_var(taskstats_seqnum);
> +		int seq = this_cpu_inc_return(taskstats_seqnum) - 1;

This looks good now.

>  		reply = genlmsg_put(skb, 0, seq, &family, 0, cmd);
>  	} else
> @@ -581,7 +580,7 @@ void taskstats_exit(struct task_struct *
>  		fill_tgid_exit(tsk);
>  	}
> 
> -	listeners = &__raw_get_cpu_var(listener_array);
> +	listeners = __this_cpu_ptr(listener_array);

I think this is wrong and should be:

-	listeners = &__raw_get_cpu_var(listener_array);
+	listeners = __this_cpu_ptr(>>>> & <<< listener_array);

Michael

Re: [cpuops inc_return V1 6/9] Taskstats: Use this_cpu_ops

[Christoph Lameter]

On Tue, 7 Dec 2010, Michael Holzheu wrote:

> I think this is wrong and should be:
>
> -	listeners = &__raw_get_cpu_var(listener_array);
> +	listeners = __this_cpu_ptr(>>>> & <<< listener_array);
>

Correct. CONFIG_TASKSTATS was not set. Thus



Subject: Taskstats: Use this_cpu_ops

Use this_cpu_inc_return in one place and avoid ugly __raw_get_cpu in another.

V3->V4:
	- Fix off by one.

V4-V4f:
	- Use &listener_array

Cc: Michael Holzheu <holzheu@linux.vnet.ibm.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 kernel/taskstats.c |    5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

Index: linux-2.6/kernel/taskstats.c
===================================================================
--- linux-2.6.orig/kernel/taskstats.c	2010-12-06 11:30:01.000000000 -0600
+++ linux-2.6/kernel/taskstats.c	2010-12-07 08:43:32.000000000 -0600
@@ -89,8 +89,7 @@ static int prepare_reply(struct genl_inf
 		return -ENOMEM;

 	if (!info) {
-		int seq = get_cpu_var(taskstats_seqnum)++;
-		put_cpu_var(taskstats_seqnum);
+		int seq = this_cpu_inc_return(taskstats_seqnum) - 1;

 		reply = genlmsg_put(skb, 0, seq, &family, 0, cmd);
 	} else
@@ -581,7 +580,7 @@ void taskstats_exit(struct task_struct *
 		fill_tgid_exit(tsk);
 	}

-	listeners = &__raw_get_cpu_var(listener_array);
+	listeners = __this_cpu_ptr(&listener_array);
 	if (list_empty(&listeners->list))
 		return;

[cpuops inc_return V1 7/9] fs: Use this_cpu_inc_return in buffer.c

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_buffer --]
[-- Type: text/plain, Size: 832 bytes --]

__this_cpu_inc can create a single instruction with the same effect
as the _get_cpu_var(..)++ construct in buffer.c.

Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 fs/buffer.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux-2.6/fs/buffer.c
===================================================================
--- linux-2.6.orig/fs/buffer.c	2010-12-06 10:56:21.000000000 -0600
+++ linux-2.6/fs/buffer.c	2010-12-06 10:56:26.000000000 -0600
@@ -3201,7 +3201,7 @@ static void recalc_bh_state(void)
 	int i;
 	int tot = 0;
 
-	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
+	if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
 		return;
 	__this_cpu_write(bh_accounting.ratelimit, 0);
 	for_each_online_cpu(i)

[cpuops inc_return V1 8/9] random: Use this_cpu_inc_return

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_random --]
[-- Type: text/plain, Size: 906 bytes --]

__this_cpu_inc can create a single instruction to do the same as
__get_cpu_var()++.

Cc: Richard Kennedy <richard@rsk.demon.co.uk>
Cc: Matt Mackall <mpm@selenic.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 drivers/char/random.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux-2.6/drivers/char/random.c
===================================================================
--- linux-2.6.orig/drivers/char/random.c	2010-12-06 11:01:06.000000000 -0600
+++ linux-2.6/drivers/char/random.c	2010-12-06 11:19:38.000000000 -0600
@@ -626,7 +626,7 @@ static void add_timer_randomness(struct
 	preempt_disable();
 	/* if over the trickle threshold, use only 1 in 4096 samples */
 	if (input_pool.entropy_count > trickle_thresh &&
-	    (__get_cpu_var(trickle_count)++ & 0xfff))
+	    ((__this_cpu_inc_return(trickle_count) - 1) & 0xfff))
 		goto out;
 
 	sample.jiffies = jiffies;

Re: [cpuops inc_return V1 8/9] random: Use this_cpu_inc_return

[Matt Mackall]

On Mon, 2010-12-06 at 11:40 -0600, Christoph Lameter wrote:
> plain text document attachment (cpuops_inc_return_random)
> __this_cpu_inc can create a single instruction to do the same as
> __get_cpu_var()++.

> -	    (__get_cpu_var(trickle_count)++ & 0xfff))
> +	    ((__this_cpu_inc_return(trickle_count) - 1) & 0xfff))

I see you've added a "- 1" to mimic the post-increment ++ semantic.

The goal of this code to add only 1 sample in every 4096, but we don't
really care which, so this semantic isn't actually important here.

-- 
Mathematics is the supreme nostalgia of our time.

Re: [cpuops inc_return V1 8/9] random: Use this_cpu_inc_return

[Christoph Lameter]

On Mon, 6 Dec 2010, Matt Mackall wrote:

> On Mon, 2010-12-06 at 11:40 -0600, Christoph Lameter wrote:
> > plain text document attachment (cpuops_inc_return_random)
> > __this_cpu_inc can create a single instruction to do the same as
> > __get_cpu_var()++.
>
> > -	    (__get_cpu_var(trickle_count)++ & 0xfff))
> > +	    ((__this_cpu_inc_return(trickle_count) - 1) & 0xfff))
>
> I see you've added a "- 1" to mimic the post-increment ++ semantic.
>
> The goal of this code to add only 1 sample in every 4096, but we don't
> really care which, so this semantic isn't actually important here.

-1 also removes the + 1 that the inc_return() semantics require because we
use xadd on x86. -1 will generate more compact code.

[cpuops inc_return V1 9/9] Xen: Use this_cpu_inc_return

[Christoph Lameter]

[-- Attachment #1: cpuops_inc_return_xen --]
[-- Type: text/plain, Size: 798 bytes --]

__this_cpu_inc_return reduces code and simplifies code.

Cc: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 drivers/xen/events.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux-2.6/drivers/xen/events.c
===================================================================
--- linux-2.6.orig/drivers/xen/events.c	2010-12-06 11:00:18.000000000 -0600
+++ linux-2.6/drivers/xen/events.c	2010-12-06 11:00:47.000000000 -0600
@@ -1109,7 +1109,7 @@ static void __xen_evtchn_do_upcall(void)
 
 		vcpu_info->evtchn_upcall_pending = 0;
 
-		if (__get_cpu_var(xed_nesting_count)++)
+		if (__this_cpu_inc_return(xed_nesting_count) - 1)
 			goto out;
 
 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */

Straggler: Connector: Use this_cpu_operations

[Christoph Lameter]

The patch was originally in the use cpuops patchset but it needs an
inc_return and is therefore dependant on an extension of the cpu ops.
Fixed up and verified that it compiles.


Subject: Connector: Use this_cpu operations

get_seq can benefit from this_cpu_operations. Address calculation is avoided
and the increment is done using an xadd.

Cc: Scott James Remnant <scott@ubuntu.com>
Cc: Mike Frysinger <vapier@gentoo.org>
Signed-off-by: Christoph Lameter <cl@linux.com>

---
 drivers/connector/cn_proc.c |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

Index: linux-2.6/drivers/connector/cn_proc.c
===================================================================
--- linux-2.6.orig/drivers/connector/cn_proc.c	2010-12-07 09:40:48.000000000 -0600
+++ linux-2.6/drivers/connector/cn_proc.c	2010-12-07 09:41:13.000000000 -0600
@@ -43,9 +43,10 @@ static DEFINE_PER_CPU(__u32, proc_event_

 static inline void get_seq(__u32 *ts, int *cpu)
 {
-	*ts = get_cpu_var(proc_event_counts)++;
+	preempt_disable();
+	*ts = __this_cpu_inc_return(proc_event_counts) -1;
 	*cpu = smp_processor_id();
-	put_cpu_var(proc_event_counts);
+	preempt_enable();
 }

 void proc_fork_connector(struct task_struct *task)