Re: 2.6.16-rc6-mm2

Ingo Molnar
2006-06-12T11:49:59Z

* Ingo Molnar <mingo@elte.hu> wrote:

>
> * Christoph Lameter <clameter@sgi.com> wrote:
>
> > Sorry that patch was still against mm1. Here is a fixed up version
> > that applies cleanly against mm2:
>
> i have applied both patches you sent in this thread but it still
> triggers tons of messages:

> trying to fix it i realized that i'd have to touch tons of
> architectures, which all duplicate this piece of code:

below is an updated patch that includes fixups for i386 - but the real
fix should be to properly reduce the per-arch local.h footprint to the
bare minimum possible, and to do this fix on the asm-generic headers.

Ingo
---
include/asm-generic/local.h | 20 ++++++++++----------
include/asm-i386/local.h | 25 ++++++++++++++-----------
include/linux/page-flags.h | 14 +++++---------
mm/page_alloc.c | 8 ++++----
4 files changed, 33 insertions(+), 34 deletions(-)

Index: linux/include/asm-generic/local.h
===================================================================
--- linux.orig/include/asm-generic/local.h
+++ linux/include/asm-generic/local.h
@@ -44,19 +44,19 @@ typedef struct
* much more efficient than these naive implementations. Note they take
* a variable (eg. mystruct.foo), not an address.
*/
-#define cpu_local_read(v) local_read(&__get_cpu_var(v))
-#define cpu_local_set(v, i) local_set(&__get_cpu_var(v), (i))
-#define cpu_local_inc(v) local_inc(&__get_cpu_var(v))
-#define cpu_local_dec(v) local_dec(&__get_cpu_var(v))
-#define cpu_local_add(i, v) local_add((i), &__get_cpu_var(v))
-#define cpu_local_sub(i, v) local_sub((i), &__get_cpu_var(v))
+#define cpu_local_read(v) local_read(&per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_set(v, i) local_set(&per_cpu(v, raw_smp_processor_id()), (i))
+#define cpu_local_inc(v) local_inc(&per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_dec(v) local_dec(&per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_add(i, v) local_add((i), &per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_sub(i, v) local_sub((i), &per_cpu(v, raw_smp_processor_id()))

/* Non-atomic increments, ie. preemption disabled and won't be touched
* in interrupt, etc. Some archs can optimize this case well.
*/
-#define __cpu_local_inc(v) __local_inc(&__get_cpu_var(v))
-#define __cpu_local_dec(v) __local_dec(&__get_cpu_var(v))
-#define __cpu_local_add(i, v) __local_add((i), &__get_cpu_var(v))
-#define __cpu_local_sub(i, v) __local_sub((i), &__get_cpu_var(v))
+#define __cpu_local_inc(v) __local_inc(&per_cpu(v, raw_smp_processor_id()))
+#define __cpu_local_dec(v) __local_dec(&per_cpu(v, raw_smp_processor_id()))
+#define __cpu_local_add(i, v) __local_add((i), &per_cpu(v, raw_smp_processor_id()))
+#define __cpu_local_sub(i, v) __local_sub((i), &per_cpu(v, raw_smp_processor_id()))

#endif /* _ASM_GENERIC_LOCAL_H */
Index: linux/include/asm-i386/local.h
===================================================================
--- linux.orig/include/asm-i386/local.h
+++ linux/include/asm-i386/local.h
@@ -53,18 +53,21 @@ static __inline__ void local_sub(long i,

/* Use these for per-cpu local_t variables: on some archs they are
* much more efficient than these naive implementations. Note they take
- * a variable, not an address.
+ * a variable (eg. mystruct.foo), not an address.
*/
-#define cpu_local_read(v) local_read(&__get_cpu_var(v))
-#define cpu_local_set(v, i) local_set(&__get_cpu_var(v), (i))
-#define cpu_local_inc(v) local_inc(&__get_cpu_var(v))
-#define cpu_local_dec(v) local_dec(&__get_cpu_var(v))
-#define cpu_local_add(i, v) local_add((i), &__get_cpu_var(v))
-#define cpu_local_sub(i, v) local_sub((i), &__get_cpu_var(v))
+#define cpu_local_read(v) local_read(&per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_set(v, i) local_set(&per_cpu(v, raw_smp_processor_id()), (i))
+#define cpu_local_inc(v) local_inc(&per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_dec(v) local_dec(&per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_add(i, v) local_add((i), &per_cpu(v, raw_smp_processor_id()))
+#define cpu_local_sub(i, v) local_sub((i), &per_cpu(v, raw_smp_processor_id()))

-#define __cpu_local_inc(v) cpu_local_inc(v)
-#define __cpu_local_dec(v) cpu_local_dec(v)
-#define __cpu_local_add(i, v) cpu_local_add((i), (v))
-#define __cpu_local_sub(i, v) cpu_local_sub((i), (v))
+/* Non-atomic increments, ie. preemption disabled and won't be touched
+ * in interrupt, etc. Some archs can optimize this case well.
+ */
+#define __cpu_local_inc(v) __local_inc(&per_cpu(v, raw_smp_processor_id()))
+#define __cpu_local_dec(v) __local_dec(&per_cpu(v, raw_smp_processor_id()))
+#define __cpu_local_add(i, v) __local_add((i), &per_cpu(v, raw_smp_processor_id()))
+#define __cpu_local_sub(i, v) __local_sub((i), &per_cpu(v, raw_smp_processor_id()))

#endif /* _ARCH_I386_LOCAL_H */
Index: linux/include/linux/page-flags.h
===================================================================
--- linux.orig/include/linux/page-flags.h
+++ linux/include/linux/page-flags.h
@@ -8,7 +8,7 @@
#include <linux/percpu.h>
#include <linux/cache.h>
#include <linux/types.h>
-
+#include <asm/local.h>
#include <asm/pgtable.h>

/*
@@ -108,10 +108,6 @@
/*
* Light weight per cpu counter implementation.
*
- * Note that these can race. We do not bother to enable preemption
- * or care about interrupt races. All we care about is to have some
- * approximate count of events.
- *
* Counters should only be incremented and no critical kernel component
* should rely on the counter values.
*
@@ -134,24 +130,24 @@ enum vm_event_item { PGPGIN, PGPGOUT, PS
};

struct vm_event_state {
- unsigned long event[NR_VM_EVENT_ITEMS];
+ local_t event[NR_VM_EVENT_ITEMS];
};

DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

static inline unsigned long get_cpu_vm_events(enum vm_event_item item)
{
- return __get_cpu_var(vm_event_states).event[item];
+ return cpu_local_read(vm_event_states.event[item]);
}

static inline void count_vm_event(enum vm_event_item item)
{
- __get_cpu_var(vm_event_states).event[item]++;
+ cpu_local_inc(vm_event_states.event[item]);
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
- __get_cpu_var(vm_event_states).event[item] += delta;
+ cpu_local_add(delta, vm_event_states.event[item]);
}

extern void all_vm_events(unsigned long *);
Index: linux/mm/page_alloc.c
===================================================================
--- linux.orig/mm/page_alloc.c
+++ linux/mm/page_alloc.c
@@ -1583,7 +1583,7 @@ static void show_node(struct zone *zone)
#endif

#ifdef CONFIG_VM_EVENT_COUNTERS
-DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
+DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{LOCAL_INIT(0)}};
EXPORT_PER_CPU_SYMBOL(vm_event_states);

static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask)
@@ -1604,7 +1604,7 @@ static void sum_vm_events(unsigned long

for (i=0; i< NR_VM_EVENT_ITEMS; i++)
- ret[i] += this->event[i];
+ ret[i] += local_read(&this->event[i]);
}
}

@@ -2881,8 +2881,8 @@ static void vm_events_fold_cpu(int cpu)
int i;

for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
- count_vm_events(i, fold_state->event[i]);
- fold_state->event[i] = 0;
+ count_vm_events(i, local_read(&fold_state->event[i]));
+ local_set(&fold_state->event[i], 0);
}
}