From: Christoph Lameter <clameter@sgi.com>
To: akpm@linux-foundation.org
Cc: linux-arch@vger.kernel.org, linux-kernel@vger.kernel.org,
Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>,
Pekka Enberg <penberg@cs.helsinki.fi>
Subject: [patch 4/7] Percpu: Add support for this_cpu_offset() to be able to create this_cpu_ptr()
Date: Wed, 31 Oct 2007 17:02:15 -0700 [thread overview]
Message-ID: <20071101000310.439787988@sgi.com> (raw)
In-Reply-To: 20071101000211.970501947@sgi.com
[-- Attachment #1: this_cpu --]
[-- Type: text/plain, Size: 6679 bytes --]
Support for this_cpu_ptr() is important for those arches that allow a faster
way to get to the per cpu area of the local processor.
Signed-off-by: Christoph Lameter <clameter@sgi.com>
---
include/asm-generic/percpu.h | 4 ++++
include/asm-ia64/percpu.h | 3 +++
include/asm-powerpc/percpu.h | 3 +++
include/asm-s390/percpu.h | 4 ++++
include/asm-sparc64/percpu.h | 2 ++
include/asm-x86/percpu_32.h | 2 ++
include/asm-x86/percpu_64.h | 4 ++++
include/linux/percpu.h | 7 +++++++
8 files changed, 29 insertions(+)
Index: linux-2.6/include/linux/percpu.h
===================================================================
--- linux-2.6.orig/include/linux/percpu.h 2007-10-31 16:41:00.907621059 -0700
+++ linux-2.6/include/linux/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -51,6 +51,13 @@
(__typeof__(ptr))(p + q); \
})
+#define this_cpu_ptr(ptr) \
+({ \
+ void *p = ptr; \
+ (__typeof__(ptr))(p + this_cpu_offset()); \
+})
+
+
extern void *__alloc_percpu(size_t size);
extern void percpu_free(void *__pdata);
Index: linux-2.6/include/asm-generic/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-generic/percpu.h 2007-10-31 16:36:43.452121172 -0700
+++ linux-2.6/include/asm-generic/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -26,6 +26,8 @@ extern unsigned long __per_cpu_offset[NR
#define __get_cpu_var(var) per_cpu(var, smp_processor_id())
#define __raw_get_cpu_var(var) per_cpu(var, raw_smp_processor_id())
+#define this_cpu_offset() __per_cpu_offset(raw_smp_processor_id())
+
/* A macro to avoid #include hell... */
#define percpu_modcopy(pcpudst, src, size) \
do { \
@@ -53,4 +55,6 @@ do { \
#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
+#define this_cpu_offset() 0
+
#endif /* _ASM_GENERIC_PERCPU_H_ */
Index: linux-2.6/include/asm-ia64/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-ia64/percpu.h 2007-10-31 16:36:43.460121335 -0700
+++ linux-2.6/include/asm-ia64/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -51,6 +51,8 @@ extern unsigned long __per_cpu_offset[NR
/* Equal to __per_cpu_offset[smp_processor_id()], but faster to access: */
DECLARE_PER_CPU(unsigned long, local_per_cpu_offset);
+#define this_cpu_offset() __ia64_per_cpu_var(local_per_cpu_offset)
+
#define per_cpu(var, cpu) (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]))
#define __get_cpu_var(var) (*RELOC_HIDE(&per_cpu__##var, __ia64_per_cpu_var(local_per_cpu_offset)))
#define __raw_get_cpu_var(var) (*RELOC_HIDE(&per_cpu__##var, __ia64_per_cpu_var(local_per_cpu_offset)))
@@ -65,6 +67,7 @@ extern void *per_cpu_init(void);
#define __get_cpu_var(var) per_cpu__##var
#define __raw_get_cpu_var(var) per_cpu__##var
#define per_cpu_init() (__phys_per_cpu_start)
+#define this_cpu_offset() 0
#endif /* SMP */
Index: linux-2.6/include/asm-powerpc/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-powerpc/percpu.h 2007-10-31 16:36:43.464121161 -0700
+++ linux-2.6/include/asm-powerpc/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -16,6 +16,8 @@
#define __my_cpu_offset() get_paca()->data_offset
#define per_cpu_offset(x) (__per_cpu_offset(x))
+#define this_cpu_offset() __my_cpu_offset()
+
/* Separate out the type, so (int[3], foo) works. */
#define DEFINE_PER_CPU(type, name) \
__attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
@@ -51,6 +53,7 @@ extern void setup_per_cpu_areas(void);
#define per_cpu(var, cpu) (*((void)(cpu), &per_cpu__##var))
#define __get_cpu_var(var) per_cpu__##var
#define __raw_get_cpu_var(var) per_cpu__##var
+#define this_cpu_offset() 0
#endif /* SMP */
Index: linux-2.6/include/asm-s390/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-s390/percpu.h 2007-10-31 16:36:43.472121072 -0700
+++ linux-2.6/include/asm-s390/percpu.h 2007-10-31 16:42:45.779370925 -0700
@@ -51,6 +51,8 @@ extern unsigned long __per_cpu_offset[NR
#define per_cpu(var,cpu) __reloc_hide(var,__per_cpu_offset[cpu])
#define per_cpu_offset(x) (__per_cpu_offset[x])
+#define this_cpu_offset() S390_lowcore.percpu_offset
+
/* A macro to avoid #include hell... */
#define percpu_modcopy(pcpudst, src, size) \
do { \
@@ -71,6 +73,8 @@ do { \
#define __raw_get_cpu_var(var) __reloc_hide(var,0)
#define per_cpu(var,cpu) __reloc_hide(var,0)
+#define this_cpu_offset() 0
+
#endif /* SMP */
#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
Index: linux-2.6/include/asm-sparc64/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-sparc64/percpu.h 2007-10-31 16:36:43.480121400 -0700
+++ linux-2.6/include/asm-sparc64/percpu.h 2007-10-31 16:42:45.779370925 -0700
@@ -5,6 +5,8 @@
register unsigned long __local_per_cpu_offset asm("g5");
+#define this_cpu_offset() __local_per_cpu_offset
+
#ifdef CONFIG_SMP
#define setup_per_cpu_areas() do { } while (0)
Index: linux-2.6/include/asm-x86/percpu_32.h
===================================================================
--- linux-2.6.orig/include/asm-x86/percpu_32.h 2007-10-31 16:36:43.484121314 -0700
+++ linux-2.6/include/asm-x86/percpu_32.h 2007-10-31 16:42:45.779370925 -0700
@@ -72,6 +72,8 @@ DECLARE_PER_CPU(unsigned long, this_cpu_
RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off)); \
}))
+#define this_cpu_offset() x86_read_percpu(this_cpu_off)
+
#define __get_cpu_var(var) __raw_get_cpu_var(var)
/* A macro to avoid #include hell... */
Index: linux-2.6/include/asm-x86/percpu_64.h
===================================================================
--- linux-2.6.orig/include/asm-x86/percpu_64.h 2007-10-31 16:36:43.492121152 -0700
+++ linux-2.6/include/asm-x86/percpu_64.h 2007-10-31 16:42:45.779370925 -0700
@@ -14,6 +14,8 @@
#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
#define __my_cpu_offset() read_pda(data_offset)
+#define this_cpu_offset() read_pda(data_offset)
+
#define per_cpu_offset(x) (__per_cpu_offset(x))
/* Separate out the type, so (int[3], foo) works. */
@@ -58,6 +60,8 @@ extern void setup_per_cpu_areas(void);
#define __get_cpu_var(var) per_cpu__##var
#define __raw_get_cpu_var(var) per_cpu__##var
+#define this_cpu_offset() 0
+
#endif /* SMP */
#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
--
WARNING: multiple messages have this Message-ID (diff)
From: Christoph Lameter <clameter@sgi.com>
To: akpm@linux-foundation.org
Cc: linux-arch@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Subject: [patch 4/7] Percpu: Add support for this_cpu_offset() to be able to create this_cpu_ptr()
Date: Wed, 31 Oct 2007 17:02:15 -0700 [thread overview]
Message-ID: <20071101000310.439787988@sgi.com> (raw)
In-Reply-To: 20071101000211.970501947@sgi.com
[-- Attachment #1: this_cpu --]
[-- Type: text/plain, Size: 6679 bytes --]
Support for this_cpu_ptr() is important for those arches that allow a faster
way to get to the per cpu area of the local processor.
Signed-off-by: Christoph Lameter <clameter@sgi.com>
---
include/asm-generic/percpu.h | 4 ++++
include/asm-ia64/percpu.h | 3 +++
include/asm-powerpc/percpu.h | 3 +++
include/asm-s390/percpu.h | 4 ++++
include/asm-sparc64/percpu.h | 2 ++
include/asm-x86/percpu_32.h | 2 ++
include/asm-x86/percpu_64.h | 4 ++++
include/linux/percpu.h | 7 +++++++
8 files changed, 29 insertions(+)
Index: linux-2.6/include/linux/percpu.h
===================================================================
--- linux-2.6.orig/include/linux/percpu.h 2007-10-31 16:41:00.907621059 -0700
+++ linux-2.6/include/linux/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -51,6 +51,13 @@
(__typeof__(ptr))(p + q); \
})
+#define this_cpu_ptr(ptr) \
+({ \
+ void *p = ptr; \
+ (__typeof__(ptr))(p + this_cpu_offset()); \
+})
+
+
extern void *__alloc_percpu(size_t size);
extern void percpu_free(void *__pdata);
Index: linux-2.6/include/asm-generic/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-generic/percpu.h 2007-10-31 16:36:43.452121172 -0700
+++ linux-2.6/include/asm-generic/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -26,6 +26,8 @@ extern unsigned long __per_cpu_offset[NR
#define __get_cpu_var(var) per_cpu(var, smp_processor_id())
#define __raw_get_cpu_var(var) per_cpu(var, raw_smp_processor_id())
+#define this_cpu_offset() __per_cpu_offset(raw_smp_processor_id())
+
/* A macro to avoid #include hell... */
#define percpu_modcopy(pcpudst, src, size) \
do { \
@@ -53,4 +55,6 @@ do { \
#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
+#define this_cpu_offset() 0
+
#endif /* _ASM_GENERIC_PERCPU_H_ */
Index: linux-2.6/include/asm-ia64/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-ia64/percpu.h 2007-10-31 16:36:43.460121335 -0700
+++ linux-2.6/include/asm-ia64/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -51,6 +51,8 @@ extern unsigned long __per_cpu_offset[NR
/* Equal to __per_cpu_offset[smp_processor_id()], but faster to access: */
DECLARE_PER_CPU(unsigned long, local_per_cpu_offset);
+#define this_cpu_offset() __ia64_per_cpu_var(local_per_cpu_offset)
+
#define per_cpu(var, cpu) (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]))
#define __get_cpu_var(var) (*RELOC_HIDE(&per_cpu__##var, __ia64_per_cpu_var(local_per_cpu_offset)))
#define __raw_get_cpu_var(var) (*RELOC_HIDE(&per_cpu__##var, __ia64_per_cpu_var(local_per_cpu_offset)))
@@ -65,6 +67,7 @@ extern void *per_cpu_init(void);
#define __get_cpu_var(var) per_cpu__##var
#define __raw_get_cpu_var(var) per_cpu__##var
#define per_cpu_init() (__phys_per_cpu_start)
+#define this_cpu_offset() 0
#endif /* SMP */
Index: linux-2.6/include/asm-powerpc/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-powerpc/percpu.h 2007-10-31 16:36:43.464121161 -0700
+++ linux-2.6/include/asm-powerpc/percpu.h 2007-10-31 16:42:45.748121446 -0700
@@ -16,6 +16,8 @@
#define __my_cpu_offset() get_paca()->data_offset
#define per_cpu_offset(x) (__per_cpu_offset(x))
+#define this_cpu_offset() __my_cpu_offset()
+
/* Separate out the type, so (int[3], foo) works. */
#define DEFINE_PER_CPU(type, name) \
__attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
@@ -51,6 +53,7 @@ extern void setup_per_cpu_areas(void);
#define per_cpu(var, cpu) (*((void)(cpu), &per_cpu__##var))
#define __get_cpu_var(var) per_cpu__##var
#define __raw_get_cpu_var(var) per_cpu__##var
+#define this_cpu_offset() 0
#endif /* SMP */
Index: linux-2.6/include/asm-s390/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-s390/percpu.h 2007-10-31 16:36:43.472121072 -0700
+++ linux-2.6/include/asm-s390/percpu.h 2007-10-31 16:42:45.779370925 -0700
@@ -51,6 +51,8 @@ extern unsigned long __per_cpu_offset[NR
#define per_cpu(var,cpu) __reloc_hide(var,__per_cpu_offset[cpu])
#define per_cpu_offset(x) (__per_cpu_offset[x])
+#define this_cpu_offset() S390_lowcore.percpu_offset
+
/* A macro to avoid #include hell... */
#define percpu_modcopy(pcpudst, src, size) \
do { \
@@ -71,6 +73,8 @@ do { \
#define __raw_get_cpu_var(var) __reloc_hide(var,0)
#define per_cpu(var,cpu) __reloc_hide(var,0)
+#define this_cpu_offset() 0
+
#endif /* SMP */
#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
Index: linux-2.6/include/asm-sparc64/percpu.h
===================================================================
--- linux-2.6.orig/include/asm-sparc64/percpu.h 2007-10-31 16:36:43.480121400 -0700
+++ linux-2.6/include/asm-sparc64/percpu.h 2007-10-31 16:42:45.779370925 -0700
@@ -5,6 +5,8 @@
register unsigned long __local_per_cpu_offset asm("g5");
+#define this_cpu_offset() __local_per_cpu_offset
+
#ifdef CONFIG_SMP
#define setup_per_cpu_areas() do { } while (0)
Index: linux-2.6/include/asm-x86/percpu_32.h
===================================================================
--- linux-2.6.orig/include/asm-x86/percpu_32.h 2007-10-31 16:36:43.484121314 -0700
+++ linux-2.6/include/asm-x86/percpu_32.h 2007-10-31 16:42:45.779370925 -0700
@@ -72,6 +72,8 @@ DECLARE_PER_CPU(unsigned long, this_cpu_
RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off)); \
}))
+#define this_cpu_offset() x86_read_percpu(this_cpu_off)
+
#define __get_cpu_var(var) __raw_get_cpu_var(var)
/* A macro to avoid #include hell... */
Index: linux-2.6/include/asm-x86/percpu_64.h
===================================================================
--- linux-2.6.orig/include/asm-x86/percpu_64.h 2007-10-31 16:36:43.492121152 -0700
+++ linux-2.6/include/asm-x86/percpu_64.h 2007-10-31 16:42:45.779370925 -0700
@@ -14,6 +14,8 @@
#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
#define __my_cpu_offset() read_pda(data_offset)
+#define this_cpu_offset() read_pda(data_offset)
+
#define per_cpu_offset(x) (__per_cpu_offset(x))
/* Separate out the type, so (int[3], foo) works. */
@@ -58,6 +60,8 @@ extern void setup_per_cpu_areas(void);
#define __get_cpu_var(var) per_cpu__##var
#define __raw_get_cpu_var(var) per_cpu__##var
+#define this_cpu_offset() 0
+
#endif /* SMP */
#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
--
next prev parent reply other threads:[~2007-11-01 0:03 UTC|newest]
Thread overview: 72+ messages / expand[flat|nested] mbox.gz Atom feed top
2007-11-01 0:02 [patch 0/7] [RFC] SLUB: Improve allocpercpu to reduce per cpu access overhead Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 0:02 ` [patch 1/7] allocpercpu: Make it a true per cpu allocator by allocating from a per cpu array Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 7:24 ` Eric Dumazet
2007-11-01 12:59 ` Christoph Lameter
2007-11-01 0:02 ` [patch 2/7] allocpercpu: Remove functions that are rarely used Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 0:02 ` [patch 3/7] Allocpercpu: Do __percpu_disguise() only if CONFIG_DEBUG_VM is set Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 7:25 ` Eric Dumazet
2007-11-01 0:02 ` Christoph Lameter [this message]
2007-11-01 0:02 ` [patch 4/7] Percpu: Add support for this_cpu_offset() to be able to create this_cpu_ptr() Christoph Lameter
2007-11-01 0:02 ` [patch 5/7] SLUB: Use allocpercpu to allocate per cpu data instead of running our own per cpu allocator Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 0:02 ` [patch 6/7] SLUB: No need to cache kmem_cache data in kmem_cache_cpu anymore Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 0:02 ` [patch 7/7] SLUB: Optimize per cpu access on the local cpu using this_cpu_ptr() Christoph Lameter
2007-11-01 0:02 ` Christoph Lameter
2007-11-01 0:24 ` [patch 0/7] [RFC] SLUB: Improve allocpercpu to reduce per cpu access overhead David Miller
2007-11-01 0:26 ` Christoph Lameter
2007-11-01 0:27 ` David Miller
2007-11-01 0:31 ` Christoph Lameter
2007-11-01 0:51 ` David Miller
2007-11-01 0:53 ` Christoph Lameter
2007-11-01 1:00 ` David Miller
2007-11-01 1:01 ` Christoph Lameter
2007-11-01 1:09 ` David Miller
2007-11-01 1:12 ` Christoph Lameter
2007-11-01 1:13 ` David Miller
2007-11-01 1:21 ` Christoph Lameter
2007-11-01 5:27 ` David Miller
2007-11-01 4:16 ` Christoph Lameter
2007-11-01 5:38 ` David Miller
2007-11-01 7:01 ` David Miller
2007-11-01 9:14 ` David Miller
2007-11-01 13:03 ` Christoph Lameter
2007-11-01 21:29 ` David Miller
2007-11-01 22:15 ` Christoph Lameter
2007-11-01 22:38 ` David Miller
2007-11-01 22:48 ` Christoph Lameter
2007-11-01 22:58 ` David Miller
2007-11-02 1:06 ` Christoph Lameter
2007-11-02 2:51 ` David Miller
2007-11-02 10:28 ` Peter Zijlstra
2007-11-02 14:35 ` Christoph Lameter
2007-11-02 15:20 ` Peter Zijlstra
2007-11-02 15:29 ` Christoph Lameter
2007-11-12 10:52 ` Herbert Xu
2007-11-12 10:52 ` Herbert Xu
2007-11-12 19:14 ` Christoph Lameter
2007-11-12 19:48 ` Eric Dumazet
2007-11-12 19:56 ` Christoph Lameter
2007-11-12 20:18 ` Eric Dumazet
2007-11-12 22:46 ` David Miller
2007-11-12 19:57 ` Luck, Tony
2007-11-12 19:57 ` Luck, Tony
2007-11-12 20:14 ` Eric Dumazet
2007-11-12 22:46 ` David Miller
2007-11-12 21:28 ` David Miller
2007-11-01 23:00 ` Eric Dumazet
2007-11-02 0:58 ` Christoph Lameter
2007-11-02 1:40 ` Christoph Lameter
2007-11-01 7:17 ` Eric Dumazet
2007-11-01 7:57 ` David Miller
2007-11-01 13:01 ` Christoph Lameter
2007-11-01 21:25 ` David Miller
2007-11-01 12:57 ` Christoph Lameter
2007-11-01 21:28 ` David Miller
2007-11-01 22:11 ` Christoph Lameter
2007-11-01 22:14 ` David Miller
2007-11-01 22:16 ` Christoph Lameter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20071101000310.439787988@sgi.com \
--to=clameter@sgi.com \
--cc=akpm@linux-foundation.org \
--cc=linux-arch@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@polymtl.ca \
--cc=penberg@cs.helsinki.fi \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.