From: Christoph Lameter <cl@linux-foundation.org>
To: akpm@linux-foundation.org
Cc: linux-kernel@vger.kernel.org,
Christoph Lameter <cl@linux-foundation.org>
Cc: linux-mm@kvack.org
Cc: rusty@rustcorp.com.au
Cc: jeremy@goop.org
Cc: ebiederm@xmission.com
Cc: travis@sgi.com
Cc: herbert@gondor.apana.org.au
Cc: xemul@openvz.org
Cc: penberg@cs.helsinki.fi
Subject: [patch 4/4] cpu alloc: Use cpu allocator instead of the builtin modules per cpu allocator
Date: Mon, 29 Sep 2008 12:35:04 -0700 [thread overview]
Message-ID: <20080929193516.500912533@quilx.com> (raw)
In-Reply-To: 20080929193500.470295078@quilx.com
[-- Attachment #1: cpu_alloc_replace_modules_per_cpu_allocator --]
[-- Type: text/plain, Size: 6672 bytes --]
Remove the builtin per cpu allocator from modules.c and use cpu_alloc instead.
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
---
include/linux/module.h | 1
kernel/module.c | 178 ++++---------------------------------------------
2 files changed, 17 insertions(+), 162 deletions(-)
Index: linux-2.6/kernel/module.c
===================================================================
--- linux-2.6.orig/kernel/module.c 2008-09-19 08:12:10.000000000 -0500
+++ linux-2.6/kernel/module.c 2008-09-19 08:16:04.000000000 -0500
@@ -337,121 +337,6 @@
return NULL;
}
-#ifdef CONFIG_SMP
-/* Number of blocks used and allocated. */
-static unsigned int pcpu_num_used, pcpu_num_allocated;
-/* Size of each block. -ve means used. */
-static int *pcpu_size;
-
-static int split_block(unsigned int i, unsigned short size)
-{
- /* Reallocation required? */
- if (pcpu_num_used + 1 > pcpu_num_allocated) {
- int *new;
-
- new = krealloc(pcpu_size, sizeof(new[0])*pcpu_num_allocated*2,
- GFP_KERNEL);
- if (!new)
- return 0;
-
- pcpu_num_allocated *= 2;
- pcpu_size = new;
- }
-
- /* Insert a new subblock */
- memmove(&pcpu_size[i+1], &pcpu_size[i],
- sizeof(pcpu_size[0]) * (pcpu_num_used - i));
- pcpu_num_used++;
-
- pcpu_size[i+1] -= size;
- pcpu_size[i] = size;
- return 1;
-}
-
-static inline unsigned int block_size(int val)
-{
- if (val < 0)
- return -val;
- return val;
-}
-
-static void *percpu_modalloc(unsigned long size, unsigned long align,
- const char *name)
-{
- unsigned long extra;
- unsigned int i;
- void *ptr;
-
- if (align > PAGE_SIZE) {
- printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
- name, align, PAGE_SIZE);
- align = PAGE_SIZE;
- }
-
- ptr = __per_cpu_start;
- for (i = 0; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) {
- /* Extra for alignment requirement. */
- extra = ALIGN((unsigned long)ptr, align) - (unsigned long)ptr;
- BUG_ON(i == 0 && extra != 0);
-
- if (pcpu_size[i] < 0 || pcpu_size[i] < extra + size)
- continue;
-
- /* Transfer extra to previous block. */
- if (pcpu_size[i-1] < 0)
- pcpu_size[i-1] -= extra;
- else
- pcpu_size[i-1] += extra;
- pcpu_size[i] -= extra;
- ptr += extra;
-
- /* Split block if warranted */
- if (pcpu_size[i] - size > sizeof(unsigned long))
- if (!split_block(i, size))
- return NULL;
-
- /* Mark allocated */
- pcpu_size[i] = -pcpu_size[i];
- return ptr;
- }
-
- printk(KERN_WARNING "Could not allocate %lu bytes percpu data\n",
- size);
- return NULL;
-}
-
-static void percpu_modfree(void *freeme)
-{
- unsigned int i;
- void *ptr = __per_cpu_start + block_size(pcpu_size[0]);
-
- /* First entry is core kernel percpu data. */
- for (i = 1; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) {
- if (ptr == freeme) {
- pcpu_size[i] = -pcpu_size[i];
- goto free;
- }
- }
- BUG();
-
- free:
- /* Merge with previous? */
- if (pcpu_size[i-1] >= 0) {
- pcpu_size[i-1] += pcpu_size[i];
- pcpu_num_used--;
- memmove(&pcpu_size[i], &pcpu_size[i+1],
- (pcpu_num_used - i) * sizeof(pcpu_size[0]));
- i--;
- }
- /* Merge with next? */
- if (i+1 < pcpu_num_used && pcpu_size[i+1] >= 0) {
- pcpu_size[i] += pcpu_size[i+1];
- pcpu_num_used--;
- memmove(&pcpu_size[i+1], &pcpu_size[i+2],
- (pcpu_num_used - (i+1)) * sizeof(pcpu_size[0]));
- }
-}
-
static unsigned int find_pcpusec(Elf_Ehdr *hdr,
Elf_Shdr *sechdrs,
const char *secstrings)
@@ -467,48 +352,6 @@
memcpy(pcpudest + per_cpu_offset(cpu), from, size);
}
-static int percpu_modinit(void)
-{
- pcpu_num_used = 2;
- pcpu_num_allocated = 2;
- pcpu_size = kmalloc(sizeof(pcpu_size[0]) * pcpu_num_allocated,
- GFP_KERNEL);
- /* Static in-kernel percpu data (used). */
- pcpu_size[0] = -(__per_cpu_end-__per_cpu_start);
- /* Free room. */
- pcpu_size[1] = PERCPU_AREA_SIZE + pcpu_size[0];
- if (pcpu_size[1] < 0) {
- printk(KERN_ERR "No per-cpu room for modules.\n");
- pcpu_num_used = 1;
- }
-
- return 0;
-}
-__initcall(percpu_modinit);
-#else /* ... !CONFIG_SMP */
-static inline void *percpu_modalloc(unsigned long size, unsigned long align,
- const char *name)
-{
- return NULL;
-}
-static inline void percpu_modfree(void *pcpuptr)
-{
- BUG();
-}
-static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,
- Elf_Shdr *sechdrs,
- const char *secstrings)
-{
- return 0;
-}
-static inline void percpu_modcopy(void *pcpudst, const void *src,
- unsigned long size)
-{
- /* pcpusec should be 0, and size of that section should be 0. */
- BUG_ON(size != 0);
-}
-#endif /* CONFIG_SMP */
-
#define MODINFO_ATTR(field) \
static void setup_modinfo_##field(struct module *mod, const char *s) \
{ \
@@ -1433,7 +1276,7 @@
module_free(mod, mod->module_init);
kfree(mod->args);
if (mod->percpu)
- percpu_modfree(mod->percpu);
+ cpu_free(mod->percpu, mod->percpu_size);
/* Free lock-classes: */
lockdep_free_key_range(mod->module_core, mod->core_size);
@@ -1833,6 +1676,7 @@
unsigned int markersstringsindex;
struct module *mod;
long err = 0;
+ unsigned long percpu_size = 0;
void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */
struct exception_table_entry *extable;
mm_segment_t old_fs;
@@ -1981,15 +1825,20 @@
if (pcpuindex) {
/* We have a special allocation for this section. */
- percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
- sechdrs[pcpuindex].sh_addralign,
- mod->name);
+ unsigned long align = sechdrs[pcpuindex].sh_addralign;
+
+ percpu_size = sechdrs[pcpuindex].sh_size;
+ percpu = cpu_alloc(percpu_size, GFP_KERNEL|__GFP_ZERO, align);
+ if (!percpu)
+ printk(KERN_WARNING "Could not allocate %lu bytes percpu data\n",
+ percpu_size);
if (!percpu) {
err = -ENOMEM;
goto free_mod;
}
sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
mod->percpu = percpu;
+ mod->percpu_size = percpu_size;
}
/* Determine total sizes, and put offsets in sh_entsize. For now
@@ -2243,7 +2092,7 @@
module_free(mod, mod->module_core);
free_percpu:
if (percpu)
- percpu_modfree(percpu);
+ cpu_free(percpu, percpu_size);
free_mod:
kfree(args);
free_hdr:
Index: linux-2.6/include/linux/module.h
===================================================================
--- linux-2.6.orig/include/linux/module.h 2008-09-19 08:12:07.000000000 -0500
+++ linux-2.6/include/linux/module.h 2008-09-19 08:12:10.000000000 -0500
@@ -323,6 +323,7 @@
/* Per-cpu data. */
void *percpu;
+ int percpu_size;
/* The command line arguments (may be mangled). People like
keeping pointers to this stuff */
--
WARNING: multiple messages have this Message-ID (diff)
From: Christoph Lameter <cl@linux-foundation.org>
To: akpm@linux-foundation.org
Cc: linux-kernel@vger.kernel.org,
Christoph Lameter <cl@linux-foundation.org>,
linux-mm@kvack.org, rusty@rustcorp.com.au, jeremy@goop.org,
ebiederm@xmission.com, travis@sgi.com,
herbert@gondor.apana.org.au, xemul@openvz.org,
penberg@cs.helsinki.fi
Subject: [patch 4/4] cpu alloc: Use cpu allocator instead of the builtin modules per cpu allocator
Date: Mon, 29 Sep 2008 12:35:04 -0700 [thread overview]
Message-ID: <20080929193516.500912533@quilx.com> (raw)
In-Reply-To: 20080929193500.470295078@quilx.com
[-- Attachment #1: cpu_alloc_replace_modules_per_cpu_allocator --]
[-- Type: text/plain, Size: 6672 bytes --]
Remove the builtin per cpu allocator from modules.c and use cpu_alloc instead.
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
---
include/linux/module.h | 1
kernel/module.c | 178 ++++---------------------------------------------
2 files changed, 17 insertions(+), 162 deletions(-)
Index: linux-2.6/kernel/module.c
===================================================================
--- linux-2.6.orig/kernel/module.c 2008-09-19 08:12:10.000000000 -0500
+++ linux-2.6/kernel/module.c 2008-09-19 08:16:04.000000000 -0500
@@ -337,121 +337,6 @@
return NULL;
}
-#ifdef CONFIG_SMP
-/* Number of blocks used and allocated. */
-static unsigned int pcpu_num_used, pcpu_num_allocated;
-/* Size of each block. -ve means used. */
-static int *pcpu_size;
-
-static int split_block(unsigned int i, unsigned short size)
-{
- /* Reallocation required? */
- if (pcpu_num_used + 1 > pcpu_num_allocated) {
- int *new;
-
- new = krealloc(pcpu_size, sizeof(new[0])*pcpu_num_allocated*2,
- GFP_KERNEL);
- if (!new)
- return 0;
-
- pcpu_num_allocated *= 2;
- pcpu_size = new;
- }
-
- /* Insert a new subblock */
- memmove(&pcpu_size[i+1], &pcpu_size[i],
- sizeof(pcpu_size[0]) * (pcpu_num_used - i));
- pcpu_num_used++;
-
- pcpu_size[i+1] -= size;
- pcpu_size[i] = size;
- return 1;
-}
-
-static inline unsigned int block_size(int val)
-{
- if (val < 0)
- return -val;
- return val;
-}
-
-static void *percpu_modalloc(unsigned long size, unsigned long align,
- const char *name)
-{
- unsigned long extra;
- unsigned int i;
- void *ptr;
-
- if (align > PAGE_SIZE) {
- printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
- name, align, PAGE_SIZE);
- align = PAGE_SIZE;
- }
-
- ptr = __per_cpu_start;
- for (i = 0; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) {
- /* Extra for alignment requirement. */
- extra = ALIGN((unsigned long)ptr, align) - (unsigned long)ptr;
- BUG_ON(i == 0 && extra != 0);
-
- if (pcpu_size[i] < 0 || pcpu_size[i] < extra + size)
- continue;
-
- /* Transfer extra to previous block. */
- if (pcpu_size[i-1] < 0)
- pcpu_size[i-1] -= extra;
- else
- pcpu_size[i-1] += extra;
- pcpu_size[i] -= extra;
- ptr += extra;
-
- /* Split block if warranted */
- if (pcpu_size[i] - size > sizeof(unsigned long))
- if (!split_block(i, size))
- return NULL;
-
- /* Mark allocated */
- pcpu_size[i] = -pcpu_size[i];
- return ptr;
- }
-
- printk(KERN_WARNING "Could not allocate %lu bytes percpu data\n",
- size);
- return NULL;
-}
-
-static void percpu_modfree(void *freeme)
-{
- unsigned int i;
- void *ptr = __per_cpu_start + block_size(pcpu_size[0]);
-
- /* First entry is core kernel percpu data. */
- for (i = 1; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) {
- if (ptr == freeme) {
- pcpu_size[i] = -pcpu_size[i];
- goto free;
- }
- }
- BUG();
-
- free:
- /* Merge with previous? */
- if (pcpu_size[i-1] >= 0) {
- pcpu_size[i-1] += pcpu_size[i];
- pcpu_num_used--;
- memmove(&pcpu_size[i], &pcpu_size[i+1],
- (pcpu_num_used - i) * sizeof(pcpu_size[0]));
- i--;
- }
- /* Merge with next? */
- if (i+1 < pcpu_num_used && pcpu_size[i+1] >= 0) {
- pcpu_size[i] += pcpu_size[i+1];
- pcpu_num_used--;
- memmove(&pcpu_size[i+1], &pcpu_size[i+2],
- (pcpu_num_used - (i+1)) * sizeof(pcpu_size[0]));
- }
-}
-
static unsigned int find_pcpusec(Elf_Ehdr *hdr,
Elf_Shdr *sechdrs,
const char *secstrings)
@@ -467,48 +352,6 @@
memcpy(pcpudest + per_cpu_offset(cpu), from, size);
}
-static int percpu_modinit(void)
-{
- pcpu_num_used = 2;
- pcpu_num_allocated = 2;
- pcpu_size = kmalloc(sizeof(pcpu_size[0]) * pcpu_num_allocated,
- GFP_KERNEL);
- /* Static in-kernel percpu data (used). */
- pcpu_size[0] = -(__per_cpu_end-__per_cpu_start);
- /* Free room. */
- pcpu_size[1] = PERCPU_AREA_SIZE + pcpu_size[0];
- if (pcpu_size[1] < 0) {
- printk(KERN_ERR "No per-cpu room for modules.\n");
- pcpu_num_used = 1;
- }
-
- return 0;
-}
-__initcall(percpu_modinit);
-#else /* ... !CONFIG_SMP */
-static inline void *percpu_modalloc(unsigned long size, unsigned long align,
- const char *name)
-{
- return NULL;
-}
-static inline void percpu_modfree(void *pcpuptr)
-{
- BUG();
-}
-static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,
- Elf_Shdr *sechdrs,
- const char *secstrings)
-{
- return 0;
-}
-static inline void percpu_modcopy(void *pcpudst, const void *src,
- unsigned long size)
-{
- /* pcpusec should be 0, and size of that section should be 0. */
- BUG_ON(size != 0);
-}
-#endif /* CONFIG_SMP */
-
#define MODINFO_ATTR(field) \
static void setup_modinfo_##field(struct module *mod, const char *s) \
{ \
@@ -1433,7 +1276,7 @@
module_free(mod, mod->module_init);
kfree(mod->args);
if (mod->percpu)
- percpu_modfree(mod->percpu);
+ cpu_free(mod->percpu, mod->percpu_size);
/* Free lock-classes: */
lockdep_free_key_range(mod->module_core, mod->core_size);
@@ -1833,6 +1676,7 @@
unsigned int markersstringsindex;
struct module *mod;
long err = 0;
+ unsigned long percpu_size = 0;
void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */
struct exception_table_entry *extable;
mm_segment_t old_fs;
@@ -1981,15 +1825,20 @@
if (pcpuindex) {
/* We have a special allocation for this section. */
- percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
- sechdrs[pcpuindex].sh_addralign,
- mod->name);
+ unsigned long align = sechdrs[pcpuindex].sh_addralign;
+
+ percpu_size = sechdrs[pcpuindex].sh_size;
+ percpu = cpu_alloc(percpu_size, GFP_KERNEL|__GFP_ZERO, align);
+ if (!percpu)
+ printk(KERN_WARNING "Could not allocate %lu bytes percpu data\n",
+ percpu_size);
if (!percpu) {
err = -ENOMEM;
goto free_mod;
}
sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
mod->percpu = percpu;
+ mod->percpu_size = percpu_size;
}
/* Determine total sizes, and put offsets in sh_entsize. For now
@@ -2243,7 +2092,7 @@
module_free(mod, mod->module_core);
free_percpu:
if (percpu)
- percpu_modfree(percpu);
+ cpu_free(percpu, percpu_size);
free_mod:
kfree(args);
free_hdr:
Index: linux-2.6/include/linux/module.h
===================================================================
--- linux-2.6.orig/include/linux/module.h 2008-09-19 08:12:07.000000000 -0500
+++ linux-2.6/include/linux/module.h 2008-09-19 08:12:10.000000000 -0500
@@ -323,6 +323,7 @@
/* Per-cpu data. */
void *percpu;
+ int percpu_size;
/* The command line arguments (may be mangled). People like
keeping pointers to this stuff */
--
next prev parent reply other threads:[~2008-09-29 19:36 UTC|newest]
Thread overview: 46+ messages / expand[flat|nested] mbox.gz Atom feed top
2008-09-29 19:35 [patch 0/4] Cpu alloc V6: Replace percpu allocator in modules.c Christoph Lameter
2008-09-29 19:35 ` Christoph Lameter
2008-09-29 19:35 ` [patch 1/4] Make the per cpu reserve configurable Christoph Lameter
2008-09-29 19:35 ` Christoph Lameter
2008-09-29 19:35 ` [patch 2/4] percpu: Rename variables PERCPU_ENOUGH_ROOM -> PERCPU_AREA_SIZE Christoph Lameter
2008-09-29 19:35 ` Christoph Lameter
2008-09-29 19:35 ` [patch 3/4] cpu alloc: The allocator Christoph Lameter
2008-09-29 19:35 ` Christoph Lameter
2008-09-30 6:35 ` Pekka Enberg
2008-09-30 6:35 ` Pekka Enberg
2008-09-30 11:38 ` Christoph Lameter
2008-09-30 11:38 ` Christoph Lameter
2008-09-30 11:48 ` Pekka Enberg
2008-09-30 11:48 ` Pekka Enberg
2008-09-30 12:12 ` Christoph Lameter
2008-09-30 12:12 ` Christoph Lameter
2008-10-03 7:33 ` Andrew Morton
2008-10-03 7:33 ` Andrew Morton
2008-10-03 7:43 ` Pekka Enberg
2008-10-03 7:43 ` Pekka Enberg
2008-10-03 8:20 ` Andrew Morton
2008-10-03 8:20 ` Andrew Morton
2008-10-03 14:15 ` Christoph Lameter
2008-10-03 14:15 ` Christoph Lameter
2008-10-03 12:48 ` Christoph Lameter
2008-10-03 12:48 ` Christoph Lameter
2008-10-05 21:10 ` Rusty Russell
2008-10-05 21:10 ` Rusty Russell
2008-10-07 13:27 ` Christoph Lameter
2008-10-07 13:27 ` Christoph Lameter
2008-09-29 19:35 ` Christoph Lameter [this message]
2008-09-29 19:35 ` [patch 4/4] cpu alloc: Use cpu allocator instead of the builtin modules per cpu allocator Christoph Lameter
2008-09-30 22:28 ` Rusty Russell
2008-09-30 22:28 ` Rusty Russell
2008-09-30 22:27 ` [patch 0/4] Cpu alloc V6: Replace percpu allocator in modules.c Rusty Russell
2008-09-30 22:27 ` Rusty Russell
2008-10-07 23:34 ` Andrew Morton
2008-10-07 23:34 ` Andrew Morton
2008-10-08 15:10 ` Christoph Lameter
2008-10-08 15:10 ` Christoph Lameter
-- strict thread matches above, loose matches on Subject: below --
2008-09-19 14:58 [patch 0/4] Cpu alloc V5: " Christoph Lameter
2008-09-19 14:59 ` [patch 4/4] cpu alloc: Use cpu allocator instead of the builtin modules per cpu allocator Christoph Lameter
2008-09-19 14:59 ` Christoph Lameter
2008-09-18 23:36 [patch 0/4] Cpu alloc V4: Replace percpu allocator in modules.c Christoph Lameter
2008-09-18 23:36 ` [patch 4/4] cpu alloc: Use cpu allocator instead of the builtin modules per cpu allocator Christoph Lameter
2008-09-19 7:39 ` Eric Dumazet
2008-09-19 12:59 ` Christoph Lameter
2008-09-19 13:23 ` Eric Dumazet
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20080929193516.500912533@quilx.com \
--to=cl@linux-foundation.org \
--cc=akpm@linux-foundation.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.