CK1 [02/13] create common functions for boot slab creation

CK1 [02/13] create common functions for boot slab creation

[Christoph Lameter]

Use a special function to create kmalloc caches and use that function in
SLAB and SLUB.

Signed-off-by: Christoph Lameter <cl@linux.com>
---
 mm/slab.c        |   48 ++++++++++++++----------------------------------
 mm/slab.h        |    5 +++++
 mm/slab_common.c |   32 ++++++++++++++++++++++++++++++++
 mm/slub.c        |   36 +++---------------------------------
 4 files changed, 54 insertions(+), 67 deletions(-)

Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-19 09:21:13.022941932 -0500
+++ linux/mm/slab.c	2012-09-19 09:21:14.422971030 -0500
@@ -1686,23 +1686,13 @@ void __init kmem_cache_init(void)
 	 * bug.
 	 */
 
-	sizes[INDEX_AC].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-	sizes[INDEX_AC].cs_cachep->name = names[INDEX_AC].name;
-	sizes[INDEX_AC].cs_cachep->size = sizes[INDEX_AC].cs_size;
-	sizes[INDEX_AC].cs_cachep->object_size = sizes[INDEX_AC].cs_size;
-	sizes[INDEX_AC].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
-	__kmem_cache_create(sizes[INDEX_AC].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
-	list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches);
+	sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
+					sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
 
-	if (INDEX_AC != INDEX_L3) {
-		sizes[INDEX_L3].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-		sizes[INDEX_L3].cs_cachep->name = names[INDEX_L3].name;
-		sizes[INDEX_L3].cs_cachep->size = sizes[INDEX_L3].cs_size;
-		sizes[INDEX_L3].cs_cachep->object_size = sizes[INDEX_L3].cs_size;
-		sizes[INDEX_L3].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
-		__kmem_cache_create(sizes[INDEX_L3].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
-		list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
-	}
+	if (INDEX_AC != INDEX_L3)
+		sizes[INDEX_L3].cs_cachep =
+			create_kmalloc_cache(names[INDEX_L3].name,
+				sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
 
 	slab_early_init = 0;
 
@@ -1714,24 +1704,14 @@ void __init kmem_cache_init(void)
 		 * Note for systems short on memory removing the alignment will
 		 * allow tighter packing of the smaller caches.
 		 */
-		if (!sizes->cs_cachep) {
-			sizes->cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-			sizes->cs_cachep->name = names->name;
-			sizes->cs_cachep->size = sizes->cs_size;
-			sizes->cs_cachep->object_size = sizes->cs_size;
-			sizes->cs_cachep->align = ARCH_KMALLOC_MINALIGN;
-			__kmem_cache_create(sizes->cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
-			list_add(&sizes->cs_cachep->list, &slab_caches);
-		}
+		if (!sizes->cs_cachep)
+			sizes->cs_cachep = create_kmalloc_cache(names->name,
+					sizes->cs_size, ARCH_KMALLOC_FLAGS);
+
 #ifdef CONFIG_ZONE_DMA
-		sizes->cs_dmacachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-		sizes->cs_dmacachep->name = names->name_dma;
-		sizes->cs_dmacachep->size = sizes->cs_size;
-		sizes->cs_dmacachep->object_size = sizes->cs_size;
-		sizes->cs_dmacachep->align = ARCH_KMALLOC_MINALIGN;
-		__kmem_cache_create(sizes->cs_dmacachep,
-			       ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC);
-		list_add(&sizes->cs_dmacachep->list, &slab_caches);
+		sizes->cs_dmacachep = create_kmalloc_cache(
+			names->name_dma, sizes->cs_size,
+			SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
 #endif
 		sizes++;
 		names++;
Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h	2012-09-19 09:21:13.014941777 -0500
+++ linux/mm/slab.h	2012-09-19 09:21:14.422971030 -0500
@@ -35,6 +35,11 @@ extern struct kmem_cache *kmem_cache;
 /* Functions provided by the slab allocators */
 extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
 
+extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
+			unsigned long flags);
+extern void create_boot_cache(struct kmem_cache *, const char *name,
+			size_t size, unsigned long flags);
+
 #ifdef CONFIG_SLUB
 struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
 	size_t align, unsigned long flags, void (*ctor)(void *));
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c	2012-09-19 09:21:13.034942193 -0500
+++ linux/mm/slab_common.c	2012-09-19 09:21:14.422971030 -0500
@@ -189,3 +189,35 @@ int slab_is_available(void)
 {
 	return slab_state >= UP;
 }
+
+/* Create a cache during boot when no slab services are available yet */
+void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
+		unsigned long flags)
+{
+	int err;
+
+	s->name = name;
+	s->size = s->object_size = size;
+	s->align = ARCH_KMALLOC_MINALIGN;
+	err = __kmem_cache_create(s, flags);
+
+	if (err)
+		panic("Creation of kmalloc slab %s size=%ld failed. Reason %d\n",
+					name, size, err);
+
+	list_add(&s->list, &slab_caches);
+	s->refcount = -1;	/* Exempt from merging for now */
+}
+
+struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
+				unsigned long flags)
+{
+	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
+
+	if (!s)
+		panic("Out of memory when creating slab %s\n", name);
+
+	create_boot_cache(s, name, size, flags);
+	s->refcount = 1;
+	return s;
+}
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2012-09-19 09:21:13.046942444 -0500
+++ linux/mm/slub.c	2012-09-19 09:21:14.422971030 -0500
@@ -3253,32 +3253,6 @@ static int __init setup_slub_nomerge(cha
 
 __setup("slub_nomerge", setup_slub_nomerge);
 
-static struct kmem_cache *__init create_kmalloc_cache(const char *name,
-						int size, unsigned int flags)
-{
-	struct kmem_cache *s;
-
-	s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-
-	s->name = name;
-	s->size = s->object_size = size;
-	s->align = ARCH_KMALLOC_MINALIGN;
-
-	/*
-	 * This function is called with IRQs disabled during early-boot on
-	 * single CPU so there's no need to take slab_mutex here.
-	 */
-	if (kmem_cache_open(s, flags))
-		goto panic;
-
-	list_add(&s->list, &slab_caches);
-	return s;
-
-panic:
-	panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
-	return NULL;
-}
-
 /*
  * Conversion table for small slabs sizes / 8 to the index in the
  * kmalloc array. This is necessary for slabs < 192 since we have non power

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [02/13] create common functions for boot slab creation

[Glauber Costa]

On 09/27/2012 12:01 AM, Christoph Lameter wrote:
> Use a special function to create kmalloc caches and use that function in
> SLAB and SLUB.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>

Seems ok.
Reviewed-by: Glauber Costa <glommer@parallels.com>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [01/13] slab: Simplify bootstrap

[Christoph Lameter]

The nodelists field in kmem_cache is pointing to the first unused
object in the array field when bootstrap is complete.

A problem with the current approach is that the statically sized
kmem_cache structure use on boot can only contain NR_CPUS entries.
If the number of nodes plus the number of cpus is greater then we
would overwrite memory following the kmem_cache_boot definition.

Increase the size of the array field to ensure that also the node
pointers fit into the array field.

Once we do that we no longer need the kmem_cache_nodelists
array and we can then also use that structure elsewhere.

Signed-off-by: Christoph Lameter <cl@linux.com>
---
 include/linux/slab_def.h |    2 +-
 mm/slab.c                |   18 +++++++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)

Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2012-09-12 10:35:59.867731044 -0500
+++ linux/include/linux/slab_def.h	2012-09-12 10:46:10.772621194 -0500
@@ -91,7 +91,7 @@ struct kmem_cache {
 	 * is statically defined, so we reserve the max number of cpus.
 	 */
 	struct kmem_list3 **nodelists;
-	struct array_cache *array[NR_CPUS];
+	struct array_cache *array[NR_CPUS + MAX_NUMNODES];
 	/*
 	 * Do not add fields after array[]
 	 */
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-12 10:46:09.980604396 -0500
+++ linux/mm/slab.c	2012-09-12 10:46:10.772621194 -0500
@@ -578,9 +578,7 @@ static struct arraycache_init initarray_
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
 /* internal cache of cache description objs */
-static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
 static struct kmem_cache kmem_cache_boot = {
-	.nodelists = kmem_cache_nodelists,
 	.batchcount = 1,
 	.limit = BOOT_CPUCACHE_ENTRIES,
 	.shared = 1,
@@ -1584,6 +1582,15 @@ static void __init set_up_list3s(struct
 }
 
 /*
+ * The memory after the last cpu cache pointer is used for the
+ * the nodelists pointer.
+ */
+static void setup_nodelists_pointer(struct kmem_cache *s)
+{
+	s->nodelists = (struct kmem_list3 **)&s->array[nr_cpu_ids];
+}
+
+/*
  * Initialisation.  Called after the page allocator have been initialised and
  * before smp_init().
  */
@@ -1597,13 +1604,15 @@ void __init kmem_cache_init(void)
 	int node;
 
 	kmem_cache = &kmem_cache_boot;
+	setup_nodelists_pointer(kmem_cache);
 
 	if (num_possible_nodes() == 1)
 		use_alien_caches = 0;
 
+
 	for (i = 0; i < NUM_INIT_LISTS; i++) {
 		kmem_list3_init(&initkmem_list3[i]);
-		if (i < MAX_NUMNODES)
+		if (i < nr_node_ids)
 			kmem_cache->nodelists[i] = NULL;
 	}
 	set_up_list3s(kmem_cache, CACHE_CACHE);
@@ -1643,7 +1652,6 @@ void __init kmem_cache_init(void)
 	list_add(&kmem_cache->list, &slab_caches);
 	kmem_cache->colour_off = cache_line_size();
 	kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
-	kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
 
 	/*
 	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
@@ -2454,7 +2462,7 @@ __kmem_cache_create (struct kmem_cache *
 	else
 		gfp = GFP_NOWAIT;
 
-	cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
+	setup_nodelists_pointer(cachep);
 #if DEBUG
 
 	/*

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [09/13] slab: rename nodelists to node

[Christoph Lameter]

Have a common naming between both slab caches for future changes.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2012-09-19 09:21:35.811415438 -0500
+++ linux/include/linux/slab_def.h	2012-09-19 09:21:37.499450510 -0500
@@ -88,16 +88,13 @@ struct kmem_cache {
 	 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
 	 * is statically defined, so we reserve the max number of cpus.
 	 */
-	struct kmem_cache_node **nodelists;
+	struct kmem_cache_node **node;
 	struct array_cache *array[NR_CPUS + MAX_NUMNODES];
 	/*
 	 * Do not add fields after array[]
 	 */
 };
 
-extern struct kmem_cache *cs_cachep[PAGE_SHIFT + MAX_ORDER];
-extern struct kmem_cache *cs_dmacachep[PAGE_SHIFT + MAX_ORDER];
-
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 void *__kmalloc(size_t size, gfp_t flags);
 
@@ -132,10 +129,10 @@ static __always_inline void *kmalloc(siz
 
 #ifdef CONFIG_ZONE_DMA
 		if (flags & GFP_DMA)
-			cachep = cs_dmacachep[i];
+			cachep = kmalloc_dma_caches[i];
 		else
 #endif
-			cachep = cs_cachep[i];
+			cachep = kmalloc_caches[i];
 
 		ret = kmem_cache_alloc_trace(size, cachep, flags);
 
@@ -178,10 +175,10 @@ static __always_inline void *kmalloc_nod
 
 #ifdef CONFIG_ZONE_DMA
 		if (flags & GFP_DMA)
-			cachep = cs_dmacachep[i];
+			cachep = kmalloc_dma_caches[i];
 		else
 #endif
-			cachep = cs_cachep[i];
+			cachep = kmalloc_caches[i];
 
 		return kmem_cache_alloc_node_trace(size, cachep, flags, node);
 	}
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-19 09:21:35.815415526 -0500
+++ linux/mm/slab.c	2012-09-19 09:21:37.503450612 -0500
@@ -291,11 +291,6 @@ static inline void clear_obj_pfmemalloc(
 	*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
 }
 
-struct kmem_cache *cs_cachep[PAGE_SHIFT + MAX_ORDER];
-EXPORT_SYMBOL(cs_cachep);
-struct kmem_cache *cs_dmacachep[PAGE_SHIFT + MAX_ORDER];
-EXPORT_SYMBOL(cs_dmacachep);
-
 /*
  * bootstrap: The caches do not work without cpuarrays anymore, but the
  * cpuarrays are allocated from the generic caches...
@@ -360,7 +355,7 @@ static void kmem_list3_init(struct kmem_
 #define MAKE_LIST(cachep, listp, slab, nodeid)				\
 	do {								\
 		INIT_LIST_HEAD(listp);					\
-		list_splice(&(cachep->nodelists[nodeid]->slab), listp);	\
+		list_splice(&(cachep->node[nodeid]->slab), listp);	\
 	} while (0)
 
 #define	MAKE_ALL_LISTS(cachep, ptr, nodeid)				\
@@ -570,7 +565,7 @@ static void slab_set_lock_classes(struct
 	struct kmem_cache_node *l3;
 	int r;
 
-	l3 = cachep->nodelists[q];
+	l3 = cachep->node[q];
 	if (!l3)
 		return;
 
@@ -613,12 +608,12 @@ static void init_node_lock_keys(int q)
 
 	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
 		struct kmem_cache_node *l3;
-		struct kmem_cache *cache = cs_cachep[i];
+		struct kmem_cache *cache = kmalloc_caches[i];
 
 		if (!cache)
 			continue;
 
-		l3 = cache->nodelists[q];
+		l3 = cache->node[q];
 		if (!l3 || OFF_SLAB(cache))
 			continue;
 
@@ -669,7 +664,7 @@ static inline struct kmem_cache *__find_
 	 * kmem_cache_create(), or __kmalloc(), before
 	 * the generic caches are initialized.
 	 */
-	BUG_ON(cs_cachep[INDEX_AC] == NULL);
+	BUG_ON(kmalloc_caches[INDEX_AC] == NULL);
 #endif
 	if (!size)
 		return ZERO_SIZE_PTR;
@@ -683,9 +678,9 @@ static inline struct kmem_cache *__find_
 	 */
 #ifdef CONFIG_ZONE_DMA
 	if (unlikely(gfpflags & GFP_DMA))
-		return cs_dmacachep[i];
+		return kmalloc_dma_caches[i];
 #endif
-	return cs_cachep[i];
+	return kmalloc_caches[i];
 }
 
 static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
@@ -893,7 +888,7 @@ static inline bool is_slab_pfmemalloc(st
 static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
 						struct array_cache *ac)
 {
-	struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()];
+	struct kmem_cache_node *l3 = cachep->node[numa_mem_id()];
 	struct slab *slabp;
 	unsigned long flags;
 
@@ -948,7 +943,7 @@ static void *__ac_get_obj(struct kmem_ca
 		 * If there are empty slabs on the slabs_free list and we are
 		 * being forced to refill the cache, mark this one !pfmemalloc.
 		 */
-		l3 = cachep->nodelists[numa_mem_id()];
+		l3 = cachep->node[numa_mem_id()];
 		if (!list_empty(&l3->slabs_free) && force_refill) {
 			struct slab *slabp = virt_to_slab(objp);
 			ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem));
@@ -1098,7 +1093,7 @@ static void free_alien_cache(struct arra
 static void __drain_alien_cache(struct kmem_cache *cachep,
 				struct array_cache *ac, int node)
 {
-	struct kmem_cache_node *rl3 = cachep->nodelists[node];
+	struct kmem_cache_node *rl3 = cachep->node[node];
 
 	if (ac->avail) {
 		spin_lock(&rl3->list_lock);
@@ -1167,7 +1162,7 @@ static inline int cache_free_alien(struc
 	if (likely(slabp->nodeid == node))
 		return 0;
 
-	l3 = cachep->nodelists[node];
+	l3 = cachep->node[node];
 	STATS_INC_NODEFREES(cachep);
 	if (l3->alien && l3->alien[nodeid]) {
 		alien = l3->alien[nodeid];
@@ -1179,24 +1174,24 @@ static inline int cache_free_alien(struc
 		ac_put_obj(cachep, alien, objp);
 		spin_unlock(&alien->lock);
 	} else {
-		spin_lock(&(cachep->nodelists[nodeid])->list_lock);
+		spin_lock(&(cachep->node[nodeid])->list_lock);
 		free_block(cachep, &objp, 1, nodeid);
-		spin_unlock(&(cachep->nodelists[nodeid])->list_lock);
+		spin_unlock(&(cachep->node[nodeid])->list_lock);
 	}
 	return 1;
 }
 #endif
 
 /*
- * Allocates and initializes nodelists for a node on each slab cache, used for
+ * Allocates and initializes node for a node on each slab cache, used for
  * either memory or cpu hotplug.  If memory is being hot-added, the kmem_list3
  * will be allocated off-node since memory is not yet online for the new node.
- * When hotplugging memory or a cpu, existing nodelists are not replaced if
+ * When hotplugging memory or a cpu, existing node are not replaced if
  * already in use.
  *
  * Must hold slab_mutex.
  */
-static int init_cache_nodelists_node(int node)
+static int init_cache_node_node(int node)
 {
 	struct kmem_cache *cachep;
 	struct kmem_cache_node *l3;
@@ -1208,7 +1203,7 @@ static int init_cache_nodelists_node(int
 		 * begin anything. Make sure some other cpu on this
 		 * node has not already allocated this
 		 */
-		if (!cachep->nodelists[node]) {
+		if (!cachep->node[node]) {
 			l3 = kmalloc_node(memsize, GFP_KERNEL, node);
 			if (!l3)
 				return -ENOMEM;
@@ -1221,14 +1216,14 @@ static int init_cache_nodelists_node(int
 			 * go.  slab_mutex is sufficient
 			 * protection here.
 			 */
-			cachep->nodelists[node] = l3;
+			cachep->node[node] = l3;
 		}
 
-		spin_lock_irq(&cachep->nodelists[node]->list_lock);
-		cachep->nodelists[node]->free_limit =
+		spin_lock_irq(&cachep->node[node]->list_lock);
+		cachep->node[node]->free_limit =
 			(1 + nr_cpus_node(node)) *
 			cachep->batchcount + cachep->num;
-		spin_unlock_irq(&cachep->nodelists[node]->list_lock);
+		spin_unlock_irq(&cachep->node[node]->list_lock);
 	}
 	return 0;
 }
@@ -1248,7 +1243,7 @@ static void __cpuinit cpuup_canceled(lon
 		/* cpu is dead; no one can alloc from it. */
 		nc = cachep->array[cpu];
 		cachep->array[cpu] = NULL;
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 
 		if (!l3)
 			goto free_array_cache;
@@ -1291,7 +1286,7 @@ free_array_cache:
 	 * shrink each nodelist to its limit.
 	 */
 	list_for_each_entry(cachep, &slab_caches, list) {
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (!l3)
 			continue;
 		drain_freelist(cachep, l3, l3->free_objects);
@@ -1311,7 +1306,7 @@ static int __cpuinit cpuup_prepare(long
 	 * kmalloc_node allows us to add the slab to the right
 	 * kmem_list3 and not this cpu's kmem_list3
 	 */
-	err = init_cache_nodelists_node(node);
+	err = init_cache_node_node(node);
 	if (err < 0)
 		goto bad;
 
@@ -1346,7 +1341,7 @@ static int __cpuinit cpuup_prepare(long
 			}
 		}
 		cachep->array[cpu] = nc;
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		BUG_ON(!l3);
 
 		spin_lock_irq(&l3->list_lock);
@@ -1446,7 +1441,7 @@ static struct notifier_block __cpuinitda
  *
  * Must hold slab_mutex.
  */
-static int __meminit drain_cache_nodelists_node(int node)
+static int __meminit drain_cache_node_node(int node)
 {
 	struct kmem_cache *cachep;
 	int ret = 0;
@@ -1454,7 +1449,7 @@ static int __meminit drain_cache_nodelis
 	list_for_each_entry(cachep, &slab_caches, list) {
 		struct kmem_cache_node *l3;
 
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (!l3)
 			continue;
 
@@ -1483,12 +1478,12 @@ static int __meminit slab_memory_callbac
 	switch (action) {
 	case MEM_GOING_ONLINE:
 		mutex_lock(&slab_mutex);
-		ret = init_cache_nodelists_node(nid);
+		ret = init_cache_node_node(nid);
 		mutex_unlock(&slab_mutex);
 		break;
 	case MEM_GOING_OFFLINE:
 		mutex_lock(&slab_mutex);
-		ret = drain_cache_nodelists_node(nid);
+		ret = drain_cache_node_node(nid);
 		mutex_unlock(&slab_mutex);
 		break;
 	case MEM_ONLINE:
@@ -1520,7 +1515,7 @@ static void __init init_list(struct kmem
 	spin_lock_init(&ptr->list_lock);
 
 	MAKE_ALL_LISTS(cachep, ptr, nodeid);
-	cachep->nodelists[nodeid] = ptr;
+	cachep->node[nodeid] = ptr;
 }
 
 /*
@@ -1532,8 +1527,8 @@ static void __init set_up_list3s(struct
 	int node;
 
 	for_each_online_node(node) {
-		cachep->nodelists[node] = &initkmem_list3[index + node];
-		cachep->nodelists[node]->next_reap = jiffies +
+		cachep->node[node] = &initkmem_list3[index + node];
+		cachep->node[node]->next_reap = jiffies +
 		    REAPTIMEOUT_LIST3 +
 		    ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
 	}
@@ -1541,11 +1536,11 @@ static void __init set_up_list3s(struct
 
 /*
  * The memory after the last cpu cache pointer is used for the
- * the nodelists pointer.
+ * the node pointer.
  */
-static void setup_nodelists_pointer(struct kmem_cache *s)
+static void setup_node_pointer(struct kmem_cache *s)
 {
-	s->nodelists = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
+	s->node = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
 }
 
 /*
@@ -1557,7 +1552,7 @@ void __init kmem_cache_init(void)
 	int i;
 
 	kmem_cache = &kmem_cache_boot;
-	setup_nodelists_pointer(kmem_cache);
+	setup_node_pointer(kmem_cache);
 
 	if (num_possible_nodes() == 1)
 		use_alien_caches = 0;
@@ -1566,7 +1561,7 @@ void __init kmem_cache_init(void)
 	for (i = 0; i < NUM_INIT_LISTS; i++) {
 		kmem_list3_init(&initkmem_list3[i]);
 		if (i < nr_node_ids)
-			kmem_cache->nodelists[i] = NULL;
+			kmem_cache->node[i] = NULL;
 	}
 	set_up_list3s(kmem_cache, CACHE_CACHE);
 
@@ -1618,11 +1613,11 @@ void __init kmem_cache_init(void)
 	 * bug.
 	 */
 
-	cs_cachep[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
+	kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
 					kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);
 
 	if (INDEX_AC != INDEX_L3)
-		cs_cachep[INDEX_L3] =
+		kmalloc_caches[INDEX_L3] =
 			create_kmalloc_cache("kmalloc-l3",
 				kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS);
 
@@ -1634,7 +1629,7 @@ void __init kmem_cache_init(void)
 		if (cs_size < KMALLOC_MIN_SIZE)
 			continue;
 
-		if (!cs_cachep[i]) {
+		if (!kmalloc_caches[i]) {
 			/*
 			 * For performance, all the general caches are L1 aligned.
 			 * This should be particularly beneficial on SMP boxes, as it
@@ -1642,12 +1637,12 @@ void __init kmem_cache_init(void)
 			 * Note for systems short on memory removing the alignment will
 			 * allow tighter packing of the smaller caches.
 			 */
-			cs_cachep[i] = create_kmalloc_cache("kmalloc",
+			kmalloc_caches[i] = create_kmalloc_cache("kmalloc",
 					cs_size, ARCH_KMALLOC_FLAGS);
 		}
 
 #ifdef CONFIG_ZONE_DMA
-		cs_dmacachep[i] = create_kmalloc_cache(
+		kmalloc_dma_caches[i] = create_kmalloc_cache(
 			"kmalloc-dma", cs_size,
 			SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
 #endif
@@ -1669,16 +1664,16 @@ void __init kmem_cache_init(void)
 
 		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-		BUG_ON(cpu_cache_get(cs_cachep[INDEX_AC])
+		BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC])
 		       != &initarray_generic.cache);
-		memcpy(ptr, cpu_cache_get(cs_cachep[INDEX_AC]),
+		memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]),
 		       sizeof(struct arraycache_init));
 		/*
 		 * Do not assume that spinlocks can be initialized via memcpy:
 		 */
 		spin_lock_init(&ptr->lock);
 
-		cs_cachep[INDEX_AC]->array[smp_processor_id()] = ptr;
+		kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr;
 	}
 	/* 5) Replace the bootstrap kmem_list3's */
 	{
@@ -1687,11 +1682,11 @@ void __init kmem_cache_init(void)
 		for_each_online_node(nid) {
 			init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
 
-			init_list(cs_cachep[INDEX_AC],
+			init_list(kmalloc_caches[INDEX_AC],
 				  &initkmem_list3[SIZE_AC + nid], nid);
 
 			if (INDEX_AC != INDEX_L3) {
-				init_list(cs_cachep[INDEX_L3],
+				init_list(kmalloc_caches[INDEX_L3],
 					  &initkmem_list3[SIZE_L3 + nid], nid);
 			}
 		}
@@ -1702,7 +1697,7 @@ void __init kmem_cache_init(void)
 	/* Create the proper names */
 	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
 		char *s;
-		struct kmem_cache *c = cs_cachep[i];
+		struct kmem_cache *c = kmalloc_caches[i];
 
 		if (!c)
 			continue;
@@ -1713,7 +1708,7 @@ void __init kmem_cache_init(void)
 		c->name = s;
 		
 #ifdef CONFIG_ZONE_DMA
-		c = cs_dmacachep[i];
+		c = kmalloc_dma_caches[i];
 		BUG_ON(!c);
 		s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
 		BUG_ON(!s);
@@ -1750,7 +1745,7 @@ void __init kmem_cache_init_late(void)
 #ifdef CONFIG_NUMA
 	/*
 	 * Register a memory hotplug callback that initializes and frees
-	 * nodelists.
+	 * node.
 	 */
 	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 #endif
@@ -1795,7 +1790,7 @@ slab_out_of_memory(struct kmem_cache *ca
 		unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
 		unsigned long active_slabs = 0, num_slabs = 0;
 
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (!l3)
 			continue;
 
@@ -2269,15 +2264,15 @@ static int __init_refok setup_cpu_cache(
 		} else {
 			int node;
 			for_each_online_node(node) {
-				cachep->nodelists[node] =
+				cachep->node[node] =
 				    kmalloc_node(sizeof(struct kmem_cache_node),
 						gfp, node);
-				BUG_ON(!cachep->nodelists[node]);
-				kmem_list3_init(cachep->nodelists[node]);
+				BUG_ON(!cachep->node[node]);
+				kmem_list3_init(cachep->node[node]);
 			}
 		}
 	}
-	cachep->nodelists[numa_mem_id()]->next_reap =
+	cachep->node[numa_mem_id()]->next_reap =
 			jiffies + REAPTIMEOUT_LIST3 +
 			((unsigned long)cachep) % REAPTIMEOUT_LIST3;
 
@@ -2388,7 +2383,7 @@ __kmem_cache_create (struct kmem_cache *
 	else
 		gfp = GFP_NOWAIT;
 
-	setup_nodelists_pointer(cachep);
+	setup_node_pointer(cachep);
 #if DEBUG
 
 	/*
@@ -2526,7 +2521,7 @@ static void check_spinlock_acquired(stru
 {
 #ifdef CONFIG_SMP
 	check_irq_off();
-	assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock);
+	assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock);
 #endif
 }
 
@@ -2534,7 +2529,7 @@ static void check_spinlock_acquired_node
 {
 #ifdef CONFIG_SMP
 	check_irq_off();
-	assert_spin_locked(&cachep->nodelists[node]->list_lock);
+	assert_spin_locked(&cachep->node[node]->list_lock);
 #endif
 }
 
@@ -2557,9 +2552,9 @@ static void do_drain(void *arg)
 
 	check_irq_off();
 	ac = cpu_cache_get(cachep);
-	spin_lock(&cachep->nodelists[node]->list_lock);
+	spin_lock(&cachep->node[node]->list_lock);
 	free_block(cachep, ac->entry, ac->avail, node);
-	spin_unlock(&cachep->nodelists[node]->list_lock);
+	spin_unlock(&cachep->node[node]->list_lock);
 	ac->avail = 0;
 }
 
@@ -2571,13 +2566,13 @@ static void drain_cpu_caches(struct kmem
 	on_each_cpu(do_drain, cachep, 1);
 	check_irq_on();
 	for_each_online_node(node) {
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (l3 && l3->alien)
 			drain_alien_cache(cachep, l3->alien);
 	}
 
 	for_each_online_node(node) {
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (l3)
 			drain_array(cachep, l3, l3->shared, 1, node);
 	}
@@ -2634,7 +2629,7 @@ static int __cache_shrink(struct kmem_ca
 
 	check_irq_on();
 	for_each_online_node(i) {
-		l3 = cachep->nodelists[i];
+		l3 = cachep->node[i];
 		if (!l3)
 			continue;
 
@@ -2681,7 +2676,7 @@ int __kmem_cache_shutdown(struct kmem_ca
 
 	/* NUMA: free the list3 structures */
 	for_each_online_node(i) {
-		l3 = cachep->nodelists[i];
+		l3 = cachep->node[i];
 		if (l3) {
 			kfree(l3->shared);
 			free_alien_cache(l3->alien);
@@ -2878,7 +2873,7 @@ static int cache_grow(struct kmem_cache
 
 	/* Take the l3 list lock to change the colour_next on this node */
 	check_irq_off();
-	l3 = cachep->nodelists[nodeid];
+	l3 = cachep->node[nodeid];
 	spin_lock(&l3->list_lock);
 
 	/* Get colour for the slab, and cal the next value. */
@@ -3076,7 +3071,7 @@ retry:
 		 */
 		batchcount = BATCHREFILL_LIMIT;
 	}
-	l3 = cachep->nodelists[node];
+	l3 = cachep->node[node];
 
 	BUG_ON(ac->avail > 0 || !l3);
 	spin_lock(&l3->list_lock);
@@ -3298,7 +3293,7 @@ static void *alternate_node_alloc(struct
 /*
  * Fallback function if there was no memory available and no objects on a
  * certain node and fall back is permitted. First we scan all the
- * available nodelists for available objects. If that fails then we
+ * available node for available objects. If that fails then we
  * perform an allocation without specifying a node. This allows the page
  * allocator to do its reclaim / fallback magic. We then insert the
  * slab into the proper nodelist and then allocate from it.
@@ -3332,8 +3327,8 @@ retry:
 		nid = zone_to_nid(zone);
 
 		if (cpuset_zone_allowed_hardwall(zone, flags) &&
-			cache->nodelists[nid] &&
-			cache->nodelists[nid]->free_objects) {
+			cache->node[nid] &&
+			cache->node[nid]->free_objects) {
 				obj = ____cache_alloc_node(cache,
 					flags | GFP_THISNODE, nid);
 				if (obj)
@@ -3393,7 +3388,7 @@ static void *____cache_alloc_node(struct
 	void *obj;
 	int x;
 
-	l3 = cachep->nodelists[nodeid];
+	l3 = cachep->node[nodeid];
 	BUG_ON(!l3);
 
 retry:
@@ -3476,7 +3471,7 @@ __cache_alloc_node(struct kmem_cache *ca
 	if (nodeid == NUMA_NO_NODE)
 		nodeid = slab_node;
 
-	if (unlikely(!cachep->nodelists[nodeid])) {
+	if (unlikely(!cachep->node[nodeid])) {
 		/* Node not bootstrapped yet */
 		ptr = fallback_alloc(cachep, flags);
 		goto out;
@@ -3590,7 +3585,7 @@ static void free_block(struct kmem_cache
 		objp = objpp[i];
 
 		slabp = virt_to_slab(objp);
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		list_del(&slabp->list);
 		check_spinlock_acquired_node(cachep, node);
 		check_slabp(cachep, slabp);
@@ -3634,7 +3629,7 @@ static void cache_flusharray(struct kmem
 	BUG_ON(!batchcount || batchcount > ac->avail);
 #endif
 	check_irq_off();
-	l3 = cachep->nodelists[node];
+	l3 = cachep->node[node];
 	spin_lock(&l3->list_lock);
 	if (l3->shared) {
 		struct array_cache *shared_array = l3->shared;
@@ -3946,7 +3941,7 @@ static int alloc_kmemlist(struct kmem_ca
 			}
 		}
 
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (l3) {
 			struct array_cache *shared = l3->shared;
 
@@ -3982,7 +3977,7 @@ static int alloc_kmemlist(struct kmem_ca
 		l3->alien = new_alien;
 		l3->free_limit = (1 + nr_cpus_node(node)) *
 					cachep->batchcount + cachep->num;
-		cachep->nodelists[node] = l3;
+		cachep->node[node] = l3;
 	}
 	return 0;
 
@@ -3991,13 +3986,13 @@ fail:
 		/* Cache is not active yet. Roll back what we did */
 		node--;
 		while (node >= 0) {
-			if (cachep->nodelists[node]) {
-				l3 = cachep->nodelists[node];
+			if (cachep->node[node]) {
+				l3 = cachep->node[node];
 
 				kfree(l3->shared);
 				free_alien_cache(l3->alien);
 				kfree(l3);
-				cachep->nodelists[node] = NULL;
+				cachep->node[node] = NULL;
 			}
 			node--;
 		}
@@ -4057,9 +4052,9 @@ static int do_tune_cpucache(struct kmem_
 		struct array_cache *ccold = new->new[i];
 		if (!ccold)
 			continue;
-		spin_lock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
+		spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
 		free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i));
-		spin_unlock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
+		spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
 		kfree(ccold);
 	}
 	kfree(new);
@@ -4180,7 +4175,7 @@ static void cache_reap(struct work_struc
 		 * have established with reasonable certainty that
 		 * we can do some work if the lock was obtained.
 		 */
-		l3 = searchp->nodelists[node];
+		l3 = searchp->node[node];
 
 		reap_alien(searchp, l3);
 
@@ -4279,7 +4274,7 @@ static int s_show(struct seq_file *m, vo
 	active_objs = 0;
 	num_slabs = 0;
 	for_each_online_node(node) {
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (!l3)
 			continue;
 
@@ -4533,7 +4528,7 @@ static int leaks_show(struct seq_file *m
 	n[1] = 0;
 
 	for_each_online_node(node) {
-		l3 = cachep->nodelists[node];
+		l3 = cachep->node[node];
 		if (!l3)
 			continue;
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [09/13] slab: rename nodelists to node

[Glauber Costa]

On 09/27/2012 12:07 AM, Christoph Lameter wrote:
> Have a common naming between both slab caches for future changes.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 
> Index: linux/include/linux/slab_def.h
> ===================================================================
> --- linux.orig/include/linux/slab_def.h	2012-09-19 09:21:35.811415438 -0500
> +++ linux/include/linux/slab_def.h	2012-09-19 09:21:37.499450510 -0500
> @@ -88,16 +88,13 @@ struct kmem_cache {
>  	 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
>  	 * is statically defined, so we reserve the max number of cpus.
>  	 */
> -	struct kmem_cache_node **nodelists;
> +	struct kmem_cache_node **node;
>  	struct array_cache *array[NR_CPUS + MAX_NUMNODES];
>  	/*
>  	 * Do not add fields after array[]
>  	 */
>  };
>  
> -extern struct kmem_cache *cs_cachep[PAGE_SHIFT + MAX_ORDER];
> -extern struct kmem_cache *cs_dmacachep[PAGE_SHIFT + MAX_ORDER];
> -
>
>  void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
>  void *__kmalloc(size_t size, gfp_t flags);
>  
> @@ -132,10 +129,10 @@ static __always_inline void *kmalloc(siz
>  
>  #ifdef CONFIG_ZONE_DMA
>  		if (flags & GFP_DMA)
> -			cachep = cs_dmacachep[i];
> +			cachep = kmalloc_dma_caches[i];
>  		else
You had just changed this to those new names in patch 7. Why don't you
change it directly to kmalloc_{,dma}_caches ?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [09/13] slab: rename nodelists to node

[Christoph Lameter]

On Fri, 28 Sep 2012, Glauber Costa wrote:

> >
> > -extern struct kmem_cache *cs_cachep[PAGE_SHIFT + MAX_ORDER];
> > -extern struct kmem_cache *cs_dmacachep[PAGE_SHIFT + MAX_ORDER];
> > -
> >
> >  void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
> >  void *__kmalloc(size_t size, gfp_t flags);
> >
> > @@ -132,10 +129,10 @@ static __always_inline void *kmalloc(siz
> >
> >  #ifdef CONFIG_ZONE_DMA
> >  		if (flags & GFP_DMA)
> > -			cachep = cs_dmacachep[i];
> > +			cachep = kmalloc_dma_caches[i];
> >  		else
> You had just changed this to those new names in patch 7. Why don't you
> change it directly to kmalloc_{,dma}_caches ?

Right. This is also not really related to what this patch ought to do.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [08/13] slab: Common name for the per node structures

[Christoph Lameter]

Rename the structure used for the per node structures in slab
to have a name that expresses that fact.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2012-09-18 12:16:12.258375911 -0500
+++ linux/include/linux/slab_def.h	2012-09-18 12:16:20.866552990 -0500
@@ -88,7 +88,7 @@ struct kmem_cache {
 	 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
 	 * is statically defined, so we reserve the max number of cpus.
 	 */
-	struct kmem_list3 **nodelists;
+	struct kmem_cache_node **nodelists;
 	struct array_cache *array[NR_CPUS + MAX_NUMNODES];
 	/*
 	 * Do not add fields after array[]
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-18 12:13:20.090834288 -0500
+++ linux/mm/slab.c	2012-09-18 12:16:20.870553116 -0500
@@ -309,7 +309,7 @@ struct arraycache_init {
 /*
  * The slab lists for all objects.
  */
-struct kmem_list3 {
+struct kmem_cache_node {
 	struct list_head slabs_partial;	/* partial list first, better asm code */
 	struct list_head slabs_full;
 	struct list_head slabs_free;
@@ -327,13 +327,13 @@ struct kmem_list3 {
  * Need this for bootstrapping a per node allocator.
  */
 #define NUM_INIT_LISTS (3 * MAX_NUMNODES)
-static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
+static struct kmem_cache_node __initdata initkmem_list3[NUM_INIT_LISTS];
 #define	CACHE_CACHE 0
 #define	SIZE_AC MAX_NUMNODES
 #define	SIZE_L3 (2 * MAX_NUMNODES)
 
 static int drain_freelist(struct kmem_cache *cache,
-			struct kmem_list3 *l3, int tofree);
+			struct kmem_cache_node *l3, int tofree);
 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
 			int node);
 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
@@ -342,9 +342,9 @@ static void cache_reap(struct work_struc
 static int slab_early_init = 1;
 
 #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
-#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3))
+#define INDEX_L3 kmalloc_index(sizeof(struct kmem_cache_node))
 
-static void kmem_list3_init(struct kmem_list3 *parent)
+static void kmem_list3_init(struct kmem_cache_node *parent)
 {
 	INIT_LIST_HEAD(&parent->slabs_full);
 	INIT_LIST_HEAD(&parent->slabs_partial);
@@ -567,7 +567,7 @@ static void slab_set_lock_classes(struct
 		int q)
 {
 	struct array_cache **alc;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	int r;
 
 	l3 = cachep->nodelists[q];
@@ -612,7 +612,7 @@ static void init_node_lock_keys(int q)
 		return;
 
 	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
-		struct kmem_list3 *l3;
+		struct kmem_cache_node *l3;
 		struct kmem_cache *cache = cs_cachep[i];
 
 		if (!cache)
@@ -893,7 +893,7 @@ static inline bool is_slab_pfmemalloc(st
 static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
 						struct array_cache *ac)
 {
-	struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()];
+	struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()];
 	struct slab *slabp;
 	unsigned long flags;
 
@@ -926,7 +926,7 @@ static void *__ac_get_obj(struct kmem_ca
 
 	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
 	if (unlikely(is_obj_pfmemalloc(objp))) {
-		struct kmem_list3 *l3;
+		struct kmem_cache_node *l3;
 
 		if (gfp_pfmemalloc_allowed(flags)) {
 			clear_obj_pfmemalloc(&objp);
@@ -1098,7 +1098,7 @@ static void free_alien_cache(struct arra
 static void __drain_alien_cache(struct kmem_cache *cachep,
 				struct array_cache *ac, int node)
 {
-	struct kmem_list3 *rl3 = cachep->nodelists[node];
+	struct kmem_cache_node *rl3 = cachep->nodelists[node];
 
 	if (ac->avail) {
 		spin_lock(&rl3->list_lock);
@@ -1119,7 +1119,7 @@ static void __drain_alien_cache(struct k
 /*
  * Called from cache_reap() to regularly drain alien caches round robin.
  */
-static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
+static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *l3)
 {
 	int node = __this_cpu_read(slab_reap_node);
 
@@ -1154,7 +1154,7 @@ static inline int cache_free_alien(struc
 {
 	struct slab *slabp = virt_to_slab(objp);
 	int nodeid = slabp->nodeid;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	struct array_cache *alien = NULL;
 	int node;
 
@@ -1199,8 +1199,8 @@ static inline int cache_free_alien(struc
 static int init_cache_nodelists_node(int node)
 {
 	struct kmem_cache *cachep;
-	struct kmem_list3 *l3;
-	const int memsize = sizeof(struct kmem_list3);
+	struct kmem_cache_node *l3;
+	const int memsize = sizeof(struct kmem_cache_node);
 
 	list_for_each_entry(cachep, &slab_caches, list) {
 		/*
@@ -1236,7 +1236,7 @@ static int init_cache_nodelists_node(int
 static void __cpuinit cpuup_canceled(long cpu)
 {
 	struct kmem_cache *cachep;
-	struct kmem_list3 *l3 = NULL;
+	struct kmem_cache_node *l3 = NULL;
 	int node = cpu_to_mem(cpu);
 	const struct cpumask *mask = cpumask_of_node(node);
 
@@ -1301,7 +1301,7 @@ free_array_cache:
 static int __cpuinit cpuup_prepare(long cpu)
 {
 	struct kmem_cache *cachep;
-	struct kmem_list3 *l3 = NULL;
+	struct kmem_cache_node *l3 = NULL;
 	int node = cpu_to_mem(cpu);
 	int err;
 
@@ -1452,7 +1452,7 @@ static int __meminit drain_cache_nodelis
 	int ret = 0;
 
 	list_for_each_entry(cachep, &slab_caches, list) {
-		struct kmem_list3 *l3;
+		struct kmem_cache_node *l3;
 
 		l3 = cachep->nodelists[node];
 		if (!l3)
@@ -1505,15 +1505,15 @@ out:
 /*
  * swap the static kmem_list3 with kmalloced memory
  */
-static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
+static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
 				int nodeid)
 {
-	struct kmem_list3 *ptr;
+	struct kmem_cache_node *ptr;
 
-	ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid);
+	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
 	BUG_ON(!ptr);
 
-	memcpy(ptr, list, sizeof(struct kmem_list3));
+	memcpy(ptr, list, sizeof(struct kmem_cache_node));
 	/*
 	 * Do not assume that spinlocks can be initialized via memcpy:
 	 */
@@ -1545,7 +1545,7 @@ static void __init set_up_list3s(struct
  */
 static void setup_nodelists_pointer(struct kmem_cache *s)
 {
-	s->nodelists = (struct kmem_list3 **)&s->array[nr_cpu_ids];
+	s->nodelists = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
 }
 
 /*
@@ -1605,7 +1605,7 @@ void __init kmem_cache_init(void)
 	 */
 	create_boot_cache(kmem_cache, "kmem_cache",
 		offsetof(struct kmem_cache, array[nr_cpu_ids]) +
-				  nr_node_ids * sizeof(struct kmem_list3 *),
+				  nr_node_ids * sizeof(struct kmem_cache_node *),
 				  SLAB_HWCACHE_ALIGN);
 
 	slab_state = PARTIAL;
@@ -1780,7 +1780,7 @@ __initcall(cpucache_init);
 static noinline void
 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
 {
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	struct slab *slabp;
 	unsigned long flags;
 	int node;
@@ -2270,7 +2270,7 @@ static int __init_refok setup_cpu_cache(
 			int node;
 			for_each_online_node(node) {
 				cachep->nodelists[node] =
-				    kmalloc_node(sizeof(struct kmem_list3),
+				    kmalloc_node(sizeof(struct kmem_cache_node),
 						gfp, node);
 				BUG_ON(!cachep->nodelists[node]);
 				kmem_list3_init(cachep->nodelists[node]);
@@ -2545,7 +2545,7 @@ static void check_spinlock_acquired_node
 #define check_spinlock_acquired_node(x, y) do { } while(0)
 #endif
 
-static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3,
 			struct array_cache *ac,
 			int force, int node);
 
@@ -2565,7 +2565,7 @@ static void do_drain(void *arg)
 
 static void drain_cpu_caches(struct kmem_cache *cachep)
 {
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	int node;
 
 	on_each_cpu(do_drain, cachep, 1);
@@ -2590,7 +2590,7 @@ static void drain_cpu_caches(struct kmem
  * Returns the actual number of slabs released.
  */
 static int drain_freelist(struct kmem_cache *cache,
-			struct kmem_list3 *l3, int tofree)
+			struct kmem_cache_node *l3, int tofree)
 {
 	struct list_head *p;
 	int nr_freed;
@@ -2628,7 +2628,7 @@ out:
 static int __cache_shrink(struct kmem_cache *cachep)
 {
 	int ret = 0, i = 0;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 
 	drain_cpu_caches(cachep);
 
@@ -2670,7 +2670,7 @@ EXPORT_SYMBOL(kmem_cache_shrink);
 int __kmem_cache_shutdown(struct kmem_cache *cachep)
 {
 	int i;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	int rc = __cache_shrink(cachep);
 
 	if (rc)
@@ -2867,7 +2867,7 @@ static int cache_grow(struct kmem_cache
 	struct slab *slabp;
 	size_t offset;
 	gfp_t local_flags;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 
 	/*
 	 * Be lazy and only check for valid flags here,  keeping it out of the
@@ -3057,7 +3057,7 @@ static void *cache_alloc_refill(struct k
 							bool force_refill)
 {
 	int batchcount;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	struct array_cache *ac;
 	int node;
 
@@ -3389,7 +3389,7 @@ static void *____cache_alloc_node(struct
 {
 	struct list_head *entry;
 	struct slab *slabp;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	void *obj;
 	int x;
 
@@ -3580,7 +3580,7 @@ static void free_block(struct kmem_cache
 		       int node)
 {
 	int i;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 
 	for (i = 0; i < nr_objects; i++) {
 		void *objp;
@@ -3626,7 +3626,7 @@ static void free_block(struct kmem_cache
 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
 {
 	int batchcount;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	int node = numa_mem_id();
 
 	batchcount = ac->batchcount;
@@ -3923,7 +3923,7 @@ EXPORT_SYMBOL(kmem_cache_size);
 static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
 {
 	int node;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	struct array_cache *new_shared;
 	struct array_cache **new_alien = NULL;
 
@@ -3968,7 +3968,7 @@ static int alloc_kmemlist(struct kmem_ca
 			free_alien_cache(new_alien);
 			continue;
 		}
-		l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node);
+		l3 = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
 		if (!l3) {
 			free_alien_cache(new_alien);
 			kfree(new_shared);
@@ -4125,7 +4125,7 @@ static int enable_cpucache(struct kmem_c
  * necessary. Note that the l3 listlock also protects the array_cache
  * if drain_array() is used on the shared array.
  */
-static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3,
 			 struct array_cache *ac, int force, int node)
 {
 	int tofree;
@@ -4164,7 +4164,7 @@ static void drain_array(struct kmem_cach
 static void cache_reap(struct work_struct *w)
 {
 	struct kmem_cache *searchp;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	int node = numa_mem_id();
 	struct delayed_work *work = to_delayed_work(w);
 
@@ -4274,7 +4274,7 @@ static int s_show(struct seq_file *m, vo
 	const char *name;
 	char *error = NULL;
 	int node;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 
 	active_objs = 0;
 	num_slabs = 0;
@@ -4517,7 +4517,7 @@ static int leaks_show(struct seq_file *m
 {
 	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
 	struct slab *slabp;
-	struct kmem_list3 *l3;
+	struct kmem_cache_node *l3;
 	const char *name;
 	unsigned long *n = m->private;
 	int node;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [08/13] slab: Common name for the per node structures

[Glauber Costa]

On 09/27/2012 12:07 AM, Christoph Lameter wrote:
> Rename the structure used for the per node structures in slab
> to have a name that expresses that fact.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 

This is unrelated to the rest of the series.

I suggest this to be applied separately.

Patch itself looks fine, it is quite mechanical, and the name change
makes sense.

> Index: linux/include/linux/slab_def.h
> ===================================================================
> --- linux.orig/include/linux/slab_def.h	2012-09-18 12:16:12.258375911 -0500
> +++ linux/include/linux/slab_def.h	2012-09-18 12:16:20.866552990 -0500
> @@ -88,7 +88,7 @@ struct kmem_cache {
>  	 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
>  	 * is statically defined, so we reserve the max number of cpus.
>  	 */
> -	struct kmem_list3 **nodelists;
> +	struct kmem_cache_node **nodelists;
>  	struct array_cache *array[NR_CPUS + MAX_NUMNODES];
>  	/*
>  	 * Do not add fields after array[]
> Index: linux/mm/slab.c
> ===================================================================
> --- linux.orig/mm/slab.c	2012-09-18 12:13:20.090834288 -0500
> +++ linux/mm/slab.c	2012-09-18 12:16:20.870553116 -0500
> @@ -309,7 +309,7 @@ struct arraycache_init {
>  /*
>   * The slab lists for all objects.
>   */
> -struct kmem_list3 {
> +struct kmem_cache_node {
>  	struct list_head slabs_partial;	/* partial list first, better asm code */
>  	struct list_head slabs_full;
>  	struct list_head slabs_free;
> @@ -327,13 +327,13 @@ struct kmem_list3 {
>   * Need this for bootstrapping a per node allocator.
>   */
>  #define NUM_INIT_LISTS (3 * MAX_NUMNODES)
> -static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
> +static struct kmem_cache_node __initdata initkmem_list3[NUM_INIT_LISTS];
>  #define	CACHE_CACHE 0
>  #define	SIZE_AC MAX_NUMNODES
>  #define	SIZE_L3 (2 * MAX_NUMNODES)
>  
>  static int drain_freelist(struct kmem_cache *cache,
> -			struct kmem_list3 *l3, int tofree);
> +			struct kmem_cache_node *l3, int tofree);
>  static void free_block(struct kmem_cache *cachep, void **objpp, int len,
>  			int node);
>  static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
> @@ -342,9 +342,9 @@ static void cache_reap(struct work_struc
>  static int slab_early_init = 1;
>  
>  #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
> -#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3))
> +#define INDEX_L3 kmalloc_index(sizeof(struct kmem_cache_node))
>  
> -static void kmem_list3_init(struct kmem_list3 *parent)
> +static void kmem_list3_init(struct kmem_cache_node *parent)
>  {
>  	INIT_LIST_HEAD(&parent->slabs_full);
>  	INIT_LIST_HEAD(&parent->slabs_partial);
> @@ -567,7 +567,7 @@ static void slab_set_lock_classes(struct
>  		int q)
>  {
>  	struct array_cache **alc;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	int r;
>  
>  	l3 = cachep->nodelists[q];
> @@ -612,7 +612,7 @@ static void init_node_lock_keys(int q)
>  		return;
>  
>  	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
> -		struct kmem_list3 *l3;
> +		struct kmem_cache_node *l3;
>  		struct kmem_cache *cache = cs_cachep[i];
>  
>  		if (!cache)
> @@ -893,7 +893,7 @@ static inline bool is_slab_pfmemalloc(st
>  static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
>  						struct array_cache *ac)
>  {
> -	struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()];
> +	struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()];
>  	struct slab *slabp;
>  	unsigned long flags;
>  
> @@ -926,7 +926,7 @@ static void *__ac_get_obj(struct kmem_ca
>  
>  	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
>  	if (unlikely(is_obj_pfmemalloc(objp))) {
> -		struct kmem_list3 *l3;
> +		struct kmem_cache_node *l3;
>  
>  		if (gfp_pfmemalloc_allowed(flags)) {
>  			clear_obj_pfmemalloc(&objp);
> @@ -1098,7 +1098,7 @@ static void free_alien_cache(struct arra
>  static void __drain_alien_cache(struct kmem_cache *cachep,
>  				struct array_cache *ac, int node)
>  {
> -	struct kmem_list3 *rl3 = cachep->nodelists[node];
> +	struct kmem_cache_node *rl3 = cachep->nodelists[node];
>  
>  	if (ac->avail) {
>  		spin_lock(&rl3->list_lock);
> @@ -1119,7 +1119,7 @@ static void __drain_alien_cache(struct k
>  /*
>   * Called from cache_reap() to regularly drain alien caches round robin.
>   */
> -static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
> +static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *l3)
>  {
>  	int node = __this_cpu_read(slab_reap_node);
>  
> @@ -1154,7 +1154,7 @@ static inline int cache_free_alien(struc
>  {
>  	struct slab *slabp = virt_to_slab(objp);
>  	int nodeid = slabp->nodeid;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	struct array_cache *alien = NULL;
>  	int node;
>  
> @@ -1199,8 +1199,8 @@ static inline int cache_free_alien(struc
>  static int init_cache_nodelists_node(int node)
>  {
>  	struct kmem_cache *cachep;
> -	struct kmem_list3 *l3;
> -	const int memsize = sizeof(struct kmem_list3);
> +	struct kmem_cache_node *l3;
> +	const int memsize = sizeof(struct kmem_cache_node);
>  
>  	list_for_each_entry(cachep, &slab_caches, list) {
>  		/*
> @@ -1236,7 +1236,7 @@ static int init_cache_nodelists_node(int
>  static void __cpuinit cpuup_canceled(long cpu)
>  {
>  	struct kmem_cache *cachep;
> -	struct kmem_list3 *l3 = NULL;
> +	struct kmem_cache_node *l3 = NULL;
>  	int node = cpu_to_mem(cpu);
>  	const struct cpumask *mask = cpumask_of_node(node);
>  
> @@ -1301,7 +1301,7 @@ free_array_cache:
>  static int __cpuinit cpuup_prepare(long cpu)
>  {
>  	struct kmem_cache *cachep;
> -	struct kmem_list3 *l3 = NULL;
> +	struct kmem_cache_node *l3 = NULL;
>  	int node = cpu_to_mem(cpu);
>  	int err;
>  
> @@ -1452,7 +1452,7 @@ static int __meminit drain_cache_nodelis
>  	int ret = 0;
>  
>  	list_for_each_entry(cachep, &slab_caches, list) {
> -		struct kmem_list3 *l3;
> +		struct kmem_cache_node *l3;
>  
>  		l3 = cachep->nodelists[node];
>  		if (!l3)
> @@ -1505,15 +1505,15 @@ out:
>  /*
>   * swap the static kmem_list3 with kmalloced memory
>   */
> -static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
> +static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
>  				int nodeid)
>  {
> -	struct kmem_list3 *ptr;
> +	struct kmem_cache_node *ptr;
>  
> -	ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid);
> +	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
>  	BUG_ON(!ptr);
>  
> -	memcpy(ptr, list, sizeof(struct kmem_list3));
> +	memcpy(ptr, list, sizeof(struct kmem_cache_node));
>  	/*
>  	 * Do not assume that spinlocks can be initialized via memcpy:
>  	 */
> @@ -1545,7 +1545,7 @@ static void __init set_up_list3s(struct
>   */
>  static void setup_nodelists_pointer(struct kmem_cache *s)
>  {
> -	s->nodelists = (struct kmem_list3 **)&s->array[nr_cpu_ids];
> +	s->nodelists = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
>  }
>  
>  /*
> @@ -1605,7 +1605,7 @@ void __init kmem_cache_init(void)
>  	 */
>  	create_boot_cache(kmem_cache, "kmem_cache",
>  		offsetof(struct kmem_cache, array[nr_cpu_ids]) +
> -				  nr_node_ids * sizeof(struct kmem_list3 *),
> +				  nr_node_ids * sizeof(struct kmem_cache_node *),
>  				  SLAB_HWCACHE_ALIGN);
>  
>  	slab_state = PARTIAL;
> @@ -1780,7 +1780,7 @@ __initcall(cpucache_init);
>  static noinline void
>  slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
>  {
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	struct slab *slabp;
>  	unsigned long flags;
>  	int node;
> @@ -2270,7 +2270,7 @@ static int __init_refok setup_cpu_cache(
>  			int node;
>  			for_each_online_node(node) {
>  				cachep->nodelists[node] =
> -				    kmalloc_node(sizeof(struct kmem_list3),
> +				    kmalloc_node(sizeof(struct kmem_cache_node),
>  						gfp, node);
>  				BUG_ON(!cachep->nodelists[node]);
>  				kmem_list3_init(cachep->nodelists[node]);
> @@ -2545,7 +2545,7 @@ static void check_spinlock_acquired_node
>  #define check_spinlock_acquired_node(x, y) do { } while(0)
>  #endif
>  
> -static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
> +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3,
>  			struct array_cache *ac,
>  			int force, int node);
>  
> @@ -2565,7 +2565,7 @@ static void do_drain(void *arg)
>  
>  static void drain_cpu_caches(struct kmem_cache *cachep)
>  {
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	int node;
>  
>  	on_each_cpu(do_drain, cachep, 1);
> @@ -2590,7 +2590,7 @@ static void drain_cpu_caches(struct kmem
>   * Returns the actual number of slabs released.
>   */
>  static int drain_freelist(struct kmem_cache *cache,
> -			struct kmem_list3 *l3, int tofree)
> +			struct kmem_cache_node *l3, int tofree)
>  {
>  	struct list_head *p;
>  	int nr_freed;
> @@ -2628,7 +2628,7 @@ out:
>  static int __cache_shrink(struct kmem_cache *cachep)
>  {
>  	int ret = 0, i = 0;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  
>  	drain_cpu_caches(cachep);
>  
> @@ -2670,7 +2670,7 @@ EXPORT_SYMBOL(kmem_cache_shrink);
>  int __kmem_cache_shutdown(struct kmem_cache *cachep)
>  {
>  	int i;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	int rc = __cache_shrink(cachep);
>  
>  	if (rc)
> @@ -2867,7 +2867,7 @@ static int cache_grow(struct kmem_cache
>  	struct slab *slabp;
>  	size_t offset;
>  	gfp_t local_flags;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  
>  	/*
>  	 * Be lazy and only check for valid flags here,  keeping it out of the
> @@ -3057,7 +3057,7 @@ static void *cache_alloc_refill(struct k
>  							bool force_refill)
>  {
>  	int batchcount;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	struct array_cache *ac;
>  	int node;
>  
> @@ -3389,7 +3389,7 @@ static void *____cache_alloc_node(struct
>  {
>  	struct list_head *entry;
>  	struct slab *slabp;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	void *obj;
>  	int x;
>  
> @@ -3580,7 +3580,7 @@ static void free_block(struct kmem_cache
>  		       int node)
>  {
>  	int i;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  
>  	for (i = 0; i < nr_objects; i++) {
>  		void *objp;
> @@ -3626,7 +3626,7 @@ static void free_block(struct kmem_cache
>  static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
>  {
>  	int batchcount;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	int node = numa_mem_id();
>  
>  	batchcount = ac->batchcount;
> @@ -3923,7 +3923,7 @@ EXPORT_SYMBOL(kmem_cache_size);
>  static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
>  {
>  	int node;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	struct array_cache *new_shared;
>  	struct array_cache **new_alien = NULL;
>  
> @@ -3968,7 +3968,7 @@ static int alloc_kmemlist(struct kmem_ca
>  			free_alien_cache(new_alien);
>  			continue;
>  		}
> -		l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node);
> +		l3 = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
>  		if (!l3) {
>  			free_alien_cache(new_alien);
>  			kfree(new_shared);
> @@ -4125,7 +4125,7 @@ static int enable_cpucache(struct kmem_c
>   * necessary. Note that the l3 listlock also protects the array_cache
>   * if drain_array() is used on the shared array.
>   */
> -static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
> +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3,
>  			 struct array_cache *ac, int force, int node)
>  {
>  	int tofree;
> @@ -4164,7 +4164,7 @@ static void drain_array(struct kmem_cach
>  static void cache_reap(struct work_struct *w)
>  {
>  	struct kmem_cache *searchp;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	int node = numa_mem_id();
>  	struct delayed_work *work = to_delayed_work(w);
>  
> @@ -4274,7 +4274,7 @@ static int s_show(struct seq_file *m, vo
>  	const char *name;
>  	char *error = NULL;
>  	int node;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  
>  	active_objs = 0;
>  	num_slabs = 0;
> @@ -4517,7 +4517,7 @@ static int leaks_show(struct seq_file *m
>  {
>  	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
>  	struct slab *slabp;
> -	struct kmem_list3 *l3;
> +	struct kmem_cache_node *l3;
>  	const char *name;
>  	unsigned long *n = m->private;
>  	int node;
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
> 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [08/13] slab: Common name for the per node structures

[Christoph Lameter]

On Fri, 28 Sep 2012, Glauber Costa wrote:

> I suggest this to be applied separately.
>
> Patch itself looks fine, it is quite mechanical, and the name change
> makes sense.

Ok I can put this patch first?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [07/13] slab: Use common kmalloc_index/kmalloc_size functions

[Christoph Lameter]

Make slab use the common functions. We can get rid of a lot
of old ugly stuff as a results. Among them the sizes
array and the weird include/linux/kmalloc_sizes file and
some pretty bad #include statements in slab_def.h.

The one thing that is different in slab is that the 32 byte
cache will also be created for arches that have page sizes
larger than 4K. There are numerous smaller allocations that
SLOB and SLUB can handle better because of their support for
smaller allocation sizes so lets keep the 32 byte slab also
for arches with > 4K pages.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-19 09:21:21.399115971 -0500
+++ linux/mm/slab.c	2012-09-19 09:21:32.531347320 -0500
@@ -291,6 +291,11 @@ static inline void clear_obj_pfmemalloc(
 	*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
 }
 
+struct kmem_cache *cs_cachep[PAGE_SHIFT + MAX_ORDER];
+EXPORT_SYMBOL(cs_cachep);
+struct kmem_cache *cs_dmacachep[PAGE_SHIFT + MAX_ORDER];
+EXPORT_SYMBOL(cs_dmacachep);
+
 /*
  * bootstrap: The caches do not work without cpuarrays anymore, but the
  * cpuarrays are allocated from the generic caches...
@@ -334,34 +339,10 @@ static void free_block(struct kmem_cache
 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
 static void cache_reap(struct work_struct *unused);
 
-/*
- * This function must be completely optimized away if a constant is passed to
- * it.  Mostly the same as what is in linux/slab.h except it returns an index.
- */
-static __always_inline int index_of(const size_t size)
-{
-	extern void __bad_size(void);
-
-	if (__builtin_constant_p(size)) {
-		int i = 0;
-
-#define CACHE(x) \
-	if (size <=x) \
-		return i; \
-	else \
-		i++;
-#include <linux/kmalloc_sizes.h>
-#undef CACHE
-		__bad_size();
-	} else
-		__bad_size();
-	return 0;
-}
-
 static int slab_early_init = 1;
 
-#define INDEX_AC index_of(sizeof(struct arraycache_init))
-#define INDEX_L3 index_of(sizeof(struct kmem_list3))
+#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
+#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3))
 
 static void kmem_list3_init(struct kmem_list3 *parent)
 {
@@ -548,30 +529,6 @@ static inline unsigned int obj_to_index(
 	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
 }
 
-/*
- * These are the default caches for kmalloc. Custom caches can have other sizes.
- */
-struct cache_sizes malloc_sizes[] = {
-#define CACHE(x) { .cs_size = (x) },
-#include <linux/kmalloc_sizes.h>
-	CACHE(ULONG_MAX)
-#undef CACHE
-};
-EXPORT_SYMBOL(malloc_sizes);
-
-/* Must match cache_sizes above. Out of line to keep cache footprint low. */
-struct cache_names {
-	char *name;
-	char *name_dma;
-};
-
-static struct cache_names __initdata cache_names[] = {
-#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
-#include <linux/kmalloc_sizes.h>
-	{NULL,}
-#undef CACHE
-};
-
 static struct arraycache_init initarray_generic =
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
@@ -649,19 +606,23 @@ static void slab_set_debugobj_lock_class
 
 static void init_node_lock_keys(int q)
 {
-	struct cache_sizes *s = malloc_sizes;
+	int i;
 
 	if (slab_state < UP)
 		return;
 
-	for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
+	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
 		struct kmem_list3 *l3;
+		struct kmem_cache *cache = cs_cachep[i];
 
-		l3 = s->cs_cachep->nodelists[q];
-		if (!l3 || OFF_SLAB(s->cs_cachep))
+		if (!cache)
 			continue;
 
-		slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
+		l3 = cache->nodelists[q];
+		if (!l3 || OFF_SLAB(cache))
+			continue;
+
+		slab_set_lock_classes(cache, &on_slab_l3_key,
 				&on_slab_alc_key, q);
 	}
 }
@@ -701,20 +662,19 @@ static inline struct array_cache *cpu_ca
 static inline struct kmem_cache *__find_general_cachep(size_t size,
 							gfp_t gfpflags)
 {
-	struct cache_sizes *csizep = malloc_sizes;
+	int i;
 
 #if DEBUG
 	/* This happens if someone tries to call
 	 * kmem_cache_create(), or __kmalloc(), before
 	 * the generic caches are initialized.
 	 */
-	BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
+	BUG_ON(cs_cachep[INDEX_AC] == NULL);
 #endif
 	if (!size)
 		return ZERO_SIZE_PTR;
 
-	while (size > csizep->cs_size)
-		csizep++;
+	i = kmalloc_index(size);
 
 	/*
 	 * Really subtle: The last entry with cs->cs_size==ULONG_MAX
@@ -723,9 +683,9 @@ static inline struct kmem_cache *__find_
 	 */
 #ifdef CONFIG_ZONE_DMA
 	if (unlikely(gfpflags & GFP_DMA))
-		return csizep->cs_dmacachep;
+		return cs_dmacachep[i];
 #endif
-	return csizep->cs_cachep;
+	return cs_cachep[i];
 }
 
 static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
@@ -1594,8 +1554,6 @@ static void setup_nodelists_pointer(stru
  */
 void __init kmem_cache_init(void)
 {
-	struct cache_sizes *sizes;
-	struct cache_names *names;
 	int i;
 
 	kmem_cache = &kmem_cache_boot;
@@ -1653,8 +1611,6 @@ void __init kmem_cache_init(void)
 	slab_state = PARTIAL;
 
 	/* 2+3) create the kmalloc caches */
-	sizes = malloc_sizes;
-	names = cache_names;
 
 	/*
 	 * Initialize the caches that provide memory for the array cache and the
@@ -1662,35 +1618,39 @@ void __init kmem_cache_init(void)
 	 * bug.
 	 */
 
-	sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
-					sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
+	cs_cachep[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
+					kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);
 
 	if (INDEX_AC != INDEX_L3)
-		sizes[INDEX_L3].cs_cachep =
-			create_kmalloc_cache(names[INDEX_L3].name,
-				sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
+		cs_cachep[INDEX_L3] =
+			create_kmalloc_cache("kmalloc-l3",
+				kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS);
 
 	slab_early_init = 0;
 
-	while (sizes->cs_size != ULONG_MAX) {
-		/*
-		 * For performance, all the general caches are L1 aligned.
-		 * This should be particularly beneficial on SMP boxes, as it
-		 * eliminates "false sharing".
-		 * Note for systems short on memory removing the alignment will
-		 * allow tighter packing of the smaller caches.
-		 */
-		if (!sizes->cs_cachep)
-			sizes->cs_cachep = create_kmalloc_cache(names->name,
-					sizes->cs_size, ARCH_KMALLOC_FLAGS);
+	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
+		size_t cs_size = kmalloc_size(i);
+
+		if (cs_size < KMALLOC_MIN_SIZE)
+			continue;
+
+		if (!cs_cachep[i]) {
+			/*
+			 * For performance, all the general caches are L1 aligned.
+			 * This should be particularly beneficial on SMP boxes, as it
+			 * eliminates "false sharing".
+			 * Note for systems short on memory removing the alignment will
+			 * allow tighter packing of the smaller caches.
+			 */
+			cs_cachep[i] = create_kmalloc_cache("kmalloc",
+					cs_size, ARCH_KMALLOC_FLAGS);
+		}
 
 #ifdef CONFIG_ZONE_DMA
-		sizes->cs_dmacachep = create_kmalloc_cache(
-			names->name_dma, sizes->cs_size,
+		cs_dmacachep[i] = create_kmalloc_cache(
+			"kmalloc-dma", cs_size,
 			SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
 #endif
-		sizes++;
-		names++;
 	}
 	/* 4) Replace the bootstrap head arrays */
 	{
@@ -1709,17 +1669,16 @@ void __init kmem_cache_init(void)
 
 		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-		BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
+		BUG_ON(cpu_cache_get(cs_cachep[INDEX_AC])
 		       != &initarray_generic.cache);
-		memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
+		memcpy(ptr, cpu_cache_get(cs_cachep[INDEX_AC]),
 		       sizeof(struct arraycache_init));
 		/*
 		 * Do not assume that spinlocks can be initialized via memcpy:
 		 */
 		spin_lock_init(&ptr->lock);
 
-		malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
-		    ptr;
+		cs_cachep[INDEX_AC]->array[smp_processor_id()] = ptr;
 	}
 	/* 5) Replace the bootstrap kmem_list3's */
 	{
@@ -1728,17 +1687,39 @@ void __init kmem_cache_init(void)
 		for_each_online_node(nid) {
 			init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
 
-			init_list(malloc_sizes[INDEX_AC].cs_cachep,
+			init_list(cs_cachep[INDEX_AC],
 				  &initkmem_list3[SIZE_AC + nid], nid);
 
 			if (INDEX_AC != INDEX_L3) {
-				init_list(malloc_sizes[INDEX_L3].cs_cachep,
+				init_list(cs_cachep[INDEX_L3],
 					  &initkmem_list3[SIZE_L3 + nid], nid);
 			}
 		}
 	}
 
 	slab_state = UP;
+
+	/* Create the proper names */
+	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
+		char *s;
+		struct kmem_cache *c = cs_cachep[i];
+
+		if (!c)
+			continue;
+
+		s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
+
+		BUG_ON(!s);
+		c->name = s;
+		
+#ifdef CONFIG_ZONE_DMA
+		c = cs_dmacachep[i];
+		BUG_ON(!c);
+		s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
+		BUG_ON(!s);
+		c->name = s;
+#endif
+	}
 }
 
 void __init kmem_cache_init_late(void)
Index: linux/include/linux/kmalloc_sizes.h
===================================================================
--- linux.orig/include/linux/kmalloc_sizes.h	2012-09-19 09:19:33.496874231 -0500
+++ /dev/null	1970-01-01 00:00:00.000000000 +0000
@@ -1,45 +0,0 @@
-#if (PAGE_SIZE == 4096)
-	CACHE(32)
-#endif
-	CACHE(64)
-#if L1_CACHE_BYTES < 64
-	CACHE(96)
-#endif
-	CACHE(128)
-#if L1_CACHE_BYTES < 128
-	CACHE(192)
-#endif
-	CACHE(256)
-	CACHE(512)
-	CACHE(1024)
-	CACHE(2048)
-	CACHE(4096)
-	CACHE(8192)
-	CACHE(16384)
-	CACHE(32768)
-	CACHE(65536)
-	CACHE(131072)
-#if KMALLOC_MAX_SIZE >= 262144
-	CACHE(262144)
-#endif
-#if KMALLOC_MAX_SIZE >= 524288
-	CACHE(524288)
-#endif
-#if KMALLOC_MAX_SIZE >= 1048576
-	CACHE(1048576)
-#endif
-#if KMALLOC_MAX_SIZE >= 2097152
-	CACHE(2097152)
-#endif
-#if KMALLOC_MAX_SIZE >= 4194304
-	CACHE(4194304)
-#endif
-#if KMALLOC_MAX_SIZE >= 8388608
-	CACHE(8388608)
-#endif
-#if KMALLOC_MAX_SIZE >= 16777216
-	CACHE(16777216)
-#endif
-#if KMALLOC_MAX_SIZE >= 33554432
-	CACHE(33554432)
-#endif
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2012-09-19 09:21:00.602683885 -0500
+++ linux/include/linux/slab_def.h	2012-09-19 09:21:32.535347400 -0500
@@ -11,8 +11,6 @@
  */
 
 #include <linux/init.h>
-#include <asm/page.h>		/* kmalloc_sizes.h needs PAGE_SIZE */
-#include <asm/cache.h>		/* kmalloc_sizes.h needs L1_CACHE_BYTES */
 #include <linux/compiler.h>
 
 /*
@@ -97,15 +95,8 @@ struct kmem_cache {
 	 */
 };
 
-/* Size description struct for general caches. */
-struct cache_sizes {
-	size_t		 	cs_size;
-	struct kmem_cache	*cs_cachep;
-#ifdef CONFIG_ZONE_DMA
-	struct kmem_cache	*cs_dmacachep;
-#endif
-};
-extern struct cache_sizes malloc_sizes[];
+extern struct kmem_cache *cs_cachep[PAGE_SHIFT + MAX_ORDER];
+extern struct kmem_cache *cs_dmacachep[PAGE_SHIFT + MAX_ORDER];
 
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 void *__kmalloc(size_t size, gfp_t flags);
@@ -132,26 +123,19 @@ static __always_inline void *kmalloc(siz
 	void *ret;
 
 	if (__builtin_constant_p(size)) {
-		int i = 0;
+		int i;
 
 		if (!size)
 			return ZERO_SIZE_PTR;
 
-#define CACHE(x) \
-		if (size <= x) \
-			goto found; \
-		else \
-			i++;
-#include <linux/kmalloc_sizes.h>
-#undef CACHE
-		return NULL;
-found:
+		i = kmalloc_index(size);
+
 #ifdef CONFIG_ZONE_DMA
 		if (flags & GFP_DMA)
-			cachep = malloc_sizes[i].cs_dmacachep;
+			cachep = cs_dmacachep[i];
 		else
 #endif
-			cachep = malloc_sizes[i].cs_cachep;
+			cachep = cs_cachep[i];
 
 		ret = kmem_cache_alloc_trace(size, cachep, flags);
 
@@ -185,26 +169,19 @@ static __always_inline void *kmalloc_nod
 	struct kmem_cache *cachep;
 
 	if (__builtin_constant_p(size)) {
-		int i = 0;
+		int i;
 
 		if (!size)
 			return ZERO_SIZE_PTR;
 
-#define CACHE(x) \
-		if (size <= x) \
-			goto found; \
-		else \
-			i++;
-#include <linux/kmalloc_sizes.h>
-#undef CACHE
-		return NULL;
-found:
+		i = kmalloc_index(size);
+
 #ifdef CONFIG_ZONE_DMA
 		if (flags & GFP_DMA)
-			cachep = malloc_sizes[i].cs_dmacachep;
+			cachep = cs_dmacachep[i];
 		else
 #endif
-			cachep = malloc_sizes[i].cs_cachep;
+			cachep = cs_cachep[i];
 
 		return kmem_cache_alloc_node_trace(size, cachep, flags, node);
 	}

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [07/13] slab: Use common kmalloc_index/kmalloc_size functions

[Glauber Costa]

On 09/27/2012 12:18 AM, Christoph Lameter wrote:
> Make slab use the common functions. We can get rid of a lot
> of old ugly stuff as a results. Among them the sizes
> array and the weird include/linux/kmalloc_sizes file and
> some pretty bad #include statements in slab_def.h.
> 
> The one thing that is different in slab is that the 32 byte
> cache will also be created for arches that have page sizes
> larger than 4K. There are numerous smaller allocations that
> SLOB and SLUB can handle better because of their support for
> smaller allocation sizes so lets keep the 32 byte slab also
> for arches with > 4K pages.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 

I believe this makes sense, and the code looks better now.

Reviewed-by: Glauber Costa <glommer@parallels.com>

One nitpick:

>  
> @@ -185,26 +169,19 @@ static __always_inline void *kmalloc_nod
>  	struct kmem_cache *cachep;
>  
>  	if (__builtin_constant_p(size)) {
> -		int i = 0;
> +		int i;
>  

Although this is technically correct, the former is correct as well, and
this end up only adding churn to the patch.

Should you decide to remove it, there is another instance of this a bit
more down.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [07/13] slab: Use common kmalloc_index/kmalloc_size functions

[Christoph Lameter]

On Fri, 28 Sep 2012, Glauber Costa wrote:

> One nitpick:
>
> >
> > @@ -185,26 +169,19 @@ static __always_inline void *kmalloc_nod
> >  	struct kmem_cache *cachep;
> >
> >  	if (__builtin_constant_p(size)) {
> > -		int i = 0;
> > +		int i;
> >
>
> Although this is technically correct, the former is correct as well, and
> this end up only adding churn to the patch.

What does the "former" refer to?

> Should you decide to remove it, there is another instance of this a bit
> more down.

Remove what?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [04/13] slab: Use the new create_boot_cache function to simplify bootstrap

[Christoph Lameter]

Simplify setup and reduce code in kmem_cache_init(). This allows us to
get rid of initarray_cache as well as the manual setup code for
the kmem_cache and kmem_cache_node arrays during bootstrap.

We introduce a new bootstrap state "PARTIAL" for slab that signals the
creation of a kmem_cache boot cache.

V1->V2: Get rid of initarray_cache as well.

Signed-off-by: Christoph Lameter <cl@linux.com>
---
 mm/slab.c |   51 ++++++++++++++++++---------------------------------
 1 file changed, 18 insertions(+), 33 deletions(-)

Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-19 09:21:14.422971030 -0500
+++ linux/mm/slab.c	2012-09-19 09:21:21.399115971 -0500
@@ -572,8 +572,6 @@ static struct cache_names __initdata cac
 #undef CACHE
 };
 
-static struct arraycache_init initarray_cache __initdata =
-    { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 static struct arraycache_init initarray_generic =
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
@@ -1596,12 +1594,9 @@ static void setup_nodelists_pointer(stru
  */
 void __init kmem_cache_init(void)
 {
-	size_t left_over;
 	struct cache_sizes *sizes;
 	struct cache_names *names;
 	int i;
-	int order;
-	int node;
 
 	kmem_cache = &kmem_cache_boot;
 	setup_nodelists_pointer(kmem_cache);
@@ -1645,36 +1640,17 @@ void __init kmem_cache_init(void)
 	 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
 	 */
 
-	node = numa_mem_id();
-
 	/* 1) create the kmem_cache */
-	INIT_LIST_HEAD(&slab_caches);
-	list_add(&kmem_cache->list, &slab_caches);
-	kmem_cache->colour_off = cache_line_size();
-	kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
 
 	/*
 	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
 	 */
-	kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
-				  nr_node_ids * sizeof(struct kmem_list3 *);
-	kmem_cache->object_size = kmem_cache->size;
-	kmem_cache->size = ALIGN(kmem_cache->object_size,
-					cache_line_size());
-	kmem_cache->reciprocal_buffer_size =
-		reciprocal_value(kmem_cache->size);
-
-	for (order = 0; order < MAX_ORDER; order++) {
-		cache_estimate(order, kmem_cache->size,
-			cache_line_size(), 0, &left_over, &kmem_cache->num);
-		if (kmem_cache->num)
-			break;
-	}
-	BUG_ON(!kmem_cache->num);
-	kmem_cache->gfporder = order;
-	kmem_cache->colour = left_over / kmem_cache->colour_off;
-	kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
-				      sizeof(struct slab), cache_line_size());
+	create_boot_cache(kmem_cache, "kmem_cache",
+		offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+				  nr_node_ids * sizeof(struct kmem_list3 *),
+				  SLAB_HWCACHE_ALIGN);
+
+	slab_state = PARTIAL;
 
 	/* 2+3) create the kmalloc caches */
 	sizes = malloc_sizes;
@@ -1722,7 +1698,6 @@ void __init kmem_cache_init(void)
 
 		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-		BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
 		memcpy(ptr, cpu_cache_get(kmem_cache),
 		       sizeof(struct arraycache_init));
 		/*
@@ -2277,7 +2252,16 @@ static int __init_refok setup_cpu_cache(
 
 	if (slab_state == DOWN) {
 		/*
-		 * Note: the first kmem_cache_create must create the cache
+		 * Note: Creation of first cache (kmem_cache).
+		 * The setup_list3s is taken care
+		 * of by the caller of __kmem_cache_create
+		 */
+		cachep->array[smp_processor_id()] = &initarray_generic.cache;
+		slab_state = PARTIAL;
+	} else
+	if (slab_state == PARTIAL) {
+		/*
+		 * Note: the second kmem_cache_create must create the cache
 		 * that's used by kmalloc(24), otherwise the creation of
 		 * further caches will BUG().
 		 */
@@ -2285,7 +2269,7 @@ static int __init_refok setup_cpu_cache(
 
 		/*
 		 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
-		 * the first cache, then we need to set up all its list3s,
+		 * the second cache, then we need to set up all its list3s,
 		 * otherwise the creation of further caches will BUG().
 		 */
 		set_up_list3s(cachep, SIZE_AC);
@@ -2294,6 +2278,7 @@ static int __init_refok setup_cpu_cache(
 		else
 			slab_state = PARTIAL_ARRAYCACHE;
 	} else {
+		/* Remaining boot caches */
 		cachep->array[smp_processor_id()] =
 			kmalloc(sizeof(struct arraycache_init), gfp);
 
@@ -2389,22 +2374,6 @@ __kmem_cache_create (struct kmem_cache *
 		size &= ~(BYTES_PER_WORD - 1);
 	}
 
-	/* calculate the final buffer alignment: */
-
-	/* 1) arch recommendation: can be overridden for debug */
-	if (flags & SLAB_HWCACHE_ALIGN) {
-		/*
-		 * Default alignment: as specified by the arch code.  Except if
-		 * an object is really small, then squeeze multiple objects into
-		 * one cacheline.
-		 */
-		ralign = cache_line_size();
-		while (size <= ralign / 2)
-			ralign /= 2;
-	} else {
-		ralign = BYTES_PER_WORD;
-	}
-
 	/*
 	 * Redzoning and user store require word alignment or possibly larger.
 	 * Note this will be overridden by architecture or caller mandated
@@ -2421,10 +2390,6 @@ __kmem_cache_create (struct kmem_cache *
 		size &= ~(REDZONE_ALIGN - 1);
 	}
 
-	/* 2) arch mandated alignment */
-	if (ralign < ARCH_SLAB_MINALIGN) {
-		ralign = ARCH_SLAB_MINALIGN;
-	}
 	/* 3) caller mandated alignment */
 	if (ralign < cachep->align) {
 		ralign = cachep->align;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [04/13] slab: Use the new create_boot_cache function to simplify bootstrap

[Glauber Costa]

On 09/27/2012 12:18 AM, Christoph Lameter wrote:
> -	node = numa_mem_id();
> -
>  	/* 1) create the kmem_cache */
> -	INIT_LIST_HEAD(&slab_caches);
> -	list_add(&kmem_cache->list, &slab_caches);
> -	kmem_cache->colour_off = cache_line_size();
> -	kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
>  
>  	/*
Don't you have to initialize this list head somewhere ?
You are deleting this code, but not putting it back anywhere.


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [04/13] slab: Use the new create_boot_cache function to simplify bootstrap

[Christoph Lameter]

On Thu, 27 Sep 2012, Glauber Costa wrote:

> On 09/27/2012 12:18 AM, Christoph Lameter wrote:
> > -	node = numa_mem_id();
> > -
> >  	/* 1) create the kmem_cache */
> > -	INIT_LIST_HEAD(&slab_caches);
> > -	list_add(&kmem_cache->list, &slab_caches);
> > -	kmem_cache->colour_off = cache_line_size();
> > -	kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
> >
> >  	/*
> Don't you have to initialize this list head somewhere ?
> You are deleting this code, but not putting it back anywhere.

Thought the declaration would do the initialization in mm/slab_common.c:

enum slab_state slab_state;
LIST_HEAD(slab_caches);
DEFINE_MUTEX(slab_mutex);
struct kmem_cache *kmem_cache;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [04/13] slab: Use the new create_boot_cache function to simplify bootstrap

[Glauber Costa]

On 09/27/2012 06:32 PM, Christoph Lameter wrote:
> On Thu, 27 Sep 2012, Glauber Costa wrote:
> 
>> On 09/27/2012 12:18 AM, Christoph Lameter wrote:
>>> -	node = numa_mem_id();
>>> -
>>>  	/* 1) create the kmem_cache */
>>> -	INIT_LIST_HEAD(&slab_caches);
>>> -	list_add(&kmem_cache->list, &slab_caches);
>>> -	kmem_cache->colour_off = cache_line_size();
>>> -	kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
>>>
>>>  	/*
>> Don't you have to initialize this list head somewhere ?
>> You are deleting this code, but not putting it back anywhere.
> 
> Thought the declaration would do the initialization in mm/slab_common.c:
> 
> enum slab_state slab_state;
> LIST_HEAD(slab_caches);
> DEFINE_MUTEX(slab_mutex);
> struct kmem_cache *kmem_cache;
> 
Fair enough. Because you were removing this in this patch, I was
expecting it to go somewhere in this patch as well. But we were actually
initializing this twice, in which case the current code seems fine.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [10/13] Do not define KMALLOC array definitions for SLOB

[Christoph Lameter]

SLOB has no support for an array of kmalloc caches. Create a section
in include/linux/slab.h that is dedicated to the kmalloc cache
definition but disabled if SLOB is selected.

slab_common.c also has functions that are not needed for slob.
Disable those as well.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2012-09-18 12:13:16.222754746 -0500
+++ linux/include/linux/slab.h	2012-09-18 12:27:10.971944526 -0500
@@ -143,21 +143,6 @@ unsigned int kmem_cache_size(struct kmem
 		(__flags), NULL)
 
 /*
- * The largest kmalloc size supported by the slab allocators is
- * 32 megabyte (2^25) or the maximum allocatable page order if that is
- * less than 32 MB.
- *
- * WARNING: Its not easy to increase this value since the allocators have
- * to do various tricks to work around compiler limitations in order to
- * ensure proper constant folding.
- */
-#define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
-				(MAX_ORDER + PAGE_SHIFT - 1) : 25)
-
-#define KMALLOC_MAX_SIZE	(1UL << KMALLOC_SHIFT_HIGH)
-#define KMALLOC_MAX_ORDER	(KMALLOC_SHIFT_HIGH - PAGE_SHIFT)
-
-/*
  * Some archs want to perform DMA into kmalloc caches and need a guaranteed
  * alignment larger than the alignment of a 64-bit integer.
  * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that.
@@ -178,8 +163,24 @@ unsigned int kmem_cache_size(struct kmem
 #endif
 
 /*
+ * The largest kmalloc size supported by the slab allocators is
+ * 32 megabyte (2^25) or the maximum allocatable page order if that is
+ * less than 32 MB.
+ *
+ * WARNING: Its not easy to increase this value since the allocators have
+ * to do various tricks to work around compiler limitations in order to
+ * ensure proper constant folding.
+ */
+#define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
+				(MAX_ORDER + PAGE_SHIFT - 1) : 25)
+
+#define KMALLOC_MAX_SIZE	(1UL << KMALLOC_SHIFT_HIGH)
+#define KMALLOC_MAX_ORDER	(KMALLOC_SHIFT_HIGH - PAGE_SHIFT)
+
+/*
  * Kmalloc subsystem.
  */
+#ifndef CONFIG_SLOB
 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
 #else
@@ -260,6 +261,7 @@ static __always_inline int kmalloc_size(
 
 	return 0;
 }
+#endif
 
 /*
  * Common kmalloc functions provided by all allocators
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c	2012-09-18 12:13:16.230754925 -0500
+++ linux/mm/slab_common.c	2012-09-18 12:16:28.354706953 -0500
@@ -218,6 +218,8 @@ int slab_is_available(void)
 	return slab_state >= UP;
 }
 
+#ifndef CONFIG_SLOB
+
 /* Create a cache during boot when no slab services are available yet */
 void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
 		unsigned long flags)
@@ -249,3 +251,5 @@ struct kmem_cache *__init create_kmalloc
 	s->refcount = 1;
 	return s;
 }
+
+#endif /* !CONFIG_SLOB */

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [10/13] Do not define KMALLOC array definitions for SLOB

[Glauber Costa]

On 09/27/2012 12:18 AM, Christoph Lameter wrote:
> SLOB has no support for an array of kmalloc caches. Create a section
> in include/linux/slab.h that is dedicated to the kmalloc cache
> definition but disabled if SLOB is selected.
> 
> slab_common.c also has functions that are not needed for slob.
> Disable those as well.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 


> Index: linux/mm/slab_common.c
> ===================================================================
> --- linux.orig/mm/slab_common.c	2012-09-18 12:13:16.230754925 -0500
> +++ linux/mm/slab_common.c	2012-09-18 12:16:28.354706953 -0500
> @@ -218,6 +218,8 @@ int slab_is_available(void)
>  	return slab_state >= UP;
>  }
>  
> +#ifndef CONFIG_SLOB
> +
>  /* Create a cache during boot when no slab services are available yet */
>  void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
>  		unsigned long flags)
I don't see why you can't fold this directly in the patch where those
things are created.

> @@ -249,3 +251,5 @@ struct kmem_cache *__init create_kmalloc
>  	s->refcount = 1;
>  	return s;
>  }
> +
> +#endif /* !CONFIG_SLOB */
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
> 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [10/13] Do not define KMALLOC array definitions for SLOB

[Christoph Lameter]

On Fri, 28 Sep 2012, Glauber Costa wrote:

> >  /* Create a cache during boot when no slab services are available yet */
> >  void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
> >  		unsigned long flags)
> I don't see why you can't fold this directly in the patch where those
> things are created.

True.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [12/13] Common names for the array of kmalloc caches

[Christoph Lameter]

Have a common name fo the kmalloc cache arrays in
SLAB and SLUB

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2012-09-20 08:58:38.645735914 -0500
+++ linux/include/linux/slab.h	2012-09-20 08:58:38.657736160 -0500
@@ -203,6 +203,11 @@ unsigned int kmem_cache_size(struct kmem
 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
 #endif
 
+extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
+#ifdef CONFIG_ZONE_DMA
+extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
+#endif
+
 /*
  * Figure out which kmalloc slab an allocation of a certain size
  * belongs to.
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c	2012-09-20 08:58:38.633735660 -0500
+++ linux/mm/slab_common.c	2012-09-24 12:41:27.373725893 -0500
@@ -252,4 +252,12 @@ struct kmem_cache *__init create_kmalloc
 	return s;
 }
 
+struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
+EXPORT_SYMBOL(kmalloc_caches);
+
+#ifdef CONFIG_ZONE_DMA
+struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
+EXPORT_SYMBOL(kmalloc_dma_caches);
+#endif
+
 #endif /* !CONFIG_SLOB */
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2012-09-20 08:58:38.645735914 -0500
+++ linux/include/linux/slub_def.h	2012-09-20 09:17:50.294766383 -0500
@@ -111,6 +111,9 @@ struct kmem_cache {
 	struct kmem_cache_node *node[MAX_NUMNODES];
 };
 
+#define KMEM_CACHE_SIZE (offsetof(struct kmem_cache, node) + \
+			nr_node_ids * sizeof(struct kmem_cache_node *))
+
 #ifdef CONFIG_ZONE_DMA
 #define SLUB_DMA __GFP_DMA
 #else
@@ -119,12 +122,6 @@ struct kmem_cache {
 #endif
 
 /*
- * We keep the general caches in an array of slab caches that are used for
- * 2^x bytes of allocations.
- */
-extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
-
-/*
  * Find the slab cache for a given combination of allocation flags and size.
  *
  * This ought to end up with a global pointer to the right cache
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2012-09-20 08:58:38.645735914 -0500
+++ linux/mm/slub.c	2012-09-24 12:41:27.377726009 -0500
@@ -3174,13 +3174,6 @@ int __kmem_cache_shutdown(struct kmem_ca
  *		Kmalloc subsystem
  *******************************************************************/
 
-struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
-EXPORT_SYMBOL(kmalloc_caches);
-
-#ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
-#endif
-
 static int __init setup_slub_min_order(char *str)
 {
 	get_option(&str, &slub_min_order);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [06/13] Common kmalloc slab index determination

[Christoph Lameter]

Extract the function to determine the index of the slab within
the array of kmalloc caches as well as a function to determine
maximum object size from the nr of the kmalloc slab.

This is used here only to simplify slub bootstrap but will
be used later also for SLAB.

Signed-off-by: Christoph Lameter <cl@linux.com> 

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2012-09-19 09:19:38.904986568 -0500
+++ linux/include/linux/slab.h	2012-09-19 09:21:27.307238804 -0500
@@ -178,6 +178,90 @@ unsigned int kmem_cache_size(struct kmem
 #endif
 
 /*
+ * Kmalloc subsystem.
+ */
+#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
+#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
+#else
+#ifdef CONFIG_SLAB
+#define KMALLOC_MIN_SIZE 32
+#else
+#define KMALLOC_MIN_SIZE 8
+#endif
+#endif
+
+#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
+
+/*
+ * Figure out which kmalloc slab an allocation of a certain size
+ * belongs to.
+ * 0 = zero alloc
+ * 1 =  65 .. 96 bytes
+ * 2 = 120 .. 192 bytes
+ * n = 2^(n-1) .. 2^n -1
+ */
+static __always_inline int kmalloc_index(size_t size)
+{
+	if (!size)
+		return 0;
+
+	if (size <= KMALLOC_MIN_SIZE)
+		return KMALLOC_SHIFT_LOW;
+
+	if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
+		return 1;
+	if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
+		return 2;
+	if (size <=          8) return 3;
+	if (size <=         16) return 4;
+	if (size <=         32) return 5;
+	if (size <=         64) return 6;
+	if (size <=        128) return 7;
+	if (size <=        256) return 8;
+	if (size <=        512) return 9;
+	if (size <=       1024) return 10;
+	if (size <=   2 * 1024) return 11;
+	if (size <=   4 * 1024) return 12;
+	if (size <=   8 * 1024) return 13;
+	if (size <=  16 * 1024) return 14;
+	if (size <=  32 * 1024) return 15;
+	if (size <=  64 * 1024) return 16;
+	if (size <= 128 * 1024) return 17;
+	if (size <= 256 * 1024) return 18;
+	if (size <= 512 * 1024) return 19;
+	if (size <= 1024 * 1024) return 20;
+	if (size <=  2 * 1024 * 1024) return 21;
+	if (size <=  4 * 1024 * 1024) return 22;
+	if (size <=  8 * 1024 * 1024) return 23;
+	if (size <=  16 * 1024 * 1024) return 24;
+	if (size <=  32 * 1024 * 1024) return 26;
+	if (size <=  64 * 1024 * 1024) return 27;
+	BUG();
+
+	/* Will never be reached. Needed because the compiler may complain */
+	return -1;
+}
+
+/*
+ * Determine size used for the nth kmalloc cache.
+ * return size or 0 if a kmalloc cache for that
+ * size does not exist
+ */
+static __always_inline int kmalloc_size(int n)
+{
+	if (n > 2)
+		return 1 << n;
+
+	if (n == 1 && KMALLOC_MIN_SIZE <= 32)
+		return 96;
+
+	if (n == 2 && KMALLOC_MIN_SIZE <= 64)
+		return 192;
+
+	return 0;
+}
+
+/*
  * Common kmalloc functions provided by all allocators
  */
 void * __must_check __krealloc(const void *, size_t, gfp_t);
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2012-09-19 09:19:38.916986820 -0500
+++ linux/include/linux/slub_def.h	2012-09-19 09:21:27.307238804 -0500
@@ -112,17 +112,6 @@ struct kmem_cache {
 };
 
 /*
- * Kmalloc subsystem.
- */
-#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
-#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
-#else
-#define KMALLOC_MIN_SIZE 8
-#endif
-
-#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
-
-/*
  * Maximum kmalloc object size handled by SLUB. Larger object allocations
  * are passed through to the page allocator. The page allocator "fastpath"
  * is relatively slow so we need this value sufficiently high so that
@@ -149,58 +138,6 @@ struct kmem_cache {
 extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
 
 /*
- * Sorry that the following has to be that ugly but some versions of GCC
- * have trouble with constant propagation and loops.
- */
-static __always_inline int kmalloc_index(size_t size)
-{
-	if (!size)
-		return 0;
-
-	if (size <= KMALLOC_MIN_SIZE)
-		return KMALLOC_SHIFT_LOW;
-
-	if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
-		return 1;
-	if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
-		return 2;
-	if (size <=          8) return 3;
-	if (size <=         16) return 4;
-	if (size <=         32) return 5;
-	if (size <=         64) return 6;
-	if (size <=        128) return 7;
-	if (size <=        256) return 8;
-	if (size <=        512) return 9;
-	if (size <=       1024) return 10;
-	if (size <=   2 * 1024) return 11;
-	if (size <=   4 * 1024) return 12;
-/*
- * The following is only needed to support architectures with a larger page
- * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
- * size we would have to go up to 128k.
- */
-	if (size <=   8 * 1024) return 13;
-	if (size <=  16 * 1024) return 14;
-	if (size <=  32 * 1024) return 15;
-	if (size <=  64 * 1024) return 16;
-	if (size <= 128 * 1024) return 17;
-	if (size <= 256 * 1024) return 18;
-	if (size <= 512 * 1024) return 19;
-	if (size <= 1024 * 1024) return 20;
-	if (size <=  2 * 1024 * 1024) return 21;
-	BUG();
-	return -1; /* Will never be reached */
-
-/*
- * What we really wanted to do and cannot do because of compiler issues is:
- *	int i;
- *	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
- *		if (size <= (1 << i))
- *			return i;
- */
-}
-
-/*
  * Find the slab cache for a given combination of allocation flags and size.
  *
  * This ought to end up with a global pointer to the right cache

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [06/13] Common kmalloc slab index determination

[Glauber Costa]

On 09/27/2012 12:20 AM, Christoph Lameter wrote:
> Extract the function to determine the index of the slab within
> the array of kmalloc caches as well as a function to determine
> maximum object size from the nr of the kmalloc slab.
> 
> This is used here only to simplify slub bootstrap but will
> be used later also for SLAB.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com> 
> 
> Index: linux/include/linux/slab.h
> ===================================================================
> --- linux.orig/include/linux/slab.h	2012-09-19 09:19:38.904986568 -0500
> +++ linux/include/linux/slab.h	2012-09-19 09:21:27.307238804 -0500
> @@ -178,6 +178,90 @@ unsigned int kmem_cache_size(struct kmem
>  #endif
>  
>  /*
> + * Kmalloc subsystem.
> + */
> +#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
> +#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
> +#else
> +#ifdef CONFIG_SLAB
> +#define KMALLOC_MIN_SIZE 32
> +#else
> +#define KMALLOC_MIN_SIZE 8
> +#endif
> +#endif
> +
> +#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
> +
> +/*
> + * Figure out which kmalloc slab an allocation of a certain size
> + * belongs to.
> + * 0 = zero alloc
> + * 1 =  65 .. 96 bytes
> + * 2 = 120 .. 192 bytes
> + * n = 2^(n-1) .. 2^n -1
> + */
> +static __always_inline int kmalloc_index(size_t size)
> +{
> +	if (!size)
> +		return 0;
> +
> +	if (size <= KMALLOC_MIN_SIZE)
> +		return KMALLOC_SHIFT_LOW;
> +
> +	if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
> +		return 1;
> +	if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
> +		return 2;
> +	if (size <=          8) return 3;
> +	if (size <=         16) return 4;
> +	if (size <=         32) return 5;
> +	if (size <=         64) return 6;
> +	if (size <=        128) return 7;
> +	if (size <=        256) return 8;
> +	if (size <=        512) return 9;
> +	if (size <=       1024) return 10;
> +	if (size <=   2 * 1024) return 11;
> +	if (size <=   4 * 1024) return 12;
> +	if (size <=   8 * 1024) return 13;
> +	if (size <=  16 * 1024) return 14;
> +	if (size <=  32 * 1024) return 15;
> +	if (size <=  64 * 1024) return 16;
> +	if (size <= 128 * 1024) return 17;
> +	if (size <= 256 * 1024) return 18;
> +	if (size <= 512 * 1024) return 19;
> +	if (size <= 1024 * 1024) return 20;
> +	if (size <=  2 * 1024 * 1024) return 21;
> +	if (size <=  4 * 1024 * 1024) return 22;
> +	if (size <=  8 * 1024 * 1024) return 23;
> +	if (size <=  16 * 1024 * 1024) return 24;
> +	if (size <=  32 * 1024 * 1024) return 26;
> +	if (size <=  64 * 1024 * 1024) return 27;
> +	BUG();
> +
> +	/* Will never be reached. Needed because the compiler may complain */
> +	return -1;
> +}
> +

That is a bunch of branches... can't we use ilog2 for that somehow ?

In any case, you skipped "return 25".

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [06/13] Common kmalloc slab index determination

[Christoph Lameter]

On Fri, 28 Sep 2012, Glauber Costa wrote:

> > +	if (size <=  8 * 1024 * 1024) return 23;
> > +	if (size <=  16 * 1024 * 1024) return 24;
> > +	if (size <=  32 * 1024 * 1024) return 26;
> > +	if (size <=  64 * 1024 * 1024) return 27;
> > +	BUG();
> > +
> > +	/* Will never be reached. Needed because the compiler may complain */
> > +	return -1;
> > +}
> > +
>
> That is a bunch of branches... can't we use ilog2 for that somehow ?

Believe me we have tried but the c compiler had issue with constant
folding. ilog2 is used for the non inlined version.

> In any case, you skipped "return 25".

Thanks. Fixed.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [05/13] Common alignment code

[Christoph Lameter]

Extract the code to do object alignment from the allocators.
Do the alignment calculations in slab_common so that the
__kmem_cache_create functions of the allocators do not have
to deal with alignment.

Signed-off-by: Christoph Lameter <cl@linux.com>
---
 mm/slab.c        |   20 --------------------
 mm/slab.h        |    3 +++
 mm/slab_common.c |   32 ++++++++++++++++++++++++++++++--
 mm/slob.c        |   10 ----------
 mm/slub.c        |   38 +-------------------------------------
 5 files changed, 34 insertions(+), 69 deletions(-)

Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h	2012-09-19 09:21:14.422971030 -0500
+++ linux/mm/slab.h	2012-09-19 09:21:24.235174758 -0500
@@ -32,6 +32,9 @@ extern struct list_head slab_caches;
 /* The slab cache that manages slab cache information */
 extern struct kmem_cache *kmem_cache;
 
+unsigned long calculate_alignment(unsigned long flags,
+		unsigned long align, unsigned long size);
+
 /* Functions provided by the slab allocators */
 extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
 
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c	2012-09-19 09:21:14.422971030 -0500
+++ linux/mm/slab_common.c	2012-09-19 09:21:24.235174758 -0500
@@ -71,6 +71,34 @@ static inline int kmem_cache_sanity_chec
 #endif
 
 /*
+ * Figure out what the alignment of the objects will be given a set of
+ * flags, a user specified alignment and the size of the objects.
+ */
+unsigned long calculate_alignment(unsigned long flags,
+		unsigned long align, unsigned long size)
+{
+	/*
+	 * If the user wants hardware cache aligned objects then follow that
+	 * suggestion if the object is sufficiently large.
+	 *
+	 * The hardware cache alignment cannot override the specified
+	 * alignment though. If that is greater then use it.
+	 */
+	if (flags & SLAB_HWCACHE_ALIGN) {
+		unsigned long ralign = cache_line_size();
+		while (size <= ralign / 2)
+			ralign /= 2;
+		align = max(align, ralign);
+	}
+
+	if (align < ARCH_SLAB_MINALIGN)
+		align = ARCH_SLAB_MINALIGN;
+
+	return ALIGN(align, sizeof(void *));
+}
+
+
+/*
  * kmem_cache_create - Create a cache.
  * @name: A string which is used in /proc/slabinfo to identify this cache.
  * @size: The size of objects to be created in this cache.
@@ -115,7 +143,7 @@ struct kmem_cache *kmem_cache_create(con
 	s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
 	if (s) {
 		s->object_size = s->size = size;
-		s->align = align;
+		s->align = calculate_alignment(flags, align, size);
 		s->ctor = ctor;
 		s->name = kstrdup(name, GFP_KERNEL);
 		if (!s->name) {
@@ -198,7 +226,7 @@ void __init create_boot_cache(struct kme
 
 	s->name = name;
 	s->size = s->object_size = size;
-	s->align = ARCH_KMALLOC_MINALIGN;
+	s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
 	err = __kmem_cache_create(s, flags);
 
 	if (err)
Index: linux/mm/slob.c
===================================================================
--- linux.orig/mm/slob.c	2012-09-19 09:21:11.354907281 -0500
+++ linux/mm/slob.c	2012-09-19 09:21:24.235174758 -0500
@@ -124,7 +124,6 @@ static inline void clear_slob_page_free(
 
 #define SLOB_UNIT sizeof(slob_t)
 #define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
-#define SLOB_ALIGN L1_CACHE_BYTES
 
 /*
  * struct slob_rcu is inserted at the tail of allocated slob blocks, which
@@ -510,20 +509,11 @@ EXPORT_SYMBOL(ksize);
 
 int __kmem_cache_create(struct kmem_cache *c, unsigned long flags)
 {
-	size_t align = c->size;
-
 	if (flags & SLAB_DESTROY_BY_RCU) {
 		/* leave room for rcu footer at the end of object */
 		c->size += sizeof(struct slob_rcu);
 	}
 	c->flags = flags;
-	/* ignore alignment unless it's forced */
-	c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
-	if (c->align < ARCH_SLAB_MINALIGN)
-		c->align = ARCH_SLAB_MINALIGN;
-	if (c->align < align)
-		c->align = align;
-
 	return 0;
 }
 
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2012-09-19 09:21:18.403053765 -0500
+++ linux/mm/slub.c	2012-09-19 09:21:24.239174845 -0500
@@ -2761,32 +2761,6 @@ static inline int calculate_order(int si
 	return -ENOSYS;
 }
 
-/*
- * Figure out what the alignment of the objects will be.
- */
-static unsigned long calculate_alignment(unsigned long flags,
-		unsigned long align, unsigned long size)
-{
-	/*
-	 * If the user wants hardware cache aligned objects then follow that
-	 * suggestion if the object is sufficiently large.
-	 *
-	 * The hardware cache alignment cannot override the specified
-	 * alignment though. If that is greater then use it.
-	 */
-	if (flags & SLAB_HWCACHE_ALIGN) {
-		unsigned long ralign = cache_line_size();
-		while (size <= ralign / 2)
-			ralign /= 2;
-		align = max(align, ralign);
-	}
-
-	if (align < ARCH_SLAB_MINALIGN)
-		align = ARCH_SLAB_MINALIGN;
-
-	return ALIGN(align, sizeof(void *));
-}
-
 static void
 init_kmem_cache_node(struct kmem_cache_node *n)
 {
@@ -2920,7 +2894,6 @@ static int calculate_sizes(struct kmem_c
 {
 	unsigned long flags = s->flags;
 	unsigned long size = s->object_size;
-	unsigned long align = s->align;
 	int order;
 
 	/*
@@ -2992,19 +2965,11 @@ static int calculate_sizes(struct kmem_c
 #endif
 
 	/*
-	 * Determine the alignment based on various parameters that the
-	 * user specified and the dynamic determination of cache line size
-	 * on bootup.
-	 */
-	align = calculate_alignment(flags, align, s->object_size);
-	s->align = align;
-
-	/*
 	 * SLUB stores one object immediately after another beginning from
 	 * offset 0. In order to align the objects we have to simply size
 	 * each object to conform to the alignment.
 	 */
-	size = ALIGN(size, align);
+	size = ALIGN(size, s->align);
 	s->size = size;
 	if (forced_order >= 0)
 		order = forced_order;
@@ -3033,7 +2998,6 @@ static int calculate_sizes(struct kmem_c
 		s->max = s->oo;
 
 	return !!oo_objects(s->oo);
-
 }
 
 static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [03/13] slub: Use a statically allocated kmem_cache boot structure for bootstrap

[Christoph Lameter]

Simplify bootstrap by statically allocated two kmem_cache structures. These are
freed after bootup is complete. Allows us to no longer worry about calculations
of sizes of kmem_cache structures during bootstrap.

V1->V2: Do not unlock mutexes that are not taken during early boot.

Reviewed-by: Glauber Costa <glommer@parallels.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
 mm/slub.c |   41 +++++++++++------------------------------
 1 file changed, 11 insertions(+), 30 deletions(-)

Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2012-09-19 09:21:14.422971030 -0500
+++ linux/mm/slub.c	2012-09-19 09:21:18.403053765 -0500
@@ -3649,9 +3649,6 @@ static void __init kmem_cache_bootstrap_
 {
 	int node;
 
-	list_add(&s->list, &slab_caches);
-	s->refcount = -1;
-
 	for_each_node_state(node, N_NORMAL_MEMORY) {
 		struct kmem_cache_node *n = get_node(s, node);
 		struct page *p;
@@ -3668,14 +3665,13 @@ static void __init kmem_cache_bootstrap_
 	}
 }
 
+static __initdata struct kmem_cache boot_kmem_cache,
+			boot_kmem_cache_node;
+
 void __init kmem_cache_init(void)
 {
 	int i;
-	int caches = 0;
-	struct kmem_cache *temp_kmem_cache;
-	int order;
-	struct kmem_cache *temp_kmem_cache_node;
-	unsigned long kmalloc_size;
+	int caches = 2;
 
 	if (debug_guardpage_minorder())
 		slub_max_order = 0;
@@ -3683,53 +3679,32 @@ void __init kmem_cache_init(void)
 	kmem_size = offsetof(struct kmem_cache, node) +
 			nr_node_ids * sizeof(struct kmem_cache_node *);
 
-	/* Allocate two kmem_caches from the page allocator */
-	kmalloc_size = ALIGN(kmem_size, cache_line_size());
-	order = get_order(2 * kmalloc_size);
-	kmem_cache = (void *)__get_free_pages(GFP_NOWAIT | __GFP_ZERO, order);
-
-	/*
-	 * Must first have the slab cache available for the allocations of the
-	 * struct kmem_cache_node's. There is special bootstrap code in
-	 * kmem_cache_open for slab_state == DOWN.
-	 */
-	kmem_cache_node = (void *)kmem_cache + kmalloc_size;
+	kmem_cache_node = &boot_kmem_cache_node;
 
-	kmem_cache_node->name = "kmem_cache_node";
-	kmem_cache_node->size = kmem_cache_node->object_size =
-		sizeof(struct kmem_cache_node);
-	kmem_cache_open(kmem_cache_node, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+	create_boot_cache(kmem_cache_node, "kmem_cache_node",
+		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
 
 	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 
 	/* Able to allocate the per node structures */
 	slab_state = PARTIAL;
 
-	temp_kmem_cache = kmem_cache;
-	kmem_cache->name = "kmem_cache";
-	kmem_cache->size = kmem_cache->object_size = kmem_size;
-	kmem_cache_open(kmem_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+	create_boot_cache(&boot_kmem_cache, "kmem_cache", kmem_size,
+		       SLAB_HWCACHE_ALIGN);
 
-	kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
-	memcpy(kmem_cache, temp_kmem_cache, kmem_size);
+	kmem_cache = kmem_cache_alloc(&boot_kmem_cache, GFP_NOWAIT);
+	memcpy(kmem_cache, &boot_kmem_cache, kmem_size);
 
 	/*
 	 * Allocate kmem_cache_node properly from the kmem_cache slab.
 	 * kmem_cache_node is separately allocated so no need to
 	 * update any list pointers.
 	 */
-	temp_kmem_cache_node = kmem_cache_node;
-
 	kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
-	memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
+	memcpy(kmem_cache_node, &boot_kmem_cache_node, kmem_size);
 
 	kmem_cache_bootstrap_fixup(kmem_cache_node);
-
-	caches++;
 	kmem_cache_bootstrap_fixup(kmem_cache);
-	caches++;
-	/* Free temporary boot structure */
-	free_pages((unsigned long)temp_kmem_cache, order);
 
 	/* Now we can use the kmem_cache to allocate kmalloc slabs */
 
@@ -3930,6 +3905,10 @@ int __kmem_cache_create(struct kmem_cach
 	if (err)
 		return err;
 
+	/* Mutex is not taken during early boot */
+	if (slab_state <= UP)
+		return 0;
+
 	mutex_unlock(&slab_mutex);
 	err = sysfs_slab_add(s);
 	mutex_lock(&slab_mutex);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [03/13] slub: Use a statically allocated kmem_cache boot structure for bootstrap

[Glauber Costa]

On 09/27/2012 12:20 AM, Christoph Lameter wrote:
> Simplify bootstrap by statically allocated two kmem_cache structures. These are
> freed after bootup is complete. Allows us to no longer worry about calculations
> of sizes of kmem_cache structures during bootstrap.
> 
> V1->V2: Do not unlock mutexes that are not taken during early boot.
> 
This V1->V2 change makes sense, btw.

> Reviewed-by: Glauber Costa <glommer@parallels.com>
> Signed-off-by: Christoph Lameter <cl@linux.com>
> ---
>  mm/slub.c |   41 +++++++++++------------------------------
>  1 file changed, 11 insertions(+), 30 deletions(-)
> 
> Index: linux/mm/slub.c
> ===================================================================
> --- linux.orig/mm/slub.c	2012-09-19 09:21:14.422971030 -0500
> +++ linux/mm/slub.c	2012-09-19 09:21:18.403053765 -0500
> @@ -3649,9 +3649,6 @@ static void __init kmem_cache_bootstrap_
>  {
>  	int node;
>  
> -	list_add(&s->list, &slab_caches);
> -	s->refcount = -1;
> -
>  	for_each_node_state(node, N_NORMAL_MEMORY) {
>  		struct kmem_cache_node *n = get_node(s, node);
>  		struct page *p;
> @@ -3668,14 +3665,13 @@ static void __init kmem_cache_bootstrap_
>  	}
>  }
>  
> +static __initdata struct kmem_cache boot_kmem_cache,
> +			boot_kmem_cache_node;
> +
>  void __init kmem_cache_init(void)
>  {
>  	int i;
> -	int caches = 0;
> -	struct kmem_cache *temp_kmem_cache;
> -	int order;
> -	struct kmem_cache *temp_kmem_cache_node;
> -	unsigned long kmalloc_size;
> +	int caches = 2;
>  
>  	if (debug_guardpage_minorder())
>  		slub_max_order = 0;
> @@ -3683,53 +3679,32 @@ void __init kmem_cache_init(void)
>  	kmem_size = offsetof(struct kmem_cache, node) +
>  			nr_node_ids * sizeof(struct kmem_cache_node *);
>  
> -	/* Allocate two kmem_caches from the page allocator */
> -	kmalloc_size = ALIGN(kmem_size, cache_line_size());
> -	order = get_order(2 * kmalloc_size);
> -	kmem_cache = (void *)__get_free_pages(GFP_NOWAIT | __GFP_ZERO, order);
> -
> -	/*
> -	 * Must first have the slab cache available for the allocations of the
> -	 * struct kmem_cache_node's. There is special bootstrap code in
> -	 * kmem_cache_open for slab_state == DOWN.
> -	 */
> -	kmem_cache_node = (void *)kmem_cache + kmalloc_size;
> +	kmem_cache_node = &boot_kmem_cache_node;
>  
> -	kmem_cache_node->name = "kmem_cache_node";
> -	kmem_cache_node->size = kmem_cache_node->object_size =
> -		sizeof(struct kmem_cache_node);
> -	kmem_cache_open(kmem_cache_node, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
> +	create_boot_cache(kmem_cache_node, "kmem_cache_node",
> +		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
>  
>  	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
>  
>  	/* Able to allocate the per node structures */
>  	slab_state = PARTIAL;
>  
> -	temp_kmem_cache = kmem_cache;
> -	kmem_cache->name = "kmem_cache";
> -	kmem_cache->size = kmem_cache->object_size = kmem_size;
> -	kmem_cache_open(kmem_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
> +	create_boot_cache(&boot_kmem_cache, "kmem_cache", kmem_size,
> +		       SLAB_HWCACHE_ALIGN);
>  
> -	kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
> -	memcpy(kmem_cache, temp_kmem_cache, kmem_size);
> +	kmem_cache = kmem_cache_alloc(&boot_kmem_cache, GFP_NOWAIT);
> +	memcpy(kmem_cache, &boot_kmem_cache, kmem_size);
>  
>  	/*
>  	 * Allocate kmem_cache_node properly from the kmem_cache slab.
>  	 * kmem_cache_node is separately allocated so no need to
>  	 * update any list pointers.
>  	 */
> -	temp_kmem_cache_node = kmem_cache_node;
> -
>  	kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
> -	memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
> +	memcpy(kmem_cache_node, &boot_kmem_cache_node, kmem_size);
>  
>  	kmem_cache_bootstrap_fixup(kmem_cache_node);
> -
> -	caches++;
>  	kmem_cache_bootstrap_fixup(kmem_cache);
> -	caches++;
> -	/* Free temporary boot structure */
> -	free_pages((unsigned long)temp_kmem_cache, order);
>  
>  	/* Now we can use the kmem_cache to allocate kmalloc slabs */
>  
> @@ -3930,6 +3905,10 @@ int __kmem_cache_create(struct kmem_cach
>  	if (err)
>  		return err;
>  
> +	/* Mutex is not taken during early boot */
> +	if (slab_state <= UP)
> +		return 0;
> +
>  	mutex_unlock(&slab_mutex);
>  	err = sysfs_slab_add(s);
>  	mutex_lock(&slab_mutex);
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
> 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [11/13] Common constants for kmalloc boundaries

[Christoph Lameter]

Standardize the constants that describe the smallest and largest
object kept in the kmalloc arrays for SLAB and SLUB.

Differentiate between the maximum size for which a slab cache is used
(KMALLOC_MAX_CACHE_SIZE) and the maximum allocatable size
(KMALLOC_MAX_SIZE, KMALLOC_MAX_ORDER).

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2012-09-19 09:21:54.283799310 -0500
+++ linux/include/linux/slab.h	2012-09-19 09:21:58.859894391 -0500
@@ -162,8 +162,9 @@ unsigned int kmem_cache_size(struct kmem
 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
 #endif
 
+#ifdef CONFIG_SLAB
 /*
- * The largest kmalloc size supported by the slab allocators is
+ * The largest kmalloc size supported by the SLAB allocators is
  * 32 megabyte (2^25) or the maximum allocatable page order if that is
  * less than 32 MB.
  *
@@ -173,9 +174,24 @@ unsigned int kmem_cache_size(struct kmem
  */
 #define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
 				(MAX_ORDER + PAGE_SHIFT - 1) : 25)
+#define KMALLOC_SHIFT_MAX	KMALLOC_SHIFT_HIGH
+#define KMALLOC_SHIFT_LOW	5
+#else
+/*
+ * SLUB allocates up to order 2 pages directly and otherwise
+ * passes the request to the page allocator.
+ */
+#define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
+#define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT)
+#define KMALLOC_SHIFT_LOW	3
+#endif
 
-#define KMALLOC_MAX_SIZE	(1UL << KMALLOC_SHIFT_HIGH)
-#define KMALLOC_MAX_ORDER	(KMALLOC_SHIFT_HIGH - PAGE_SHIFT)
+/* Maximum allocatable size */
+#define KMALLOC_MAX_SIZE	(1UL << KMALLOC_SHIFT_MAX)
+/* Maximum size for which we actually use a slab cache */
+#define KMALLOC_MAX_CACHE_SIZE	(1UL << KMALLOC_SHIFT_HIGH)
+/* Maximum order allocatable via the slab allocagtor */
+#define KMALLOC_MAX_ORDER	(KMALLOC_SHIFT_MAX - PAGE_SHIFT)
 
 /*
  * Kmalloc subsystem.
@@ -184,14 +200,8 @@ unsigned int kmem_cache_size(struct kmem
 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
 #else
-#ifdef CONFIG_SLAB
-#define KMALLOC_MIN_SIZE 32
-#else
-#define KMALLOC_MIN_SIZE 8
+#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
 #endif
-#endif
-
-#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
 
 /*
  * Figure out which kmalloc slab an allocation of a certain size
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2012-09-19 09:21:27.307238804 -0500
+++ linux/include/linux/slub_def.h	2012-09-19 09:21:58.863894464 -0500
@@ -111,19 +111,6 @@ struct kmem_cache {
 	struct kmem_cache_node *node[MAX_NUMNODES];
 };
 
-/*
- * Maximum kmalloc object size handled by SLUB. Larger object allocations
- * are passed through to the page allocator. The page allocator "fastpath"
- * is relatively slow so we need this value sufficiently high so that
- * performance critical objects are allocated through the SLUB fastpath.
- *
- * This should be dropped to PAGE_SIZE / 2 once the page allocator
- * "fastpath" becomes competitive with the slab allocator fastpaths.
- */
-#define SLUB_MAX_SIZE (2 * PAGE_SIZE)
-
-#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
-
 #ifdef CONFIG_ZONE_DMA
 #define SLUB_DMA __GFP_DMA
 #else
@@ -135,7 +122,7 @@ struct kmem_cache {
  * We keep the general caches in an array of slab caches that are used for
  * 2^x bytes of allocations.
  */
-extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
+extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
 
 /*
  * Find the slab cache for a given combination of allocation flags and size.
@@ -204,7 +191,7 @@ static __always_inline void *kmalloc_lar
 static __always_inline void *kmalloc(size_t size, gfp_t flags)
 {
 	if (__builtin_constant_p(size)) {
-		if (size > SLUB_MAX_SIZE)
+		if (size > KMALLOC_MAX_CACHE_SIZE)
 			return kmalloc_large(size, flags);
 
 		if (!(flags & SLUB_DMA)) {
@@ -240,7 +227,7 @@ kmem_cache_alloc_node_trace(struct kmem_
 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
 	if (__builtin_constant_p(size) &&
-		size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) {
+		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) {
 			struct kmem_cache *s = kmalloc_slab(size);
 
 		if (!s)
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2012-09-19 09:21:24.239174845 -0500
+++ linux/mm/slub.c	2012-09-19 09:21:58.863894464 -0500
@@ -2777,7 +2777,7 @@ init_kmem_cache_node(struct kmem_cache_n
 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
 {
 	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
-			SLUB_PAGE_SHIFT * sizeof(struct kmem_cache_cpu));
+			KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
 
 	/*
 	 * Must align to double word boundary for the double cmpxchg
@@ -3174,11 +3174,11 @@ int __kmem_cache_shutdown(struct kmem_ca
  *		Kmalloc subsystem
  *******************************************************************/
 
-struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
+struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
 EXPORT_SYMBOL(kmalloc_caches);
 
 #ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
+static struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
 #endif
 
 static int __init setup_slub_min_order(char *str)
@@ -3280,7 +3280,7 @@ void *__kmalloc(size_t size, gfp_t flags
 	struct kmem_cache *s;
 	void *ret;
 
-	if (unlikely(size > SLUB_MAX_SIZE))
+	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
 		return kmalloc_large(size, flags);
 
 	s = get_slab(size, flags);
@@ -3316,7 +3316,7 @@ void *__kmalloc_node(size_t size, gfp_t
 	struct kmem_cache *s;
 	void *ret;
 
-	if (unlikely(size > SLUB_MAX_SIZE)) {
+	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
 		ret = kmalloc_large_node(size, flags, node);
 
 		trace_kmalloc_node(_RET_IP_, ret,
@@ -3721,7 +3721,7 @@ void __init kmem_cache_init(void)
 		caches++;
 	}
 
-	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
+	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
 		kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
 		caches++;
 	}
@@ -3739,7 +3739,7 @@ void __init kmem_cache_init(void)
 		BUG_ON(!kmalloc_caches[2]->name);
 	}
 
-	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
+	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
 		char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
 
 		BUG_ON(!s);
@@ -3751,7 +3751,7 @@ void __init kmem_cache_init(void)
 #endif
 
 #ifdef CONFIG_ZONE_DMA
-	for (i = 0; i < SLUB_PAGE_SHIFT; i++) {
+	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
 		struct kmem_cache *s = kmalloc_caches[i];
 
 		if (s && s->size) {
@@ -3925,7 +3925,7 @@ void *__kmalloc_track_caller(size_t size
 	struct kmem_cache *s;
 	void *ret;
 
-	if (unlikely(size > SLUB_MAX_SIZE))
+	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
 		return kmalloc_large(size, gfpflags);
 
 	s = get_slab(size, gfpflags);
@@ -3948,7 +3948,7 @@ void *__kmalloc_node_track_caller(size_t
 	struct kmem_cache *s;
 	void *ret;
 
-	if (unlikely(size > SLUB_MAX_SIZE)) {
+	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
 		ret = kmalloc_large_node(size, gfpflags, node);
 
 		trace_kmalloc_node(caller, ret,
@@ -4307,7 +4307,7 @@ static void resiliency_test(void)
 {
 	u8 *p;
 
-	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || SLUB_PAGE_SHIFT < 10);
+	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
 
 	printk(KERN_ERR "SLUB resiliency testing\n");
 	printk(KERN_ERR "-----------------------\n");

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

CK1 [13/13] Common function to create the kmalloc array

[Christoph Lameter]

The kmalloc array is created in similar ways in both SLAB
and SLUB. Create a common function and have both allocators
call that function.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2012-09-20 08:58:38.621735414 -0500
+++ linux/mm/slab.c	2012-09-20 08:58:38.673736492 -0500
@@ -1623,30 +1623,6 @@ void __init kmem_cache_init(void)
 
 	slab_early_init = 0;
 
-	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
-		size_t cs_size = kmalloc_size(i);
-
-		if (cs_size < KMALLOC_MIN_SIZE)
-			continue;
-
-		if (!kmalloc_caches[i]) {
-			/*
-			 * For performance, all the general caches are L1 aligned.
-			 * This should be particularly beneficial on SMP boxes, as it
-			 * eliminates "false sharing".
-			 * Note for systems short on memory removing the alignment will
-			 * allow tighter packing of the smaller caches.
-			 */
-			kmalloc_caches[i] = create_kmalloc_cache("kmalloc",
-					cs_size, ARCH_KMALLOC_FLAGS);
-		}
-
-#ifdef CONFIG_ZONE_DMA
-		kmalloc_dma_caches[i] = create_kmalloc_cache(
-			"kmalloc-dma", cs_size,
-			SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
-#endif
-	}
 	/* 4) Replace the bootstrap head arrays */
 	{
 		struct array_cache *ptr;
@@ -1692,29 +1668,7 @@ void __init kmem_cache_init(void)
 		}
 	}
 
-	slab_state = UP;
-
-	/* Create the proper names */
-	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
-		char *s;
-		struct kmem_cache *c = kmalloc_caches[i];
-
-		if (!c)
-			continue;
-
-		s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
-
-		BUG_ON(!s);
-		c->name = s;
-		
-#ifdef CONFIG_ZONE_DMA
-		c = kmalloc_dma_caches[i];
-		BUG_ON(!c);
-		s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
-		BUG_ON(!s);
-		c->name = s;
-#endif
-	}
+	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
 }
 
 void __init kmem_cache_init_late(void)
Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h	2012-09-20 08:58:38.565734246 -0500
+++ linux/mm/slab.h	2012-09-20 08:58:38.673736492 -0500
@@ -35,6 +35,12 @@ extern struct kmem_cache *kmem_cache;
 unsigned long calculate_alignment(unsigned long flags,
 		unsigned long align, unsigned long size);
 
+#ifndef CONFIG_SLOB
+/* Kmalloc array related functions */
+void create_kmalloc_caches(gfp_t);
+#endif
+
+
 /* Functions provided by the slab allocators */
 extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
 
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c	2012-09-20 08:58:38.657736160 -0500
+++ linux/mm/slab_common.c	2012-09-20 08:59:38.554983198 -0500
@@ -260,4 +260,58 @@ struct kmem_cache *kmalloc_dma_caches[KM
 EXPORT_SYMBOL(kmalloc_dma_caches);
 #endif
 
+/*
+ * Create the kmalloc array. Some of the regular kmalloc arrays
+ * may already have been created because they were needed to
+ * enable allocations for slab creation.
+ */
+void __init create_kmalloc_caches(gfp_t flags)
+{
+	int i;
+
+	/* Caches that are not of the two-to-the-power-of size */
+	if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
+		kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
+
+	if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2])
+		kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
+
+	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
+		if (!kmalloc_caches[i])
+			kmalloc_caches[i] = create_kmalloc_cache(NULL,
+				       			1 << i, flags);
+
+	/* Kmalloc array is now usable */
+	slab_state = UP;
+
+	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
+		struct kmem_cache *s = kmalloc_caches[i];
+		char *n;
+
+		if (s) {
+			n = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
+
+			BUG_ON(!n);
+			s->name = n;
+		}
+	}
+
+#ifdef CONFIG_ZONE_DMA
+	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
+		struct kmem_cache *s = kmalloc_caches[i];
+
+		if (s) {
+			int size = kmalloc_size(i);
+			char *n = kasprintf(GFP_NOWAIT,
+				 "dma-kmalloc-%d", size);
+
+			BUG_ON(!n);
+			kmalloc_dma_caches[i] = create_kmalloc_cache(n,
+				size, SLAB_CACHE_DMA | flags);
+		}
+	}
+#endif
+}
+
+
 #endif /* !CONFIG_SLOB */
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2012-09-20 08:58:38.661736249 -0500
+++ linux/mm/slub.c	2012-09-20 08:58:38.673736492 -0500
@@ -3628,7 +3628,6 @@ static __initdata struct kmem_cache boot
 void __init kmem_cache_init(void)
 {
 	int i;
-	int caches = 2;
 
 	if (debug_guardpage_minorder())
 		slub_max_order = 0;
@@ -3703,64 +3702,16 @@ void __init kmem_cache_init(void)
 			size_index[size_index_elem(i)] = 8;
 	}
 
-	/* Caches that are not of the two-to-the-power-of size */
-	if (KMALLOC_MIN_SIZE <= 32) {
-		kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
-		caches++;
-	}
-
-	if (KMALLOC_MIN_SIZE <= 64) {
-		kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
-		caches++;
-	}
-
-	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
-		kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
-		caches++;
-	}
-
-	slab_state = UP;
-
-	/* Provide the correct kmalloc names now that the caches are up */
-	if (KMALLOC_MIN_SIZE <= 32) {
-		kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
-		BUG_ON(!kmalloc_caches[1]->name);
-	}
-
-	if (KMALLOC_MIN_SIZE <= 64) {
-		kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
-		BUG_ON(!kmalloc_caches[2]->name);
-	}
-
-	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
-		char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
-
-		BUG_ON(!s);
-		kmalloc_caches[i]->name = s;
-	}
+	create_kmalloc_caches(0);
 
 #ifdef CONFIG_SMP
 	register_cpu_notifier(&slab_notifier);
 #endif
 
-#ifdef CONFIG_ZONE_DMA
-	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
-		struct kmem_cache *s = kmalloc_caches[i];
-
-		if (s && s->size) {
-			char *name = kasprintf(GFP_NOWAIT,
-				 "dma-kmalloc-%d", s->object_size);
-
-			BUG_ON(!name);
-			kmalloc_dma_caches[i] = create_kmalloc_cache(name,
-				s->object_size, SLAB_CACHE_DMA);
-		}
-	}
-#endif
 	printk(KERN_INFO
-		"SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
+		"SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d,"
 		" CPUs=%d, Nodes=%d\n",
-		caches, cache_line_size(),
+		cache_line_size(),
 		slub_min_order, slub_max_order, slub_min_objects,
 		nr_cpu_ids, nr_node_ids);
 }

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: CK1 [13/13] Common function to create the kmalloc array

[Glauber Costa]

On 09/27/2012 12:29 AM, Christoph Lameter wrote:
> The kmalloc array is created in similar ways in both SLAB
> and SLUB. Create a common function and have both allocators
> call that function.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 

The code looks fine and becomes natural at this point, after the
previous preparation changes.

Reviewed-by: Glauber Costa <glommer@parallels.com>


> Index: linux/mm/slab.c
> ===================================================================
> --- linux.orig/mm/slab.c	2012-09-20 08:58:38.621735414 -0500
> +++ linux/mm/slab.c	2012-09-20 08:58:38.673736492 -0500
> @@ -1623,30 +1623,6 @@ void __init kmem_cache_init(void)
>  
>  	slab_early_init = 0;
>  
> -	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
> -		size_t cs_size = kmalloc_size(i);
> -
> -		if (cs_size < KMALLOC_MIN_SIZE)
> -			continue;
> -
> -		if (!kmalloc_caches[i]) {
> -			/*
> -			 * For performance, all the general caches are L1 aligned.
> -			 * This should be particularly beneficial on SMP boxes, as it
> -			 * eliminates "false sharing".
> -			 * Note for systems short on memory removing the alignment will
> -			 * allow tighter packing of the smaller caches.
> -			 */
> -			kmalloc_caches[i] = create_kmalloc_cache("kmalloc",
> -					cs_size, ARCH_KMALLOC_FLAGS);
> -		}
> -
> -#ifdef CONFIG_ZONE_DMA
> -		kmalloc_dma_caches[i] = create_kmalloc_cache(
> -			"kmalloc-dma", cs_size,
> -			SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
> -#endif
> -	}
>  	/* 4) Replace the bootstrap head arrays */
>  	{
>  		struct array_cache *ptr;
> @@ -1692,29 +1668,7 @@ void __init kmem_cache_init(void)
>  		}
>  	}
>  
> -	slab_state = UP;
> -
> -	/* Create the proper names */
> -	for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
> -		char *s;
> -		struct kmem_cache *c = kmalloc_caches[i];
> -
> -		if (!c)
> -			continue;
> -
> -		s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
> -
> -		BUG_ON(!s);
> -		c->name = s;
> -		
> -#ifdef CONFIG_ZONE_DMA
> -		c = kmalloc_dma_caches[i];
> -		BUG_ON(!c);
> -		s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
> -		BUG_ON(!s);
> -		c->name = s;
> -#endif
> -	}
> +	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
>  }
>  
>  void __init kmem_cache_init_late(void)
> Index: linux/mm/slab.h
> ===================================================================
> --- linux.orig/mm/slab.h	2012-09-20 08:58:38.565734246 -0500
> +++ linux/mm/slab.h	2012-09-20 08:58:38.673736492 -0500
> @@ -35,6 +35,12 @@ extern struct kmem_cache *kmem_cache;
>  unsigned long calculate_alignment(unsigned long flags,
>  		unsigned long align, unsigned long size);
>  
> +#ifndef CONFIG_SLOB
> +/* Kmalloc array related functions */
> +void create_kmalloc_caches(gfp_t);
> +#endif
> +
> +

What is the big problem with having the declaration for slob as well?
An ifdef around a single function signature looks very ugly to me.

>  /* Functions provided by the slab allocators */
>  extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
>  
> Index: linux/mm/slab_common.c
> ===================================================================
> --- linux.orig/mm/slab_common.c	2012-09-20 08:58:38.657736160 -0500
> +++ linux/mm/slab_common.c	2012-09-20 08:59:38.554983198 -0500
> @@ -260,4 +260,58 @@ struct kmem_cache *kmalloc_dma_caches[KM
>  EXPORT_SYMBOL(kmalloc_dma_caches);
>  #endif
>  
> +/*
> + * Create the kmalloc array. Some of the regular kmalloc arrays
> + * may already have been created because they were needed to
> + * enable allocations for slab creation.
> + */
> +void __init create_kmalloc_caches(gfp_t flags)
> +{
> +	int i;
> +
> +	/* Caches that are not of the two-to-the-power-of size */
> +	if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
> +		kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
> +
> +	if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2])
> +		kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
> +
> +	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
> +		if (!kmalloc_caches[i])
> +			kmalloc_caches[i] = create_kmalloc_cache(NULL,
> +				       			1 << i, flags);
> +
> +	/* Kmalloc array is now usable */
> +	slab_state = UP;
> +
> +	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
> +		struct kmem_cache *s = kmalloc_caches[i];
> +		char *n;
> +
> +		if (s) {
> +			n = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
> +
> +			BUG_ON(!n);
> +			s->name = n;
> +		}

Isn't it better to write:

	if (!s)
		continue;

	s->name = kasprintf(...);
	BUG_ON(!s->name);
?


> +	}
> +
> +#ifdef CONFIG_ZONE_DMA
> +	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
> +		struct kmem_cache *s = kmalloc_caches[i];
> +
> +		if (s) {
> +			int size = kmalloc_size(i);
> +			char *n = kasprintf(GFP_NOWAIT,
> +				 "dma-kmalloc-%d", size);
> +
> +			BUG_ON(!n);
> +			kmalloc_dma_caches[i] = create_kmalloc_cache(n,
> +				size, SLAB_CACHE_DMA | flags);
> +		}
> +	}
> +#endif
> +}
> +
> +
>  #endif /* !CONFIG_SLOB */
> Index: linux/mm/slub.c
> ===================================================================
> --- linux.orig/mm/slub.c	2012-09-20 08:58:38.661736249 -0500
> +++ linux/mm/slub.c	2012-09-20 08:58:38.673736492 -0500
> @@ -3628,7 +3628,6 @@ static __initdata struct kmem_cache boot
>  void __init kmem_cache_init(void)
>  {
>  	int i;
> -	int caches = 2;
>  
>  	if (debug_guardpage_minorder())
>  		slub_max_order = 0;
> @@ -3703,64 +3702,16 @@ void __init kmem_cache_init(void)
>  			size_index[size_index_elem(i)] = 8;
>  	}
>  
> -	/* Caches that are not of the two-to-the-power-of size */
> -	if (KMALLOC_MIN_SIZE <= 32) {
> -		kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
> -		caches++;
> -	}
> -
> -	if (KMALLOC_MIN_SIZE <= 64) {
> -		kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
> -		caches++;
> -	}
> -
> -	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
> -		kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
> -		caches++;
> -	}
> -
> -	slab_state = UP;
> -
> -	/* Provide the correct kmalloc names now that the caches are up */
> -	if (KMALLOC_MIN_SIZE <= 32) {
> -		kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
> -		BUG_ON(!kmalloc_caches[1]->name);
> -	}
> -
> -	if (KMALLOC_MIN_SIZE <= 64) {
> -		kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
> -		BUG_ON(!kmalloc_caches[2]->name);
> -	}
> -
> -	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
> -		char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
> -
> -		BUG_ON(!s);
> -		kmalloc_caches[i]->name = s;
> -	}
> +	create_kmalloc_caches(0);
>  
>  #ifdef CONFIG_SMP
>  	register_cpu_notifier(&slab_notifier);
>  #endif
>  
> -#ifdef CONFIG_ZONE_DMA
> -	for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
> -		struct kmem_cache *s = kmalloc_caches[i];
> -
> -		if (s && s->size) {
> -			char *name = kasprintf(GFP_NOWAIT,
> -				 "dma-kmalloc-%d", s->object_size);
> -
> -			BUG_ON(!name);
> -			kmalloc_dma_caches[i] = create_kmalloc_cache(name,
> -				s->object_size, SLAB_CACHE_DMA);
> -		}
> -	}
> -#endif
>  	printk(KERN_INFO
> -		"SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
> +		"SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d,"
>  		" CPUs=%d, Nodes=%d\n",
> -		caches, cache_line_size(),
> +		cache_line_size(),
>  		slub_min_order, slub_max_order, slub_min_objects,
>  		nr_cpu_ids, nr_node_ids);
>  }
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
> 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>