[3.11 2/4] slob: Rework #ifdeffery in slab.h

[3.11 2/4] slob: Rework #ifdeffery in slab.h

[Christoph Lameter]

Make the SLOB specific stuff harmonize more with the way the other allocators
do it. Create the typical kmalloc constants for that purpose. SLOB does not
support it but the constants help us avoid #ifdefs.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2013-06-14 12:25:33.443671057 -0500
+++ linux/include/linux/slab.h	2013-06-14 13:19:19.597081306 -0500
@@ -169,11 +169,7 @@ struct kmem_cache {
 	struct list_head list;	/* List of all slab caches on the system */
 };
 
-#define KMALLOC_MAX_SIZE (1UL << 30)
-
-#include <linux/slob_def.h>
-
-#else /* CONFIG_SLOB */
+#endif /* CONFIG_SLOB */
 
 /*
  * Kmalloc array related definitions
@@ -195,7 +191,9 @@ struct kmem_cache {
 #ifndef KMALLOC_SHIFT_LOW
 #define KMALLOC_SHIFT_LOW	5
 #endif
-#else
+#endif
+
+#ifdef CONFIG_SLUB
 /*
  * SLUB allocates up to order 2 pages directly and otherwise
  * passes the request to the page allocator.
@@ -207,6 +205,19 @@ struct kmem_cache {
 #endif
 #endif
 
+#ifdef CONFIG_SLOB
+/*
+ * SLOB passes all page size and larger requests to the page allocator.
+ * No kmalloc array is necessary since objects of different sizes can
+ * be allocated from the same page.
+ */
+#define KMALLOC_SHIFT_MAX	30
+#define KMALLOC_SHIFT_HIGH	PAGE_SHIFT
+#ifndef KMALLOC_SHIFT_LOW
+#define KMALLOC_SHIFT_LOW	3
+#endif
+#endif
+
 /* Maximum allocatable size */
 #define KMALLOC_MAX_SIZE	(1UL << KMALLOC_SHIFT_MAX)
 /* Maximum size for which we actually use a slab cache */
@@ -221,6 +232,7 @@ struct kmem_cache {
 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
 #endif
 
+#ifndef CONFIG_SLOB
 extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
 #ifdef CONFIG_ZONE_DMA
 extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
@@ -275,13 +287,18 @@ static __always_inline int kmalloc_index
 	/* Will never be reached. Needed because the compiler may complain */
 	return -1;
 }
+#endif /* !CONFIG_SLOB */
 
 #ifdef CONFIG_SLAB
 #include <linux/slab_def.h>
-#elif defined(CONFIG_SLUB)
+#endif
+
+#ifdef CONFIG_SLUB
 #include <linux/slub_def.h>
-#else
-#error "Unknown slab allocator"
+#endif
+
+#ifdef CONFIG_SLOB
+#include <linux/slob_def.h>
 #endif
 
 /*
@@ -291,6 +308,7 @@ static __always_inline int kmalloc_index
  */
 static __always_inline int kmalloc_size(int n)
 {
+#ifndef CONFIG_SLOB
 	if (n > 2)
 		return 1 << n;
 
@@ -299,10 +317,9 @@ static __always_inline int kmalloc_size(
 
 	if (n == 2 && KMALLOC_MIN_SIZE <= 64)
 		return 192;
-
+#endif
 	return 0;
 }
-#endif /* !CONFIG_SLOB */
 
 /*
  * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[3.11 1/4] slub: Make cpu partial slab support configurable V2

[Christoph Lameter]

cpu partial support can introduce level of indeterminism that is not wanted
in certain context (like a realtime kernel). Make it configurable.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2013-06-14 09:50:58.190453865 -0500
+++ linux/include/linux/slub_def.h	2013-06-14 09:50:58.186453794 -0500
@@ -73,7 +73,9 @@ struct kmem_cache {
 	int size;		/* The size of an object including meta data */
 	int object_size;	/* The size of an object without meta data */
 	int offset;		/* Free pointer offset. */
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 	int cpu_partial;	/* Number of per cpu partial objects to keep around */
+#endif
 	struct kmem_cache_order_objects oo;
 
 	/* Allocation and freeing of slabs */
@@ -104,6 +106,15 @@ struct kmem_cache {
 	struct kmem_cache_node *node[MAX_NUMNODES];
 };
 
+static inline int kmem_cache_cpu_partial(struct kmem_cache *s)
+{
+#ifdef CONFIG_SLUB_CPU_PARTIAL
+	return s->cpu_partial;
+#else
+	return 0;
+#endif
+}
+
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 void *__kmalloc(size_t size, gfp_t flags);
 
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c	2013-06-14 09:50:58.190453865 -0500
+++ linux/mm/slub.c	2013-06-14 09:50:58.186453794 -0500
@@ -1573,7 +1573,8 @@ static void *get_partial_node(struct kme
 			put_cpu_partial(s, page, 0);
 			stat(s, CPU_PARTIAL_NODE);
 		}
-		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
+		if (kmem_cache_debug(s) ||
+			       available > kmem_cache_cpu_partial(s) / 2)
 			break;
 
 	}
@@ -1884,6 +1885,7 @@ redo:
 static void unfreeze_partials(struct kmem_cache *s,
 		struct kmem_cache_cpu *c)
 {
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 	struct kmem_cache_node *n = NULL, *n2 = NULL;
 	struct page *page, *discard_page = NULL;
 
@@ -1938,6 +1940,7 @@ static void unfreeze_partials(struct kme
 		discard_slab(s, page);
 		stat(s, FREE_SLAB);
 	}
+#endif
 }
 
 /*
@@ -1951,6 +1954,7 @@ static void unfreeze_partials(struct kme
  */
 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
 {
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 	struct page *oldpage;
 	int pages;
 	int pobjects;
@@ -1987,6 +1991,7 @@ static void put_cpu_partial(struct kmem_
 		page->next = oldpage;
 
 	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
+#endif
 }
 
 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
@@ -2495,6 +2500,7 @@ static void __slab_free(struct kmem_cach
 		new.inuse--;
 		if ((!new.inuse || !prior) && !was_frozen) {
 
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 			if (!kmem_cache_debug(s) && !prior)
 
 				/*
@@ -2503,7 +2509,9 @@ static void __slab_free(struct kmem_cach
 				 */
 				new.frozen = 1;
 
-			else { /* Needs to be taken off a list */
+			else
+#endif
+		       		{ /* Needs to be taken off a list */
 
 	                        n = get_node(s, page_to_nid(page));
 				/*
@@ -2525,6 +2533,7 @@ static void __slab_free(struct kmem_cach
 		"__slab_free"));
 
 	if (likely(!n)) {
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 
 		/*
 		 * If we just froze the page then put it onto the
@@ -2534,6 +2543,7 @@ static void __slab_free(struct kmem_cach
 			put_cpu_partial(s, page, 1);
 			stat(s, CPU_PARTIAL_FREE);
 		}
+#endif
 		/*
 		 * The list lock was not taken therefore no list
 		 * activity can be necessary.
@@ -3041,7 +3051,7 @@ static int kmem_cache_open(struct kmem_c
 	 * list to avoid pounding the page allocator excessively.
 	 */
 	set_min_partial(s, ilog2(s->size) / 2);
-
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 	/*
 	 * cpu_partial determined the maximum number of objects kept in the
 	 * per cpu partial lists of a processor.
@@ -3069,6 +3079,7 @@ static int kmem_cache_open(struct kmem_c
 		s->cpu_partial = 13;
 	else
 		s->cpu_partial = 30;
+#endif
 
 #ifdef CONFIG_NUMA
 	s->remote_node_defrag_ratio = 1000;
@@ -4424,7 +4435,7 @@ SLAB_ATTR(order);
 
 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
 {
-	return sprintf(buf, "%lu\n", s->min_partial);
+	return sprintf(buf, "%u\n", kmem_cache_cpu_partial(s));
 }
 
 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
@@ -4444,7 +4455,7 @@ SLAB_ATTR(min_partial);
 
 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
 {
-	return sprintf(buf, "%u\n", s->cpu_partial);
+	return sprintf(buf, "%u\n", kmem_cache_cpu_partial(s));
 }
 
 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
@@ -4458,10 +4469,13 @@ static ssize_t cpu_partial_store(struct
 		return err;
 	if (objects && kmem_cache_debug(s))
 		return -EINVAL;
-
+#ifdef CONFIG_SLUB_CPU_PARTIAL
 	s->cpu_partial = objects;
 	flush_all(s);
 	return length;
+#else
+	return -ENOSYS;
+#endif
 }
 SLAB_ATTR(cpu_partial);
 
Index: linux/init/Kconfig
===================================================================
--- linux.orig/init/Kconfig	2013-06-14 09:50:58.190453865 -0500
+++ linux/init/Kconfig	2013-06-14 09:50:58.186453794 -0500
@@ -1559,6 +1559,17 @@ config SLOB
 
 endchoice
 
+config SLUB_CPU_PARTIAL
+	default y
+	depends on SLUB
+	bool "SLUB per cpu partial cache"
+	help
+	  Per cpu partial caches accellerate objects allocation and freeing
+	  that is local to a processor at the price of more indeterminism
+	  in the latency of the free. On overflow these caches will be cleared
+	  which requires the taking of locks that may cause latency spikes.
+	  Typically one would choose no for a realtime system.
+
 config MMAP_ALLOW_UNINITIALIZED
 	bool "Allow mmapped anonymous memory to be uninitialized"
 	depends on EXPERT && !MMU

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[3.11 4/4] Move kmalloc definitions to slab.h

[Christoph Lameter]

All the kmallocs are mostly doing the same. Unify them.

slob_def.h becomes empty. So remove it.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2013-06-14 14:34:26.148925322 -0500
+++ linux/include/linux/slab.h	2013-06-14 14:35:12.509741180 -0500
@@ -4,6 +4,8 @@
  * (C) SGI 2006, Christoph Lameter
  * 	Cleaned up and restructured to ease the addition of alternative
  * 	implementations of SLAB allocators.
+ * (C) Linux Foundation 2008-2013
+ *      Unified interface for all slab allocators
  */
 
 #ifndef _LINUX_SLAB_H
@@ -12,6 +14,7 @@
 #include <linux/gfp.h>
 #include <linux/types.h>
 #include <linux/workqueue.h>
+#include <linux/kmemleak.h>
 
 
 /*
@@ -329,10 +332,71 @@ kmem_cache_alloc_node_trace(struct kmem_
 #include <linux/slub_def.h>
 #endif
 
-#ifdef CONFIG_SLOB
-#include <linux/slob_def.h>
+static __always_inline void *
+kmalloc_order(size_t size, gfp_t flags, unsigned int order)
+{
+	void *ret;
+
+	flags |= (__GFP_COMP | __GFP_KMEMCG);
+	ret = (void *) __get_free_pages(flags, order);
+	kmemleak_alloc(ret, size, 1, flags);
+	return ret;
+}
+
+#ifdef CONFIG_TRACING
+extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);
+#else
+static __always_inline void *
+kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
+{
+	return kmalloc_order(size, flags, order);
+}
 #endif
 
+static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
+{
+	unsigned int order = get_order(size);
+	return kmalloc_order_trace(size, flags, order);
+}
+
+#ifdef CONFIG_TRACING
+extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t);
+#else
+static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s,
+		gfp_t flags)
+{
+	return kmem_cache_alloc(s, flags);
+}
+#endif
+
+/**
+ * kmalloc - allocate memory
+ * @size: how many bytes of memory are required.
+ * @flags: the type of memory to allocate (see kcalloc).
+ *
+ * kmalloc is the normal method of allocating memory
+ * for objects smaller than page size in the kernel.
+ */
+static __always_inline void *kmalloc(size_t size, gfp_t flags)
+{
+	if (__builtin_constant_p(size)) {
+		if (size > KMALLOC_MAX_CACHE_SIZE)
+			return kmalloc_large(size, flags);
+#ifndef CONFIG_SLOB
+		if (!(flags & GFP_DMA)) {
+			int index = kmalloc_index(size);
+
+			if (!index)
+				return ZERO_SIZE_PTR;
+
+			return kmem_cache_alloc_trace(kmalloc_caches[index],
+					flags);
+		}
+#endif
+	}
+	return __kmalloc(size, flags);
+}
+
 /*
  * Determine size used for the nth kmalloc cache.
  * return size or 0 if a kmalloc cache for that
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2013-06-14 14:34:26.148925322 -0500
+++ linux/include/linux/slab_def.h	2013-06-14 14:34:26.144925252 -0500
@@ -102,44 +102,4 @@ struct kmem_cache {
 	 */
 };
 
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);
-#else
-static __always_inline void *
-kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
-{
-	return kmem_cache_alloc(cachep, flags);
-}
-#endif
-
-static __always_inline void *kmalloc(size_t size, gfp_t flags)
-{
-	struct kmem_cache *cachep;
-	void *ret;
-
-	if (__builtin_constant_p(size)) {
-		int i;
-
-		if (!size)
-			return ZERO_SIZE_PTR;
-
-		if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))
-			return NULL;
-
-		i = kmalloc_index(size);
-
-#ifdef CONFIG_ZONE_DMA
-		if (flags & GFP_DMA)
-			cachep = kmalloc_dma_caches[i];
-		else
-#endif
-			cachep = kmalloc_caches[i];
-
-		ret = kmem_cache_alloc_trace(cachep, flags, size);
-
-		return ret;
-	}
-	return __kmalloc(size, flags);
-}
-
 #endif	/* _LINUX_SLAB_DEF_H */
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2013-06-14 14:34:26.148925322 -0500
+++ linux/include/linux/slub_def.h	2013-06-14 14:34:26.144925252 -0500
@@ -12,8 +12,6 @@
 #include <linux/workqueue.h>
 #include <linux/kobject.h>
 
-#include <linux/kmemleak.h>
-
 enum stat_item {
 	ALLOC_FASTPATH,		/* Allocation from cpu slab */
 	ALLOC_SLOWPATH,		/* Allocation by getting a new cpu slab */
@@ -115,17 +113,6 @@ static inline int kmem_cache_cpu_partial
 #endif
 }
 
-static __always_inline void *
-kmalloc_order(size_t size, gfp_t flags, unsigned int order)
-{
-	void *ret;
-
-	flags |= (__GFP_COMP | __GFP_KMEMCG);
-	ret = (void *) __get_free_pages(flags, order);
-	kmemleak_alloc(ret, size, 1, flags);
-	return ret;
-}
-
 /**
  * Calling this on allocated memory will check that the memory
  * is expected to be in use, and print warnings if not.
@@ -139,47 +126,4 @@ static inline bool verify_mem_not_delete
 }
 #endif
 
-#ifdef CONFIG_TRACING
-extern void *
-kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size);
-extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);
-#else
-static __always_inline void *
-kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
-{
-	return kmem_cache_alloc(s, gfpflags);
-}
-
-static __always_inline void *
-kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
-{
-	return kmalloc_order(size, flags, order);
-}
-#endif
-
-static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
-{
-	unsigned int order = get_order(size);
-	return kmalloc_order_trace(size, flags, order);
-}
-
-static __always_inline void *kmalloc(size_t size, gfp_t flags)
-{
-	if (__builtin_constant_p(size)) {
-		if (size > KMALLOC_MAX_CACHE_SIZE)
-			return kmalloc_large(size, flags);
-
-		if (!(flags & GFP_DMA)) {
-			int index = kmalloc_index(size);
-
-			if (!index)
-				return ZERO_SIZE_PTR;
-
-			return kmem_cache_alloc_trace(kmalloc_caches[index],
-					flags, size);
-		}
-	}
-	return __kmalloc(size, flags);
-}
-
 #endif /* _LINUX_SLUB_DEF_H */
Index: linux/include/linux/slob_def.h
===================================================================
--- linux.orig/include/linux/slob_def.h	2013-06-14 14:04:09.000000000 -0500
+++ /dev/null	1970-01-01 00:00:00.000000000 +0000
@@ -1,17 +0,0 @@
-#ifndef __LINUX_SLOB_DEF_H
-#define __LINUX_SLOB_DEF_H
-
-/*
- * kmalloc - allocate memory
- * @size: how many bytes of memory are required.
- * @flags: the type of memory to allocate (see kcalloc).
- *
- * kmalloc is the normal method of allocating memory
- * in the kernel.
- */
-static __always_inline void *kmalloc(size_t size, gfp_t flags)
-{
-	return __kmalloc_node(size, flags, NUMA_NO_NODE);
-}
-
-#endif /* __LINUX_SLOB_DEF_H */

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[3.11 3/4] Move kmalloc_node functions to common code

[Christoph Lameter]

The kmalloc_node functions of all slab allcoators are similar now so
lets move them into slab.h. This requires some function naming changes
in slob.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2013-06-14 13:40:52.424106451 -0500
+++ linux/include/linux/slab.h	2013-06-14 14:45:24.000000000 -0500
@@ -289,6 +289,38 @@ static __always_inline int kmalloc_index
 }
 #endif /* !CONFIG_SLOB */
 
+void *__kmalloc(size_t size, gfp_t flags);
+void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags);
+
+#ifdef CONFIG_NUMA
+void *__kmalloc_node(size_t size, gfp_t flags, int node);
+void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
+#else
+static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
+{
+	return __kmalloc(size, flags);
+}
+
+static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)
+{
+	return kmem_cache_alloc(s, flags);
+}
+#endif
+
+#ifdef CONFIG_TRACING
+extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
+					   gfp_t gfpflags,
+					   int node);
+#else
+static __always_inline void *
+kmem_cache_alloc_node_trace(struct kmem_cache *s,
+			      gfp_t gfpflags,
+			      int node)
+{
+	return kmem_cache_alloc_node(s, gfpflags, node);
+}
+#endif
+
 #ifdef CONFIG_SLAB
 #include <linux/slab_def.h>
 #endif
@@ -321,6 +353,23 @@ static __always_inline int kmalloc_size(
 	return 0;
 }
 
+static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
+{
+#ifndef CONFIG_SLOB
+	if (__builtin_constant_p(size) &&
+		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {
+		int i = kmalloc_index(size);
+
+		if (!i)
+			return ZERO_SIZE_PTR;
+
+		return kmem_cache_alloc_node_trace(kmalloc_caches[i],
+			       			flags, node);
+	}
+#endif
+	return __kmalloc_node(size, flags, node);
+}
+
 /*
  * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
  * Intended for arches that get misalignment faults even for 64 bit integer
@@ -441,36 +490,6 @@ static inline void *kcalloc(size_t n, si
 	return kmalloc_array(n, size, flags | __GFP_ZERO);
 }
 
-#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB)
-/**
- * kmalloc_node - allocate memory from a specific node
- * @size: how many bytes of memory are required.
- * @flags: the type of memory to allocate (see kcalloc).
- * @node: node to allocate from.
- *
- * kmalloc() for non-local nodes, used to allocate from a specific node
- * if available. Equivalent to kmalloc() in the non-NUMA single-node
- * case.
- */
-static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	return kmalloc(size, flags);
-}
-
-static inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	return __kmalloc(size, flags);
-}
-
-void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
-
-static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,
-					gfp_t flags, int node)
-{
-	return kmem_cache_alloc(cachep, flags);
-}
-#endif /* !CONFIG_NUMA && !CONFIG_SLOB */
-
 /*
  * kmalloc_track_caller is a special version of kmalloc that records the
  * calling function of the routine calling it for slab leak tracking instead
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2013-06-14 13:40:52.424106451 -0500
+++ linux/include/linux/slub_def.h	2013-06-14 14:45:24.000000000 -0500
@@ -115,9 +115,6 @@ static inline int kmem_cache_cpu_partial
 #endif
 }
 
-void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
-void *__kmalloc(size_t size, gfp_t flags);
-
 static __always_inline void *
 kmalloc_order(size_t size, gfp_t flags, unsigned int order)
 {
@@ -185,38 +182,4 @@ static __always_inline void *kmalloc(siz
 	return __kmalloc(size, flags);
 }
 
-#ifdef CONFIG_NUMA
-void *__kmalloc_node(size_t size, gfp_t flags, int node);
-void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
-					   gfp_t gfpflags,
-					   int node, size_t size);
-#else
-static __always_inline void *
-kmem_cache_alloc_node_trace(struct kmem_cache *s,
-			      gfp_t gfpflags,
-			      int node, size_t size)
-{
-	return kmem_cache_alloc_node(s, gfpflags, node);
-}
-#endif
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	if (__builtin_constant_p(size) &&
-		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {
-		int index = kmalloc_index(size);
-
-		if (!index)
-			return ZERO_SIZE_PTR;
-
-		return kmem_cache_alloc_node_trace(kmalloc_caches[index],
-			       flags, node, size);
-	}
-	return __kmalloc_node(size, flags, node);
-}
-#endif
-
 #endif /* _LINUX_SLUB_DEF_H */
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2013-06-14 13:40:52.424106451 -0500
+++ linux/include/linux/slab_def.h	2013-06-14 14:45:24.000000000 -0500
@@ -102,9 +102,6 @@ struct kmem_cache {
 	 */
 };
 
-void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
-void *__kmalloc(size_t size, gfp_t flags);
-
 #ifdef CONFIG_TRACING
 extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);
 #else
@@ -145,53 +142,4 @@ static __always_inline void *kmalloc(siz
 	return __kmalloc(size, flags);
 }
 
-#ifdef CONFIG_NUMA
-extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
-extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
-					 gfp_t flags,
-					 int nodeid,
-					 size_t size);
-#else
-static __always_inline void *
-kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
-			    gfp_t flags,
-			    int nodeid,
-			    size_t size)
-{
-	return kmem_cache_alloc_node(cachep, flags, nodeid);
-}
-#endif
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	struct kmem_cache *cachep;
-
-	if (__builtin_constant_p(size)) {
-		int i;
-
-		if (!size)
-			return ZERO_SIZE_PTR;
-
-		if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))
-			return NULL;
-
-		i = kmalloc_index(size);
-
-#ifdef CONFIG_ZONE_DMA
-		if (flags & GFP_DMA)
-			cachep = kmalloc_dma_caches[i];
-		else
-#endif
-			cachep = kmalloc_caches[i];
-
-		return kmem_cache_alloc_node_trace(cachep, flags, node, size);
-	}
-	return __kmalloc_node(size, flags, node);
-}
-
-#endif	/* CONFIG_NUMA */
-
 #endif	/* _LINUX_SLAB_DEF_H */
Index: linux/include/linux/slob_def.h
===================================================================
--- linux.orig/include/linux/slob_def.h	2013-06-14 13:40:52.424106451 -0500
+++ linux/include/linux/slob_def.h	2013-06-14 14:45:24.000000000 -0500
@@ -1,24 +1,7 @@
 #ifndef __LINUX_SLOB_DEF_H
 #define __LINUX_SLOB_DEF_H
 
-#include <linux/numa.h>
-
-void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep,
-					      gfp_t flags)
-{
-	return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE);
-}
-
-void *__kmalloc_node(size_t size, gfp_t flags, int node);
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	return __kmalloc_node(size, flags, node);
-}
-
-/**
+/*
  * kmalloc - allocate memory
  * @size: how many bytes of memory are required.
  * @flags: the type of memory to allocate (see kcalloc).
@@ -31,9 +14,4 @@ static __always_inline void *kmalloc(siz
 	return __kmalloc_node(size, flags, NUMA_NO_NODE);
 }
 
-static __always_inline void *__kmalloc(size_t size, gfp_t flags)
-{
-	return kmalloc(size, flags);
-}
-
 #endif /* __LINUX_SLOB_DEF_H */
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2013-06-14 13:40:52.424106451 -0500
+++ linux/mm/slab.c	2013-06-14 13:40:52.420106378 -0500
@@ -3681,7 +3681,7 @@ __do_kmalloc_node(size_t size, gfp_t fla
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
-	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
+	return kmem_cache_alloc_node_trace(cachep, flags, node);
 }
 
 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
Index: linux/mm/slob.c
===================================================================
--- linux.orig/mm/slob.c	2013-06-14 13:14:08.000000000 -0500
+++ linux/mm/slob.c	2013-06-14 14:44:56.812030812 -0500
@@ -462,11 +462,11 @@ __do_kmalloc_node(size_t size, gfp_t gfp
 	return ret;
 }
 
-void *__kmalloc_node(size_t size, gfp_t gfp, int node)
+void *__kmalloc(size_t size, gfp_t gfp)
 {
-	return __do_kmalloc_node(size, gfp, node, _RET_IP_);
+	return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, _RET_IP_);
 }
-EXPORT_SYMBOL(__kmalloc_node);
+EXPORT_SYMBOL(__kmalloc);
 
 #ifdef CONFIG_TRACING
 void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller)
@@ -534,7 +534,7 @@ int __kmem_cache_create(struct kmem_cach
 	return 0;
 }
 
-void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
+void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 {
 	void *b;
 
@@ -560,7 +560,27 @@ void *kmem_cache_alloc_node(struct kmem_
 	kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags);
 	return b;
 }
+EXPORT_SYMBOL(slob_alloc_node);
+
+void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
+{
+	return slob_alloc_node(cachep, flags, NUMA_NO_NODE);
+}
+EXPORT_SYMBOL(kmem_cache_alloc);
+
+#ifdef CONFIG_NUMA
+void *__kmalloc_node(size_t size, gfp_t gfp, int node)
+{
+	return __do_kmalloc_node(size, gfp, node, _RET_IP_);
+}
+EXPORT_SYMBOL(__kmalloc_node);
+
+void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node)
+{
+	return slob_alloc_node(cachep, gfp, node);
+}
 EXPORT_SYMBOL(kmem_cache_alloc_node);
+#endif
 
 static void __kmem_cache_free(void *b, int size)
 {

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Pekka Enberg]

On 06/14/2013 10:55 PM, Christoph Lameter wrote:
> cpu partial support can introduce level of indeterminism that is not wanted
> in certain context (like a realtime kernel). Make it configurable.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>

The changelog is way too vague. Numbers? Anyone who would want to
use this in real world scenarios, please speak up!

			Pekka

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Christoph Lameter]

On Tue, 18 Jun 2013, Pekka Enberg wrote:

> The changelog is way too vague. Numbers? Anyone who would want to
> use this in real world scenarios, please speak up!

Steve?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 1165 bytes --]

On Tue, 18 Jun 2013 14:17:25 +0000
Christoph Lameter <cl@linux.com> wrote:

> On Tue, 18 Jun 2013, Pekka Enberg wrote:
> 
> > The changelog is way too vague. Numbers? Anyone who would want to
> > use this in real world scenarios, please speak up!
> 
> Steve?

Steve's out this morning so I'll take a stab at it.

This was an RT request. When we switched over to SLUB we saw an
immediate overall performance boost over SLAB but encountered some
249 microsecond latency spikes when testing on large systems
(40-core/256GB RAM). Latency traces showed that our spikes were
SLUB's cpu_partial processing in unfreeze_partials(). 

We hacked up a boot script that would traverse the /sys/kernel/slab
tree and write a zero to all the 'cpu_partial' entries (turning them
off) but asked Christoph if he had a way to configure cpu_partial
processing out, since running the script at boot did not actually catch
all instances of cpu_partial. 

I'm sure it would be better to actually do cpu_partial processing in
small chunks to avoid latency spikes in latency sensitive applications
but for the short-term it's just easier to turn it off. 

Clark

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Pekka Enberg]

On Tue, Jun 18, 2013 at 6:21 PM, Clark Williams <williams@redhat.com> wrote:
> I'm sure it would be better to actually do cpu_partial processing in
> small chunks to avoid latency spikes in latency sensitive applications

Sounds like a patch I'd be much more interested in applying...

                        Pekka

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Pekka Enberg]

On Fri, Jun 14, 2013 at 11:06 PM, Christoph Lameter <cl@linux.com> wrote:
> The kmalloc_node functions of all slab allcoators are similar now so
> lets move them into slab.h. This requires some function naming changes
> in slob.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>

I'm seeing this after "make defconfig" on x86-64:

  CC      mm/slub.o
mm/slub.c:2445:7: error: conflicting types for ‘kmem_cache_alloc_node_trace’
include/linux/slab.h:311:14: note: previous declaration of
‘kmem_cache_alloc_node_trace’ was here
mm/slub.c:2455:1: error: conflicting types for ‘kmem_cache_alloc_node_trace’
include/linux/slab.h:311:14: note: previous declaration of
‘kmem_cache_alloc_node_trace’ was here
make[1]: *** [mm/slub.o] Error 1
make: *** [mm/slub.o] Error 2

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Christoph Lameter]

[-- Attachment #1: Type: TEXT/PLAIN, Size: 2922 bytes --]

On Tue, 18 Jun 2013, Pekka Enberg wrote:

> I'm seeing this after "make defconfig" on x86-64:
>
>   CC      mm/slub.o
> mm/slub.c:2445:7: error: conflicting types for i? 1/2 kmem_cache_alloc_node_tracei? 1/2 
> include/linux/slab.h:311:14: note: previous declaration of
> i? 1/2 kmem_cache_alloc_node_tracei? 1/2  was here
> mm/slub.c:2455:1: error: conflicting types for i? 1/2 kmem_cache_alloc_node_tracei? 1/2 
> include/linux/slab.h:311:14: note: previous declaration of
> i? 1/2 kmem_cache_alloc_node_tracei? 1/2  was here
> make[1]: *** [mm/slub.o] Error 1
> make: *** [mm/slub.o] Error 2

Gosh I dropped the size_t parameter from these functions. CONFIG_TRACING
needs these.


Subject: Fix kmem_cache_alloc*_trace parameters

The size parameter is needed.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2013-06-18 11:45:22.472313944 -0500
+++ linux/include/linux/slab.h	2013-06-18 11:53:29.816926981 -0500
@@ -313,12 +313,12 @@ static __always_inline void *kmem_cache_
 #ifdef CONFIG_TRACING
 extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
 					   gfp_t gfpflags,
-					   int node);
+					   int node, size_t size);
 #else
 static __always_inline void *
 kmem_cache_alloc_node_trace(struct kmem_cache *s,
 			      gfp_t gfpflags,
-			      int node)
+			      int node, size_t size)
 {
 	return kmem_cache_alloc_node(s, gfpflags, node);
 }
@@ -360,10 +360,10 @@ static __always_inline void *kmalloc_lar
 }

 #ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t);
+extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);
 #else
 static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s,
-		gfp_t flags)
+		gfp_t flags, size_t size)
 {
 	return kmem_cache_alloc(s, flags);
 }
@@ -390,7 +390,7 @@ static __always_inline void *kmalloc(siz
 				return ZERO_SIZE_PTR;

 			return kmem_cache_alloc_trace(kmalloc_caches[index],
-					flags);
+					flags, size);
 		}
 #endif
 	}
@@ -428,7 +428,7 @@ static __always_inline void *kmalloc_nod
 			return ZERO_SIZE_PTR;

 		return kmem_cache_alloc_node_trace(kmalloc_caches[i],
-			       			flags, node);
+			       			flags, node, size);
 	}
 #endif
 	return __kmalloc_node(size, flags, node);
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2013-06-18 11:45:15.984199533 -0500
+++ linux/mm/slab.c	2013-06-18 11:54:23.857883936 -0500
@@ -3681,7 +3681,7 @@ __do_kmalloc_node(size_t size, gfp_t fla
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
-	return kmem_cache_alloc_node_trace(cachep, flags, node);
+	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
 }

 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Joonsoo Kim]

Hello, Christoph.

On Fri, Jun 14, 2013 at 07:55:13PM +0000, Christoph Lameter wrote:
> cpu partial support can introduce level of indeterminism that is not wanted
> in certain context (like a realtime kernel). Make it configurable.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 
> Index: linux/include/linux/slub_def.h
> ===================================================================
> --- linux.orig/include/linux/slub_def.h	2013-06-14 09:50:58.190453865 -0500
> +++ linux/include/linux/slub_def.h	2013-06-14 09:50:58.186453794 -0500
> @@ -73,7 +73,9 @@ struct kmem_cache {
>  	int size;		/* The size of an object including meta data */
>  	int object_size;	/* The size of an object without meta data */
>  	int offset;		/* Free pointer offset. */
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  	int cpu_partial;	/* Number of per cpu partial objects to keep around */
> +#endif
>  	struct kmem_cache_order_objects oo;
>  
>  	/* Allocation and freeing of slabs */
> @@ -104,6 +106,15 @@ struct kmem_cache {
>  	struct kmem_cache_node *node[MAX_NUMNODES];
>  };

How about maintaining cpu_partial when !CONFIG_SLUB_CPU_PARTIAL?
It makes code less churn and doesn't have much overhead.
At bottom, my implementation with cpu_partial is attached. It uses less '#ifdef'.

>  
> +static inline int kmem_cache_cpu_partial(struct kmem_cache *s)
> +{
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
> +	return s->cpu_partial;
> +#else
> +	return 0;
> +#endif
> +}
> +
>  void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
>  void *__kmalloc(size_t size, gfp_t flags);
>  
> Index: linux/mm/slub.c
> ===================================================================
> --- linux.orig/mm/slub.c	2013-06-14 09:50:58.190453865 -0500
> +++ linux/mm/slub.c	2013-06-14 09:50:58.186453794 -0500
> @@ -1573,7 +1573,8 @@ static void *get_partial_node(struct kme
>  			put_cpu_partial(s, page, 0);
>  			stat(s, CPU_PARTIAL_NODE);
>  		}
> -		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
> +		if (kmem_cache_debug(s) ||
> +			       available > kmem_cache_cpu_partial(s) / 2)
>  			break;
>  
>  	}
> @@ -1884,6 +1885,7 @@ redo:
>  static void unfreeze_partials(struct kmem_cache *s,
>  		struct kmem_cache_cpu *c)
>  {
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  	struct kmem_cache_node *n = NULL, *n2 = NULL;
>  	struct page *page, *discard_page = NULL;
>  
> @@ -1938,6 +1940,7 @@ static void unfreeze_partials(struct kme
>  		discard_slab(s, page);
>  		stat(s, FREE_SLAB);
>  	}
> +#endif
>  }
>  
>  /*
> @@ -1951,6 +1954,7 @@ static void unfreeze_partials(struct kme
>   */
>  static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
>  {
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  	struct page *oldpage;
>  	int pages;
>  	int pobjects;
> @@ -1987,6 +1991,7 @@ static void put_cpu_partial(struct kmem_
>  		page->next = oldpage;
>  
>  	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
> +#endif
>  }
>  
>  static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
> @@ -2495,6 +2500,7 @@ static void __slab_free(struct kmem_cach
>  		new.inuse--;
>  		if ((!new.inuse || !prior) && !was_frozen) {
>  
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  			if (!kmem_cache_debug(s) && !prior)
>  
>  				/*
> @@ -2503,7 +2509,9 @@ static void __slab_free(struct kmem_cach
>  				 */
>  				new.frozen = 1;
>  
> -			else { /* Needs to be taken off a list */
> +			else
> +#endif
> +		       		{ /* Needs to be taken off a list */
>  
>  	                        n = get_node(s, page_to_nid(page));
>  				/*
> @@ -2525,6 +2533,7 @@ static void __slab_free(struct kmem_cach
>  		"__slab_free"));
>  
>  	if (likely(!n)) {
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  
>  		/*
>  		 * If we just froze the page then put it onto the
> @@ -2534,6 +2543,7 @@ static void __slab_free(struct kmem_cach
>  			put_cpu_partial(s, page, 1);
>  			stat(s, CPU_PARTIAL_FREE);
>  		}
> +#endif
>  		/*
>  		 * The list lock was not taken therefore no list
>  		 * activity can be necessary.
> @@ -3041,7 +3051,7 @@ static int kmem_cache_open(struct kmem_c
>  	 * list to avoid pounding the page allocator excessively.
>  	 */
>  	set_min_partial(s, ilog2(s->size) / 2);
> -
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  	/*
>  	 * cpu_partial determined the maximum number of objects kept in the
>  	 * per cpu partial lists of a processor.
> @@ -3069,6 +3079,7 @@ static int kmem_cache_open(struct kmem_c
>  		s->cpu_partial = 13;
>  	else
>  		s->cpu_partial = 30;
> +#endif
>  
>  #ifdef CONFIG_NUMA
>  	s->remote_node_defrag_ratio = 1000;
> @@ -4424,7 +4435,7 @@ SLAB_ATTR(order);
>  
>  static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
>  {
> -	return sprintf(buf, "%lu\n", s->min_partial);
> +	return sprintf(buf, "%u\n", kmem_cache_cpu_partial(s));
>  }

min_partial is not related to cpu_partial.

>  
>  static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
> @@ -4444,7 +4455,7 @@ SLAB_ATTR(min_partial);
>  
>  static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
>  {
> -	return sprintf(buf, "%u\n", s->cpu_partial);
> +	return sprintf(buf, "%u\n", kmem_cache_cpu_partial(s));
>  }
>  
>  static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
> @@ -4458,10 +4469,13 @@ static ssize_t cpu_partial_store(struct
>  		return err;
>  	if (objects && kmem_cache_debug(s))
>  		return -EINVAL;
> -
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  	s->cpu_partial = objects;
>  	flush_all(s);
>  	return length;
> +#else
> +	return -ENOSYS;
> +#endif
>  }
>  SLAB_ATTR(cpu_partial);
>  
> Index: linux/init/Kconfig
> ===================================================================
> --- linux.orig/init/Kconfig	2013-06-14 09:50:58.190453865 -0500
> +++ linux/init/Kconfig	2013-06-14 09:50:58.186453794 -0500
> @@ -1559,6 +1559,17 @@ config SLOB
>  
>  endchoice
>  
> +config SLUB_CPU_PARTIAL
> +	default y
> +	depends on SLUB
> +	bool "SLUB per cpu partial cache"
> +	help
> +	  Per cpu partial caches accellerate objects allocation and freeing
> +	  that is local to a processor at the price of more indeterminism
> +	  in the latency of the free. On overflow these caches will be cleared
> +	  which requires the taking of locks that may cause latency spikes.
> +	  Typically one would choose no for a realtime system.
> +
>  config MMAP_ALLOW_UNINITIALIZED
>  	bool "Allow mmapped anonymous memory to be uninitialized"
>  	depends on EXPERT && !MMU
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>


-----------------------8<---------------------------
diff --git a/init/Kconfig b/init/Kconfig
index 2d9b831..a7ec1ec 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1559,6 +1559,17 @@ config SLOB
 
 endchoice
 
+config SLUB_CPU_PARTIAL
+	default y
+	depends on SLUB
+	bool "SLUB per cpu partial cache"
+	help
+	  Per cpu partial caches accellerate objects allocation and freeing
+	  that is local to a processor at the price of more indeterminism
+	  in the latency of the free. On overflow these caches will be cleared
+	  which requires the taking of locks that may cause latency spikes.
+	  Typically one would choose no for a realtime system.
+
 config MMAP_ALLOW_UNINITIALIZED
 	bool "Allow mmapped anonymous memory to be uninitialized"
 	depends on EXPERT && !MMU
diff --git a/mm/slub.c b/mm/slub.c
index 57707f0..c820f9d 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -123,6 +123,15 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
 #endif
 }
 
+static inline int kmem_cache_has_cpu_partial(struct kmem_cache *s)
+{
+#ifdef CONFIG_SLUB_PARTIAL
+	return !kmem_cache_debug(s);
+#else
+	return false;
+#endif
+}
+
 /*
  * Issues still to be resolved:
  *
@@ -1573,7 +1582,8 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 			put_cpu_partial(s, page, 0);
 			stat(s, CPU_PARTIAL_NODE);
 		}
-		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
+		if (!kmem_cache_has_cpu_partial(s)
+			|| available > s->cpu_partial / 2)
 			break;
 
 static void unfreeze_partials(struct kmem_cache *s,
 		struct kmem_cache_cpu *c)
 {
+#ifdef CONFIG_SLUB_PARTIAL
 	struct kmem_cache_node *n = NULL, *n2 = NULL;
 	struct page *page, *discard_page = NULL;
 
@@ -1938,6 +1949,7 @@ static void unfreeze_partials(struct kmem_cache *s,
 		discard_slab(s, page);
 		stat(s, FREE_SLAB);
 	}
+#endif
 }
 
 /*
@@ -1951,6 +1963,7 @@ static void unfreeze_partials(struct kmem_cache *s,
  */
 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
 {
+#ifdef CONFIG_SLUB_PARTIAL
 	struct page *oldpage;
 	int pages;
 	int pobjects;
@@ -1987,6 +2000,7 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
 		page->next = oldpage;
 
 	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
+#endif
 }
 
 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
@@ -2495,7 +2509,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 		new.inuse--;
 		if ((!new.inuse || !prior) && !was_frozen) {
 
-			if (!kmem_cache_debug(s) && !prior)
+			if (kmem_cache_has_cpu_partial(s) && !prior)
 
 				/*
 				 * Slab was on no list before and will be partially empty
@@ -3059,7 +3073,7 @@ static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
 	 *    per node list when we run out of per cpu objects. We only fetch 50%
 	 *    to keep some capacity around for frees.
 	 */
-	if (kmem_cache_debug(s))
+	if (!kmem_cache_has_cpu_partial(s))
 		s->cpu_partial = 0;
 	else if (s->size >= PAGE_SIZE)
 		s->cpu_partial = 2;
@@ -4456,7 +4470,7 @@ static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
 	err = strict_strtoul(buf, 10, &objects);
 	if (err)
 		return err;
-	if (objects && kmem_cache_debug(s))
+	if (objects && !kmem_cache_has_cpu_partial(s))
 		return -EINVAL;
 
 	s->cpu_partial = objects;
-- 
1.7.9.5


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Joonsoo Kim]

On Fri, Jun 14, 2013 at 08:06:36PM +0000, Christoph Lameter wrote:
> The kmalloc_node functions of all slab allcoators are similar now so
> lets move them into slab.h. This requires some function naming changes
> in slob.
> 
> Signed-off-by: Christoph Lameter <cl@linux.com>
> 
> Index: linux/include/linux/slab.h
> ===================================================================
> --- linux.orig/include/linux/slab.h	2013-06-14 13:40:52.424106451 -0500
> +++ linux/include/linux/slab.h	2013-06-14 14:45:24.000000000 -0500
> @@ -289,6 +289,38 @@ static __always_inline int kmalloc_index
>  }
>  #endif /* !CONFIG_SLOB */
>  
> +void *__kmalloc(size_t size, gfp_t flags);
> +void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags);
> +
> +#ifdef CONFIG_NUMA
> +void *__kmalloc_node(size_t size, gfp_t flags, int node);
> +void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
> +#else
> +static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
> +{
> +	return __kmalloc(size, flags);
> +}
> +
> +static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)
> +{
> +	return kmem_cache_alloc(s, flags);
> +}
> +#endif
> +
> +#ifdef CONFIG_TRACING
> +extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
> +					   gfp_t gfpflags,
> +					   int node);
> +#else
> +static __always_inline void *
> +kmem_cache_alloc_node_trace(struct kmem_cache *s,
> +			      gfp_t gfpflags,
> +			      int node)
> +{
> +	return kmem_cache_alloc_node(s, gfpflags, node);
> +}
> +#endif
> +
>  #ifdef CONFIG_SLAB
>  #include <linux/slab_def.h>
>  #endif
> @@ -321,6 +353,23 @@ static __always_inline int kmalloc_size(
>  	return 0;
>  }
>  
> +static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
> +{
> +#ifndef CONFIG_SLOB
> +	if (__builtin_constant_p(size) &&
> +		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {

s/SLAB_CACHE_DMA/GFP_DMA

> +		int i = kmalloc_index(size);
> +
> +		if (!i)
> +			return ZERO_SIZE_PTR;
> +
> +		return kmem_cache_alloc_node_trace(kmalloc_caches[i],
> +			       			flags, node);
> +	}
> +#endif
> +	return __kmalloc_node(size, flags, node);
> +}
> +
>  /*
>   * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
>   * Intended for arches that get misalignment faults even for 64 bit integer
> @@ -441,36 +490,6 @@ static inline void *kcalloc(size_t n, si
>  	return kmalloc_array(n, size, flags | __GFP_ZERO);
>  }
>  
> -#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB)
> -/**
> - * kmalloc_node - allocate memory from a specific node
> - * @size: how many bytes of memory are required.
> - * @flags: the type of memory to allocate (see kcalloc).
> - * @node: node to allocate from.
> - *
> - * kmalloc() for non-local nodes, used to allocate from a specific node
> - * if available. Equivalent to kmalloc() in the non-NUMA single-node
> - * case.
> - */
> -static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
> -{
> -	return kmalloc(size, flags);
> -}
> -
> -static inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
> -{
> -	return __kmalloc(size, flags);
> -}
> -
> -void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
> -
> -static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,
> -					gfp_t flags, int node)
> -{
> -	return kmem_cache_alloc(cachep, flags);
> -}
> -#endif /* !CONFIG_NUMA && !CONFIG_SLOB */
> -
>  /*
>   * kmalloc_track_caller is a special version of kmalloc that records the
>   * calling function of the routine calling it for slab leak tracking instead
> Index: linux/include/linux/slub_def.h
> ===================================================================
> --- linux.orig/include/linux/slub_def.h	2013-06-14 13:40:52.424106451 -0500
> +++ linux/include/linux/slub_def.h	2013-06-14 14:45:24.000000000 -0500
> @@ -115,9 +115,6 @@ static inline int kmem_cache_cpu_partial
>  #endif
>  }
>  
> -void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
> -void *__kmalloc(size_t size, gfp_t flags);
> -
>  static __always_inline void *
>  kmalloc_order(size_t size, gfp_t flags, unsigned int order)
>  {
> @@ -185,38 +182,4 @@ static __always_inline void *kmalloc(siz
>  	return __kmalloc(size, flags);
>  }
>  
> -#ifdef CONFIG_NUMA
> -void *__kmalloc_node(size_t size, gfp_t flags, int node);
> -void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
> -
> -#ifdef CONFIG_TRACING
> -extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
> -					   gfp_t gfpflags,
> -					   int node, size_t size);
> -#else
> -static __always_inline void *
> -kmem_cache_alloc_node_trace(struct kmem_cache *s,
> -			      gfp_t gfpflags,
> -			      int node, size_t size)
> -{
> -	return kmem_cache_alloc_node(s, gfpflags, node);
> -}
> -#endif
> -
> -static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
> -{
> -	if (__builtin_constant_p(size) &&
> -		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {
> -		int index = kmalloc_index(size);
> -
> -		if (!index)
> -			return ZERO_SIZE_PTR;
> -
> -		return kmem_cache_alloc_node_trace(kmalloc_caches[index],
> -			       flags, node, size);
> -	}
> -	return __kmalloc_node(size, flags, node);
> -}
> -#endif
> -
>  #endif /* _LINUX_SLUB_DEF_H */
> Index: linux/include/linux/slab_def.h
> ===================================================================
> --- linux.orig/include/linux/slab_def.h	2013-06-14 13:40:52.424106451 -0500
> +++ linux/include/linux/slab_def.h	2013-06-14 14:45:24.000000000 -0500
> @@ -102,9 +102,6 @@ struct kmem_cache {
>  	 */
>  };
>  
> -void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
> -void *__kmalloc(size_t size, gfp_t flags);
> -
>  #ifdef CONFIG_TRACING
>  extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);
>  #else
> @@ -145,53 +142,4 @@ static __always_inline void *kmalloc(siz
>  	return __kmalloc(size, flags);
>  }
>  
> -#ifdef CONFIG_NUMA
> -extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
> -extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
> -
> -#ifdef CONFIG_TRACING
> -extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
> -					 gfp_t flags,
> -					 int nodeid,
> -					 size_t size);
> -#else
> -static __always_inline void *
> -kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
> -			    gfp_t flags,
> -			    int nodeid,
> -			    size_t size)
> -{
> -	return kmem_cache_alloc_node(cachep, flags, nodeid);
> -}
> -#endif
> -
> -static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
> -{
> -	struct kmem_cache *cachep;
> -
> -	if (__builtin_constant_p(size)) {
> -		int i;
> -
> -		if (!size)
> -			return ZERO_SIZE_PTR;
> -
> -		if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))
> -			return NULL;
> -
> -		i = kmalloc_index(size);
> -
> -#ifdef CONFIG_ZONE_DMA
> -		if (flags & GFP_DMA)
> -			cachep = kmalloc_dma_caches[i];
> -		else
> -#endif
> -			cachep = kmalloc_caches[i];
> -
> -		return kmem_cache_alloc_node_trace(cachep, flags, node, size);
> -	}
> -	return __kmalloc_node(size, flags, node);
> -}
> -
> -#endif	/* CONFIG_NUMA */
> -
>  #endif	/* _LINUX_SLAB_DEF_H */
> Index: linux/include/linux/slob_def.h
> ===================================================================
> --- linux.orig/include/linux/slob_def.h	2013-06-14 13:40:52.424106451 -0500
> +++ linux/include/linux/slob_def.h	2013-06-14 14:45:24.000000000 -0500
> @@ -1,24 +1,7 @@
>  #ifndef __LINUX_SLOB_DEF_H
>  #define __LINUX_SLOB_DEF_H
>  
> -#include <linux/numa.h>
> -
> -void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
> -
> -static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep,
> -					      gfp_t flags)
> -{
> -	return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE);
> -}
> -
> -void *__kmalloc_node(size_t size, gfp_t flags, int node);
> -
> -static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
> -{
> -	return __kmalloc_node(size, flags, node);
> -}
> -
> -/**
> +/*
>   * kmalloc - allocate memory
>   * @size: how many bytes of memory are required.
>   * @flags: the type of memory to allocate (see kcalloc).
> @@ -31,9 +14,4 @@ static __always_inline void *kmalloc(siz
>  	return __kmalloc_node(size, flags, NUMA_NO_NODE);
>  }
>  
> -static __always_inline void *__kmalloc(size_t size, gfp_t flags)
> -{
> -	return kmalloc(size, flags);
> -}
> -
>  #endif /* __LINUX_SLOB_DEF_H */
> Index: linux/mm/slab.c
> ===================================================================
> --- linux.orig/mm/slab.c	2013-06-14 13:40:52.424106451 -0500
> +++ linux/mm/slab.c	2013-06-14 13:40:52.420106378 -0500
> @@ -3681,7 +3681,7 @@ __do_kmalloc_node(size_t size, gfp_t fla
>  	cachep = kmalloc_slab(size, flags);
>  	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
>  		return cachep;
> -	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
> +	return kmem_cache_alloc_node_trace(cachep, flags, node);
>  }
>  
>  #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
> Index: linux/mm/slob.c
> ===================================================================
> --- linux.orig/mm/slob.c	2013-06-14 13:14:08.000000000 -0500
> +++ linux/mm/slob.c	2013-06-14 14:44:56.812030812 -0500
> @@ -462,11 +462,11 @@ __do_kmalloc_node(size_t size, gfp_t gfp
>  	return ret;
>  }
>  
> -void *__kmalloc_node(size_t size, gfp_t gfp, int node)
> +void *__kmalloc(size_t size, gfp_t gfp)
>  {
> -	return __do_kmalloc_node(size, gfp, node, _RET_IP_);
> +	return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, _RET_IP_);
>  }
> -EXPORT_SYMBOL(__kmalloc_node);
> +EXPORT_SYMBOL(__kmalloc);
>  
>  #ifdef CONFIG_TRACING
>  void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller)
> @@ -534,7 +534,7 @@ int __kmem_cache_create(struct kmem_cach
>  	return 0;
>  }
>  
> -void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
> +void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
>  {
>  	void *b;
>  
> @@ -560,7 +560,27 @@ void *kmem_cache_alloc_node(struct kmem_
>  	kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags);
>  	return b;
>  }
> +EXPORT_SYMBOL(slob_alloc_node);
> +
> +void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> +{
> +	return slob_alloc_node(cachep, flags, NUMA_NO_NODE);
> +}
> +EXPORT_SYMBOL(kmem_cache_alloc);
> +
> +#ifdef CONFIG_NUMA
> +void *__kmalloc_node(size_t size, gfp_t gfp, int node)
> +{
> +	return __do_kmalloc_node(size, gfp, node, _RET_IP_);
> +}
> +EXPORT_SYMBOL(__kmalloc_node);
> +
> +void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node)
> +{
> +	return slob_alloc_node(cachep, gfp, node);
> +}
>  EXPORT_SYMBOL(kmem_cache_alloc_node);
> +#endif
>  
>  static void __kmem_cache_free(void *b, int size)
>  {
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Christoph Lameter]

On Wed, 19 Jun 2013, Joonsoo Kim wrote:

> How about maintaining cpu_partial when !CONFIG_SLUB_CPU_PARTIAL?
> It makes code less churn and doesn't have much overhead.
> At bottom, my implementation with cpu_partial is attached. It uses less '#ifdef'.

Looks good. I am fine with it.

Acked-by: Christoph Lameter <cl@linux.com>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Christoph Lameter]

On Wed, 19 Jun 2013, Joonsoo Kim wrote:

> > +#ifndef CONFIG_SLOB
> > +	if (__builtin_constant_p(size) &&
> > +		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {
>
> s/SLAB_CACHE_DMA/GFP_DMA

Ok. Could you remove the rest of the email in the future? Its difficult to
find your comment in the long diff.


Subject: slab.h: Use the correct GFP flag.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2013-06-18 11:53:29.000000000 -0500
+++ linux/include/linux/slab.h	2013-06-19 09:31:58.303069398 -0500
@@ -421,7 +421,7 @@ static __always_inline void *kmalloc_nod
 {
 #ifndef CONFIG_SLOB
 	if (__builtin_constant_p(size) &&
-		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {
+		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {
 		int i = kmalloc_index(size);

 		if (!i)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Joonsoo Kim]

On Wed, Jun 19, 2013 at 02:29:29PM +0000, Christoph Lameter wrote:
> On Wed, 19 Jun 2013, Joonsoo Kim wrote:
> 
> > How about maintaining cpu_partial when !CONFIG_SLUB_CPU_PARTIAL?
> > It makes code less churn and doesn't have much overhead.
> > At bottom, my implementation with cpu_partial is attached. It uses less '#ifdef'.
> 
> Looks good. I am fine with it.
> 
> Acked-by: Christoph Lameter <cl@linux.com>

Thanks!

Hello, Pekka.
I attach a right formatted patch with acked by Christoph and
signed off by me.

It is based on v3.10-rc6 and top of a patch
"slub: do not put a slab to cpu partial list when cpu_partial is 0".

> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@kvack.org.  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>


-----------------8<-----------------------------------------------

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Joonsoo Kim]

On Wed, Jun 19, 2013 at 02:33:57PM +0000, Christoph Lameter wrote:
> On Wed, 19 Jun 2013, Joonsoo Kim wrote:
> 
> > > +#ifndef CONFIG_SLOB
> > > +	if (__builtin_constant_p(size) &&
> > > +		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {
> >
> > s/SLAB_CACHE_DMA/GFP_DMA
> 
> Ok. Could you remove the rest of the email in the future? Its difficult to
> find your comment in the long diff.

Okay!

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Wanpeng Li]

On Thu, Jun 20, 2013 at 10:50:56AM +0900, Joonsoo Kim wrote:
>On Wed, Jun 19, 2013 at 02:29:29PM +0000, Christoph Lameter wrote:
>> On Wed, 19 Jun 2013, Joonsoo Kim wrote:
>> 
>> > How about maintaining cpu_partial when !CONFIG_SLUB_CPU_PARTIAL?
>> > It makes code less churn and doesn't have much overhead.
>> > At bottom, my implementation with cpu_partial is attached. It uses less '#ifdef'.
>> 
>> Looks good. I am fine with it.
>> 
>> Acked-by: Christoph Lameter <cl@linux.com>
>
>Thanks!
>
>Hello, Pekka.
>I attach a right formatted patch with acked by Christoph and
>signed off by me.
>
>It is based on v3.10-rc6 and top of a patch
>"slub: do not put a slab to cpu partial list when cpu_partial is 0".
>
>> 
>> --
>> To unsubscribe, send a message with 'unsubscribe linux-mm' in
>> the body to majordomo@kvack.org.  For more info on Linux MM,
>> see: http://www.linux-mm.org/ .
>> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
>
>
>-----------------8<-----------------------------------------------
>>From a3257adcff89fd89a7ecb26c1247eec511302807 Mon Sep 17 00:00:00 2001
>From: Joonsoo Kim <iamjoonsoo.kim@lge.com>
>Date: Wed, 19 Jun 2013 14:05:52 +0900
>Subject: [PATCH] slub: Make cpu partial slab support configurable
>
>cpu partial support can introduce level of indeterminism that is not
>wanted in certain context (like a realtime kernel). Make it configurable.
>
>This patch is based on Christoph Lameter's
>"slub: Make cpu partial slab support configurable V2".
>

As you know, actually cpu_partial is the maximum number of objects kept 
in the per cpu slab and cpu partial lists of a processor instead of 
just the maximum number of objects kept in cpu partial lists of a
processor. The allocation will always fallback to slow path if not 
config SLUB_CPU_PARTIAL, whether it will lead to more latency?

Regards,
Wanpeng Li 

>Acked-by: Christoph Lameter <cl@linux.com>
>Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
>
>diff --git a/init/Kconfig b/init/Kconfig
>index 2d9b831..a7ec1ec 100644
>--- a/init/Kconfig
>+++ b/init/Kconfig
>@@ -1559,6 +1559,17 @@ config SLOB
>
> endchoice
>
>+config SLUB_CPU_PARTIAL
>+	default y
>+	depends on SLUB
>+	bool "SLUB per cpu partial cache"
>+	help
>+	  Per cpu partial caches accellerate objects allocation and freeing
>+	  that is local to a processor at the price of more indeterminism
>+	  in the latency of the free. On overflow these caches will be cleared
>+	  which requires the taking of locks that may cause latency spikes.
>+	  Typically one would choose no for a realtime system.
>+
> config MMAP_ALLOW_UNINITIALIZED
> 	bool "Allow mmapped anonymous memory to be uninitialized"
> 	depends on EXPERT && !MMU
>diff --git a/mm/slub.c b/mm/slub.c
>index 7033b4f..a670d22 100644
>--- a/mm/slub.c
>+++ b/mm/slub.c
>@@ -123,6 +123,15 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
> #endif
> }
>
>+static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
>+{
>+#ifdef CONFIG_SLUB_CPU_PARTIAL
>+	return !kmem_cache_debug(s);
>+#else
>+	return false;
>+#endif
>+}
>+
> /*
>  * Issues still to be resolved:
>  *
>@@ -1573,7 +1582,8 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
> 			put_cpu_partial(s, page, 0);
> 			stat(s, CPU_PARTIAL_NODE);
> 		}
>-		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
>+		if (!kmem_cache_has_cpu_partial(s)
>+			|| available > s->cpu_partial / 2)
> 			break;
>
> 	}
>@@ -1884,6 +1894,7 @@ redo:
> static void unfreeze_partials(struct kmem_cache *s,
> 		struct kmem_cache_cpu *c)
> {
>+#ifdef CONFIG_SLUB_CPU_PARTIAL
> 	struct kmem_cache_node *n = NULL, *n2 = NULL;
> 	struct page *page, *discard_page = NULL;
>
>@@ -1938,6 +1949,7 @@ static void unfreeze_partials(struct kmem_cache *s,
> 		discard_slab(s, page);
> 		stat(s, FREE_SLAB);
> 	}
>+#endif
> }
>
> /*
>@@ -1951,6 +1963,7 @@ static void unfreeze_partials(struct kmem_cache *s,
>  */
> static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
> {
>+#ifdef CONFIG_SLUB_CPU_PARTIAL
> 	struct page *oldpage;
> 	int pages;
> 	int pobjects;
>@@ -1990,6 +2003,7 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
> 		page->next = oldpage;
>
> 	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
>+#endif
> }
>
> static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
>@@ -2498,7 +2512,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
> 		new.inuse--;
> 		if ((!new.inuse || !prior) && !was_frozen) {
>
>-			if (!kmem_cache_debug(s) && !prior)
>+			if (kmem_cache_has_cpu_partial(s) && !prior)
>
> 				/*
> 				 * Slab was on no list before and will be partially empty
>@@ -3062,7 +3076,7 @@ static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
> 	 *    per node list when we run out of per cpu objects. We only fetch 50%
> 	 *    to keep some capacity around for frees.
> 	 */
>-	if (kmem_cache_debug(s))
>+	if (!kmem_cache_has_cpu_partial(s))
> 		s->cpu_partial = 0;
> 	else if (s->size >= PAGE_SIZE)
> 		s->cpu_partial = 2;
>@@ -4459,7 +4473,7 @@ static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
> 	err = strict_strtoul(buf, 10, &objects);
> 	if (err)
> 		return err;
>-	if (objects && kmem_cache_debug(s))
>+	if (objects && !kmem_cache_has_cpu_partial(s))
> 		return -EINVAL;
>
> 	s->cpu_partial = objects;
>-- 
>1.7.9.5
>
>--
>To unsubscribe, send a message with 'unsubscribe linux-mm' in
>the body to majordomo@kvack.org.  For more info on Linux MM,
>see: http://www.linux-mm.org/ .
>Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Wanpeng Li]

On Thu, Jun 20, 2013 at 10:50:56AM +0900, Joonsoo Kim wrote:
>On Wed, Jun 19, 2013 at 02:29:29PM +0000, Christoph Lameter wrote:
>> On Wed, 19 Jun 2013, Joonsoo Kim wrote:
>> 
>> > How about maintaining cpu_partial when !CONFIG_SLUB_CPU_PARTIAL?
>> > It makes code less churn and doesn't have much overhead.
>> > At bottom, my implementation with cpu_partial is attached. It uses less '#ifdef'.
>> 
>> Looks good. I am fine with it.
>> 
>> Acked-by: Christoph Lameter <cl@linux.com>
>
>Thanks!
>
>Hello, Pekka.
>I attach a right formatted patch with acked by Christoph and
>signed off by me.
>
>It is based on v3.10-rc6 and top of a patch
>"slub: do not put a slab to cpu partial list when cpu_partial is 0".
>
>> 
>> --
>> To unsubscribe, send a message with 'unsubscribe linux-mm' in
>> the body to majordomo@kvack.org.  For more info on Linux MM,
>> see: http://www.linux-mm.org/ .
>> Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
>
>
>-----------------8<-----------------------------------------------
>>From a3257adcff89fd89a7ecb26c1247eec511302807 Mon Sep 17 00:00:00 2001
>From: Joonsoo Kim <iamjoonsoo.kim@lge.com>
>Date: Wed, 19 Jun 2013 14:05:52 +0900
>Subject: [PATCH] slub: Make cpu partial slab support configurable
>
>cpu partial support can introduce level of indeterminism that is not
>wanted in certain context (like a realtime kernel). Make it configurable.
>
>This patch is based on Christoph Lameter's
>"slub: Make cpu partial slab support configurable V2".
>

As you know, actually cpu_partial is the maximum number of objects kept 
in the per cpu slab and cpu partial lists of a processor instead of 
just the maximum number of objects kept in cpu partial lists of a
processor. The allocation will always fallback to slow path if not 
config SLUB_CPU_PARTIAL, whether it will lead to more latency?

Regards,
Wanpeng Li 

>Acked-by: Christoph Lameter <cl@linux.com>
>Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
>
>diff --git a/init/Kconfig b/init/Kconfig
>index 2d9b831..a7ec1ec 100644
>--- a/init/Kconfig
>+++ b/init/Kconfig
>@@ -1559,6 +1559,17 @@ config SLOB
>
> endchoice
>
>+config SLUB_CPU_PARTIAL
>+	default y
>+	depends on SLUB
>+	bool "SLUB per cpu partial cache"
>+	help
>+	  Per cpu partial caches accellerate objects allocation and freeing
>+	  that is local to a processor at the price of more indeterminism
>+	  in the latency of the free. On overflow these caches will be cleared
>+	  which requires the taking of locks that may cause latency spikes.
>+	  Typically one would choose no for a realtime system.
>+
> config MMAP_ALLOW_UNINITIALIZED
> 	bool "Allow mmapped anonymous memory to be uninitialized"
> 	depends on EXPERT && !MMU
>diff --git a/mm/slub.c b/mm/slub.c
>index 7033b4f..a670d22 100644
>--- a/mm/slub.c
>+++ b/mm/slub.c
>@@ -123,6 +123,15 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
> #endif
> }
>
>+static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
>+{
>+#ifdef CONFIG_SLUB_CPU_PARTIAL
>+	return !kmem_cache_debug(s);
>+#else
>+	return false;
>+#endif
>+}
>+
> /*
>  * Issues still to be resolved:
>  *
>@@ -1573,7 +1582,8 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
> 			put_cpu_partial(s, page, 0);
> 			stat(s, CPU_PARTIAL_NODE);
> 		}
>-		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
>+		if (!kmem_cache_has_cpu_partial(s)
>+			|| available > s->cpu_partial / 2)
> 			break;
>
> 	}
>@@ -1884,6 +1894,7 @@ redo:
> static void unfreeze_partials(struct kmem_cache *s,
> 		struct kmem_cache_cpu *c)
> {
>+#ifdef CONFIG_SLUB_CPU_PARTIAL
> 	struct kmem_cache_node *n = NULL, *n2 = NULL;
> 	struct page *page, *discard_page = NULL;
>
>@@ -1938,6 +1949,7 @@ static void unfreeze_partials(struct kmem_cache *s,
> 		discard_slab(s, page);
> 		stat(s, FREE_SLAB);
> 	}
>+#endif
> }
>
> /*
>@@ -1951,6 +1963,7 @@ static void unfreeze_partials(struct kmem_cache *s,
>  */
> static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
> {
>+#ifdef CONFIG_SLUB_CPU_PARTIAL
> 	struct page *oldpage;
> 	int pages;
> 	int pobjects;
>@@ -1990,6 +2003,7 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
> 		page->next = oldpage;
>
> 	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
>+#endif
> }
>
> static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
>@@ -2498,7 +2512,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
> 		new.inuse--;
> 		if ((!new.inuse || !prior) && !was_frozen) {
>
>-			if (!kmem_cache_debug(s) && !prior)
>+			if (kmem_cache_has_cpu_partial(s) && !prior)
>
> 				/*
> 				 * Slab was on no list before and will be partially empty
>@@ -3062,7 +3076,7 @@ static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
> 	 *    per node list when we run out of per cpu objects. We only fetch 50%
> 	 *    to keep some capacity around for frees.
> 	 */
>-	if (kmem_cache_debug(s))
>+	if (!kmem_cache_has_cpu_partial(s))
> 		s->cpu_partial = 0;
> 	else if (s->size >= PAGE_SIZE)
> 		s->cpu_partial = 2;
>@@ -4459,7 +4473,7 @@ static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
> 	err = strict_strtoul(buf, 10, &objects);
> 	if (err)
> 		return err;
>-	if (objects && kmem_cache_debug(s))
>+	if (objects && !kmem_cache_has_cpu_partial(s))
> 		return -EINVAL;
>
> 	s->cpu_partial = objects;
>-- 
>1.7.9.5
>
>--
>To unsubscribe, send a message with 'unsubscribe linux-mm' in
>the body to majordomo@kvack.org.  For more info on Linux MM,
>see: http://www.linux-mm.org/ .
>Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Joonsoo Kim]

On Thu, Jun 20, 2013 at 10:53:36AM +0800, Wanpeng Li wrote:
> On Thu, Jun 20, 2013 at 10:50:56AM +0900, Joonsoo Kim wrote:
> >On Wed, Jun 19, 2013 at 02:29:29PM +0000, Christoph Lameter wrote:
> >
> >
> >-----------------8<-----------------------------------------------
> >>From a3257adcff89fd89a7ecb26c1247eec511302807 Mon Sep 17 00:00:00 2001
> >From: Joonsoo Kim <iamjoonsoo.kim@lge.com>
> >Date: Wed, 19 Jun 2013 14:05:52 +0900
> >Subject: [PATCH] slub: Make cpu partial slab support configurable
> >
> >cpu partial support can introduce level of indeterminism that is not
> >wanted in certain context (like a realtime kernel). Make it configurable.
> >
> >This patch is based on Christoph Lameter's
> >"slub: Make cpu partial slab support configurable V2".
> >
> 
> As you know, actually cpu_partial is the maximum number of objects kept 
> in the per cpu slab and cpu partial lists of a processor instead of 
> just the maximum number of objects kept in cpu partial lists of a
> processor. The allocation will always fallback to slow path if not 
> config SLUB_CPU_PARTIAL, whether it will lead to more latency?

No, the SLUB maintain a cpu slab even if s->cpu_partial is 0.
It is a violation of definition of cpu_partial as you pointed out, but,
current implementation do this way.

Thanks.

> 
> Regards,
> Wanpeng Li 
> 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Joonsoo Kim]

On Thu, Jun 20, 2013 at 10:50:56AM +0900, Joonsoo Kim wrote:
> Hello, Pekka.
> I attach a right formatted patch with acked by Christoph and
> signed off by me.
> 
> It is based on v3.10-rc6 and top of a patch
> "slub: do not put a slab to cpu partial list when cpu_partial is 0".

One more change is needed in __slab_free(), so I attach v2.

------------------8<-------------------------------------

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Steven Rostedt]

On Tue, 2013-06-18 at 18:25 +0300, Pekka Enberg wrote:
> On Tue, Jun 18, 2013 at 6:21 PM, Clark Williams <williams@redhat.com> wrote:
> > I'm sure it would be better to actually do cpu_partial processing in
> > small chunks to avoid latency spikes in latency sensitive applications
> 
> Sounds like a patch I'd be much more interested in applying...

Is this going to happen, otherwise we would really like a fix for RT.

-- Steve


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Christoph Lameter]

On Tue, 25 Jun 2013, Steven Rostedt wrote:

> On Tue, 2013-06-18 at 18:25 +0300, Pekka Enberg wrote:
> > On Tue, Jun 18, 2013 at 6:21 PM, Clark Williams <williams@redhat.com> wrote:
> > > I'm sure it would be better to actually do cpu_partial processing in
> > > small chunks to avoid latency spikes in latency sensitive applications
> >
> > Sounds like a patch I'd be much more interested in applying...
>
> Is this going to happen, otherwise we would really like a fix for RT.

Forget it. Just switch cpu_partial processing off. It will be in small
chunks then.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 970 bytes --]

On Mon, 1 Jul 2013 18:16:35 +0000
Christoph Lameter <cl@linux.com> wrote:

> On Tue, 25 Jun 2013, Steven Rostedt wrote:
> 
> > On Tue, 2013-06-18 at 18:25 +0300, Pekka Enberg wrote:
> > > On Tue, Jun 18, 2013 at 6:21 PM, Clark Williams <williams@redhat.com> wrote:
> > > > I'm sure it would be better to actually do cpu_partial processing in
> > > > small chunks to avoid latency spikes in latency sensitive applications
> > >
> > > Sounds like a patch I'd be much more interested in applying...
> >
> > Is this going to happen, otherwise we would really like a fix for RT.
> 
> Forget it. Just switch cpu_partial processing off. It will be in small
> chunks then.
> 

What's your recommended method for switching cpu_partial processing
off? 

I'm not all that keen on repeatedly traversing /sys/kernel/slab looking
for 'cpu_partial' entries, mainly because if you do it at boot time
(i.e. from a startup script) you miss some of the entries. 


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Christoph Lameter]

On Tue, 2 Jul 2013, Clark Williams wrote:

> What's your recommended method for switching cpu_partial processing
> off?
>
> I'm not all that keen on repeatedly traversing /sys/kernel/slab looking
> for 'cpu_partial' entries, mainly because if you do it at boot time
> (i.e. from a startup script) you miss some of the entries.

Merge the patch that makes a config option and compile it out of the
kernel?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Clark Williams]

[-- Attachment #1: Type: text/plain, Size: 584 bytes --]

On Tue, 2 Jul 2013 16:47:01 +0000
Christoph Lameter <cl@linux.com> wrote:

> On Tue, 2 Jul 2013, Clark Williams wrote:
> 
> > What's your recommended method for switching cpu_partial processing
> > off?
> >
> > I'm not all that keen on repeatedly traversing /sys/kernel/slab looking
> > for 'cpu_partial' entries, mainly because if you do it at boot time
> > (i.e. from a startup script) you miss some of the entries.
> 
> Merge the patch that makes a config option and compile it out of the
> kernel?
> 

Ah, ok that makes sense. I thought you meant an alternative.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Pekka Enberg]

On Thu, Jun 20, 2013 at 8:50 AM, Joonsoo Kim <iamjoonsoo.kim@lge.com> wrote:
> On Thu, Jun 20, 2013 at 10:50:56AM +0900, Joonsoo Kim wrote:
>> Hello, Pekka.
>> I attach a right formatted patch with acked by Christoph and
>> signed off by me.
>>
>> It is based on v3.10-rc6 and top of a patch
>> "slub: do not put a slab to cpu partial list when cpu_partial is 0".
>
> One more change is needed in __slab_free(), so I attach v2.
>
> ------------------8<-------------------------------------
> From 22fef26a9775745a041ca8820971d475714ee351 Mon Sep 17 00:00:00 2001
> From: Joonsoo Kim <iamjoonsoo.kim@lge.com>
> Date: Wed, 19 Jun 2013 14:05:52 +0900
> Subject: [PATCH v2] slub: Make cpu partial slab support configurable
>
> cpu partial support can introduce level of indeterminism that is not
> wanted in certain context (like a realtime kernel). Make it configurable.
>
> This patch is based on Christoph Lameter's
> "slub: Make cpu partial slab support configurable V2".
>
> Acked-by: Christoph Lameter <cl@linux.com>
> Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>

Applied, thanks!

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Pekka Enberg]

On Tue, Jun 18, 2013 at 8:02 PM, Christoph Lameter <cl@linux.com> wrote:
> On Tue, 18 Jun 2013, Pekka Enberg wrote:
>
>> I'm seeing this after "make defconfig" on x86-64:
>>
>>   CC      mm/slub.o
>> mm/slub.c:2445:7: error: conflicting types for �kmem_cache_alloc_node_trace�
>> include/linux/slab.h:311:14: note: previous declaration of
>> �kmem_cache_alloc_node_trace� was here
>> mm/slub.c:2455:1: error: conflicting types for �kmem_cache_alloc_node_trace�
>> include/linux/slab.h:311:14: note: previous declaration of
>> �kmem_cache_alloc_node_trace� was here
>> make[1]: *** [mm/slub.o] Error 1
>> make: *** [mm/slub.o] Error 2
>
> Gosh I dropped the size_t parameter from these functions. CONFIG_TRACING
> needs these.
>
> Subject: Fix kmem_cache_alloc*_trace parameters
>
> The size parameter is needed.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>

Can you please resend as a single patch against slab/next that compiles?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 3/4] Move kmalloc_node functions to common code

[Christoph Lameter]

On Sun, 7 Jul 2013, Pekka Enberg wrote:

> Can you please resend as a single patch against slab/next that compiles?


Subject: Move kmalloc_node functions to common code

The kmalloc_node functions of all slab allcoators are similar now so
lets move them into slab.h. This requires some function naming changes
in slob.

Signed-off-by: Christoph Lameter <cl@linux.com>

Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h	2013-07-08 13:42:01.335620737 -0500
+++ linux/include/linux/slab.h	2013-07-08 13:42:57.500567780 -0500
@@ -289,6 +289,38 @@ static __always_inline int kmalloc_index
 }
 #endif /* !CONFIG_SLOB */

+void *__kmalloc(size_t size, gfp_t flags);
+void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags);
+
+#ifdef CONFIG_NUMA
+void *__kmalloc_node(size_t size, gfp_t flags, int node);
+void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
+#else
+static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
+{
+	return __kmalloc(size, flags);
+}
+
+static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)
+{
+	return kmem_cache_alloc(s, flags);
+}
+#endif
+
+#ifdef CONFIG_TRACING
+extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
+					   gfp_t gfpflags,
+					   int node);
+#else
+static __always_inline void *
+kmem_cache_alloc_node_trace(struct kmem_cache *s,
+			      gfp_t gfpflags,
+			      int node)
+{
+	return kmem_cache_alloc_node(s, gfpflags, node);
+}
+#endif
+
 #ifdef CONFIG_SLAB
 #include <linux/slab_def.h>
 #endif
@@ -321,6 +353,23 @@ static __always_inline int kmalloc_size(
 	return 0;
 }

+static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
+{
+#ifndef CONFIG_SLOB
+	if (__builtin_constant_p(size) &&
+		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {
+		int i = kmalloc_index(size);
+
+		if (!i)
+			return ZERO_SIZE_PTR;
+
+		return kmem_cache_alloc_node_trace(kmalloc_caches[i],
+			       			flags, node);
+	}
+#endif
+	return __kmalloc_node(size, flags, node);
+}
+
 /*
  * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
  * Intended for arches that get misalignment faults even for 64 bit integer
@@ -451,36 +500,6 @@ static inline void *kcalloc(size_t n, si
 	return kmalloc_array(n, size, flags | __GFP_ZERO);
 }

-#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB)
-/**
- * kmalloc_node - allocate memory from a specific node
- * @size: how many bytes of memory are required.
- * @flags: the type of memory to allocate (see kmalloc).
- * @node: node to allocate from.
- *
- * kmalloc() for non-local nodes, used to allocate from a specific node
- * if available. Equivalent to kmalloc() in the non-NUMA single-node
- * case.
- */
-static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	return kmalloc(size, flags);
-}
-
-static inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	return __kmalloc(size, flags);
-}
-
-void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
-
-static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,
-					gfp_t flags, int node)
-{
-	return kmem_cache_alloc(cachep, flags);
-}
-#endif /* !CONFIG_NUMA && !CONFIG_SLOB */
-
 /*
  * kmalloc_track_caller is a special version of kmalloc that records the
  * calling function of the routine calling it for slab leak tracking instead
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h	2013-07-08 13:42:01.335620737 -0500
+++ linux/include/linux/slub_def.h	2013-07-08 13:42:01.331620669 -0500
@@ -104,9 +104,6 @@ struct kmem_cache {
 	struct kmem_cache_node *node[MAX_NUMNODES];
 };

-void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
-void *__kmalloc(size_t size, gfp_t flags);
-
 static __always_inline void *
 kmalloc_order(size_t size, gfp_t flags, unsigned int order)
 {
@@ -174,38 +171,4 @@ static __always_inline void *kmalloc(siz
 	return __kmalloc(size, flags);
 }

-#ifdef CONFIG_NUMA
-void *__kmalloc_node(size_t size, gfp_t flags, int node);
-void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
-					   gfp_t gfpflags,
-					   int node, size_t size);
-#else
-static __always_inline void *
-kmem_cache_alloc_node_trace(struct kmem_cache *s,
-			      gfp_t gfpflags,
-			      int node, size_t size)
-{
-	return kmem_cache_alloc_node(s, gfpflags, node);
-}
-#endif
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	if (__builtin_constant_p(size) &&
-		size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) {
-		int index = kmalloc_index(size);
-
-		if (!index)
-			return ZERO_SIZE_PTR;
-
-		return kmem_cache_alloc_node_trace(kmalloc_caches[index],
-			       flags, node, size);
-	}
-	return __kmalloc_node(size, flags, node);
-}
-#endif
-
 #endif /* _LINUX_SLUB_DEF_H */
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h	2013-07-08 13:42:01.335620737 -0500
+++ linux/include/linux/slab_def.h	2013-07-08 13:42:01.331620669 -0500
@@ -102,9 +102,6 @@ struct kmem_cache {
 	 */
 };

-void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
-void *__kmalloc(size_t size, gfp_t flags);
-
 #ifdef CONFIG_TRACING
 extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);
 #else
@@ -145,53 +142,4 @@ static __always_inline void *kmalloc(siz
 	return __kmalloc(size, flags);
 }

-#ifdef CONFIG_NUMA
-extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
-extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
-					 gfp_t flags,
-					 int nodeid,
-					 size_t size);
-#else
-static __always_inline void *
-kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
-			    gfp_t flags,
-			    int nodeid,
-			    size_t size)
-{
-	return kmem_cache_alloc_node(cachep, flags, nodeid);
-}
-#endif
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	struct kmem_cache *cachep;
-
-	if (__builtin_constant_p(size)) {
-		int i;
-
-		if (!size)
-			return ZERO_SIZE_PTR;
-
-		if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))
-			return NULL;
-
-		i = kmalloc_index(size);
-
-#ifdef CONFIG_ZONE_DMA
-		if (flags & GFP_DMA)
-			cachep = kmalloc_dma_caches[i];
-		else
-#endif
-			cachep = kmalloc_caches[i];
-
-		return kmem_cache_alloc_node_trace(cachep, flags, node, size);
-	}
-	return __kmalloc_node(size, flags, node);
-}
-
-#endif	/* CONFIG_NUMA */
-
 #endif	/* _LINUX_SLAB_DEF_H */
Index: linux/include/linux/slob_def.h
===================================================================
--- linux.orig/include/linux/slob_def.h	2013-07-08 13:42:01.335620737 -0500
+++ linux/include/linux/slob_def.h	2013-07-08 13:43:40.613294737 -0500
@@ -1,31 +1,9 @@
 #ifndef __LINUX_SLOB_DEF_H
 #define __LINUX_SLOB_DEF_H

-#include <linux/numa.h>
-
-void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep,
-					      gfp_t flags)
-{
-	return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE);
-}
-
-void *__kmalloc_node(size_t size, gfp_t flags, int node);
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
-	return __kmalloc_node(size, flags, node);
-}
-
 static __always_inline void *kmalloc(size_t size, gfp_t flags)
 {
 	return __kmalloc_node(size, flags, NUMA_NO_NODE);
 }

-static __always_inline void *__kmalloc(size_t size, gfp_t flags)
-{
-	return kmalloc(size, flags);
-}
-
 #endif /* __LINUX_SLOB_DEF_H */
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c	2013-07-08 13:42:01.335620737 -0500
+++ linux/mm/slab.c	2013-07-08 13:42:01.331620669 -0500
@@ -3686,7 +3686,7 @@ __do_kmalloc_node(size_t size, gfp_t fla
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
-	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
+	return kmem_cache_alloc_node_trace(cachep, flags, node);
 }

 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
Index: linux/mm/slob.c
===================================================================
--- linux.orig/mm/slob.c	2013-07-08 13:42:01.335620737 -0500
+++ linux/mm/slob.c	2013-07-08 13:42:01.331620669 -0500
@@ -462,11 +462,11 @@ __do_kmalloc_node(size_t size, gfp_t gfp
 	return ret;
 }

-void *__kmalloc_node(size_t size, gfp_t gfp, int node)
+void *__kmalloc(size_t size, gfp_t gfp)
 {
-	return __do_kmalloc_node(size, gfp, node, _RET_IP_);
+	return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, _RET_IP_);
 }
-EXPORT_SYMBOL(__kmalloc_node);
+EXPORT_SYMBOL(__kmalloc);

 #ifdef CONFIG_TRACING
 void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller)
@@ -534,7 +534,7 @@ int __kmem_cache_create(struct kmem_cach
 	return 0;
 }

-void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
+void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 {
 	void *b;

@@ -560,7 +560,27 @@ void *kmem_cache_alloc_node(struct kmem_
 	kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags);
 	return b;
 }
+EXPORT_SYMBOL(slob_alloc_node);
+
+void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
+{
+	return slob_alloc_node(cachep, flags, NUMA_NO_NODE);
+}
+EXPORT_SYMBOL(kmem_cache_alloc);
+
+#ifdef CONFIG_NUMA
+void *__kmalloc_node(size_t size, gfp_t gfp, int node)
+{
+	return __do_kmalloc_node(size, gfp, node, _RET_IP_);
+}
+EXPORT_SYMBOL(__kmalloc_node);
+
+void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node)
+{
+	return slob_alloc_node(cachep, gfp, node);
+}
 EXPORT_SYMBOL(kmem_cache_alloc_node);
+#endif

 static void __kmem_cache_free(void *b, int size)
 {

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Steven Rostedt]

On Tue, 2013-07-02 at 16:47 +0000, Christoph Lameter wrote:
> On Tue, 2 Jul 2013, Clark Williams wrote:
> 
> > What's your recommended method for switching cpu_partial processing
> > off?
> >
> > I'm not all that keen on repeatedly traversing /sys/kernel/slab looking
> > for 'cpu_partial' entries, mainly because if you do it at boot time
> > (i.e. from a startup script) you miss some of the entries.
> 
> Merge the patch that makes a config option and compile it out of the
> kernel?

When I run a stress test of the box (kernel compile along with
hackbench), with that patch applied, the system hangs for long periods
of time. I have no idea why, but the oom killer would trigger
constantly.

Anyway, I'm thinking of just applying this patch. I think it would work
for -rt.

-- Steve

diff --git a/mm/slub.c b/mm/slub.c
index 75a8ffd..a288e72 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -124,6 +124,18 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
 #endif
 }
 
+#ifdef CONFIG_PREEMPT_RT_FULL
+static inline int kmem_cache_debug_rt(struct kmem_cache *s)
+{
+	return 1;
+}
+#else
+static inline int kmem_cache_debug_rt(struct kmem_cache *s)
+{
+	return kmem_cache_debug(s);
+}
+#endif
+
 /*
  * Issues still to be resolved:
  *
@@ -3147,7 +3159,7 @@ static int kmem_cache_open(struct kmem_cache *s,
 	 *    per node list when we run out of per cpu objects. We only fetch 50%
 	 *    to keep some capacity around for frees.
 	 */
-	if (kmem_cache_debug(s))
+	if (kmem_cache_debug_rt(s))
 		s->cpu_partial = 0;
 	else if (s->size >= PAGE_SIZE)
 		s->cpu_partial = 2;


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Pekka Enberg]

Hi Steven,

On Wed, Jul 17, 2013 at 5:46 AM, Steven Rostedt <rostedt@goodmis.org> wrote:
> When I run a stress test of the box (kernel compile along with
> hackbench), with that patch applied, the system hangs for long periods
> of time. I have no idea why, but the oom killer would trigger
> constantly.

Are you using this patch or the modified version by Joonsoo that
landed in Linus' tree?

                        Pekka

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Steven Rostedt]

On Wed, 2013-07-17 at 10:04 +0300, Pekka Enberg wrote:
> Hi Steven,
> 
> On Wed, Jul 17, 2013 at 5:46 AM, Steven Rostedt <rostedt@goodmis.org> wrote:
> > When I run a stress test of the box (kernel compile along with
> > hackbench), with that patch applied, the system hangs for long periods
> > of time. I have no idea why, but the oom killer would trigger
> > constantly.
> 
> Are you using this patch or the modified version by Joonsoo that
> landed in Linus' tree?

I was using the patch that Christoph posted. It was also attached to the
-rt kernel for 3.6. I could have backported the patch poorly too.

-- Steve


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Christoph Lameter]

On Wed, 17 Jul 2013, Steven Rostedt wrote:

> I was using the patch that Christoph posted. It was also attached to the
> -rt kernel for 3.6. I could have backported the patch poorly too.

Could you try upstream?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Steven Rostedt]

On Wed, 2013-07-17 at 15:04 +0000, Christoph Lameter wrote:
> On Wed, 17 Jul 2013, Steven Rostedt wrote:
> 
> > I was using the patch that Christoph posted. It was also attached to the
> > -rt kernel for 3.6. I could have backported the patch poorly too.
> 
> Could you try upstream?

Yeah, I'm running it now.

-- Steve


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [3.11 1/4] slub: Make cpu partial slab support configurable V2

[Steven Rostedt]

On Wed, 2013-07-17 at 11:15 -0400, Steven Rostedt wrote:
> On Wed, 2013-07-17 at 15:04 +0000, Christoph Lameter wrote:
> > On Wed, 17 Jul 2013, Steven Rostedt wrote:
> > 
> > > I was using the patch that Christoph posted. It was also attached to the
> > > -rt kernel for 3.6. I could have backported the patch poorly too.
> > 
> > Could you try upstream?
> 
> Yeah, I'm running it now.

It's been running fine for about an hour, where my backport locked up in
minutes.

Thus, I'm not blaming Christoph here. It could be either -rt related, or
3.6 backport related.

-- Steve


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>