Re: [PATCH] slob: poor man's NUMA, take 2.

[Paul Mundt <lethal@linux-sh.org>]

On Tue, Jun 12, 2007 at 10:28:57PM -0500, Matt Mackall wrote:
> On Wed, Jun 13, 2007 at 12:12:03PM +0900, Paul Mundt wrote:
> > +static inline void *slob_new_page(gfp_t gfp, int order, int node)
> > +{
> > +	void *page;
> > +
> > +#ifdef CONFIG_NUMA
> > +	if (node != -1)
> > +		page = alloc_pages_node(node, gfp, order);
> > +	else
> > +#endif
> > +		page = alloc_pages(gfp, order);
> > +
> > +	if (!page)
> > +		return NULL;
> > +
> > +	return page_address(page);
> 
> We might want to leave the inlining decision here to the compiler. The
> ifdef may change that decision..
> 
> > -void *__kmalloc(size_t size, gfp_t gfp)
> > +static void *slob_node_alloc(size_t size, gfp_t gfp, int node)
> 
> See my comment in the last message.
> 
Here's an updated copy with the node variants always defined.

I've left the nid=-1 case in as the default for the non-node variants, as
this is the approach also used by SLUB. alloc_pages() is special cased
for NUMA, and takes the memory policy under advisement when doing the
allocation, so the page ends up in a reasonable place.

Signed-off-by: Paul Mundt <lethal@linux-sh.org>

--

 include/linux/slab.h |   11 +++++++--
 mm/slob.c            |   59 ++++++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/include/linux/slab.h b/include/linux/slab.h
index a015236..97d9b0a 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -63,7 +63,7 @@ int kmem_ptr_validate(struct kmem_cache *cachep, const void *ptr);
 		sizeof(struct __struct), __alignof__(struct __struct),\
 		(__flags), NULL, NULL)
 
-#ifdef CONFIG_NUMA
+#if defined(CONFIG_NUMA) || defined(CONFIG_SLOB)
 extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
 #else
 static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,
@@ -190,7 +190,14 @@ static inline void *kzalloc(size_t size, gfp_t flags)
 }
 #endif
 
-#ifndef CONFIG_NUMA
+#if defined(CONFIG_SLOB)
+extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
+
+static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
+{
+	return __kmalloc_node(size, flags, node);
+}
+#elif !defined(CONFIG_NUMA)
 static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
 	return kmalloc(size, flags);
diff --git a/mm/slob.c b/mm/slob.c
index 06e5e72..07e3730 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -204,6 +204,23 @@ static int slob_last(slob_t *s)
 	return !((unsigned long)slob_next(s) & ~PAGE_MASK);
 }
 
+static void *slob_new_page(gfp_t gfp, int order, int node)
+{
+	void *page;
+
+#ifdef CONFIG_NUMA
+	if (node != -1)
+		page = alloc_pages_node(node, gfp, order);
+	else
+#endif
+		page = alloc_pages(gfp, order);
+
+	if (!page)
+		return NULL;
+
+	return page_address(page);
+}
+
 /*
  * Allocate a slob block within a given slob_page sp.
  */
@@ -258,7 +275,7 @@ static void *slob_page_alloc(struct slob_page *sp, size_t size, int align)
 /*
  * slob_alloc: entry point into the slob allocator.
  */
-static void *slob_alloc(size_t size, gfp_t gfp, int align)
+static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)
 {
 	struct slob_page *sp;
 	slob_t *b = NULL;
@@ -267,6 +284,15 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align)
 	spin_lock_irqsave(&slob_lock, flags);
 	/* Iterate through each partially free page, try to find room */
 	list_for_each_entry(sp, &free_slob_pages, list) {
+#ifdef CONFIG_NUMA
+		/*
+		 * If there's a node specification, search for a partial
+		 * page with a matching node id in the freelist.
+		 */
+		if (node != -1 && page_to_nid(&sp->page) != node)
+			continue;
+#endif
+
 		if (sp->units >= SLOB_UNITS(size)) {
 			b = slob_page_alloc(sp, size, align);
 			if (b)
@@ -277,7 +303,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align)
 
 	/* Not enough space: must allocate a new page */
 	if (!b) {
-		b = (slob_t *)__get_free_page(gfp);
+		b = slob_new_page(gfp, 0, node);
 		if (!b)
 			return 0;
 		sp = (struct slob_page *)virt_to_page(b);
@@ -381,22 +407,20 @@ out:
 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long)
 #endif
 
-
-void *__kmalloc(size_t size, gfp_t gfp)
+void *__kmalloc_node(size_t size, gfp_t gfp, int node)
 {
 	int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
 
 	if (size < PAGE_SIZE - align) {
 		unsigned int *m;
-		m = slob_alloc(size + align, gfp, align);
+		m = slob_alloc(size + align, gfp, align, node);
 		if (m)
 			*m = size;
 		return (void *)m + align;
 	} else {
 		void *ret;
 
-		ret = (void *) __get_free_pages(gfp | __GFP_COMP,
-						get_order(size));
+		ret = slob_new_page(gfp | __GFP_COMP, get_order(size), node);
 		if (ret) {
 			struct page *page;
 			page = virt_to_page(ret);
@@ -405,6 +429,12 @@ void *__kmalloc(size_t size, gfp_t gfp)
 		return ret;
 	}
 }
+EXPORT_SYMBOL(__kmalloc_node);
+
+void *__kmalloc(size_t size, gfp_t gfp)
+{
+	return __kmalloc_node(size, gfp, -1);
+}
 EXPORT_SYMBOL(__kmalloc);
 
 /**
@@ -455,7 +485,6 @@ void kfree(const void *block)
 	} else
 		put_page(&sp->page);
 }
-
 EXPORT_SYMBOL(kfree);
 
 /* can't use ksize for kmem_cache_alloc memory, only kmalloc */
@@ -487,7 +516,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
 {
 	struct kmem_cache *c;
 
-	c = slob_alloc(sizeof(struct kmem_cache), flags, 0);
+	c = slob_alloc(sizeof(struct kmem_cache), flags, 0, -1);
 
 	if (c) {
 		c->name = name;
@@ -517,20 +546,26 @@ void kmem_cache_destroy(struct kmem_cache *c)
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
 
-void *kmem_cache_alloc(struct kmem_cache *c, gfp_t flags)
+void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 {
 	void *b;
 
 	if (c->size < PAGE_SIZE)
-		b = slob_alloc(c->size, flags, c->align);
+		b = slob_alloc(c->size, flags, c->align, node);
 	else
-		b = (void *)__get_free_pages(flags, get_order(c->size));
+		b = slob_new_page(flags, get_order(c->size), node);
 
 	if (c->ctor)
 		c->ctor(b, c, 0);
 
 	return b;
 }
+EXPORT_SYMBOL(kmem_cache_alloc_node);
+
+void *kmem_cache_alloc(struct kmem_cache *c, gfp_t flags)
+{
+	return kmem_cache_alloc_node(c, flags, -1);
+}
 EXPORT_SYMBOL(kmem_cache_alloc);
 
 void *kmem_cache_zalloc(struct kmem_cache *c, gfp_t flags)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>