[PATCH] SLUB: use have_arch_cmpxchg()

[Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>]

SLUB: use have_arch_cmpxchg()

* Christoph Lameter (clameter@sgi.com) wrote:
> If we really want to do this then the implementation of all of these 
> components need to result in competitive performance on all platforms.
> 

Patch 2 of 2

Use have_arch_cmpxchg() to select between irq disable and cmpxchg_local.

Pro:
- Should provide the same performances on architectures that does not provide
  fast local_cmpxchg.
- Improved performances on architectures that provide fast local_cmpxchg.

Cons:
- Does not help code readability, i.e.:

if (have_arch_cmpxchg())
	preempt_disable();
else
	local_irq_save(flags);

It applies on top of the cmpxchg_local() slub patch.
It depends on the Add have_arch_cmpxchg() patch.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
---
 mm/slub.c |   65 +++++++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 46 insertions(+), 19 deletions(-)

Index: slab/mm/slub.c
===================================================================
--- slab.orig/mm/slub.c	2007-08-22 10:28:22.000000000 -0400
+++ slab/mm/slub.c	2007-08-22 10:51:53.000000000 -0400
@@ -1450,7 +1450,8 @@ static void *__slab_alloc(struct kmem_ca
 	void **freelist = NULL;
 	unsigned long flags;
 
-	local_irq_save(flags);
+	if (have_arch_cmpxchg())
+		local_irq_save(flags);
 	c = get_cpu_slab(s, smp_processor_id());
 	if (!c->page)
 		/* Slab was flushed */
@@ -1482,7 +1483,8 @@ out_object:
 	c->freelist = object[c->offset];
 out:
 	slab_unlock(c->page);
-	local_irq_restore(flags);
+	if (have_arch_cmpxchg())
+		local_irq_restore(flags);
 	if (unlikely((gfpflags & __GFP_ZERO)))
 		memset(object, 0, c->objsize);
 	return object;
@@ -1525,7 +1527,8 @@ new_slab:
 		c->page = new;
 		goto load_freelist;
 	}
-	local_irq_restore(flags);
+	if (have_arch_cmpxchg())
+		local_irq_restore(flags);
 	return NULL;
 debug:
 	object = c->page->freelist;
@@ -1553,8 +1556,12 @@ static void __always_inline *slab_alloc(
 {
 	void **object;
 	struct kmem_cache_cpu *c;
+	unsigned long flags;
 
-	preempt_disable();
+	if (have_arch_cmpxchg())
+		preempt_disable();
+	else
+		local_irq_save(flags);
 	c = get_cpu_slab(s, raw_smp_processor_id());
 	if (unlikely(!node_match(c, node)))
 		goto slow;
@@ -1564,15 +1571,21 @@ redo:
 	if (unlikely(!object))
 		goto slow;
 
-	if (unlikely(cmpxchg_local(&c->freelist, object,
-			object[c->offset]) != object))
-		goto redo;
-	preempt_enable();
+	if (have_arch_cmpxchg()) {
+		if (unlikely(cmpxchg_local(&c->freelist, object,
+				object[c->offset]) != object))
+			goto redo;
+		preempt_enable();
+	} else {
+		c->freelist = object[c->offset];
+		local_irq_restore(flags);
+	}
 	if (unlikely((gfpflags & __GFP_ZERO)))
 		memset(object, 0, c->objsize);
 	return object;
 slow:
-	preempt_enable();
+	if (have_arch_cmpxchg())
+		preempt_enable();
 	return __slab_alloc(s, gfpflags, node, addr);
 }
 
@@ -1605,7 +1618,8 @@ static void __slab_free(struct kmem_cach
 	void **object = (void *)x;
 	unsigned long flags;
 
-	local_irq_save(flags);
+	if (have_arch_cmpxchg())
+		local_irq_save(flags);
 	slab_lock(page);
 
 	if (unlikely(SlabDebug(page)))
@@ -1631,7 +1645,8 @@ checks_ok:
 
 out_unlock:
 	slab_unlock(page);
-	local_irq_restore(flags);
+	if (have_arch_cmpxchg())
+		local_irq_restore(flags);
 	return;
 
 slab_empty:
@@ -1642,7 +1657,8 @@ slab_empty:
 		remove_partial(s, page);
 
 	slab_unlock(page);
-	local_irq_restore(flags);
+	if (have_arch_cmpxchg())
+		local_irq_restore(flags);
 	discard_slab(s, page);
 	return;
 
@@ -1669,27 +1685,38 @@ static void __always_inline slab_free(st
 	void **object = (void *)x;
 	void **freelist;
 	struct kmem_cache_cpu *c;
+	unsigned long flags;
 
-	preempt_disable();
+	if (have_arch_cmpxchg())
+		preempt_disable();
+	else
+		local_irq_save(flags);
 	c = get_cpu_slab(s, raw_smp_processor_id());
 	if (unlikely(c->node < 0))
 		goto slow;
 
 redo:
 	freelist = c->freelist;
-	barrier();	/* Read freelist before page, wrt local interrupts */
+	if (have_arch_cmpxchg())
+		barrier(); /* Read freelist before page, wrt local interrupts */
 	if (unlikely(page != c->page))
 		goto slow;
 
 	object[c->offset] = freelist;
 
-	if (unlikely(cmpxchg_local(&c->freelist,
-			freelist, object) != freelist))
-		goto redo;
-	preempt_enable();
+	if (have_arch_cmpxchg()) {
+		if (unlikely(cmpxchg_local(&c->freelist,
+				freelist, object) != freelist))
+			goto redo;
+		preempt_enable();
+	} else {
+		c->freelist = object;
+		local_irq_restore(flags);
+	}
 	return;
 slow:
-	preempt_enable();
+	if (have_arch_cmpxchg())
+		preempt_enable();
 	__slab_free(s, page, x, addr, c->offset);
 }
 
-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68