Re: [patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and 80486

[Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>]

* Christoph Lameter (clameter@sgi.com) wrote:
> @@ -1638,18 +1668,26 @@ static void __always_inline slab_free(st
>  			struct page *page, void *x, void *addr)
>  {
>  	void **object = (void *)x;
> -	unsigned long flags;
> +	void **freelist;
>  	struct kmem_cache_cpu *c;
>  
> -	local_irq_save(flags);
> -	c = get_cpu_slab(s, smp_processor_id());
> -	if (likely(page == c->page && c->node >= 0)) {
> -		object[c->offset] = c->freelist;
> -		c->freelist = object;
> -	} else
> -		__slab_free(s, page, x, addr, c->offset);
> +	c = get_cpu_slab(s, raw_smp_processor_id());
> +	if (unlikely(c->node >= 0))
> +		goto slow;
> +
> +redo:
> +	freelist = c->freelist;
> +	smp_rmb();
> +	if (unlikely(page != c->page))
> +		goto slow;
>  
> -	local_irq_restore(flags);
> +	object[c->offset] = freelist;
> +
> +	if (unlikely(cmpxchg_local(&c->freelist, freelist, object) != freelist))

If you don't plan to disable preemption here, cmpxchg_local should
probably not be used.

> +		goto redo;
> +	return;
> +slow:
> +	__slab_free(s, page, x, addr, c->offset);
>  }
>  
>  void kmem_cache_free(struct kmem_cache *s, void *x)
> Index: linux-2.6.23-rc1/include/linux/slub_def.h
> ===================================================================
> --- linux-2.6.23-rc1.orig/include/linux/slub_def.h	2007-07-27 19:30:03.000000000 -0700
> +++ linux-2.6.23-rc1/include/linux/slub_def.h	2007-07-27 21:15:27.000000000 -0700
> @@ -12,11 +12,11 @@
>  #include <linux/kobject.h>
>  
>  struct kmem_cache_cpu {
> -	void **freelist;
> -	struct page *page;
> -	int node;
> -	unsigned int offset;
> -	unsigned int objsize;
> +	void **freelist;	/* Updated through atomic ops */
> +	struct page *page;	/* Updated with interrupts disabled */
> +	int node;		/* Updated with interrupts disabled */
> +	unsigned int offset;	/* Set up on kmem_cache_create() */
> +	unsigned int objsize;	/* Set up on kmem_cache_create() */
>  };
>  
>  struct kmem_cache_node {

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68