Re: [patch 01/23] Fall back on interrupt disable in cmpxchg8b on 80386 and 80486

Mathieu Desnoyers
2007-08-20T22:29:41Z

* Christoph Lameter (clameter@sgi.com) wrote:
> @@ -1638,18 +1668,26 @@ static void __always_inline slab_free(st
> struct page *page, void *x, void *addr)
> {
> void **object = (void *)x;
> - unsigned long flags;
> + void **freelist;
> struct kmem_cache_cpu *c;
>
> - local_irq_save(flags);
> - c = get_cpu_slab(s, smp_processor_id());
> - if (likely(page == c->page && c->node >= 0)) {
> - object[c->offset] = c->freelist;
> - c->freelist = object;
> - } else
> - __slab_free(s, page, x, addr, c->offset);
> + c = get_cpu_slab(s, raw_smp_processor_id());
> + if (unlikely(c->node >= 0))
> + goto slow;
> +
> +redo:
> + freelist = c->freelist;
> + smp_rmb();
> + if (unlikely(page != c->page))
> + goto slow;
>
> - local_irq_restore(flags);
> + object[c->offset] = freelist;
> +
> + if (unlikely(cmpxchg_local(&c->freelist, freelist, object) != freelist))

If you don't plan to disable preemption here, cmpxchg_local should
probably not be used.

> + goto redo;
> + return;
> +slow:
> + __slab_free(s, page, x, addr, c->offset);
> }
>
> void kmem_cache_free(struct kmem_cache *s, void *x)
> Index: linux-2.6.23-rc1/include/linux/slub_def.h
> ===================================================================
> --- linux-2.6.23-rc1.orig/include/linux/slub_def.h 2007-07-27 19:30:03.000000000 -0700
> +++ linux-2.6.23-rc1/include/linux/slub_def.h 2007-07-27 21:15:27.000000000 -0700
> @@ -12,11 +12,11 @@
> #include <linux/kobject.h>
>
> struct kmem_cache_cpu {
> - void **freelist;
> - struct page *page;
> - int node;
> - unsigned int offset;
> - unsigned int objsize;
> + void **freelist; /* Updated through atomic ops */
> + struct page *page; /* Updated with interrupts disabled */
> + int node; /* Updated with interrupts disabled */
> + unsigned int offset; /* Set up on kmem_cache_create() */
> + unsigned int objsize; /* Set up on kmem_cache_create() */
> };
>
> struct kmem_cache_node {