* [slubllv6 01/17] slub: Push irq disable into allocate_slab()
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:13 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 02/17] slub: Do not use frozen page flag but a bit in the page counters Christoph Lameter
` (15 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: push_irq_disable --]
[-- Type: text/plain, Size: 1522 bytes --]
Do the irq handling in allocate_slab() instead of __slab_alloc().
__slab_alloc() is already cluttered and allocate_slab() is already
fiddling around with gfp flags.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-19 13:59:24.271151647 -0500
+++ linux-2.6/mm/slub.c 2011-05-19 13:59:24.981151647 -0500
@@ -1187,6 +1187,11 @@ static struct page *allocate_slab(struct
struct kmem_cache_order_objects oo = s->oo;
gfp_t alloc_gfp;
+ flags &= gfp_allowed_mask;
+
+ if (flags & __GFP_WAIT)
+ local_irq_enable();
+
flags |= s->allocflags;
/*
@@ -1203,12 +1208,15 @@ static struct page *allocate_slab(struct
* Try a lower order alloc if possible
*/
page = alloc_slab_page(flags, node, oo);
- if (!page)
- return NULL;
-
stat(s, ORDER_FALLBACK);
}
+ if (flags & __GFP_WAIT)
+ local_irq_disable();
+
+ if (!page)
+ return NULL;
+
if (kmemcheck_enabled
&& !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
int pages = 1 << oo_order(oo);
@@ -1849,15 +1857,8 @@ new_slab:
goto load_freelist;
}
- gfpflags &= gfp_allowed_mask;
- if (gfpflags & __GFP_WAIT)
- local_irq_enable();
-
page = new_slab(s, gfpflags, node);
- if (gfpflags & __GFP_WAIT)
- local_irq_disable();
-
if (page) {
c = __this_cpu_ptr(s->cpu_slab);
stat(s, ALLOC_SLAB);
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 01/17] slub: Push irq disable into allocate_slab()
2011-05-26 18:14 ` [slubllv6 01/17] slub: Push irq disable into allocate_slab() Christoph Lameter
@ 2011-05-26 22:13 ` David Rientjes
2011-05-31 16:55 ` Christoph Lameter
0 siblings, 1 reply; 32+ messages in thread
From: David Rientjes @ 2011-05-26 22:13 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
On Thu, 26 May 2011, Christoph Lameter wrote:
> Do the irq handling in allocate_slab() instead of __slab_alloc().
>
> __slab_alloc() is already cluttered and allocate_slab() is already
> fiddling around with gfp flags.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> ---
> mm/slub.c | 21 +++++++++++----------
> 1 file changed, 11 insertions(+), 10 deletions(-)
>
> Index: linux-2.6/mm/slub.c
> ===================================================================
> --- linux-2.6.orig/mm/slub.c 2011-05-19 13:59:24.271151647 -0500
> +++ linux-2.6/mm/slub.c 2011-05-19 13:59:24.981151647 -0500
> @@ -1187,6 +1187,11 @@ static struct page *allocate_slab(struct
> struct kmem_cache_order_objects oo = s->oo;
> gfp_t alloc_gfp;
>
> + flags &= gfp_allowed_mask;
> +
> + if (flags & __GFP_WAIT)
> + local_irq_enable();
> +
> flags |= s->allocflags;
>
> /*
> @@ -1203,12 +1208,15 @@ static struct page *allocate_slab(struct
> * Try a lower order alloc if possible
> */
> page = alloc_slab_page(flags, node, oo);
> - if (!page)
> - return NULL;
> -
> stat(s, ORDER_FALLBACK);
> }
>
> + if (flags & __GFP_WAIT)
> + local_irq_disable();
> +
> + if (!page)
> + return NULL;
> +
This changes the meaning of ORDER_FALLBACK from its previous meaning,
which was "number of times the preferred order could not be allocated and
then minimum order could be allocated" to "number of times the preferred
order could not be allocated, regardless of whether the minimum order
allocation was successful." The former is the true meaning of the word
"fallback," so is this semantics change avoidable? Otherwise it seems
like the statistic should be renamed (NEW_SLAB_FAIL?)
> if (kmemcheck_enabled
> && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
> int pages = 1 << oo_order(oo);
> @@ -1849,15 +1857,8 @@ new_slab:
> goto load_freelist;
> }
>
> - gfpflags &= gfp_allowed_mask;
> - if (gfpflags & __GFP_WAIT)
> - local_irq_enable();
> -
> page = new_slab(s, gfpflags, node);
>
> - if (gfpflags & __GFP_WAIT)
> - local_irq_disable();
> -
> if (page) {
> c = __this_cpu_ptr(s->cpu_slab);
> stat(s, ALLOC_SLAB);
>
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 01/17] slub: Push irq disable into allocate_slab()
2011-05-26 22:13 ` David Rientjes
@ 2011-05-31 16:55 ` Christoph Lameter
0 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-31 16:55 UTC (permalink / raw)
To: David Rientjes
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
On Thu, 26 May 2011, David Rientjes wrote:
> > + if (flags & __GFP_WAIT)
> > + local_irq_disable();
> > +
> > + if (!page)
> > + return NULL;
> > +
>
> This changes the meaning of ORDER_FALLBACK from its previous meaning,
> which was "number of times the preferred order could not be allocated and
> then minimum order could be allocated" to "number of times the preferred
> order could not be allocated, regardless of whether the minimum order
> allocation was successful." The former is the true meaning of the word
> "fallback," so is this semantics change avoidable? Otherwise it seems
> like the statistic should be renamed (NEW_SLAB_FAIL?)
Ok that needs to be fixed:
Subject: slub: Push irq disable into allocate_slab()
Do the irq handling in allocate_slab() instead of __slab_alloc().
__slab_alloc() is already cluttered and allocate_slab() is already
fiddling around with gfp flags.
v6->v7:
Only increment ORDER_FALLBACK if we get a page during fallback
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 23 +++++++++++++----------
1 file changed, 13 insertions(+), 10 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-26 16:13:58.085604969 -0500
+++ linux-2.6/mm/slub.c 2011-05-31 09:42:08.102989621 -0500
@@ -1187,6 +1187,11 @@ static struct page *allocate_slab(struct
struct kmem_cache_order_objects oo = s->oo;
gfp_t alloc_gfp;
+ flags &= gfp_allowed_mask;
+
+ if (flags & __GFP_WAIT)
+ local_irq_enable();
+
flags |= s->allocflags;
/*
@@ -1203,12 +1208,17 @@ static struct page *allocate_slab(struct
* Try a lower order alloc if possible
*/
page = alloc_slab_page(flags, node, oo);
- if (!page)
- return NULL;
- stat(s, ORDER_FALLBACK);
+ if (page)
+ stat(s, ORDER_FALLBACK);
}
+ if (flags & __GFP_WAIT)
+ local_irq_disable();
+
+ if (!page)
+ return NULL;
+
if (kmemcheck_enabled
&& !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
int pages = 1 << oo_order(oo);
@@ -1849,15 +1859,8 @@ new_slab:
goto load_freelist;
}
- gfpflags &= gfp_allowed_mask;
- if (gfpflags & __GFP_WAIT)
- local_irq_enable();
-
page = new_slab(s, gfpflags, node);
- if (gfpflags & __GFP_WAIT)
- local_irq_disable();
-
if (page) {
c = __this_cpu_ptr(s->cpu_slab);
stat(s, ALLOC_SLAB);
^ permalink raw reply [flat|nested] 32+ messages in thread
* [slubllv6 02/17] slub: Do not use frozen page flag but a bit in the page counters
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
2011-05-26 18:14 ` [slubllv6 01/17] slub: Push irq disable into allocate_slab() Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:13 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 03/17] slub: Move page->frozen handling near where the page->freelist handling occurs Christoph Lameter
` (14 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: frozen_field --]
[-- Type: text/plain, Size: 3256 bytes --]
Do not use a page flag for the frozen bit. It needs to be part
of the state that is handled with cmpxchg_double(). So use a bit
in the counter struct in the page struct for that purpose.
Also all page start out as frozen pages so set the bit
when the page is allocated.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/mm_types.h | 5 +++--
include/linux/page-flags.h | 2 --
mm/slub.c | 12 ++++++------
3 files changed, 9 insertions(+), 10 deletions(-)
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h 2011-05-19 13:54:29.271151992 -0500
+++ linux-2.6/include/linux/mm_types.h 2011-05-19 13:59:28.551151643 -0500
@@ -41,8 +41,9 @@ struct page {
* & limit reverse map searches.
*/
struct { /* SLUB */
- u16 inuse;
- u16 objects;
+ unsigned inuse:16;
+ unsigned objects:15;
+ unsigned frozen:1;
};
};
union {
Index: linux-2.6/include/linux/page-flags.h
===================================================================
--- linux-2.6.orig/include/linux/page-flags.h 2011-05-19 13:54:29.281151992 -0500
+++ linux-2.6/include/linux/page-flags.h 2011-05-19 13:59:28.551151643 -0500
@@ -212,8 +212,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEAR
__PAGEFLAG(SlobFree, slob_free)
-__PAGEFLAG(SlubFrozen, slub_frozen)
-
/*
* Private page markings that may be used by the filesystem that owns the page
* for its own purposes.
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-19 13:59:24.981151647 -0500
+++ linux-2.6/mm/slub.c 2011-05-19 13:59:28.561151643 -0500
@@ -166,7 +166,7 @@ static inline int kmem_cache_debug(struc
#define OO_SHIFT 16
#define OO_MASK ((1 << OO_SHIFT) - 1)
-#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
+#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
/* Internal SLUB flags */
#define __OBJECT_POISON 0x80000000UL /* Poison object */
@@ -1025,7 +1025,7 @@ static noinline int free_debug_processin
}
/* Special debug activities for freeing objects */
- if (!PageSlubFrozen(page) && !page->freelist)
+ if (!page->frozen && !page->freelist)
remove_full(s, page);
if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr);
@@ -1421,7 +1421,7 @@ static inline int lock_and_freeze_slab(s
{
if (slab_trylock(page)) {
__remove_partial(n, page);
- __SetPageSlubFrozen(page);
+ page->frozen = 1;
return 1;
}
return 0;
@@ -1535,7 +1535,7 @@ static void unfreeze_slab(struct kmem_ca
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
- __ClearPageSlubFrozen(page);
+ page->frozen = 0;
if (page->inuse) {
if (page->freelist) {
@@ -1866,7 +1866,7 @@ new_slab:
flush_slab(s, c);
slab_lock(page);
- __SetPageSlubFrozen(page);
+ page->frozen = 1;
c->node = page_to_nid(page);
c->page = page;
goto load_freelist;
@@ -2048,7 +2048,7 @@ static void __slab_free(struct kmem_cach
page->freelist = object;
page->inuse--;
- if (unlikely(PageSlubFrozen(page))) {
+ if (unlikely(page->frozen)) {
stat(s, FREE_FROZEN);
goto out_unlock;
}
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 02/17] slub: Do not use frozen page flag but a bit in the page counters
2011-05-26 18:14 ` [slubllv6 02/17] slub: Do not use frozen page flag but a bit in the page counters Christoph Lameter
@ 2011-05-26 22:13 ` David Rientjes
2011-05-31 16:57 ` Christoph Lameter
0 siblings, 1 reply; 32+ messages in thread
From: David Rientjes @ 2011-05-26 22:13 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
On Thu, 26 May 2011, Christoph Lameter wrote:
> Do not use a page flag for the frozen bit. It needs to be part
> of the state that is handled with cmpxchg_double(). So use a bit
> in the counter struct in the page struct for that purpose.
>
I don't think the new limit on the number of objects per page, 32767,
should cause a problem in practice.
> Also all page start out as frozen pages so set the bit
> when the page is allocated.
>
This comment is probably intended for the next patch in the series, which
modifies new_slab() to set the bit on allocation. There's no new behavior
introduced by this patch.
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
>
> ---
> include/linux/mm_types.h | 5 +++--
> include/linux/page-flags.h | 2 --
> mm/slub.c | 12 ++++++------
> 3 files changed, 9 insertions(+), 10 deletions(-)
>
> Index: linux-2.6/include/linux/mm_types.h
> ===================================================================
> --- linux-2.6.orig/include/linux/mm_types.h 2011-05-19 13:54:29.271151992 -0500
> +++ linux-2.6/include/linux/mm_types.h 2011-05-19 13:59:28.551151643 -0500
> @@ -41,8 +41,9 @@ struct page {
> * & limit reverse map searches.
> */
> struct { /* SLUB */
> - u16 inuse;
> - u16 objects;
> + unsigned inuse:16;
> + unsigned objects:15;
> + unsigned frozen:1;
> };
> };
> union {
> Index: linux-2.6/include/linux/page-flags.h
> ===================================================================
> --- linux-2.6.orig/include/linux/page-flags.h 2011-05-19 13:54:29.281151992 -0500
> +++ linux-2.6/include/linux/page-flags.h 2011-05-19 13:59:28.551151643 -0500
> @@ -212,8 +212,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEAR
>
> __PAGEFLAG(SlobFree, slob_free)
>
> -__PAGEFLAG(SlubFrozen, slub_frozen)
> -
The PG_slub_frozen entry in enum pageflags also needs to be removed.
> /*
> * Private page markings that may be used by the filesystem that owns the page
> * for its own purposes.
> Index: linux-2.6/mm/slub.c
> ===================================================================
> --- linux-2.6.orig/mm/slub.c 2011-05-19 13:59:24.981151647 -0500
> +++ linux-2.6/mm/slub.c 2011-05-19 13:59:28.561151643 -0500
> @@ -166,7 +166,7 @@ static inline int kmem_cache_debug(struc
>
> #define OO_SHIFT 16
> #define OO_MASK ((1 << OO_SHIFT) - 1)
> -#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
> +#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
>
> /* Internal SLUB flags */
> #define __OBJECT_POISON 0x80000000UL /* Poison object */
> @@ -1025,7 +1025,7 @@ static noinline int free_debug_processin
> }
>
> /* Special debug activities for freeing objects */
> - if (!PageSlubFrozen(page) && !page->freelist)
> + if (!page->frozen && !page->freelist)
> remove_full(s, page);
> if (s->flags & SLAB_STORE_USER)
> set_track(s, object, TRACK_FREE, addr);
> @@ -1421,7 +1421,7 @@ static inline int lock_and_freeze_slab(s
> {
> if (slab_trylock(page)) {
> __remove_partial(n, page);
> - __SetPageSlubFrozen(page);
> + page->frozen = 1;
> return 1;
> }
> return 0;
> @@ -1535,7 +1535,7 @@ static void unfreeze_slab(struct kmem_ca
> {
> struct kmem_cache_node *n = get_node(s, page_to_nid(page));
>
> - __ClearPageSlubFrozen(page);
> + page->frozen = 0;
> if (page->inuse) {
>
> if (page->freelist) {
> @@ -1866,7 +1866,7 @@ new_slab:
> flush_slab(s, c);
>
> slab_lock(page);
> - __SetPageSlubFrozen(page);
> + page->frozen = 1;
> c->node = page_to_nid(page);
> c->page = page;
> goto load_freelist;
> @@ -2048,7 +2048,7 @@ static void __slab_free(struct kmem_cach
> page->freelist = object;
> page->inuse--;
>
> - if (unlikely(PageSlubFrozen(page))) {
> + if (unlikely(page->frozen)) {
> stat(s, FREE_FROZEN);
> goto out_unlock;
> }
>
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 02/17] slub: Do not use frozen page flag but a bit in the page counters
2011-05-26 22:13 ` David Rientjes
@ 2011-05-31 16:57 ` Christoph Lameter
0 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-31 16:57 UTC (permalink / raw)
To: David Rientjes
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
On Thu, 26 May 2011, David Rientjes wrote:
> > Also all page start out as frozen pages so set the bit
> > when the page is allocated.
> >
>
> This comment is probably intended for the next patch in the series, which
> modifies new_slab() to set the bit on allocation. There's no new behavior
> introduced by this patch.
Right.
> The PG_slub_frozen entry in enum pageflags also needs to be removed.
Ok.
Subject: slub: Do not use frozen page flag but a bit in the page counters
Do not use a page flag for the frozen bit. It needs to be part
of the state that is handled with cmpxchg_double(). So use a bit
in the counter struct in the page struct for that purpose.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/mm_types.h | 5 +++--
include/linux/page-flags.h | 5 -----
mm/slub.c | 12 ++++++------
3 files changed, 9 insertions(+), 13 deletions(-)
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h 2011-05-31 09:40:57.402990070 -0500
+++ linux-2.6/include/linux/mm_types.h 2011-05-31 09:42:53.632989323 -0500
@@ -41,8 +41,9 @@ struct page {
* & limit reverse map searches.
*/
struct { /* SLUB */
- u16 inuse;
- u16 objects;
+ unsigned inuse:16;
+ unsigned objects:15;
+ unsigned frozen:1;
};
};
union {
Index: linux-2.6/include/linux/page-flags.h
===================================================================
--- linux-2.6.orig/include/linux/page-flags.h 2011-05-31 09:40:57.392990068 -0500
+++ linux-2.6/include/linux/page-flags.h 2011-05-31 09:43:25.402989115 -0500
@@ -124,9 +124,6 @@ enum pageflags {
/* SLOB */
PG_slob_free = PG_private,
-
- /* SLUB */
- PG_slub_frozen = PG_active,
};
#ifndef __GENERATING_BOUNDS_H
@@ -212,8 +209,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEAR
__PAGEFLAG(SlobFree, slob_free)
-__PAGEFLAG(SlubFrozen, slub_frozen)
-
/*
* Private page markings that may be used by the filesystem that owns the page
* for its own purposes.
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-31 09:42:08.102989621 -0500
+++ linux-2.6/mm/slub.c 2011-05-31 09:42:53.632989323 -0500
@@ -166,7 +166,7 @@ static inline int kmem_cache_debug(struc
#define OO_SHIFT 16
#define OO_MASK ((1 << OO_SHIFT) - 1)
-#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
+#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
/* Internal SLUB flags */
#define __OBJECT_POISON 0x80000000UL /* Poison object */
@@ -1025,7 +1025,7 @@ static noinline int free_debug_processin
}
/* Special debug activities for freeing objects */
- if (!PageSlubFrozen(page) && !page->freelist)
+ if (!page->frozen && !page->freelist)
remove_full(s, page);
if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr);
@@ -1424,7 +1424,7 @@ static inline int lock_and_freeze_slab(s
{
if (slab_trylock(page)) {
__remove_partial(n, page);
- __SetPageSlubFrozen(page);
+ page->frozen = 1;
return 1;
}
return 0;
@@ -1538,7 +1538,7 @@ static void unfreeze_slab(struct kmem_ca
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
- __ClearPageSlubFrozen(page);
+ page->frozen = 0;
if (page->inuse) {
if (page->freelist) {
@@ -1868,7 +1868,7 @@ new_slab:
flush_slab(s, c);
slab_lock(page);
- __SetPageSlubFrozen(page);
+ page->frozen = 1;
c->node = page_to_nid(page);
c->page = page;
goto load_freelist;
@@ -2048,7 +2048,7 @@ static void __slab_free(struct kmem_cach
page->freelist = object;
page->inuse--;
- if (unlikely(PageSlubFrozen(page))) {
+ if (unlikely(page->frozen)) {
stat(s, FREE_FROZEN);
goto out_unlock;
}
^ permalink raw reply [flat|nested] 32+ messages in thread
* [slubllv6 03/17] slub: Move page->frozen handling near where the page->freelist handling occurs
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
2011-05-26 18:14 ` [slubllv6 01/17] slub: Push irq disable into allocate_slab() Christoph Lameter
2011-05-26 18:14 ` [slubllv6 02/17] slub: Do not use frozen page flag but a bit in the page counters Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:13 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 04/17] x86: Add support for cmpxchg_double Christoph Lameter
` (13 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: frozen_move --]
[-- Type: text/plain, Size: 1878 bytes --]
This is necessary because the frozen bit has to be handled in the same cmpxchg_double
with the freelist and the counters.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:08.000000000 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:40:37.644875159 -0500
@@ -1284,6 +1284,7 @@ static struct page *new_slab(struct kmem
page->freelist = start;
page->inuse = 0;
+ page->frozen = 1;
out:
return page;
}
@@ -1422,7 +1423,6 @@ static inline int lock_and_freeze_slab(s
{
if (slab_trylock(page)) {
__remove_partial(n, page);
- page->frozen = 1;
return 1;
}
return 0;
@@ -1536,7 +1536,6 @@ static void unfreeze_slab(struct kmem_ca
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
- page->frozen = 0;
if (page->inuse) {
if (page->freelist) {
@@ -1669,6 +1668,7 @@ static void deactivate_slab(struct kmem_
}
c->page = NULL;
c->tid = next_tid(c->tid);
+ page->frozen = 0;
unfreeze_slab(s, page, tail);
}
@@ -1829,6 +1829,8 @@ static void *__slab_alloc(struct kmem_ca
stat(s, ALLOC_REFILL);
load_freelist:
+ VM_BUG_ON(!page->frozen);
+
object = page->freelist;
if (unlikely(!object))
goto another_slab;
@@ -1853,6 +1855,7 @@ new_slab:
page = get_partial(s, gfpflags, node);
if (page) {
stat(s, ALLOC_FROM_PARTIAL);
+ page->frozen = 1;
c->node = page_to_nid(page);
c->page = page;
goto load_freelist;
@@ -2373,6 +2376,7 @@ static void early_kmem_cache_node_alloc(
BUG_ON(!n);
page->freelist = get_freepointer(kmem_cache_node, n);
page->inuse++;
+ page->frozen = 0;
kmem_cache_node->node[node] = n;
#ifdef CONFIG_SLUB_DEBUG
init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 04/17] x86: Add support for cmpxchg_double
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (2 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 03/17] slub: Move page->frozen handling near where the page->freelist handling occurs Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 05/17] mm: Rearrange struct page Christoph Lameter
` (12 subsequent siblings)
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, tj, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: cmpxchg_double_x86 --]
[-- Type: text/plain, Size: 5729 bytes --]
A simple implementation that only supports the word size and does not
have a fallback mode (would require a spinlock).
And 32 and 64 bit support for cmpxchg_double. cmpxchg double uses
the cmpxchg8b or cmpxchg16b instruction on x86 processors to compare
and swap 2 machine words. This allows lockless algorithms to move more
context information through critical sections.
Set a flag CONFIG_CMPXCHG_DOUBLE to signal the support of that feature
during kernel builds.
Cc: tj@kernel.org
Signed-off-by: Christoph Lameter <cl@linux.com>
---
arch/x86/Kconfig.cpu | 3 ++
arch/x86/include/asm/cmpxchg_32.h | 46 ++++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/cmpxchg_64.h | 45 +++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/cpufeature.h | 1
4 files changed, 95 insertions(+)
Index: linux-2.6/arch/x86/include/asm/cmpxchg_64.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/cmpxchg_64.h 2011-05-16 11:40:36.421463498 -0500
+++ linux-2.6/arch/x86/include/asm/cmpxchg_64.h 2011-05-16 11:46:34.781463079 -0500
@@ -151,4 +151,49 @@ extern void __cmpxchg_wrong_size(void);
cmpxchg_local((ptr), (o), (n)); \
})
+#define cmpxchg16b(ptr, o1, o2, n1, n2) \
+({ \
+ char __ret; \
+ __typeof__(o2) __junk; \
+ __typeof__(*(ptr)) __old1 = (o1); \
+ __typeof__(o2) __old2 = (o2); \
+ __typeof__(*(ptr)) __new1 = (n1); \
+ __typeof__(o2) __new2 = (n2); \
+ asm volatile(LOCK_PREFIX_HERE "lock; cmpxchg16b (%%rsi);setz %1" \
+ : "=d"(__junk), "=a"(__ret) \
+ : "S"(ptr), "b"(__new1), "c"(__new2), \
+ "a"(__old1), "d"(__old2)); \
+ __ret; })
+
+
+#define cmpxchg16b_local(ptr, o1, o2, n1, n2) \
+({ \
+ char __ret; \
+ __typeof__(o2) __junk; \
+ __typeof__(*(ptr)) __old1 = (o1); \
+ __typeof__(o2) __old2 = (o2); \
+ __typeof__(*(ptr)) __new1 = (n1); \
+ __typeof__(o2) __new2 = (n2); \
+ asm volatile("cmpxchg16b (%%rsi)\n\t\tsetz %1\n\t" \
+ : "=d"(__junk)_, "=a"(__ret) \
+ : "S"((ptr)), "b"(__new1), "c"(__new2), \
+ "a"(__old1), "d"(__old2)); \
+ __ret; })
+
+#define cmpxchg_double(ptr, o1, o2, n1, n2) \
+({ \
+ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
+ VM_BUG_ON((unsigned long)(ptr) % 16); \
+ cmpxchg16b((ptr), (o1), (o2), (n1), (n2)); \
+})
+
+#define cmpxchg_double_local(ptr, o1, o2, n1, n2) \
+({ \
+ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
+ VM_BUG_ON((unsigned long)(ptr) % 16); \
+ cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2)); \
+})
+
+#define system_has_cmpxchg_double() cpu_has_cx16
+
#endif /* _ASM_X86_CMPXCHG_64_H */
Index: linux-2.6/arch/x86/include/asm/cmpxchg_32.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/cmpxchg_32.h 2011-05-16 11:40:36.431463498 -0500
+++ linux-2.6/arch/x86/include/asm/cmpxchg_32.h 2011-05-16 11:46:34.781463079 -0500
@@ -280,4 +280,50 @@ static inline unsigned long cmpxchg_386(
#endif
+#define cmpxchg8b(ptr, o1, o2, n1, n2) \
+({ \
+ char __ret; \
+ __typeof__(o2) __dummy; \
+ __typeof__(*(ptr)) __old1 = (o1); \
+ __typeof__(o2) __old2 = (o2); \
+ __typeof__(*(ptr)) __new1 = (n1); \
+ __typeof__(o2) __new2 = (n2); \
+ asm volatile(LOCK_PREFIX_HERE "lock; cmpxchg8b (%%esi); setz %1"\
+ : "d="(__dummy), "=a" (__ret) \
+ : "S" ((ptr)), "a" (__old1), "d"(__old2), \
+ "b" (__new1), "c" (__new2) \
+ : "memory"); \
+ __ret; })
+
+
+#define cmpxchg8b_local(ptr, o1, o2, n1, n2) \
+({ \
+ char __ret; \
+ __typeof__(o2) __dummy; \
+ __typeof__(*(ptr)) __old1 = (o1); \
+ __typeof__(o2) __old2 = (o2); \
+ __typeof__(*(ptr)) __new1 = (n1); \
+ __typeof__(o2) __new2 = (n2); \
+ asm volatile("cmpxchg8b (%%esi); tsetz %1" \
+ : "d="(__dummy), "=a"(__ret) \
+ : "S" ((ptr)), "a" (__old), "d"(__old2), \
+ "b" (__new1), "c" (__new2), \
+ : "memory"); \
+ __ret; })
+
+
+#define cmpxchg_double(ptr, o1, o2, n1, n2) \
+({ \
+ BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
+ VM_BUG_ON((unsigned long)(ptr) % 8); \
+ cmpxchg8b((ptr), (o1), (o2), (n1), (n2)); \
+})
+
+#define cmpxchg_double_local(ptr, o1, o2, n1, n2) \
+({ \
+ BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
+ VM_BUG_ON((unsigned long)(ptr) % 8); \
+ cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2)); \
+})
+
#endif /* _ASM_X86_CMPXCHG_32_H */
Index: linux-2.6/arch/x86/Kconfig.cpu
===================================================================
--- linux-2.6.orig/arch/x86/Kconfig.cpu 2011-05-16 11:40:36.401463498 -0500
+++ linux-2.6/arch/x86/Kconfig.cpu 2011-05-16 11:46:34.781463079 -0500
@@ -308,6 +308,9 @@ config X86_CMPXCHG
config CMPXCHG_LOCAL
def_bool X86_64 || (X86_32 && !M386)
+config CMPXCHG_DOUBLE
+ def_bool X86_64 || (X86_32 && !M386)
+
config X86_L1_CACHE_SHIFT
int
default "7" if MPENTIUM4 || MPSC
Index: linux-2.6/arch/x86/include/asm/cpufeature.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/cpufeature.h 2011-05-16 11:40:36.411463498 -0500
+++ linux-2.6/arch/x86/include/asm/cpufeature.h 2011-05-16 11:46:34.801463079 -0500
@@ -286,6 +286,7 @@ extern const char * const x86_power_flag
#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR)
#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ)
#define cpu_has_perfctr_core boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
+#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16)
#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
# define cpu_has_invlpg 1
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 05/17] mm: Rearrange struct page
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (3 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 04/17] x86: Add support for cmpxchg_double Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:14 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 06/17] slub: Add cmpxchg_double_slab() Christoph Lameter
` (11 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: resort_struct_page --]
[-- Type: text/plain, Size: 4982 bytes --]
We need to be able to use cmpxchg_double on the freelist and object count
field in struct page. Rearrange the fields in struct page according to
doubleword entities so that the freelist pointer comes before the counters.
Do the rearranging with a future in mind where we use more doubleword
atomics to avoid locking of updates to flags/mapping or lru pointers.
Create another union to allow access to counters in struct page as a
single unsigned long value.
The doublewords must be properly aligned for cmpxchg_double to work.
Sadly this increases the size of page struct by one word on some architectures.
But as a resultpage structs are now cacheline aligned on x86_64.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/mm_types.h | 89 +++++++++++++++++++++++++++++++----------------
1 file changed, 60 insertions(+), 29 deletions(-)
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h 2011-05-06 12:03:46.000000000 -0500
+++ linux-2.6/include/linux/mm_types.h 2011-05-06 12:50:40.000000000 -0500
@@ -30,52 +30,74 @@ struct address_space;
* moment. Note that we have no way to track which tasks are using
* a page, though if it is a pagecache page, rmap structures can tell us
* who is mapping it.
+ *
+ * The objects in struct page are organized in double word blocks in
+ * order to allows us to use atomic double word operations on portions
+ * of struct page. That is currently only used by slub but the arrangement
+ * allows the use of atomic double word operations on the flags/mapping
+ * and lru list pointers also.
*/
struct page {
+ /* First double word block */
unsigned long flags; /* Atomic flags, some possibly
* updated asynchronously */
- atomic_t _count; /* Usage count, see below. */
+ struct address_space *mapping; /* If low bit clear, points to
+ * inode address_space, or NULL.
+ * If page mapped as anonymous
+ * memory, low bit is set, and
+ * it points to anon_vma object:
+ * see PAGE_MAPPING_ANON below.
+ */
+ /* Second double word */
union {
- atomic_t _mapcount; /* Count of ptes mapped in mms,
- * to show when page is mapped
- * & limit reverse map searches.
+ struct {
+ pgoff_t index; /* Our offset within mapping. */
+ atomic_t _mapcount; /* Count of ptes mapped in mms,
+ * to show when page is mapped
+ * & limit reverse map searches.
+ */
+ atomic_t _count; /* Usage count, see below. */
+ };
+
+ struct { /* SLUB cmpxchg_double area */
+ void *freelist;
+ union {
+ unsigned long counters;
+ struct {
+ unsigned inuse:16;
+ unsigned objects:15;
+ unsigned frozen:1;
+ /*
+ * Kernel may make use of this field even when slub
+ * uses the rest of the double word!
*/
- struct { /* SLUB */
- unsigned inuse:16;
- unsigned objects:15;
- unsigned frozen:1;
+ atomic_t _count;
+ };
+ };
};
};
+
+ /* Third double word block */
+ struct list_head lru; /* Pageout list, eg. active_list
+ * protected by zone->lru_lock !
+ */
+
+ /* Remainder is not double word aligned */
union {
- struct {
- unsigned long private; /* Mapping-private opaque data:
+ unsigned long private; /* Mapping-private opaque data:
* usually used for buffer_heads
* if PagePrivate set; used for
* swp_entry_t if PageSwapCache;
* indicates order in the buddy
* system if PG_buddy is set.
*/
- struct address_space *mapping; /* If low bit clear, points to
- * inode address_space, or NULL.
- * If page mapped as anonymous
- * memory, low bit is set, and
- * it points to anon_vma object:
- * see PAGE_MAPPING_ANON below.
- */
- };
#if USE_SPLIT_PTLOCKS
- spinlock_t ptl;
+ spinlock_t ptl;
#endif
- struct kmem_cache *slab; /* SLUB: Pointer to slab */
- struct page *first_page; /* Compound tail pages */
+ struct kmem_cache *slab; /* SLUB: Pointer to slab */
+ struct page *first_page; /* Compound tail pages */
};
- union {
- pgoff_t index; /* Our offset within mapping. */
- void *freelist; /* SLUB: freelist req. slab lock */
- };
- struct list_head lru; /* Pageout list, eg. active_list
- * protected by zone->lru_lock !
- */
+
/*
* On machines where all RAM is mapped into kernel address space,
* we can simply calculate the virtual address. On machines with
@@ -101,7 +123,16 @@ struct page {
*/
void *shadow;
#endif
-};
+}
+/*
+ * If another subsystem starts using the double word pairing for atomic
+ * operations on struct page then it must change the #if to ensure
+ * proper alignment of the page struct.
+ */
+#if defined(CONFIG_SLUB) && defined(CONFIG_CMPXCHG_LOCAL)
+ __attribute__((__aligned__(2*sizeof(unsigned long))))
+#endif
+;
/*
* A region containing a mapping of a non-memory backed file under NOMMU
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 05/17] mm: Rearrange struct page
2011-05-26 18:14 ` [slubllv6 05/17] mm: Rearrange struct page Christoph Lameter
@ 2011-05-26 22:14 ` David Rientjes
0 siblings, 0 replies; 32+ messages in thread
From: David Rientjes @ 2011-05-26 22:14 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
On Thu, 26 May 2011, Christoph Lameter wrote:
> We need to be able to use cmpxchg_double on the freelist and object count
> field in struct page. Rearrange the fields in struct page according to
> doubleword entities so that the freelist pointer comes before the counters.
> Do the rearranging with a future in mind where we use more doubleword
> atomics to avoid locking of updates to flags/mapping or lru pointers.
>
> Create another union to allow access to counters in struct page as a
> single unsigned long value.
>
> The doublewords must be properly aligned for cmpxchg_double to work.
> Sadly this increases the size of page struct by one word on some architectures.
> But as a resultpage structs are now cacheline aligned on x86_64.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> ---
> include/linux/mm_types.h | 89 +++++++++++++++++++++++++++++++----------------
> 1 file changed, 60 insertions(+), 29 deletions(-)
>
> Index: linux-2.6/include/linux/mm_types.h
> ===================================================================
> --- linux-2.6.orig/include/linux/mm_types.h 2011-05-06 12:03:46.000000000 -0500
> +++ linux-2.6/include/linux/mm_types.h 2011-05-06 12:50:40.000000000 -0500
> @@ -30,52 +30,74 @@ struct address_space;
> * moment. Note that we have no way to track which tasks are using
> * a page, though if it is a pagecache page, rmap structures can tell us
> * who is mapping it.
> + *
> + * The objects in struct page are organized in double word blocks in
> + * order to allows us to use atomic double word operations on portions
> + * of struct page. That is currently only used by slub but the arrangement
> + * allows the use of atomic double word operations on the flags/mapping
> + * and lru list pointers also.
> */
> struct page {
> + /* First double word block */
> unsigned long flags; /* Atomic flags, some possibly
> * updated asynchronously */
> - atomic_t _count; /* Usage count, see below. */
> + struct address_space *mapping; /* If low bit clear, points to
> + * inode address_space, or NULL.
> + * If page mapped as anonymous
> + * memory, low bit is set, and
> + * it points to anon_vma object:
> + * see PAGE_MAPPING_ANON below.
> + */
> + /* Second double word */
> union {
> - atomic_t _mapcount; /* Count of ptes mapped in mms,
> - * to show when page is mapped
> - * & limit reverse map searches.
> + struct {
> + pgoff_t index; /* Our offset within mapping. */
> + atomic_t _mapcount; /* Count of ptes mapped in mms,
> + * to show when page is mapped
> + * & limit reverse map searches.
> + */
> + atomic_t _count; /* Usage count, see below. */
> + };
> +
> + struct { /* SLUB cmpxchg_double area */
> + void *freelist;
> + union {
> + unsigned long counters;
> + struct {
> + unsigned inuse:16;
> + unsigned objects:15;
> + unsigned frozen:1;
> + /*
> + * Kernel may make use of this field even when slub
> + * uses the rest of the double word!
> */
> - struct { /* SLUB */
> - unsigned inuse:16;
> - unsigned objects:15;
> - unsigned frozen:1;
> + atomic_t _count;
> + };
> + };
> };
> };
> +
> + /* Third double word block */
> + struct list_head lru; /* Pageout list, eg. active_list
> + * protected by zone->lru_lock !
> + */
> +
> + /* Remainder is not double word aligned */
> union {
> - struct {
> - unsigned long private; /* Mapping-private opaque data:
> + unsigned long private; /* Mapping-private opaque data:
Space before ident.
> * usually used for buffer_heads
> * if PagePrivate set; used for
> * swp_entry_t if PageSwapCache;
> * indicates order in the buddy
> * system if PG_buddy is set.
> */
> - struct address_space *mapping; /* If low bit clear, points to
> - * inode address_space, or NULL.
> - * If page mapped as anonymous
> - * memory, low bit is set, and
> - * it points to anon_vma object:
> - * see PAGE_MAPPING_ANON below.
> - */
> - };
> #if USE_SPLIT_PTLOCKS
> - spinlock_t ptl;
> + spinlock_t ptl;
> #endif
> - struct kmem_cache *slab; /* SLUB: Pointer to slab */
> - struct page *first_page; /* Compound tail pages */
> + struct kmem_cache *slab; /* SLUB: Pointer to slab */
> + struct page *first_page; /* Compound tail pages */
> };
> - union {
> - pgoff_t index; /* Our offset within mapping. */
> - void *freelist; /* SLUB: freelist req. slab lock */
> - };
> - struct list_head lru; /* Pageout list, eg. active_list
> - * protected by zone->lru_lock !
> - */
> +
> /*
> * On machines where all RAM is mapped into kernel address space,
> * we can simply calculate the virtual address. On machines with
> @@ -101,7 +123,16 @@ struct page {
> */
> void *shadow;
> #endif
> -};
> +}
> +/*
> + * If another subsystem starts using the double word pairing for atomic
> + * operations on struct page then it must change the #if to ensure
> + * proper alignment of the page struct.
> + */
> +#if defined(CONFIG_SLUB) && defined(CONFIG_CMPXCHG_LOCAL)
> + __attribute__((__aligned__(2*sizeof(unsigned long))))
> +#endif
> +;
>
> /*
> * A region containing a mapping of a non-memory backed file under NOMMU
Ah, the joys of rebasing during the merge window. This doesn't apply
because of ca16d140af91 ("mm: don't access vm_flags as 'int'") merged
earlier today.
^ permalink raw reply [flat|nested] 32+ messages in thread
* [slubllv6 06/17] slub: Add cmpxchg_double_slab()
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (4 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 05/17] mm: Rearrange struct page Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:14 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 07/17] slub: explicit list_lock taking Christoph Lameter
` (10 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: cmpxchg_double_slab --]
[-- Type: text/plain, Size: 3784 bytes --]
Add a function that operates on the second doubleword in the page struct
and manipulates the object counters, the freelist and the frozen attribute.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slub_def.h | 1 +
mm/slub.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 46 insertions(+)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-16 11:46:33.591463082 -0500
+++ linux-2.6/mm/slub.c 2011-05-16 11:46:51.181463060 -0500
@@ -131,6 +131,9 @@ static inline int kmem_cache_debug(struc
/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST
+/* Enable to log cmpxchg failures */
+#undef SLUB_DEBUG_CMPXCHG
+
/*
* Mininum number of partial slabs. These will be left on the partial
* lists even if they are empty. kmem_cache_shrink may reclaim them.
@@ -170,6 +173,7 @@ static inline int kmem_cache_debug(struc
/* Internal SLUB flags */
#define __OBJECT_POISON 0x80000000UL /* Poison object */
+#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
static int kmem_size = sizeof(struct kmem_cache);
@@ -354,6 +358,37 @@ static void get_map(struct kmem_cache *s
set_bit(slab_index(p, s, addr), map);
}
+static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
+ void *freelist_old, unsigned long counters_old,
+ void *freelist_new, unsigned long counters_new,
+ const char *n)
+{
+#ifdef CONFIG_CMPXCHG_DOUBLE
+ if (s->flags & __CMPXCHG_DOUBLE) {
+ if (cmpxchg_double(&page->freelist,
+ freelist_old, counters_old,
+ freelist_new, counters_new))
+ return 1;
+ } else
+#endif
+ {
+ if (page->freelist == freelist_old && page->counters == counters_old) {
+ page->freelist = freelist_new;
+ page->counters = counters_new;
+ return 1;
+ }
+ }
+
+ cpu_relax();
+ stat(s, CMPXCHG_DOUBLE_FAIL);
+
+#ifdef SLUB_DEBUG_CMPXCHG
+ printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
+#endif
+
+ return 0;
+}
+
/*
* Debug settings:
*/
@@ -2596,6 +2631,12 @@ static int kmem_cache_open(struct kmem_c
}
}
+#ifdef CONFIG_CMPXCHG_DOUBLE
+ if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
+ /* Enable fast mode */
+ s->flags |= __CMPXCHG_DOUBLE;
+#endif
+
/*
* The larger the object size is, the more pages we want on the partial
* list to avoid pounding the page allocator excessively.
@@ -4493,6 +4534,8 @@ STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
STAT_ATTR(ORDER_FALLBACK, order_fallback);
+STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
+STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
#endif
static struct attribute *slab_attrs[] = {
@@ -4550,6 +4593,8 @@ static struct attribute *slab_attrs[] =
&deactivate_to_tail_attr.attr,
&deactivate_remote_frees_attr.attr,
&order_fallback_attr.attr,
+ &cmpxchg_double_fail_attr.attr,
+ &cmpxchg_double_cpu_fail_attr.attr,
#endif
#ifdef CONFIG_FAILSLAB
&failslab_attr.attr,
Index: linux-2.6/include/linux/slub_def.h
===================================================================
--- linux-2.6.orig/include/linux/slub_def.h 2011-05-16 11:40:35.371463499 -0500
+++ linux-2.6/include/linux/slub_def.h 2011-05-16 11:46:51.181463060 -0500
@@ -33,6 +33,7 @@ enum stat_item {
DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
ORDER_FALLBACK, /* Number of times fallback was necessary */
CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
+ CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */
NR_SLUB_STAT_ITEMS };
struct kmem_cache_cpu {
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 06/17] slub: Add cmpxchg_double_slab()
2011-05-26 18:14 ` [slubllv6 06/17] slub: Add cmpxchg_double_slab() Christoph Lameter
@ 2011-05-26 22:14 ` David Rientjes
2011-05-31 17:05 ` Christoph Lameter
0 siblings, 1 reply; 32+ messages in thread
From: David Rientjes @ 2011-05-26 22:14 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: Type: TEXT/PLAIN, Size: 2918 bytes --]
On Thu, 26 May 2011, Christoph Lameter wrote:
> Add a function that operates on the second doubleword in the page struct
> and manipulates the object counters, the freelist and the frozen attribute.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> ---
> include/linux/slub_def.h | 1 +
> mm/slub.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 46 insertions(+)
>
> Index: linux-2.6/mm/slub.c
> ===================================================================
> --- linux-2.6.orig/mm/slub.c 2011-05-16 11:46:33.591463082 -0500
> +++ linux-2.6/mm/slub.c 2011-05-16 11:46:51.181463060 -0500
> @@ -131,6 +131,9 @@ static inline int kmem_cache_debug(struc
> /* Enable to test recovery from slab corruption on boot */
> #undef SLUB_RESILIENCY_TEST
>
> +/* Enable to log cmpxchg failures */
> +#undef SLUB_DEBUG_CMPXCHG
> +
> /*
> * Mininum number of partial slabs. These will be left on the partial
> * lists even if they are empty. kmem_cache_shrink may reclaim them.
> @@ -170,6 +173,7 @@ static inline int kmem_cache_debug(struc
>
> /* Internal SLUB flags */
> #define __OBJECT_POISON 0x80000000UL /* Poison object */
> +#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
>
> static int kmem_size = sizeof(struct kmem_cache);
>
> @@ -354,6 +358,37 @@ static void get_map(struct kmem_cache *s
> set_bit(slab_index(p, s, addr), map);
> }
>
> +static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
> + void *freelist_old, unsigned long counters_old,
> + void *freelist_new, unsigned long counters_new,
> + const char *n)
This is defined only under CONFIG_SLUB_DEBUG, which is surely a mistake:
mm/slub.c: In function ‘acquire_slab’:
mm/slub.c:1460: error: implicit declaration of function ‘cmpxchg_double_slab’
> +{
> +#ifdef CONFIG_CMPXCHG_DOUBLE
> + if (s->flags & __CMPXCHG_DOUBLE) {
> + if (cmpxchg_double(&page->freelist,
> + freelist_old, counters_old,
> + freelist_new, counters_new))
> + return 1;
> + } else
> +#endif
> + {
> + if (page->freelist == freelist_old && page->counters == counters_old) {
> + page->freelist = freelist_new;
> + page->counters = counters_new;
> + return 1;
> + }
> + }
> +
> + cpu_relax();
> + stat(s, CMPXCHG_DOUBLE_FAIL);
> +
> +#ifdef SLUB_DEBUG_CMPXCHG
> + printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
> +#endif
> +
> + return 0;
> +}
> +
> /*
> * Debug settings:
> */
> @@ -2596,6 +2631,12 @@ static int kmem_cache_open(struct kmem_c
> }
> }
>
> +#ifdef CONFIG_CMPXCHG_DOUBLE
> + if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
> + /* Enable fast mode */
> + s->flags |= __CMPXCHG_DOUBLE;
> +#endif
> +
Not sure why this is excluded for SLAB_DEBUG_FLAGS? If it really should
be, then we should clear __CMPXCHG_DOUBLE if debugging features are
enabled through sysfs as well.
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 06/17] slub: Add cmpxchg_double_slab()
2011-05-26 22:14 ` David Rientjes
@ 2011-05-31 17:05 ` Christoph Lameter
0 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-31 17:05 UTC (permalink / raw)
To: David Rientjes
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: Type: TEXT/PLAIN, Size: 6330 bytes --]
On Thu, 26 May 2011, David Rientjes wrote:
> > +static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
> > + void *freelist_old, unsigned long counters_old,
> > + void *freelist_new, unsigned long counters_new,
> > + const char *n)
>
> This is defined only under CONFIG_SLUB_DEBUG, which is surely a mistake:
>
> mm/slub.c: In function ‘acquire_slab’:
> mm/slub.c:1460: error: implicit declaration of function ‘cmpxchg_double_slab’
Ok.
> > +#ifdef CONFIG_CMPXCHG_DOUBLE
> > + if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
> > + /* Enable fast mode */
> > + s->flags |= __CMPXCHG_DOUBLE;
> > +#endif
> > +
>
> Not sure why this is excluded for SLAB_DEBUG_FLAGS? If it really should
> be, then we should clear __CMPXCHG_DOUBLE if debugging features are
> enabled through sysfs as well.
Right. Fixed up patch follows:
Subject: slub: Add cmpxchg_double_slab()
Add a function that operates on the second doubleword in the page struct
and manipulates the object counters, the freelist and the frozen attribute.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slub_def.h | 1
mm/slub.c | 65 +++++++++++++++++++++++++++++++++++++++++++----
2 files changed, 61 insertions(+), 5 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-31 11:57:59.622937422 -0500
+++ linux-2.6/mm/slub.c 2011-05-31 12:03:16.652935392 -0500
@@ -131,6 +131,9 @@ static inline int kmem_cache_debug(struc
/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST
+/* Enable to log cmpxchg failures */
+#undef SLUB_DEBUG_CMPXCHG
+
/*
* Mininum number of partial slabs. These will be left on the partial
* lists even if they are empty. kmem_cache_shrink may reclaim them.
@@ -170,6 +173,7 @@ static inline int kmem_cache_debug(struc
/* Internal SLUB flags */
#define __OBJECT_POISON 0x80000000UL /* Poison object */
+#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
static int kmem_size = sizeof(struct kmem_cache);
@@ -338,6 +342,37 @@ static inline int oo_objects(struct kmem
return x.x & OO_MASK;
}
+static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
+ void *freelist_old, unsigned long counters_old,
+ void *freelist_new, unsigned long counters_new,
+ const char *n)
+{
+#ifdef CONFIG_CMPXCHG_DOUBLE
+ if (s->flags & __CMPXCHG_DOUBLE) {
+ if (cmpxchg_double(&page->freelist,
+ freelist_old, counters_old,
+ freelist_new, counters_new))
+ return 1;
+ } else
+#endif
+ {
+ if (page->freelist == freelist_old && page->counters == counters_old) {
+ page->freelist = freelist_new;
+ page->counters = counters_new;
+ return 1;
+ }
+ }
+
+ cpu_relax();
+ stat(s, CMPXCHG_DOUBLE_FAIL);
+
+#ifdef SLUB_DEBUG_CMPXCHG
+ printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
+#endif
+
+ return 0;
+}
+
#ifdef CONFIG_SLUB_DEBUG
/*
* Determine a map of object in use on a page.
@@ -2600,6 +2635,12 @@ static int kmem_cache_open(struct kmem_c
}
}
+#ifdef CONFIG_CMPXCHG_DOUBLE
+ if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
+ /* Enable fast mode */
+ s->flags |= __CMPXCHG_DOUBLE;
+#endif
+
/*
* The larger the object size is, the more pages we want on the partial
* list to avoid pounding the page allocator excessively.
@@ -4252,8 +4293,10 @@ static ssize_t sanity_checks_store(struc
const char *buf, size_t length)
{
s->flags &= ~SLAB_DEBUG_FREE;
- if (buf[0] == '1')
+ if (buf[0] == '1') {
+ s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_DEBUG_FREE;
+ }
return length;
}
SLAB_ATTR(sanity_checks);
@@ -4267,8 +4310,10 @@ static ssize_t trace_store(struct kmem_c
size_t length)
{
s->flags &= ~SLAB_TRACE;
- if (buf[0] == '1')
+ if (buf[0] == '1') {
+ s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_TRACE;
+ }
return length;
}
SLAB_ATTR(trace);
@@ -4285,8 +4330,10 @@ static ssize_t red_zone_store(struct kme
return -EBUSY;
s->flags &= ~SLAB_RED_ZONE;
- if (buf[0] == '1')
+ if (buf[0] == '1') {
+ s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_RED_ZONE;
+ }
calculate_sizes(s, -1);
return length;
}
@@ -4304,8 +4351,10 @@ static ssize_t poison_store(struct kmem_
return -EBUSY;
s->flags &= ~SLAB_POISON;
- if (buf[0] == '1')
+ if (buf[0] == '1') {
+ s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_POISON;
+ }
calculate_sizes(s, -1);
return length;
}
@@ -4323,8 +4372,10 @@ static ssize_t store_user_store(struct k
return -EBUSY;
s->flags &= ~SLAB_STORE_USER;
- if (buf[0] == '1')
+ if (buf[0] == '1') {
+ s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_STORE_USER;
+ }
calculate_sizes(s, -1);
return length;
}
@@ -4497,6 +4548,8 @@ STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
STAT_ATTR(ORDER_FALLBACK, order_fallback);
+STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
+STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
#endif
static struct attribute *slab_attrs[] = {
@@ -4554,6 +4607,8 @@ static struct attribute *slab_attrs[] =
&deactivate_to_tail_attr.attr,
&deactivate_remote_frees_attr.attr,
&order_fallback_attr.attr,
+ &cmpxchg_double_fail_attr.attr,
+ &cmpxchg_double_cpu_fail_attr.attr,
#endif
#ifdef CONFIG_FAILSLAB
&failslab_attr.attr,
Index: linux-2.6/include/linux/slub_def.h
===================================================================
--- linux-2.6.orig/include/linux/slub_def.h 2011-05-31 11:50:01.762940481 -0500
+++ linux-2.6/include/linux/slub_def.h 2011-05-31 11:58:01.742937411 -0500
@@ -33,6 +33,7 @@ enum stat_item {
DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
ORDER_FALLBACK, /* Number of times fallback was necessary */
CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
+ CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */
NR_SLUB_STAT_ITEMS };
struct kmem_cache_cpu {
^ permalink raw reply [flat|nested] 32+ messages in thread
* [slubllv6 07/17] slub: explicit list_lock taking
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (5 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 06/17] slub: Add cmpxchg_double_slab() Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:15 ` David Rientjes
2011-05-27 19:40 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 08/17] slub: Pass kmem_cache struct to lock and freeze slab Christoph Lameter
` (9 subsequent siblings)
16 siblings, 2 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: unlock_list_ops --]
[-- Type: text/plain, Size: 6092 bytes --]
The allocator fastpath rework does change the usage of the list_lock.
Remove the list_lock processing from the functions that hide them from the
critical sections and move them into those critical sections.
This in turn simplifies the support functions (no __ variant needed anymore)
and simplifies the lock handling on bootstrap.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 74 ++++++++++++++++++++++++++++++--------------------------------
1 file changed, 36 insertions(+), 38 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:43.764875120 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:40:44.814875113 -0500
@@ -917,25 +917,21 @@ static inline void slab_free_hook(struct
/*
* Tracking of fully allocated slabs for debugging purposes.
*/
-static void add_full(struct kmem_cache_node *n, struct page *page)
+static void add_full(struct kmem_cache *s,
+ struct kmem_cache_node *n, struct page *page)
{
- spin_lock(&n->list_lock);
+ if (!(s->flags & SLAB_STORE_USER))
+ return;
+
list_add(&page->lru, &n->full);
- spin_unlock(&n->list_lock);
}
static void remove_full(struct kmem_cache *s, struct page *page)
{
- struct kmem_cache_node *n;
-
if (!(s->flags & SLAB_STORE_USER))
return;
- n = get_node(s, page_to_nid(page));
-
- spin_lock(&n->list_lock);
list_del(&page->lru);
- spin_unlock(&n->list_lock);
}
/* Tracking of the number of slabs for debugging purposes */
@@ -1060,8 +1056,13 @@ static noinline int free_debug_processin
}
/* Special debug activities for freeing objects */
- if (!page->frozen && !page->freelist)
+ if (!page->frozen && !page->freelist) {
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+ spin_lock(&n->list_lock);
remove_full(s, page);
+ spin_unlock(&n->list_lock);
+ }
if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0);
@@ -1420,36 +1421,26 @@ static __always_inline int slab_trylock(
/*
* Management of partially allocated slabs
*/
-static void add_partial(struct kmem_cache_node *n,
+static inline void add_partial(struct kmem_cache_node *n,
struct page *page, int tail)
{
- spin_lock(&n->list_lock);
n->nr_partial++;
if (tail)
list_add_tail(&page->lru, &n->partial);
else
list_add(&page->lru, &n->partial);
- spin_unlock(&n->list_lock);
}
-static inline void __remove_partial(struct kmem_cache_node *n,
+static inline void remove_partial(struct kmem_cache_node *n,
struct page *page)
{
list_del(&page->lru);
n->nr_partial--;
}
-static void remove_partial(struct kmem_cache *s, struct page *page)
-{
- struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
- spin_lock(&n->list_lock);
- __remove_partial(n, page);
- spin_unlock(&n->list_lock);
-}
-
/*
- * Lock slab and remove from the partial list.
+ * Lock slab, remove from the partial list and put the object into the
+ * per cpu freelist.
*
* Must hold list_lock.
*/
@@ -1457,7 +1448,7 @@ static inline int lock_and_freeze_slab(s
struct page *page)
{
if (slab_trylock(page)) {
- __remove_partial(n, page);
+ remove_partial(n, page);
return 1;
}
return 0;
@@ -1574,12 +1565,17 @@ static void unfreeze_slab(struct kmem_ca
if (page->inuse) {
if (page->freelist) {
+ spin_lock(&n->list_lock);
add_partial(n, page, tail);
+ spin_unlock(&n->list_lock);
stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
} else {
stat(s, DEACTIVATE_FULL);
- if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
- add_full(n, page);
+ if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) {
+ spin_lock(&n->list_lock);
+ add_full(s, n, page);
+ spin_unlock(&n->list_lock);
+ }
}
slab_unlock(page);
} else {
@@ -1595,7 +1591,9 @@ static void unfreeze_slab(struct kmem_ca
* kmem_cache_shrink can reclaim any empty slabs from
* the partial list.
*/
+ spin_lock(&n->list_lock);
add_partial(n, page, 1);
+ spin_unlock(&n->list_lock);
slab_unlock(page);
} else {
slab_unlock(page);
@@ -2097,7 +2095,11 @@ static void __slab_free(struct kmem_cach
* then add it.
*/
if (unlikely(!prior)) {
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+ spin_lock(&n->list_lock);
add_partial(get_node(s, page_to_nid(page)), page, 1);
+ spin_unlock(&n->list_lock);
stat(s, FREE_ADD_PARTIAL);
}
@@ -2111,7 +2113,11 @@ slab_empty:
/*
* Slab still on the partial list.
*/
- remove_partial(s, page);
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+ spin_lock(&n->list_lock);
+ remove_partial(n, page);
+ spin_unlock(&n->list_lock);
stat(s, FREE_REMOVE_PARTIAL);
}
slab_unlock(page);
@@ -2393,7 +2399,6 @@ static void early_kmem_cache_node_alloc(
{
struct page *page;
struct kmem_cache_node *n;
- unsigned long flags;
BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
@@ -2420,14 +2425,7 @@ static void early_kmem_cache_node_alloc(
init_kmem_cache_node(n, kmem_cache_node);
inc_slabs_node(kmem_cache_node, node, page->objects);
- /*
- * lockdep requires consistent irq usage for each lock
- * so even though there cannot be a race this early in
- * the boot sequence, we still disable irqs.
- */
- local_irq_save(flags);
add_partial(n, page, 0);
- local_irq_restore(flags);
}
static void free_kmem_cache_nodes(struct kmem_cache *s)
@@ -2711,7 +2709,7 @@ static void free_partial(struct kmem_cac
spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry_safe(page, h, &n->partial, lru) {
if (!page->inuse) {
- __remove_partial(n, page);
+ remove_partial(n, page);
discard_slab(s, page);
} else {
list_slab_objects(s, page,
@@ -3049,7 +3047,7 @@ int kmem_cache_shrink(struct kmem_cache
* may have freed the last object and be
* waiting to release the slab.
*/
- __remove_partial(n, page);
+ remove_partial(n, page);
slab_unlock(page);
discard_slab(s, page);
} else {
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 07/17] slub: explicit list_lock taking
2011-05-26 18:14 ` [slubllv6 07/17] slub: explicit list_lock taking Christoph Lameter
@ 2011-05-26 22:15 ` David Rientjes
2011-05-31 17:11 ` Christoph Lameter
2011-05-27 19:40 ` David Rientjes
1 sibling, 1 reply; 32+ messages in thread
From: David Rientjes @ 2011-05-26 22:15 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: Type: TEXT/PLAIN, Size: 7167 bytes --]
On Thu, 26 May 2011, Christoph Lameter wrote:
> The allocator fastpath rework does change the usage of the list_lock.
> Remove the list_lock processing from the functions that hide them from the
> critical sections and move them into those critical sections.
>
> This in turn simplifies the support functions (no __ variant needed anymore)
> and simplifies the lock handling on bootstrap.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> ---
> mm/slub.c | 74 ++++++++++++++++++++++++++++++--------------------------------
> 1 file changed, 36 insertions(+), 38 deletions(-)
>
> Index: linux-2.6/mm/slub.c
> ===================================================================
> --- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:43.764875120 -0500
> +++ linux-2.6/mm/slub.c 2011-05-24 09:40:44.814875113 -0500
> @@ -917,25 +917,21 @@ static inline void slab_free_hook(struct
> /*
> * Tracking of fully allocated slabs for debugging purposes.
> */
> -static void add_full(struct kmem_cache_node *n, struct page *page)
> +static void add_full(struct kmem_cache *s,
> + struct kmem_cache_node *n, struct page *page)
This also needs an update to the CONFIG_SLUB_DEBUG=n definition, not only
for number of arguments but also the order:
mm/slub.c:1781: error: too many arguments to function ‘add_full’
and
mm/slub.c:1781: warning: passing argument 1 of ‘add_full’ from incompatible pointer type
mm/slub.c:1187: note: expected ‘struct kmem_cache_node *’ but argument is of type ‘struct kmem_cache *’
mm/slub.c:1781: warning: passing argument 2 of ‘add_full’ from incompatible pointer type
mm/slub.c:1187: note: expected ‘struct page *’ but argument is of type ‘struct kmem_cache_node *’
> {
> - spin_lock(&n->list_lock);
> + if (!(s->flags & SLAB_STORE_USER))
> + return;
> +
> list_add(&page->lru, &n->full);
> - spin_unlock(&n->list_lock);
> }
>
> static void remove_full(struct kmem_cache *s, struct page *page)
> {
> - struct kmem_cache_node *n;
> -
> if (!(s->flags & SLAB_STORE_USER))
> return;
>
> - n = get_node(s, page_to_nid(page));
> -
> - spin_lock(&n->list_lock);
> list_del(&page->lru);
> - spin_unlock(&n->list_lock);
> }
>
> /* Tracking of the number of slabs for debugging purposes */
> @@ -1060,8 +1056,13 @@ static noinline int free_debug_processin
> }
>
> /* Special debug activities for freeing objects */
> - if (!page->frozen && !page->freelist)
> + if (!page->frozen && !page->freelist) {
> + struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> +
> + spin_lock(&n->list_lock);
> remove_full(s, page);
> + spin_unlock(&n->list_lock);
> + }
> if (s->flags & SLAB_STORE_USER)
> set_track(s, object, TRACK_FREE, addr);
> trace(s, page, object, 0);
> @@ -1420,36 +1421,26 @@ static __always_inline int slab_trylock(
> /*
> * Management of partially allocated slabs
> */
> -static void add_partial(struct kmem_cache_node *n,
> +static inline void add_partial(struct kmem_cache_node *n,
> struct page *page, int tail)
> {
> - spin_lock(&n->list_lock);
> n->nr_partial++;
> if (tail)
> list_add_tail(&page->lru, &n->partial);
> else
> list_add(&page->lru, &n->partial);
> - spin_unlock(&n->list_lock);
> }
>
> -static inline void __remove_partial(struct kmem_cache_node *n,
> +static inline void remove_partial(struct kmem_cache_node *n,
> struct page *page)
> {
> list_del(&page->lru);
> n->nr_partial--;
> }
>
> -static void remove_partial(struct kmem_cache *s, struct page *page)
> -{
> - struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> -
> - spin_lock(&n->list_lock);
> - __remove_partial(n, page);
> - spin_unlock(&n->list_lock);
> -}
> -
> /*
> - * Lock slab and remove from the partial list.
> + * Lock slab, remove from the partial list and put the object into the
> + * per cpu freelist.
> *
> * Must hold list_lock.
> */
> @@ -1457,7 +1448,7 @@ static inline int lock_and_freeze_slab(s
> struct page *page)
> {
> if (slab_trylock(page)) {
> - __remove_partial(n, page);
> + remove_partial(n, page);
> return 1;
> }
> return 0;
> @@ -1574,12 +1565,17 @@ static void unfreeze_slab(struct kmem_ca
> if (page->inuse) {
>
> if (page->freelist) {
> + spin_lock(&n->list_lock);
> add_partial(n, page, tail);
> + spin_unlock(&n->list_lock);
> stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
> } else {
> stat(s, DEACTIVATE_FULL);
> - if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
> - add_full(n, page);
> + if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) {
> + spin_lock(&n->list_lock);
> + add_full(s, n, page);
> + spin_unlock(&n->list_lock);
> + }
> }
> slab_unlock(page);
> } else {
> @@ -1595,7 +1591,9 @@ static void unfreeze_slab(struct kmem_ca
> * kmem_cache_shrink can reclaim any empty slabs from
> * the partial list.
> */
> + spin_lock(&n->list_lock);
> add_partial(n, page, 1);
> + spin_unlock(&n->list_lock);
> slab_unlock(page);
> } else {
> slab_unlock(page);
> @@ -2097,7 +2095,11 @@ static void __slab_free(struct kmem_cach
> * then add it.
> */
> if (unlikely(!prior)) {
> + struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> +
> + spin_lock(&n->list_lock);
> add_partial(get_node(s, page_to_nid(page)), page, 1);
> + spin_unlock(&n->list_lock);
> stat(s, FREE_ADD_PARTIAL);
> }
>
> @@ -2111,7 +2113,11 @@ slab_empty:
> /*
> * Slab still on the partial list.
> */
> - remove_partial(s, page);
> + struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> +
> + spin_lock(&n->list_lock);
> + remove_partial(n, page);
> + spin_unlock(&n->list_lock);
> stat(s, FREE_REMOVE_PARTIAL);
> }
> slab_unlock(page);
> @@ -2393,7 +2399,6 @@ static void early_kmem_cache_node_alloc(
> {
> struct page *page;
> struct kmem_cache_node *n;
> - unsigned long flags;
>
> BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
>
> @@ -2420,14 +2425,7 @@ static void early_kmem_cache_node_alloc(
> init_kmem_cache_node(n, kmem_cache_node);
> inc_slabs_node(kmem_cache_node, node, page->objects);
>
> - /*
> - * lockdep requires consistent irq usage for each lock
> - * so even though there cannot be a race this early in
> - * the boot sequence, we still disable irqs.
> - */
> - local_irq_save(flags);
> add_partial(n, page, 0);
> - local_irq_restore(flags);
> }
>
> static void free_kmem_cache_nodes(struct kmem_cache *s)
> @@ -2711,7 +2709,7 @@ static void free_partial(struct kmem_cac
> spin_lock_irqsave(&n->list_lock, flags);
> list_for_each_entry_safe(page, h, &n->partial, lru) {
> if (!page->inuse) {
> - __remove_partial(n, page);
> + remove_partial(n, page);
> discard_slab(s, page);
> } else {
> list_slab_objects(s, page,
> @@ -3049,7 +3047,7 @@ int kmem_cache_shrink(struct kmem_cache
> * may have freed the last object and be
> * waiting to release the slab.
> */
> - __remove_partial(n, page);
> + remove_partial(n, page);
> slab_unlock(page);
> discard_slab(s, page);
> } else {
>
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 07/17] slub: explicit list_lock taking
2011-05-26 22:15 ` David Rientjes
@ 2011-05-31 17:11 ` Christoph Lameter
0 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-31 17:11 UTC (permalink / raw)
To: David Rientjes
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: Type: TEXT/PLAIN, Size: 7980 bytes --]
On Thu, 26 May 2011, David Rientjes wrote:
> > -static void add_full(struct kmem_cache_node *n, struct page *page)
> > +static void add_full(struct kmem_cache *s,
> > + struct kmem_cache_node *n, struct page *page)
>
> This also needs an update to the CONFIG_SLUB_DEBUG=n definition, not only
> for number of arguments but also the order:
>
> mm/slub.c:1781: error: too many arguments to function ‘add_full’
>
> and
>
> mm/slub.c:1781: warning: passing argument 1 of ‘add_full’ from incompatible pointer type
> mm/slub.c:1187: note: expected ‘struct kmem_cache_node *’ but argument is of type ‘struct kmem_cache *’
> mm/slub.c:1781: warning: passing argument 2 of ‘add_full’ from incompatible pointer type
> mm/slub.c:1187: note: expected ‘struct page *’ but argument is of type ‘struct kmem_cache_node *’
Correct.
Subject: slub: explicit list_lock taking
The allocator fastpath rework does change the usage of the list_lock.
Remove the list_lock processing from the functions that hide them from the
critical sections and move them into those critical sections.
This in turn simplifies the support functions (no __ variant needed anymore)
and simplifies the lock handling on bootstrap.
Inline add_partial since it becomes pretty simple.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 89 ++++++++++++++++++++++++++++++++++----------------------------
1 file changed, 49 insertions(+), 40 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-31 10:14:00.812977367 -0500
+++ linux-2.6/mm/slub.c 2011-05-31 10:14:03.852977349 -0500
@@ -916,26 +916,27 @@ static inline void slab_free_hook(struct
/*
* Tracking of fully allocated slabs for debugging purposes.
+ *
+ * list_lock must be held.
*/
-static void add_full(struct kmem_cache_node *n, struct page *page)
+static void add_full(struct kmem_cache *s,
+ struct kmem_cache_node *n, struct page *page)
{
- spin_lock(&n->list_lock);
+ if (!(s->flags & SLAB_STORE_USER))
+ return;
+
list_add(&page->lru, &n->full);
- spin_unlock(&n->list_lock);
}
+/*
+ * list_lock must be held.
+ */
static void remove_full(struct kmem_cache *s, struct page *page)
{
- struct kmem_cache_node *n;
-
if (!(s->flags & SLAB_STORE_USER))
return;
- n = get_node(s, page_to_nid(page));
-
- spin_lock(&n->list_lock);
list_del(&page->lru);
- spin_unlock(&n->list_lock);
}
/* Tracking of the number of slabs for debugging purposes */
@@ -1060,8 +1061,13 @@ static noinline int free_debug_processin
}
/* Special debug activities for freeing objects */
- if (!page->frozen && !page->freelist)
+ if (!page->frozen && !page->freelist) {
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+ spin_lock(&n->list_lock);
remove_full(s, page);
+ spin_unlock(&n->list_lock);
+ }
if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0);
@@ -1170,7 +1176,8 @@ static inline int slab_pad_check(struct
{ return 1; }
static inline int check_object(struct kmem_cache *s, struct page *page,
void *object, u8 val) { return 1; }
-static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
+static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
+ struct page *page) {}
static inline unsigned long kmem_cache_flags(unsigned long objsize,
unsigned long flags, const char *name,
void (*ctor)(void *))
@@ -1420,38 +1427,33 @@ static __always_inline int slab_trylock(
}
/*
- * Management of partially allocated slabs
+ * Management of partially allocated slabs.
+ *
+ * list_lock must be held.
*/
-static void add_partial(struct kmem_cache_node *n,
+static inline void add_partial(struct kmem_cache_node *n,
struct page *page, int tail)
{
- spin_lock(&n->list_lock);
n->nr_partial++;
if (tail)
list_add_tail(&page->lru, &n->partial);
else
list_add(&page->lru, &n->partial);
- spin_unlock(&n->list_lock);
}
-static inline void __remove_partial(struct kmem_cache_node *n,
+/*
+ * list_lock must be held.
+ */
+static inline void remove_partial(struct kmem_cache_node *n,
struct page *page)
{
list_del(&page->lru);
n->nr_partial--;
}
-static void remove_partial(struct kmem_cache *s, struct page *page)
-{
- struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
- spin_lock(&n->list_lock);
- __remove_partial(n, page);
- spin_unlock(&n->list_lock);
-}
-
/*
- * Lock slab and remove from the partial list.
+ * Lock slab, remove from the partial list and put the object into the
+ * per cpu freelist.
*
* Must hold list_lock.
*/
@@ -1459,7 +1461,7 @@ static inline int lock_and_freeze_slab(s
struct page *page)
{
if (slab_trylock(page)) {
- __remove_partial(n, page);
+ remove_partial(n, page);
return 1;
}
return 0;
@@ -1576,12 +1578,17 @@ static void unfreeze_slab(struct kmem_ca
if (page->inuse) {
if (page->freelist) {
+ spin_lock(&n->list_lock);
add_partial(n, page, tail);
+ spin_unlock(&n->list_lock);
stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
} else {
stat(s, DEACTIVATE_FULL);
- if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
- add_full(n, page);
+ if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) {
+ spin_lock(&n->list_lock);
+ add_full(s, n, page);
+ spin_unlock(&n->list_lock);
+ }
}
slab_unlock(page);
} else {
@@ -1597,7 +1604,9 @@ static void unfreeze_slab(struct kmem_ca
* kmem_cache_shrink can reclaim any empty slabs from
* the partial list.
*/
+ spin_lock(&n->list_lock);
add_partial(n, page, 1);
+ spin_unlock(&n->list_lock);
slab_unlock(page);
} else {
slab_unlock(page);
@@ -2099,7 +2108,11 @@ static void __slab_free(struct kmem_cach
* then add it.
*/
if (unlikely(!prior)) {
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+ spin_lock(&n->list_lock);
add_partial(get_node(s, page_to_nid(page)), page, 1);
+ spin_unlock(&n->list_lock);
stat(s, FREE_ADD_PARTIAL);
}
@@ -2113,7 +2126,11 @@ slab_empty:
/*
* Slab still on the partial list.
*/
- remove_partial(s, page);
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+ spin_lock(&n->list_lock);
+ remove_partial(n, page);
+ spin_unlock(&n->list_lock);
stat(s, FREE_REMOVE_PARTIAL);
}
slab_unlock(page);
@@ -2395,7 +2412,6 @@ static void early_kmem_cache_node_alloc(
{
struct page *page;
struct kmem_cache_node *n;
- unsigned long flags;
BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
@@ -2422,14 +2438,7 @@ static void early_kmem_cache_node_alloc(
init_kmem_cache_node(n, kmem_cache_node);
inc_slabs_node(kmem_cache_node, node, page->objects);
- /*
- * lockdep requires consistent irq usage for each lock
- * so even though there cannot be a race this early in
- * the boot sequence, we still disable irqs.
- */
- local_irq_save(flags);
add_partial(n, page, 0);
- local_irq_restore(flags);
}
static void free_kmem_cache_nodes(struct kmem_cache *s)
@@ -2713,7 +2722,7 @@ static void free_partial(struct kmem_cac
spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry_safe(page, h, &n->partial, lru) {
if (!page->inuse) {
- __remove_partial(n, page);
+ remove_partial(n, page);
discard_slab(s, page);
} else {
list_slab_objects(s, page,
@@ -3051,7 +3060,7 @@ int kmem_cache_shrink(struct kmem_cache
* may have freed the last object and be
* waiting to release the slab.
*/
- __remove_partial(n, page);
+ remove_partial(n, page);
slab_unlock(page);
discard_slab(s, page);
} else {
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [slubllv6 07/17] slub: explicit list_lock taking
2011-05-26 18:14 ` [slubllv6 07/17] slub: explicit list_lock taking Christoph Lameter
2011-05-26 22:15 ` David Rientjes
@ 2011-05-27 19:40 ` David Rientjes
1 sibling, 0 replies; 32+ messages in thread
From: David Rientjes @ 2011-05-27 19:40 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
On Thu, 26 May 2011, Christoph Lameter wrote:
> The allocator fastpath rework does change the usage of the list_lock.
> Remove the list_lock processing from the functions that hide them from the
> critical sections and move them into those critical sections.
>
Sounds good, but I'd suggest adding comments to the now list_lock-free
functions that specify that the lock must be held (when
slab_state != DOWN).
> This in turn simplifies the support functions (no __ variant needed anymore)
> and simplifies the lock handling on bootstrap.
>
Please also specify that you're inlining add_partial().
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> ---
> mm/slub.c | 74 ++++++++++++++++++++++++++++++--------------------------------
> 1 file changed, 36 insertions(+), 38 deletions(-)
>
> Index: linux-2.6/mm/slub.c
> ===================================================================
> --- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:43.764875120 -0500
> +++ linux-2.6/mm/slub.c 2011-05-24 09:40:44.814875113 -0500
> @@ -917,25 +917,21 @@ static inline void slab_free_hook(struct
> /*
> * Tracking of fully allocated slabs for debugging purposes.
> */
> -static void add_full(struct kmem_cache_node *n, struct page *page)
> +static void add_full(struct kmem_cache *s,
> + struct kmem_cache_node *n, struct page *page)
> {
> - spin_lock(&n->list_lock);
> + if (!(s->flags & SLAB_STORE_USER))
> + return;
> +
> list_add(&page->lru, &n->full);
> - spin_unlock(&n->list_lock);
> }
>
> static void remove_full(struct kmem_cache *s, struct page *page)
> {
> - struct kmem_cache_node *n;
> -
> if (!(s->flags & SLAB_STORE_USER))
> return;
>
> - n = get_node(s, page_to_nid(page));
> -
> - spin_lock(&n->list_lock);
> list_del(&page->lru);
> - spin_unlock(&n->list_lock);
> }
>
> /* Tracking of the number of slabs for debugging purposes */
> @@ -1060,8 +1056,13 @@ static noinline int free_debug_processin
> }
>
> /* Special debug activities for freeing objects */
> - if (!page->frozen && !page->freelist)
> + if (!page->frozen && !page->freelist) {
> + struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> +
> + spin_lock(&n->list_lock);
> remove_full(s, page);
> + spin_unlock(&n->list_lock);
> + }
> if (s->flags & SLAB_STORE_USER)
> set_track(s, object, TRACK_FREE, addr);
> trace(s, page, object, 0);
> @@ -1420,36 +1421,26 @@ static __always_inline int slab_trylock(
> /*
> * Management of partially allocated slabs
> */
> -static void add_partial(struct kmem_cache_node *n,
> +static inline void add_partial(struct kmem_cache_node *n,
> struct page *page, int tail)
> {
> - spin_lock(&n->list_lock);
> n->nr_partial++;
> if (tail)
> list_add_tail(&page->lru, &n->partial);
> else
> list_add(&page->lru, &n->partial);
> - spin_unlock(&n->list_lock);
> }
>
> -static inline void __remove_partial(struct kmem_cache_node *n,
> +static inline void remove_partial(struct kmem_cache_node *n,
> struct page *page)
> {
> list_del(&page->lru);
> n->nr_partial--;
> }
>
> -static void remove_partial(struct kmem_cache *s, struct page *page)
> -{
> - struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> -
> - spin_lock(&n->list_lock);
> - __remove_partial(n, page);
> - spin_unlock(&n->list_lock);
> -}
> -
> /*
> - * Lock slab and remove from the partial list.
> + * Lock slab, remove from the partial list and put the object into the
> + * per cpu freelist.
> *
> * Must hold list_lock.
> */
Comment looks like it got updated prematurely in this patch.
> @@ -1457,7 +1448,7 @@ static inline int lock_and_freeze_slab(s
> struct page *page)
> {
> if (slab_trylock(page)) {
> - __remove_partial(n, page);
> + remove_partial(n, page);
> return 1;
> }
> return 0;
> @@ -1574,12 +1565,17 @@ static void unfreeze_slab(struct kmem_ca
> if (page->inuse) {
>
> if (page->freelist) {
> + spin_lock(&n->list_lock);
> add_partial(n, page, tail);
> + spin_unlock(&n->list_lock);
> stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
> } else {
> stat(s, DEACTIVATE_FULL);
> - if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
> - add_full(n, page);
> + if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) {
> + spin_lock(&n->list_lock);
> + add_full(s, n, page);
> + spin_unlock(&n->list_lock);
> + }
> }
> slab_unlock(page);
> } else {
> @@ -1595,7 +1591,9 @@ static void unfreeze_slab(struct kmem_ca
> * kmem_cache_shrink can reclaim any empty slabs from
> * the partial list.
> */
> + spin_lock(&n->list_lock);
> add_partial(n, page, 1);
> + spin_unlock(&n->list_lock);
> slab_unlock(page);
> } else {
> slab_unlock(page);
> @@ -2097,7 +2095,11 @@ static void __slab_free(struct kmem_cach
> * then add it.
> */
> if (unlikely(!prior)) {
> + struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> +
> + spin_lock(&n->list_lock);
> add_partial(get_node(s, page_to_nid(page)), page, 1);
> + spin_unlock(&n->list_lock);
> stat(s, FREE_ADD_PARTIAL);
> }
>
Second call to get_node() is unnecessary, you can pass n.
> @@ -2111,7 +2113,11 @@ slab_empty:
> /*
> * Slab still on the partial list.
> */
> - remove_partial(s, page);
> + struct kmem_cache_node *n = get_node(s, page_to_nid(page));
> +
> + spin_lock(&n->list_lock);
> + remove_partial(n, page);
> + spin_unlock(&n->list_lock);
> stat(s, FREE_REMOVE_PARTIAL);
> }
> slab_unlock(page);
> @@ -2393,7 +2399,6 @@ static void early_kmem_cache_node_alloc(
> {
> struct page *page;
> struct kmem_cache_node *n;
> - unsigned long flags;
>
> BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
>
> @@ -2420,14 +2425,7 @@ static void early_kmem_cache_node_alloc(
> init_kmem_cache_node(n, kmem_cache_node);
> inc_slabs_node(kmem_cache_node, node, page->objects);
>
> - /*
> - * lockdep requires consistent irq usage for each lock
> - * so even though there cannot be a race this early in
> - * the boot sequence, we still disable irqs.
> - */
> - local_irq_save(flags);
> add_partial(n, page, 0);
> - local_irq_restore(flags);
> }
>
> static void free_kmem_cache_nodes(struct kmem_cache *s)
This is no longer true? It wouldn't have been changed with this patch.
^ permalink raw reply [flat|nested] 32+ messages in thread
* [slubllv6 08/17] slub: Pass kmem_cache struct to lock and freeze slab
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (6 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 07/17] slub: explicit list_lock taking Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-27 19:43 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 09/17] slub: Rework allocator fastpaths Christoph Lameter
` (8 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: pass_kmem_cache_to_lock_and_freeze --]
[-- Type: text/plain, Size: 1957 bytes --]
We need more information about the slab for the cmpxchg implementation.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-16 11:46:58.311463052 -0500
+++ linux-2.6/mm/slub.c 2011-05-16 11:47:01.591463049 -0500
@@ -1444,8 +1444,8 @@ static inline void remove_partial(struct
*
* Must hold list_lock.
*/
-static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
- struct page *page)
+static inline int lock_and_freeze_slab(struct kmem_cache *s,
+ struct kmem_cache_node *n, struct page *page)
{
if (slab_trylock(page)) {
remove_partial(n, page);
@@ -1457,7 +1457,8 @@ static inline int lock_and_freeze_slab(s
/*
* Try to allocate a partial slab from a specific node.
*/
-static struct page *get_partial_node(struct kmem_cache_node *n)
+static struct page *get_partial_node(struct kmem_cache *s,
+ struct kmem_cache_node *n)
{
struct page *page;
@@ -1472,7 +1473,7 @@ static struct page *get_partial_node(str
spin_lock(&n->list_lock);
list_for_each_entry(page, &n->partial, lru)
- if (lock_and_freeze_slab(n, page))
+ if (lock_and_freeze_slab(s, n, page))
goto out;
page = NULL;
out:
@@ -1523,7 +1524,7 @@ static struct page *get_any_partial(stru
if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
n->nr_partial > s->min_partial) {
- page = get_partial_node(n);
+ page = get_partial_node(s, n);
if (page) {
put_mems_allowed();
return page;
@@ -1543,7 +1544,7 @@ static struct page *get_partial(struct k
struct page *page;
int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
- page = get_partial_node(get_node(s, searchnode));
+ page = get_partial_node(s, get_node(s, searchnode));
if (page || node != NUMA_NO_NODE)
return page;
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 09/17] slub: Rework allocator fastpaths
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (7 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 08/17] slub: Pass kmem_cache struct to lock and freeze slab Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 10/17] slub: Invert locking and avoid slab lock Christoph Lameter
` (7 subsequent siblings)
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: rework_fastpaths --]
[-- Type: text/plain, Size: 13909 bytes --]
Rework the allocation paths so that updates of the page freelist, frozen state
and number of objects use cmpxchg_double_slab().
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 408 ++++++++++++++++++++++++++++++++++++++++++--------------------
1 file changed, 279 insertions(+), 129 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-26 09:10:34.965543901 -0500
+++ linux-2.6/mm/slub.c 2011-05-26 09:10:47.715543820 -0500
@@ -987,11 +987,6 @@ static noinline int alloc_debug_processi
if (!check_slab(s, page))
goto bad;
- if (!on_freelist(s, page, object)) {
- object_err(s, page, object, "Object already allocated");
- goto bad;
- }
-
if (!check_valid_pointer(s, page, object)) {
object_err(s, page, object, "Freelist Pointer check fails");
goto bad;
@@ -1055,14 +1050,6 @@ static noinline int free_debug_processin
goto fail;
}
- /* Special debug activities for freeing objects */
- if (!page->frozen && !page->freelist) {
- struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
- spin_lock(&n->list_lock);
- remove_full(s, page);
- spin_unlock(&n->list_lock);
- }
if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0);
@@ -1447,11 +1434,52 @@ static inline void remove_partial(struct
static inline int lock_and_freeze_slab(struct kmem_cache *s,
struct kmem_cache_node *n, struct page *page)
{
- if (slab_trylock(page)) {
- remove_partial(n, page);
+ void *freelist;
+ unsigned long counters;
+ struct page new;
+
+
+ if (!slab_trylock(page))
+ return 0;
+
+ /*
+ * Zap the freelist and set the frozen bit.
+ * The old freelist is the list of objects for the
+ * per cpu allocation list.
+ */
+ do {
+ freelist = page->freelist;
+ counters = page->counters;
+ new.counters = counters;
+ new.inuse = page->objects;
+
+ VM_BUG_ON(new.frozen);
+ new.frozen = 1;
+
+ } while (!cmpxchg_double_slab(s, page,
+ freelist, counters,
+ NULL, new.counters,
+ "lock and freeze"));
+
+ remove_partial(n, page);
+
+ if (freelist) {
+ /* Populate the per cpu freelist */
+ this_cpu_write(s->cpu_slab->freelist, freelist);
+ this_cpu_write(s->cpu_slab->page, page);
+ this_cpu_write(s->cpu_slab->node, page_to_nid(page));
return 1;
+ } else {
+ /*
+ * Slab page came from the wrong list. No object to allocate
+ * from. Put it onto the correct list and continue partial
+ * scan.
+ */
+ printk(KERN_ERR "SLUB: %s : Page without available objects on"
+ " partial list\n", s->name);
+ slab_unlock(page);
+ return 0;
}
- return 0;
}
/*
@@ -1551,59 +1579,6 @@ static struct page *get_partial(struct k
return get_any_partial(s, flags);
}
-/*
- * Move a page back to the lists.
- *
- * Must be called with the slab lock held.
- *
- * On exit the slab lock will have been dropped.
- */
-static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
- __releases(bitlock)
-{
- struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
- if (page->inuse) {
-
- if (page->freelist) {
- spin_lock(&n->list_lock);
- add_partial(n, page, tail);
- spin_unlock(&n->list_lock);
- stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
- } else {
- stat(s, DEACTIVATE_FULL);
- if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) {
- spin_lock(&n->list_lock);
- add_full(s, n, page);
- spin_unlock(&n->list_lock);
- }
- }
- slab_unlock(page);
- } else {
- stat(s, DEACTIVATE_EMPTY);
- if (n->nr_partial < s->min_partial) {
- /*
- * Adding an empty slab to the partial slabs in order
- * to avoid page allocator overhead. This slab needs
- * to come after the other slabs with objects in
- * so that the others get filled first. That way the
- * size of the partial list stays small.
- *
- * kmem_cache_shrink can reclaim any empty slabs from
- * the partial list.
- */
- spin_lock(&n->list_lock);
- add_partial(n, page, 1);
- spin_unlock(&n->list_lock);
- slab_unlock(page);
- } else {
- slab_unlock(page);
- stat(s, FREE_SLAB);
- discard_slab(s, page);
- }
- }
-}
-
#ifdef CONFIG_PREEMPT
/*
* Calculate the next globally unique transaction for disambiguiation
@@ -1673,37 +1648,158 @@ void init_kmem_cache_cpus(struct kmem_ca
/*
* Remove the cpu slab
*/
+
+/*
+ * Remove the cpu slab
+ */
static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
- __releases(bitlock)
{
+ enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
struct page *page = c->page;
- int tail = 1;
+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+ int lock = 0;
+ enum slab_modes l = M_NONE, m = M_NONE;
+ void *freelist;
+ void *nextfree;
+ int tail = 0;
+ struct page new;
+ struct page old;
- if (page->freelist)
+ if (page->freelist) {
stat(s, DEACTIVATE_REMOTE_FREES);
+ tail = 1;
+ }
+
+ c->tid = next_tid(c->tid);
+ c->page = NULL;
+ freelist = c->freelist;
+ c->freelist = NULL;
+
/*
- * Merge cpu freelist into slab freelist. Typically we get here
- * because both freelists are empty. So this is unlikely
- * to occur.
+ * Stage one: Free all available per cpu objects back
+ * to the page freelist while it is still frozen. Leave the
+ * last one.
+ *
+ * There is no need to take the list->lock because the page
+ * is still frozen.
*/
- while (unlikely(c->freelist)) {
- void **object;
+ while (freelist && (nextfree = get_freepointer(s, freelist))) {
+ void *prior;
+ unsigned long counters;
+
+ do {
+ prior = page->freelist;
+ counters = page->counters;
+ set_freepointer(s, freelist, prior);
+ new.counters = counters;
+ new.inuse--;
+ VM_BUG_ON(!new.frozen);
+
+ } while (!cmpxchg_double_slab(s, page,
+ prior, counters,
+ freelist, new.counters,
+ "drain percpu freelist"));
- tail = 0; /* Hot objects. Put the slab first */
+ freelist = nextfree;
+ }
- /* Retrieve object from cpu_freelist */
- object = c->freelist;
- c->freelist = get_freepointer(s, c->freelist);
+ /*
+ * Stage two: Ensure that the page is unfrozen while the
+ * list presence reflects the actual number of objects
+ * during unfreeze.
+ *
+ * We setup the list membership and then perform a cmpxchg
+ * with the count. If there is a mismatch then the page
+ * is not unfrozen but the page is on the wrong list.
+ *
+ * Then we restart the process which may have to remove
+ * the page from the list that we just put it on again
+ * because the number of objects in the slab may have
+ * changed.
+ */
+redo:
- /* And put onto the regular freelist */
- set_freepointer(s, object, page->freelist);
- page->freelist = object;
- page->inuse--;
+ old.freelist = page->freelist;
+ old.counters = page->counters;
+ VM_BUG_ON(!old.frozen);
+
+ /* Determine target state of the slab */
+ new.counters = old.counters;
+ if (freelist) {
+ new.inuse--;
+ set_freepointer(s, freelist, old.freelist);
+ new.freelist = freelist;
+ } else
+ new.freelist = old.freelist;
+
+ new.frozen = 0;
+
+ if (!new.inuse && n->nr_partial < s->min_partial)
+ m = M_FREE;
+ else if (new.freelist) {
+ m = M_PARTIAL;
+ if (!lock) {
+ lock = 1;
+ /*
+ * Taking the spinlock removes the possiblity
+ * that acquire_slab() will see a slab page that
+ * is frozen
+ */
+ spin_lock(&n->list_lock);
+ }
+ } else {
+ m = M_FULL;
+ if (kmem_cache_debug(s) && !lock) {
+ lock = 1;
+ /*
+ * This also ensures that the scanning of full
+ * slabs from diagnostic functions will not see
+ * any frozen slabs.
+ */
+ spin_lock(&n->list_lock);
+ }
+ }
+
+ if (l != m) {
+
+ if (l == M_PARTIAL)
+
+ remove_partial(n, page);
+
+ else if (l == M_FULL)
+
+ remove_full(s, page);
+
+ if (m == M_PARTIAL) {
+
+ add_partial(n, page, tail);
+ stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
+
+ } else if (m == M_FULL) {
+
+ stat(s, DEACTIVATE_FULL);
+ add_full(s, n, page);
+
+ }
+ }
+
+ l = m;
+ if (!cmpxchg_double_slab(s, page,
+ old.freelist, old.counters,
+ new.freelist, new.counters,
+ "unfreezing slab"))
+ goto redo;
+
+ slab_unlock(page);
+
+ if (lock)
+ spin_unlock(&n->list_lock);
+
+ if (m == M_FREE) {
+ stat(s, DEACTIVATE_EMPTY);
+ discard_slab(s, page);
+ stat(s, FREE_SLAB);
}
- c->page = NULL;
- c->tid = next_tid(c->tid);
- page->frozen = 0;
- unfreeze_slab(s, page, tail);
}
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
@@ -1838,6 +1934,8 @@ static void *__slab_alloc(struct kmem_ca
void **object;
struct page *page;
unsigned long flags;
+ struct page new;
+ unsigned long counters;
local_irq_save(flags);
#ifdef CONFIG_PREEMPT
@@ -1860,25 +1958,33 @@ static void *__slab_alloc(struct kmem_ca
if (unlikely(!node_match(c, node)))
goto another_slab;
- stat(s, ALLOC_REFILL);
+ stat(s, ALLOC_SLOWPATH);
+
+ do {
+ object = page->freelist;
+ counters = page->counters;
+ new.counters = counters;
+ new.inuse = page->objects;
+ VM_BUG_ON(!new.frozen);
+
+ } while (!cmpxchg_double_slab(s, page,
+ object, counters,
+ NULL, new.counters,
+ "__slab_alloc"));
load_freelist:
VM_BUG_ON(!page->frozen);
- object = page->freelist;
if (unlikely(!object))
goto another_slab;
- if (kmem_cache_debug(s))
- goto debug;
- c->freelist = get_freepointer(s, object);
- page->inuse = page->objects;
- page->freelist = NULL;
+ stat(s, ALLOC_REFILL);
slab_unlock(page);
+
+ c->freelist = get_freepointer(s, object);
c->tid = next_tid(c->tid);
local_irq_restore(flags);
- stat(s, ALLOC_SLOWPATH);
return object;
another_slab:
@@ -1888,9 +1994,10 @@ new_slab:
page = get_partial(s, gfpflags, node);
if (page) {
stat(s, ALLOC_FROM_PARTIAL);
- page->frozen = 1;
- c->node = page_to_nid(page);
- c->page = page;
+ object = c->freelist;
+
+ if (kmem_cache_debug(s))
+ goto debug;
goto load_freelist;
}
@@ -1898,12 +2005,19 @@ new_slab:
if (page) {
c = __this_cpu_ptr(s->cpu_slab);
- stat(s, ALLOC_SLAB);
if (c->page)
flush_slab(s, c);
+ /*
+ * No other reference to the page yet so we can
+ * muck around with it freely without cmpxchg
+ */
+ object = page->freelist;
+ page->freelist = NULL;
+ page->inuse = page->objects;
+
+ stat(s, ALLOC_SLAB);
slab_lock(page);
- page->frozen = 1;
c->node = page_to_nid(page);
c->page = page;
goto load_freelist;
@@ -1912,12 +2026,12 @@ new_slab:
slab_out_of_memory(s, gfpflags, node);
local_irq_restore(flags);
return NULL;
+
debug:
- if (!alloc_debug_processing(s, page, object, addr))
- goto another_slab;
+ if (!object || !alloc_debug_processing(s, page, object, addr))
+ goto new_slab;
- page->inuse++;
- page->freelist = get_freepointer(s, object);
+ c->freelist = get_freepointer(s, object);
deactivate_slab(s, c);
c->page = NULL;
c->node = NUMA_NO_NODE;
@@ -2069,6 +2183,11 @@ static void __slab_free(struct kmem_cach
{
void *prior;
void **object = (void *)x;
+ int was_frozen;
+ int inuse;
+ struct page new;
+ unsigned long counters;
+ struct kmem_cache_node *n = NULL;
unsigned long flags;
local_irq_save(flags);
@@ -2078,32 +2197,65 @@ static void __slab_free(struct kmem_cach
if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
goto out_unlock;
- prior = page->freelist;
- set_freepointer(s, object, prior);
- page->freelist = object;
- page->inuse--;
-
- if (unlikely(page->frozen)) {
- stat(s, FREE_FROZEN);
- goto out_unlock;
- }
+ do {
+ prior = page->freelist;
+ counters = page->counters;
+ set_freepointer(s, object, prior);
+ new.counters = counters;
+ was_frozen = new.frozen;
+ new.inuse--;
+ if ((!new.inuse || !prior) && !was_frozen && !n) {
+ n = get_node(s, page_to_nid(page));
+ /*
+ * Speculatively acquire the list_lock.
+ * If the cmpxchg does not succeed then we may
+ * drop the list_lock without any processing.
+ *
+ * Otherwise the list_lock will synchronize with
+ * other processors updating the list of slabs.
+ */
+ spin_lock(&n->list_lock);
+ }
+ inuse = new.inuse;
- if (unlikely(!page->inuse))
- goto slab_empty;
+ } while (!cmpxchg_double_slab(s, page,
+ prior, counters,
+ object, new.counters,
+ "__slab_free"));
+
+ if (likely(!n)) {
+ /*
+ * The list lock was not taken therefore no list
+ * activity can be necessary.
+ */
+ if (was_frozen)
+ stat(s, FREE_FROZEN);
+ goto out_unlock;
+ }
/*
- * Objects left in the slab. If it was not on the partial list before
- * then add it.
+ * was_frozen may have been set after we acquired the list_lock in
+ * an earlier loop. So we need to check it here again.
*/
- if (unlikely(!prior)) {
- struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+ if (was_frozen)
+ stat(s, FREE_FROZEN);
+ else {
+ if (unlikely(!inuse && n->nr_partial > s->min_partial))
+ goto slab_empty;
- spin_lock(&n->list_lock);
- add_partial(get_node(s, page_to_nid(page)), page, 1);
- spin_unlock(&n->list_lock);
- stat(s, FREE_ADD_PARTIAL);
+ /*
+ * Objects left in the slab. If it was not on the partial list before
+ * then add it.
+ */
+ if (unlikely(!prior)) {
+ remove_full(s, page);
+ add_partial(n, page, 0);
+ stat(s, FREE_ADD_PARTIAL);
+ }
}
+ spin_unlock(&n->list_lock);
+
out_unlock:
slab_unlock(page);
local_irq_restore(flags);
@@ -2114,13 +2266,11 @@ slab_empty:
/*
* Slab still on the partial list.
*/
- struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
- spin_lock(&n->list_lock);
remove_partial(n, page);
- spin_unlock(&n->list_lock);
stat(s, FREE_REMOVE_PARTIAL);
}
+
+ spin_unlock(&n->list_lock);
slab_unlock(page);
local_irq_restore(flags);
stat(s, FREE_SLAB);
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 10/17] slub: Invert locking and avoid slab lock
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (8 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 09/17] slub: Rework allocator fastpaths Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 11/17] slub: Disable interrupts in free_debug processing Christoph Lameter
` (6 subsequent siblings)
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: slab_lock_subsume --]
[-- Type: text/plain, Size: 10362 bytes --]
Locking slabs is no longer necesary if the arch supports cmpxchg operations
and if no debuggin features are used on a slab. If the arch does not support
cmpxchg then we fallback to use the slab lock to do a cmpxchg like operation.
The patch also changes the lock order. Slab locks are subsumed to the node lock
now. With that approach slab_trylocking is no longer necessary.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 131 +++++++++++++++++++++++++-------------------------------------
1 file changed, 53 insertions(+), 78 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:49.564875083 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:40:54.274875052 -0500
@@ -2,10 +2,11 @@
* SLUB: A slab allocator that limits cache line use instead of queuing
* objects in per cpu and per node lists.
*
- * The allocator synchronizes using per slab locks and only
- * uses a centralized lock to manage a pool of partial slabs.
+ * The allocator synchronizes using per slab locks or atomic operatios
+ * and only uses a centralized lock to manage a pool of partial slabs.
*
* (C) 2007 SGI, Christoph Lameter
+ * (C) 2011 Linux Foundation, Christoph Lameter
*/
#include <linux/mm.h>
@@ -32,15 +33,27 @@
/*
* Lock order:
- * 1. slab_lock(page)
- * 2. slab->list_lock
- *
- * The slab_lock protects operations on the object of a particular
- * slab and its metadata in the page struct. If the slab lock
- * has been taken then no allocations nor frees can be performed
- * on the objects in the slab nor can the slab be added or removed
- * from the partial or full lists since this would mean modifying
- * the page_struct of the slab.
+ * 1. slub_lock (Global Semaphore)
+ * 2. node->list_lock
+ * 3. slab_lock(page) (Only on some arches and for debugging)
+ *
+ * slub_lock
+ *
+ * The role of the slub_lock is to protect the list of all the slabs
+ * and to synchronize major metadata changes to slab cache structures.
+ *
+ * The slab_lock is only used for debugging and on arches that do not
+ * have the ability to do a cmpxchg_double. It only protects the second
+ * double word in the page struct. Meaning
+ * A. page->freelist -> List of object free in a page
+ * B. page->counters -> Counters of objects
+ * C. page->frozen -> frozen state
+ *
+ * If a slab is frozen then it is exempt from list management. It is not
+ * on any list. The processor that froze the slab is the one who can
+ * perform list operations on the page. Other processors may put objects
+ * onto the freelist but the processor that froze the slab is the only
+ * one that can retrieve the objects from the page's freelist.
*
* The list_lock protects the partial and full list on each node and
* the partial slab counter. If taken then no new slabs may be added or
@@ -53,20 +66,6 @@
* slabs, operations can continue without any centralized lock. F.e.
* allocating a long series of objects that fill up slabs does not require
* the list lock.
- *
- * The lock order is sometimes inverted when we are trying to get a slab
- * off a list. We take the list_lock and then look for a page on the list
- * to use. While we do that objects in the slabs may be freed. We can
- * only operate on the slab if we have also taken the slab_lock. So we use
- * a slab_trylock() on the slab. If trylock was successful then no frees
- * can occur anymore and we can use the slab for allocations etc. If the
- * slab_trylock() does not succeed then frees are in progress in the slab and
- * we must stay away from it for a while since we may cause a bouncing
- * cacheline if we try to acquire the lock. So go onto the next slab.
- * If all pages are busy then we may allocate a new slab instead of reusing
- * a partial slab. A new slab has no one operating on it and thus there is
- * no danger of cacheline contention.
- *
* Interrupts are disabled during allocation and deallocation in order to
* make the slab allocator safe to use in the context of an irq. In addition
* interrupts are disabled to ensure that the processor does not change
@@ -346,7 +345,7 @@ static inline int oo_objects(struct kmem
/*
* Determine a map of object in use on a page.
*
- * Slab lock or node listlock must be held to guarantee that the page does
+ * Node listlock must be held to guarantee that the page does
* not vanish from under us.
*/
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
@@ -358,6 +357,19 @@ static void get_map(struct kmem_cache *s
set_bit(slab_index(p, s, addr), map);
}
+/*
+ * Per slab locking using the pagelock
+ */
+static __always_inline void slab_lock(struct page *page)
+{
+ bit_spin_lock(PG_locked, &page->flags);
+}
+
+static __always_inline void slab_unlock(struct page *page)
+{
+ __bit_spin_unlock(PG_locked, &page->flags);
+}
+
static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
void *freelist_old, unsigned long counters_old,
void *freelist_new, unsigned long counters_new,
@@ -372,11 +384,14 @@ static inline bool cmpxchg_double_slab(s
} else
#endif
{
+ slab_lock(page);
if (page->freelist == freelist_old && page->counters == counters_old) {
page->freelist = freelist_new;
page->counters = counters_new;
+ slab_unlock(page);
return 1;
}
+ slab_unlock(page);
}
cpu_relax();
@@ -808,10 +823,11 @@ static int check_slab(struct kmem_cache
static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{
int nr = 0;
- void *fp = page->freelist;
+ void *fp;
void *object = NULL;
unsigned long max_objects;
+ fp = page->freelist;
while (fp && nr <= page->objects) {
if (fp == search)
return 1;
@@ -1019,6 +1035,8 @@ bad:
static noinline int free_debug_processing(struct kmem_cache *s,
struct page *page, void *object, unsigned long addr)
{
+ slab_lock(page);
+
if (!check_slab(s, page))
goto fail;
@@ -1054,10 +1072,12 @@ static noinline int free_debug_processin
set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0);
init_object(s, object, SLUB_RED_INACTIVE);
+ slab_unlock(page);
return 1;
fail:
slab_fix(s, "Object at 0x%p not freed", object);
+ slab_unlock(page);
return 0;
}
@@ -1385,27 +1405,6 @@ static void discard_slab(struct kmem_cac
}
/*
- * Per slab locking using the pagelock
- */
-static __always_inline void slab_lock(struct page *page)
-{
- bit_spin_lock(PG_locked, &page->flags);
-}
-
-static __always_inline void slab_unlock(struct page *page)
-{
- __bit_spin_unlock(PG_locked, &page->flags);
-}
-
-static __always_inline int slab_trylock(struct page *page)
-{
- int rc = 1;
-
- rc = bit_spin_trylock(PG_locked, &page->flags);
- return rc;
-}
-
-/*
* Management of partially allocated slabs
*/
static inline void add_partial(struct kmem_cache_node *n,
@@ -1431,17 +1430,13 @@ static inline void remove_partial(struct
*
* Must hold list_lock.
*/
-static inline int lock_and_freeze_slab(struct kmem_cache *s,
+static inline int acquire_slab(struct kmem_cache *s,
struct kmem_cache_node *n, struct page *page)
{
void *freelist;
unsigned long counters;
struct page new;
-
- if (!slab_trylock(page))
- return 0;
-
/*
* Zap the freelist and set the frozen bit.
* The old freelist is the list of objects for the
@@ -1477,7 +1472,6 @@ static inline int lock_and_freeze_slab(s
*/
printk(KERN_ERR "SLUB: %s : Page without available objects on"
" partial list\n", s->name);
- slab_unlock(page);
return 0;
}
}
@@ -1501,7 +1495,7 @@ static struct page *get_partial_node(str
spin_lock(&n->list_lock);
list_for_each_entry(page, &n->partial, lru)
- if (lock_and_freeze_slab(s, n, page))
+ if (acquire_slab(s, n, page))
goto out;
page = NULL;
out:
@@ -1790,8 +1784,6 @@ redo:
"unfreezing slab"))
goto redo;
- slab_unlock(page);
-
if (lock)
spin_unlock(&n->list_lock);
@@ -1805,7 +1797,6 @@ redo:
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
{
stat(s, CPUSLAB_FLUSH);
- slab_lock(c->page);
deactivate_slab(s, c);
}
@@ -1954,7 +1945,6 @@ static void *__slab_alloc(struct kmem_ca
if (!page)
goto new_slab;
- slab_lock(page);
if (unlikely(!node_match(c, node)))
goto another_slab;
@@ -1980,8 +1970,6 @@ load_freelist:
stat(s, ALLOC_REFILL);
- slab_unlock(page);
-
c->freelist = get_freepointer(s, object);
c->tid = next_tid(c->tid);
local_irq_restore(flags);
@@ -2017,7 +2005,6 @@ new_slab:
page->inuse = page->objects;
stat(s, ALLOC_SLAB);
- slab_lock(page);
c->node = page_to_nid(page);
c->page = page;
goto load_freelist;
@@ -2191,7 +2178,6 @@ static void __slab_free(struct kmem_cach
unsigned long flags;
local_irq_save(flags);
- slab_lock(page);
stat(s, FREE_SLOWPATH);
if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
@@ -2257,7 +2243,6 @@ static void __slab_free(struct kmem_cach
spin_unlock(&n->list_lock);
out_unlock:
- slab_unlock(page);
local_irq_restore(flags);
return;
@@ -2271,7 +2256,6 @@ slab_empty:
}
spin_unlock(&n->list_lock);
- slab_unlock(page);
local_irq_restore(flags);
stat(s, FREE_SLAB);
discard_slab(s, page);
@@ -3192,14 +3176,8 @@ int kmem_cache_shrink(struct kmem_cache
* list_lock. page->inuse here is the upper limit.
*/
list_for_each_entry_safe(page, t, &n->partial, lru) {
- if (!page->inuse && slab_trylock(page)) {
- /*
- * Must hold slab lock here because slab_free
- * may have freed the last object and be
- * waiting to release the slab.
- */
+ if (!page->inuse) {
remove_partial(n, page);
- slab_unlock(page);
discard_slab(s, page);
} else {
list_move(&page->lru,
@@ -3787,12 +3765,9 @@ static int validate_slab(struct kmem_cac
static void validate_slab_slab(struct kmem_cache *s, struct page *page,
unsigned long *map)
{
- if (slab_trylock(page)) {
- validate_slab(s, page, map);
- slab_unlock(page);
- } else
- printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
- s->name, page);
+ slab_lock(page);
+ validate_slab(s, page, map);
+ slab_unlock(page);
}
static int validate_slab_node(struct kmem_cache *s,
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 11/17] slub: Disable interrupts in free_debug processing
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (9 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 10/17] slub: Invert locking and avoid slab lock Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 12/17] slub: Avoid disabling interrupts in free slowpath Christoph Lameter
` (5 subsequent siblings)
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: irqoff_in_free_debug_processing --]
[-- Type: text/plain, Size: 1466 bytes --]
We will be calling free_debug_processing with interrupts disabled
in some case when the later patches are applied. Some of the
functions called by free_debug_processing expect interrupts to be
off.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:54.274875052 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:40:57.434875035 -0500
@@ -1035,6 +1035,10 @@ bad:
static noinline int free_debug_processing(struct kmem_cache *s,
struct page *page, void *object, unsigned long addr)
{
+ unsigned long flags;
+ int rc = 0;
+
+ local_irq_save(flags);
slab_lock(page);
if (!check_slab(s, page))
@@ -1051,7 +1055,7 @@ static noinline int free_debug_processin
}
if (!check_object(s, page, object, SLUB_RED_ACTIVE))
- return 0;
+ goto out;
if (unlikely(s != page->slab)) {
if (!PageSlab(page)) {
@@ -1072,13 +1076,15 @@ static noinline int free_debug_processin
set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0);
init_object(s, object, SLUB_RED_INACTIVE);
+ rc = 1;
+out:
slab_unlock(page);
- return 1;
+ local_irq_restore(flags);
+ return rc;
fail:
slab_fix(s, "Object at 0x%p not freed", object);
- slab_unlock(page);
- return 0;
+ goto out;
}
static int __init setup_slub_debug(char *str)
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 12/17] slub: Avoid disabling interrupts in free slowpath
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (10 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 11/17] slub: Disable interrupts in free_debug processing Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 22:16 ` David Rientjes
2011-05-26 18:14 ` [slubllv6 13/17] slub: Get rid of the another_slab label Christoph Lameter
` (4 subsequent siblings)
16 siblings, 1 reply; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: slab_free_without_irqoff --]
[-- Type: text/plain, Size: 2060 bytes --]
Disabling interrupts can be avoided now. However, list operation still require
disabling interrupts since allocations can occur from interrupt
contexts and there is no way to perform atomic list operations.
The acquition of the list_lock therefore has to disable interrupts as well.
Dropping interrupt handling significantly simplifies the slowpath.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:57.434875035 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:41:00.194875015 -0500
@@ -2183,11 +2183,10 @@ static void __slab_free(struct kmem_cach
struct kmem_cache_node *n = NULL;
unsigned long flags;
- local_irq_save(flags);
stat(s, FREE_SLOWPATH);
if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
- goto out_unlock;
+ return;
do {
prior = page->freelist;
@@ -2206,7 +2205,7 @@ static void __slab_free(struct kmem_cach
* Otherwise the list_lock will synchronize with
* other processors updating the list of slabs.
*/
- spin_lock(&n->list_lock);
+ spin_lock_irqsave(&n->list_lock, flags);
}
inuse = new.inuse;
@@ -2222,7 +2221,7 @@ static void __slab_free(struct kmem_cach
*/
if (was_frozen)
stat(s, FREE_FROZEN);
- goto out_unlock;
+ return;
}
/*
@@ -2245,11 +2244,7 @@ static void __slab_free(struct kmem_cach
stat(s, FREE_ADD_PARTIAL);
}
}
-
- spin_unlock(&n->list_lock);
-
-out_unlock:
- local_irq_restore(flags);
+ spin_unlock_irqrestore(&n->list_lock, flags);
return;
slab_empty:
@@ -2261,8 +2256,7 @@ slab_empty:
stat(s, FREE_REMOVE_PARTIAL);
}
- spin_unlock(&n->list_lock);
- local_irq_restore(flags);
+ spin_unlock_irqrestore(&n->list_lock, flags);
stat(s, FREE_SLAB);
discard_slab(s, page);
}
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 12/17] slub: Avoid disabling interrupts in free slowpath
2011-05-26 18:14 ` [slubllv6 12/17] slub: Avoid disabling interrupts in free slowpath Christoph Lameter
@ 2011-05-26 22:16 ` David Rientjes
2011-05-31 17:12 ` Christoph Lameter
0 siblings, 1 reply; 32+ messages in thread
From: David Rientjes @ 2011-05-26 22:16 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: Type: TEXT/PLAIN, Size: 2529 bytes --]
On Thu, 26 May 2011, Christoph Lameter wrote:
> Disabling interrupts can be avoided now. However, list operation still require
> disabling interrupts since allocations can occur from interrupt
> contexts and there is no way to perform atomic list operations.
>
> The acquition of the list_lock therefore has to disable interrupts as well.
>
> Dropping interrupt handling significantly simplifies the slowpath.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> ---
> mm/slub.c | 16 +++++-----------
> 1 file changed, 5 insertions(+), 11 deletions(-)
>
> Index: linux-2.6/mm/slub.c
> ===================================================================
> --- linux-2.6.orig/mm/slub.c 2011-05-24 09:40:57.434875035 -0500
> +++ linux-2.6/mm/slub.c 2011-05-24 09:41:00.194875015 -0500
> @@ -2183,11 +2183,10 @@ static void __slab_free(struct kmem_cach
> struct kmem_cache_node *n = NULL;
> unsigned long flags;
>
> - local_irq_save(flags);
> stat(s, FREE_SLOWPATH);
>
> if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
> - goto out_unlock;
> + return;
>
> do {
> prior = page->freelist;
> @@ -2206,7 +2205,7 @@ static void __slab_free(struct kmem_cach
> * Otherwise the list_lock will synchronize with
> * other processors updating the list of slabs.
> */
> - spin_lock(&n->list_lock);
> + spin_lock_irqsave(&n->list_lock, flags);
> }
> inuse = new.inuse;
>
> @@ -2222,7 +2221,7 @@ static void __slab_free(struct kmem_cach
> */
> if (was_frozen)
> stat(s, FREE_FROZEN);
> - goto out_unlock;
> + return;
> }
>
> /*
> @@ -2245,11 +2244,7 @@ static void __slab_free(struct kmem_cach
> stat(s, FREE_ADD_PARTIAL);
> }
> }
> -
> - spin_unlock(&n->list_lock);
> -
> -out_unlock:
> - local_irq_restore(flags);
> + spin_unlock_irqrestore(&n->list_lock, flags);
We know that flags will be initialized at this point in the code, but the
compiler doesn't and it emits this:
mm/slub.c: In function ‘__slab_free’:
mm/slub.c:2198: warning: ‘flags’ may be used uninitialized in this function
so it'll need uninitialized_var().
> return;
>
> slab_empty:
> @@ -2261,8 +2256,7 @@ slab_empty:
> stat(s, FREE_REMOVE_PARTIAL);
> }
>
> - spin_unlock(&n->list_lock);
> - local_irq_restore(flags);
> + spin_unlock_irqrestore(&n->list_lock, flags);
> stat(s, FREE_SLAB);
> discard_slab(s, page);
> }
>
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [slubllv6 12/17] slub: Avoid disabling interrupts in free slowpath
2011-05-26 22:16 ` David Rientjes
@ 2011-05-31 17:12 ` Christoph Lameter
0 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-31 17:12 UTC (permalink / raw)
To: David Rientjes
Cc: Pekka Enberg, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: Type: TEXT/PLAIN, Size: 1919 bytes --]
On Thu, 26 May 2011, David Rientjes wrote:
> We know that flags will be initialized at this point in the code, but the
> compiler doesn't and it emits this:
>
> mm/slub.c: In function ‘__slab_free’:
> mm/slub.c:2198: warning: ‘flags’ may be used uninitialized in this function
>
> so it'll need uninitialized_var().
Subject: slub: Disable interrupts in free_debug processing
We will be calling free_debug_processing with interrupts disabled
in some case when the later patches are applied. Some of the
functions called by free_debug_processing expect interrupts to be
off.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-31 10:20:05.472975031 -0500
+++ linux-2.6/mm/slub.c 2011-05-31 10:20:09.792975006 -0500
@@ -1040,6 +1040,10 @@ bad:
static noinline int free_debug_processing(struct kmem_cache *s,
struct page *page, void *object, unsigned long addr)
{
+ unsigned long flags;
+ int rc = 0;
+
+ local_irq_save(flags);
slab_lock(page);
if (!check_slab(s, page))
@@ -1056,7 +1060,7 @@ static noinline int free_debug_processin
}
if (!check_object(s, page, object, SLUB_RED_ACTIVE))
- return 0;
+ goto out;
if (unlikely(s != page->slab)) {
if (!PageSlab(page)) {
@@ -1077,13 +1081,15 @@ static noinline int free_debug_processin
set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0);
init_object(s, object, SLUB_RED_INACTIVE);
+ rc = 1;
+out:
slab_unlock(page);
- return 1;
+ local_irq_restore(flags);
+ return rc;
fail:
slab_fix(s, "Object at 0x%p not freed", object);
- slab_unlock(page);
- return 0;
+ goto out;
}
static int __init setup_slub_debug(char *str)
^ permalink raw reply [flat|nested] 32+ messages in thread
* [slubllv6 13/17] slub: Get rid of the another_slab label
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (11 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 12/17] slub: Avoid disabling interrupts in free slowpath Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 14/17] slub: Add statistics for the case that the current slab does not match the node Christoph Lameter
` (3 subsequent siblings)
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: eliminate_another_slab --]
[-- Type: text/plain, Size: 1107 bytes --]
We can avoid deactivate slab in special cases if we do the
deactivation of slabs in each code flow that leads to new_slab.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:41:00.194875015 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:41:03.744874994 -0500
@@ -1951,8 +1951,10 @@ static void *__slab_alloc(struct kmem_ca
if (!page)
goto new_slab;
- if (unlikely(!node_match(c, node)))
- goto another_slab;
+ if (unlikely(!node_match(c, node))) {
+ deactivate_slab(s, c);
+ goto new_slab;
+ }
stat(s, ALLOC_SLOWPATH);
@@ -1972,7 +1974,7 @@ load_freelist:
VM_BUG_ON(!page->frozen);
if (unlikely(!object))
- goto another_slab;
+ goto new_slab;
stat(s, ALLOC_REFILL);
@@ -1981,9 +1983,6 @@ load_freelist:
local_irq_restore(flags);
return object;
-another_slab:
- deactivate_slab(s, c);
-
new_slab:
page = get_partial(s, gfpflags, node);
if (page) {
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 14/17] slub: Add statistics for the case that the current slab does not match the node
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (12 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 13/17] slub: Get rid of the another_slab label Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 15/17] slub: fast release on full slab Christoph Lameter
` (2 subsequent siblings)
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: node_mismatch --]
[-- Type: text/plain, Size: 2050 bytes --]
Slub reloads the per cpu slab if the page does not satisfy the NUMA condition. Track
those reloads since doing so has a performance impact.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slub_def.h | 1 +
mm/slub.c | 3 +++
2 files changed, 4 insertions(+)
Index: linux-2.6/include/linux/slub_def.h
===================================================================
--- linux-2.6.orig/include/linux/slub_def.h 2011-05-24 09:40:43.764875120 -0500
+++ linux-2.6/include/linux/slub_def.h 2011-05-24 09:41:07.574874967 -0500
@@ -24,6 +24,7 @@ enum stat_item {
ALLOC_FROM_PARTIAL, /* Cpu slab acquired from partial list */
ALLOC_SLAB, /* Cpu slab acquired from page allocator */
ALLOC_REFILL, /* Refill cpu slab from slab freelist */
+ ALLOC_NODE_MISMATCH, /* Switching cpu slab */
FREE_SLAB, /* Slab freed to the page allocator */
CPUSLAB_FLUSH, /* Abandoning of the cpu slab */
DEACTIVATE_FULL, /* Cpu slab was full when deactivated */
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:41:03.744874994 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:41:07.574874967 -0500
@@ -1952,6 +1952,7 @@ static void *__slab_alloc(struct kmem_ca
goto new_slab;
if (unlikely(!node_match(c, node))) {
+ stat(s, ALLOC_NODE_MISMATCH);
deactivate_slab(s, c);
goto new_slab;
}
@@ -4651,6 +4652,7 @@ STAT_ATTR(FREE_REMOVE_PARTIAL, free_remo
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
+STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
@@ -4710,6 +4712,7 @@ static struct attribute *slab_attrs[] =
&alloc_from_partial_attr.attr,
&alloc_slab_attr.attr,
&alloc_refill_attr.attr,
+ &alloc_node_mismatch_attr.attr,
&free_slab_attr.attr,
&cpuslab_flush_attr.attr,
&deactivate_full_attr.attr,
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 15/17] slub: fast release on full slab
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (13 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 14/17] slub: Add statistics for the case that the current slab does not match the node Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 16/17] slub: Not necessary to check for empty slab on load_freelist Christoph Lameter
2011-05-26 18:14 ` [slubllv6 17/17] slub: slabinfo update for cmpxchg handling Christoph Lameter
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: slab_alloc_fast_release --]
[-- Type: text/plain, Size: 3041 bytes --]
Make deactivation occur implicitly while checking out the current freelist.
This avoids one cmpxchg operation on a slab that is now fully in use.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slub_def.h | 1 +
mm/slub.c | 21 +++++++++++++++++++--
2 files changed, 20 insertions(+), 2 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:41:07.574874967 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:41:11.294874943 -0500
@@ -1963,9 +1963,21 @@ static void *__slab_alloc(struct kmem_ca
object = page->freelist;
counters = page->counters;
new.counters = counters;
- new.inuse = page->objects;
VM_BUG_ON(!new.frozen);
+ /*
+ * If there is no object left then we use this loop to
+ * deactivate the slab which is simple since no objects
+ * are left in the slab and therefore we do not need to
+ * put the page back onto the partial list.
+ *
+ * If there are objects left then we retrieve them
+ * and use them to refill the per cpu queue.
+ */
+
+ new.inuse = page->objects;
+ new.frozen = object != NULL;
+
} while (!cmpxchg_double_slab(s, page,
object, counters,
NULL, new.counters,
@@ -1974,8 +1986,11 @@ static void *__slab_alloc(struct kmem_ca
load_freelist:
VM_BUG_ON(!page->frozen);
- if (unlikely(!object))
+ if (unlikely(!object)) {
+ c->page = NULL;
+ stat(s, DEACTIVATE_BYPASS);
goto new_slab;
+ }
stat(s, ALLOC_REFILL);
@@ -4660,6 +4675,7 @@ STAT_ATTR(DEACTIVATE_EMPTY, deactivate_e
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
+STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
STAT_ATTR(ORDER_FALLBACK, order_fallback);
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
@@ -4720,6 +4736,7 @@ static struct attribute *slab_attrs[] =
&deactivate_to_head_attr.attr,
&deactivate_to_tail_attr.attr,
&deactivate_remote_frees_attr.attr,
+ &deactivate_bypass_attr.attr,
&order_fallback_attr.attr,
&cmpxchg_double_fail_attr.attr,
&cmpxchg_double_cpu_fail_attr.attr,
Index: linux-2.6/include/linux/slub_def.h
===================================================================
--- linux-2.6.orig/include/linux/slub_def.h 2011-05-24 09:41:07.574874967 -0500
+++ linux-2.6/include/linux/slub_def.h 2011-05-24 09:41:11.294874943 -0500
@@ -32,6 +32,7 @@ enum stat_item {
DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */
DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */
DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
+ DEACTIVATE_BYPASS, /* Implicit deactivation */
ORDER_FALLBACK, /* Number of times fallback was necessary */
CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 16/17] slub: Not necessary to check for empty slab on load_freelist
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (14 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 15/17] slub: fast release on full slab Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
2011-05-26 18:14 ` [slubllv6 17/17] slub: slabinfo update for cmpxchg handling Christoph Lameter
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: goto_load_freelist --]
[-- Type: text/plain, Size: 870 bytes --]
load_freelist is now only branched to only if there are objects available.
So no need to check the object variable for NULL.
---
mm/slub.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2011-05-24 09:41:11.294874943 -0500
+++ linux-2.6/mm/slub.c 2011-05-24 09:41:15.454874919 -0500
@@ -1983,9 +1983,6 @@ static void *__slab_alloc(struct kmem_ca
NULL, new.counters,
"__slab_alloc"));
-load_freelist:
- VM_BUG_ON(!page->frozen);
-
if (unlikely(!object)) {
c->page = NULL;
stat(s, DEACTIVATE_BYPASS);
@@ -1994,6 +1991,8 @@ load_freelist:
stat(s, ALLOC_REFILL);
+load_freelist:
+ VM_BUG_ON(!page->frozen);
c->freelist = get_freepointer(s, object);
c->tid = next_tid(c->tid);
local_irq_restore(flags);
^ permalink raw reply [flat|nested] 32+ messages in thread* [slubllv6 17/17] slub: slabinfo update for cmpxchg handling
2011-05-26 18:14 [slubllv6 00/17] SLUB: Lockless freelists for objects V6 Christoph Lameter
` (15 preceding siblings ...)
2011-05-26 18:14 ` [slubllv6 16/17] slub: Not necessary to check for empty slab on load_freelist Christoph Lameter
@ 2011-05-26 18:14 ` Christoph Lameter
16 siblings, 0 replies; 32+ messages in thread
From: Christoph Lameter @ 2011-05-26 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: David Rientjes, Eric Dumazet, H. Peter Anvin, linux-kernel,
Thomas Gleixner
[-- Attachment #1: update_slabinfo --]
[-- Type: text/plain, Size: 6085 bytes --]
Update the statistics handling and the slabinfo tool to include the new
statistics in the reports it generates.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
tools/slub/slabinfo.c | 57 ++++++++++++++++++++++++++++++++++----------------
1 file changed, 39 insertions(+), 18 deletions(-)
Index: linux-2.6/tools/slub/slabinfo.c
===================================================================
--- linux-2.6.orig/tools/slub/slabinfo.c 2011-05-24 09:36:51.534876607 -0500
+++ linux-2.6/tools/slub/slabinfo.c 2011-05-24 09:41:18.114874899 -0500
@@ -2,8 +2,9 @@
* Slabinfo: Tool to get reports about slabs
*
* (C) 2007 sgi, Christoph Lameter
+ * (C) 2011 Linux Foundation, Christoph Lameter
*
- * Compile by:
+ * Compile with:
*
* gcc -o slabinfo slabinfo.c
*/
@@ -39,6 +40,8 @@ struct slabinfo {
unsigned long cpuslab_flush, deactivate_full, deactivate_empty;
unsigned long deactivate_to_head, deactivate_to_tail;
unsigned long deactivate_remote_frees, order_fallback;
+ unsigned long cmpxchg_double_cpu_fail, cmpxchg_double_fail;
+ unsigned long alloc_node_mismatch, deactivate_bypass;
int numa[MAX_NODES];
int numa_partial[MAX_NODES];
} slabinfo[MAX_SLABS];
@@ -99,7 +102,7 @@ static void fatal(const char *x, ...)
static void usage(void)
{
- printf("slabinfo 5/7/2007. (c) 2007 sgi.\n\n"
+ printf("slabinfo 4/15/2011. (c) 2007 sgi/(c) 2011 Linux Foundation.\n\n"
"slabinfo [-ahnpvtsz] [-d debugopts] [slab-regexp]\n"
"-a|--aliases Show aliases\n"
"-A|--activity Most active slabs first\n"
@@ -293,7 +296,7 @@ int line = 0;
static void first_line(void)
{
if (show_activity)
- printf("Name Objects Alloc Free %%Fast Fallb O\n");
+ printf("Name Objects Alloc Free %%Fast Fallb O CmpX UL\n");
else
printf("Name Objects Objsize Space "
"Slabs/Part/Cpu O/S O %%Fr %%Ef Flg\n");
@@ -379,14 +382,14 @@ static void show_tracking(struct slabinf
printf("\n%s: Kernel object allocation\n", s->name);
printf("-----------------------------------------------------------------------\n");
if (read_slab_obj(s, "alloc_calls"))
- printf(buffer);
+ printf("%s", buffer);
else
printf("No Data\n");
printf("\n%s: Kernel object freeing\n", s->name);
printf("------------------------------------------------------------------------\n");
if (read_slab_obj(s, "free_calls"))
- printf(buffer);
+ printf("%s", buffer);
else
printf("No Data\n");
@@ -400,7 +403,7 @@ static void ops(struct slabinfo *s)
if (read_slab_obj(s, "ops")) {
printf("\n%s: kmem_cache operations\n", s->name);
printf("--------------------------------------------\n");
- printf(buffer);
+ printf("%s", buffer);
} else
printf("\n%s has no kmem_cache operations\n", s->name);
}
@@ -462,19 +465,32 @@ static void slab_stats(struct slabinfo *
if (s->cpuslab_flush)
printf("Flushes %8lu\n", s->cpuslab_flush);
- if (s->alloc_refill)
- printf("Refill %8lu\n", s->alloc_refill);
-
total = s->deactivate_full + s->deactivate_empty +
- s->deactivate_to_head + s->deactivate_to_tail;
+ s->deactivate_to_head + s->deactivate_to_tail + s->deactivate_bypass;
- if (total)
- printf("Deactivate Full=%lu(%lu%%) Empty=%lu(%lu%%) "
- "ToHead=%lu(%lu%%) ToTail=%lu(%lu%%)\n",
- s->deactivate_full, (s->deactivate_full * 100) / total,
- s->deactivate_empty, (s->deactivate_empty * 100) / total,
- s->deactivate_to_head, (s->deactivate_to_head * 100) / total,
+ if (total) {
+ printf("\nSlab Deactivation Ocurrences %%\n");
+ printf("-------------------------------------------------\n");
+ printf("Slab full %7lu %3lu%%\n",
+ s->deactivate_full, (s->deactivate_full * 100) / total);
+ printf("Slab empty %7lu %3lu%%\n",
+ s->deactivate_empty, (s->deactivate_empty * 100) / total);
+ printf("Moved to head of partial list %7lu %3lu%%\n",
+ s->deactivate_to_head, (s->deactivate_to_head * 100) / total);
+ printf("Moved to tail of partial list %7lu %3lu%%\n",
s->deactivate_to_tail, (s->deactivate_to_tail * 100) / total);
+ printf("Deactivation bypass %7lu %3lu%%\n",
+ s->deactivate_bypass, (s->deactivate_bypass * 100) / total);
+ printf("Refilled from foreign frees %7lu %3lu%%\n",
+ s->alloc_refill, (s->alloc_refill * 100) / total);
+ printf("Node mismatch %7lu %3lu%%\n",
+ s->alloc_node_mismatch, (s->alloc_node_mismatch * 100) / total);
+ }
+
+ if (s->cmpxchg_double_fail || s->cmpxchg_double_cpu_fail)
+ printf("\nCmpxchg_double Looping\n------------------------\n");
+ printf("Locked Cmpxchg Double redos %lu\nUnlocked Cmpxchg Double redos %lu\n",
+ s->cmpxchg_double_fail, s->cmpxchg_double_cpu_fail);
}
static void report(struct slabinfo *s)
@@ -573,12 +589,13 @@ static void slabcache(struct slabinfo *s
total_alloc = s->alloc_fastpath + s->alloc_slowpath;
total_free = s->free_fastpath + s->free_slowpath;
- printf("%-21s %8ld %10ld %10ld %3ld %3ld %5ld %1d\n",
+ printf("%-21s %8ld %10ld %10ld %3ld %3ld %5ld %1d %4ld %4ld\n",
s->name, s->objects,
total_alloc, total_free,
total_alloc ? (s->alloc_fastpath * 100 / total_alloc) : 0,
total_free ? (s->free_fastpath * 100 / total_free) : 0,
- s->order_fallback, s->order);
+ s->order_fallback, s->order, s->cmpxchg_double_fail,
+ s->cmpxchg_double_cpu_fail);
}
else
printf("%-21s %8ld %7d %8s %14s %4d %1d %3ld %3ld %s\n",
@@ -1190,6 +1207,10 @@ static void read_slab_dir(void)
slab->deactivate_to_tail = get_obj("deactivate_to_tail");
slab->deactivate_remote_frees = get_obj("deactivate_remote_frees");
slab->order_fallback = get_obj("order_fallback");
+ slab->cmpxchg_double_cpu_fail = get_obj("cmpxchg_double_cpu_fail");
+ slab->cmpxchg_double_fail = get_obj("cmpxchg_double_fail");
+ slab->alloc_node_mismatch = get_obj("alloc_node_mismatch");
+ slab->deactivate_bypass = get_obj("deactivate_bypass");
chdir("..");
if (slab->name[0] == ':')
alias_targets++;
^ permalink raw reply [flat|nested] 32+ messages in thread