diff for duplicates of <20100713101747.2835.45722.sendpatchset@danny.redhat> diff --git a/a/1.txt b/N1/1.txt index baa7250..0eb97b9 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -432,9 +432,3 @@ index 7bb7940..7a5d6dc 100644 if (page_to_nid(page) != node) { -- 1.7.1.1 - --- -To unsubscribe, send a message with 'unsubscribe linux-mm' in -the body to majordomo@kvack.org. For more info on Linux MM, -see: http://www.linux-mm.org/ . -Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a> diff --git a/a/content_digest b/N1/content_digest index 868835d..c99794f 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -449,12 +449,6 @@ " \tBUG_ON(!page);\n" " \tif (page_to_nid(page) != node) {\n" "-- \n" - "1.7.1.1\n" - "\n" - "--\n" - "To unsubscribe, send a message with 'unsubscribe linux-mm' in\n" - "the body to majordomo@kvack.org. For more info on Linux MM,\n" - "see: http://www.linux-mm.org/ .\n" - "Don't email: <a href=mailto:\"dont@kvack.org\"> email@kvack.org </a>" + 1.7.1.1 -2c52454c6c1e44a8a2b2c54dda1a500bf94ff40d94903453122918a044bbccae +88ed1e5f5119a60fcf1cd773ca24263f4c658f4b147dc0d1cc2f1e4d9315139f
diff --git a/a/1.txt b/N2/1.txt index baa7250..8b13789 100644 --- a/a/1.txt +++ b/N2/1.txt @@ -1,440 +1 @@ ->From fba0bdebc34d3db41a2c975eb38e9548ea5c2ed1 Mon Sep 17 00:00:00 2001 -From: Xiaotian Feng <dfeng@redhat.com> -Date: Tue, 13 Jul 2010 10:40:05 +0800 -Subject: [PATCH 05/30] mm: sl[au]b: add knowledge of reserve pages -Restrict objects from reserve slabs (ALLOC_NO_WATERMARKS) to allocation -contexts that are entitled to it. This is done to ensure reserve pages don't -leak out and get consumed. - -The basic pattern used for all # allocators is the following, for each active -slab page we store if it came from an emergency allocation. When we find it -did, make sure the current allocation context would have been able to allocate -page from the emergency reserves as well. In that case allow the allocation. If -not, force a new slab allocation. When that works the memory pressure has -lifted enough to allow this context to get an object, otherwise fail the -allocation. - -[mszeredi@suse.cz: Fix use of uninitialized variable in cache_grow] -[dfeng@redhat.com: Minor fix related with SLABDEBUG] -Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> -Signed-off-by: Miklos Szeredi <mszeredi@suse.cz> -Signed-off-by: Suresh Jayaraman <sjayaraman@suse.de> -Signed-off-by: Xiaotian Feng <dfeng@redhat.com> ---- - include/linux/slub_def.h | 1 + - mm/slab.c | 62 +++++++++++++++++++++++++++++++++++++++------ - mm/slob.c | 16 +++++++++++- - mm/slub.c | 42 ++++++++++++++++++++++++++----- - 4 files changed, 104 insertions(+), 17 deletions(-) - -diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h -index 6447a72..9ef61f4 100644 ---- a/include/linux/slub_def.h -+++ b/include/linux/slub_def.h -@@ -39,6 +39,7 @@ struct kmem_cache_cpu { - void **freelist; /* Pointer to first free per cpu object */ - struct page *page; /* The slab from which we are allocating */ - int node; /* The node of the page (or -1 for debug) */ -+ int reserve; /* Did the current page come from the reserve */ - #ifdef CONFIG_SLUB_STATS - unsigned stat[NR_SLUB_STAT_ITEMS]; - #endif -diff --git a/mm/slab.c b/mm/slab.c -index 4e9c46f..d8cd757 100644 ---- a/mm/slab.c -+++ b/mm/slab.c -@@ -120,6 +120,8 @@ - #include <asm/tlbflush.h> - #include <asm/page.h> - -+#include "internal.h" -+ - /* - * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON. - * 0 for faster, smaller code (especially in the critical paths). -@@ -244,7 +246,8 @@ struct array_cache { - unsigned int avail; - unsigned int limit; - unsigned int batchcount; -- unsigned int touched; -+ unsigned int touched:1, -+ reserve:1; - spinlock_t lock; - void *entry[]; /* - * Must have this definition in here for the proper -@@ -680,6 +683,27 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) - return cachep->array[smp_processor_id()]; - } - -+/* -+ * If the last page came from the reserves, and the current allocation context -+ * does not have access to them, force an allocation to test the watermarks. -+ */ -+static inline int slab_force_alloc(struct kmem_cache *cachep, gfp_t flags) -+{ -+ if (unlikely(cpu_cache_get(cachep)->reserve) && -+ !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS)) -+ return 1; -+ -+ return 0; -+} -+ -+static inline void slab_set_reserve(struct kmem_cache *cachep, int reserve) -+{ -+ struct array_cache *ac = cpu_cache_get(cachep); -+ -+ if (unlikely(ac->reserve != reserve)) -+ ac->reserve = reserve; -+} -+ - static inline struct kmem_cache *__find_general_cachep(size_t size, - gfp_t gfpflags) - { -@@ -886,6 +910,7 @@ static struct array_cache *alloc_arraycache(int node, int entries, - nc->limit = entries; - nc->batchcount = batchcount; - nc->touched = 0; -+ nc->reserve = 0; - spin_lock_init(&nc->lock); - } - return nc; -@@ -1674,7 +1699,8 @@ __initcall(cpucache_init); - * did not request dmaable memory, we might get it, but that - * would be relatively rare and ignorable. - */ --static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) -+static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid, -+ int *reserve) - { - struct page *page; - int nr_pages; -@@ -1696,6 +1722,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) - if (!page) - return NULL; - -+ *reserve = page->reserve; - nr_pages = (1 << cachep->gfporder); - if (cachep->flags & SLAB_RECLAIM_ACCOUNT) - add_zone_page_state(page_zone(page), -@@ -2128,6 +2155,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) - cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; - cpu_cache_get(cachep)->batchcount = 1; - cpu_cache_get(cachep)->touched = 0; -+ cpu_cache_get(cachep)->reserve = 0; - cachep->batchcount = 1; - cachep->limit = BOOT_CPUCACHE_ENTRIES; - return 0; -@@ -2813,6 +2841,7 @@ static int cache_grow(struct kmem_cache *cachep, - size_t offset; - gfp_t local_flags; - struct kmem_list3 *l3; -+ int reserve = -1; - - /* - * Be lazy and only check for valid flags here, keeping it out of the -@@ -2851,7 +2880,7 @@ static int cache_grow(struct kmem_cache *cachep, - * 'nodeid'. - */ - if (!objp) -- objp = kmem_getpages(cachep, local_flags, nodeid); -+ objp = kmem_getpages(cachep, local_flags, nodeid, &reserve); - if (!objp) - goto failed; - -@@ -2868,6 +2897,8 @@ static int cache_grow(struct kmem_cache *cachep, - if (local_flags & __GFP_WAIT) - local_irq_disable(); - check_irq_off(); -+ if (reserve != -1) -+ slab_set_reserve(cachep, reserve); - spin_lock(&l3->list_lock); - - /* Make slab active. */ -@@ -3002,7 +3033,8 @@ bad: - #define check_slabp(x,y) do { } while(0) - #endif - --static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) -+static void *cache_alloc_refill(struct kmem_cache *cachep, -+ gfp_t flags, int must_refill) - { - int batchcount; - struct kmem_list3 *l3; -@@ -3012,6 +3044,8 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) - retry: - check_irq_off(); - node = numa_mem_id(); -+ if (unlikely(must_refill)) -+ goto force_grow; - ac = cpu_cache_get(cachep); - batchcount = ac->batchcount; - if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { -@@ -3081,11 +3115,14 @@ alloc_done: - - if (unlikely(!ac->avail)) { - int x; -+force_grow: - x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL); - - /* cache_grow can reenable interrupts, then ac could change. */ - ac = cpu_cache_get(cachep); -- if (!x && ac->avail == 0) /* no objects in sight? abort */ -+ -+ /* no objects in sight? abort */ -+ if (!x && (ac->avail == 0 || must_refill)) - return NULL; - - if (!ac->avail) /* objects refilled by interrupt? */ -@@ -3175,17 +3212,18 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) - { - void *objp; - struct array_cache *ac; -+ int must_refill = slab_force_alloc(cachep, flags); - - check_irq_off(); - - ac = cpu_cache_get(cachep); -- if (likely(ac->avail)) { -+ if (likely(ac->avail && !must_refill)) { - STATS_INC_ALLOCHIT(cachep); - ac->touched = 1; - objp = ac->entry[--ac->avail]; - } else { - STATS_INC_ALLOCMISS(cachep); -- objp = cache_alloc_refill(cachep, flags); -+ objp = cache_alloc_refill(cachep, flags, must_refill); - /* - * the 'ac' may be updated by cache_alloc_refill(), - * and kmemleak_erase() requires its correct value. -@@ -3243,7 +3281,7 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) - struct zone *zone; - enum zone_type high_zoneidx = gfp_zone(flags); - void *obj = NULL; -- int nid; -+ int nid, reserve; - - if (flags & __GFP_THISNODE) - return NULL; -@@ -3280,10 +3318,12 @@ retry: - if (local_flags & __GFP_WAIT) - local_irq_enable(); - kmem_flagcheck(cache, flags); -- obj = kmem_getpages(cache, local_flags, numa_mem_id()); -+ obj = kmem_getpages(cache, local_flags, numa_mem_id(), -+ &reserve); - if (local_flags & __GFP_WAIT) - local_irq_disable(); - if (obj) { -+ slab_set_reserve(cache, reserve); - /* - * Insert into the appropriate per node queues - */ -@@ -3323,6 +3363,9 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, - l3 = cachep->nodelists[nodeid]; - BUG_ON(!l3); - -+ if (unlikely(slab_force_alloc(cachep, flags))) -+ goto force_grow; -+ - retry: - check_irq_off(); - spin_lock(&l3->list_lock); -@@ -3360,6 +3403,7 @@ retry: - - must_grow: - spin_unlock(&l3->list_lock); -+force_grow: - x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL); - if (x) - goto retry; -diff --git a/mm/slob.c b/mm/slob.c -index 3f19a34..b84b611 100644 ---- a/mm/slob.c -+++ b/mm/slob.c -@@ -71,6 +71,7 @@ - #include <trace/events/kmem.h> - - #include <asm/atomic.h> -+#include "internal.h" - - /* - * slob_block has a field 'units', which indicates size of block if +ve, -@@ -193,6 +194,11 @@ struct slob_rcu { - static DEFINE_SPINLOCK(slob_lock); - - /* -+ * tracks the reserve state for the allocator. -+ */ -+static int slob_reserve; -+ -+/* - * Encode the given size and next info into a free slob block s. - */ - static void set_slob(slob_t *s, slobidx_t size, slob_t *next) -@@ -242,7 +248,7 @@ static int slob_last(slob_t *s) - - static void *slob_new_pages(gfp_t gfp, int order, int node) - { -- void *page; -+ struct page *page; - - #ifdef CONFIG_NUMA - if (node != -1) -@@ -254,6 +260,8 @@ static void *slob_new_pages(gfp_t gfp, int order, int node) - if (!page) - return NULL; - -+ slob_reserve = page->reserve; -+ - return page_address(page); - } - -@@ -326,6 +334,11 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) - slob_t *b = NULL; - unsigned long flags; - -+ if (unlikely(slob_reserve)) { -+ if (!(gfp_to_alloc_flags(gfp) & ALLOC_NO_WATERMARKS)) -+ goto grow; -+ } -+ - if (size < SLOB_BREAK1) - slob_list = &free_slob_small; - else if (size < SLOB_BREAK2) -@@ -364,6 +377,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) - } - spin_unlock_irqrestore(&slob_lock, flags); - -+grow: - /* Not enough space: must allocate a new page */ - if (!b) { - b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); -diff --git a/mm/slub.c b/mm/slub.c -index 7bb7940..7a5d6dc 100644 ---- a/mm/slub.c -+++ b/mm/slub.c -@@ -27,6 +27,8 @@ - #include <linux/memory.h> - #include <linux/math64.h> - #include <linux/fault-inject.h> -+#include "internal.h" -+ - - /* - * Lock order: -@@ -1139,7 +1141,8 @@ static void setup_object(struct kmem_cache *s, struct page *page, - s->ctor(object); - } - --static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) -+static -+struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node, int *reserve) - { - struct page *page; - void *start; -@@ -1153,6 +1156,8 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) - if (!page) - goto out; - -+ *reserve = page->reserve; -+ - inc_slabs_node(s, page_to_nid(page), page->objects); - page->slab = s; - page->flags |= 1 << PG_slab; -@@ -1606,10 +1611,20 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - { - void **object; - struct page *new; -+ int reserve; - - /* We handle __GFP_ZERO in the caller */ - gfpflags &= ~__GFP_ZERO; - -+ if (unlikely(c->reserve)) { -+ /* -+ * If the current slab is a reserve slab and the current -+ * allocation context does not allow access to the reserves we -+ * must force an allocation to test the current levels. -+ */ -+ if (!(gfp_to_alloc_flags(gfpflags) & ALLOC_NO_WATERMARKS)) -+ goto grow_slab; -+ } - if (!c->page) - goto new_slab; - -@@ -1623,8 +1638,8 @@ load_freelist: - object = c->page->freelist; - if (unlikely(!object)) - goto another_slab; -- if (unlikely(SLABDEBUG && PageSlubDebug(c->page))) -- goto debug; -+ if (unlikely(SLABDEBUG && PageSlubDebug(c->page) || c->reserve)) -+ goto slow_path; - - c->freelist = get_freepointer(s, object); - c->page->inuse = c->page->objects; -@@ -1646,16 +1661,18 @@ new_slab: - goto load_freelist; - } - -+grow_slab: - if (gfpflags & __GFP_WAIT) - local_irq_enable(); - -- new = new_slab(s, gfpflags, node); -+ new = new_slab(s, gfpflags, node, &reserve); - - if (gfpflags & __GFP_WAIT) - local_irq_disable(); - - if (new) { - c = __this_cpu_ptr(s->cpu_slab); -+ c->reserve = reserve; - stat(s, ALLOC_SLAB); - if (c->page) - flush_slab(s, c); -@@ -1667,10 +1684,20 @@ new_slab: - if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) - slab_out_of_memory(s, gfpflags, node); - return NULL; --debug: -- if (!alloc_debug_processing(s, c->page, object, addr)) -+ -+slow_path: -+ if (!c->reserve && !alloc_debug_processing(s, c->page, object, addr)) - goto another_slab; - -+ /* -+ * Avoid the slub fast path in slab_alloc() by not setting -+ * c->freelist and the fast path in slab_free() by making -+ * node_match() fail by setting c->node to -1. -+ * -+ * We use this for for debug and reserve checks which need -+ * to be done for each allocation. -+ */ -+ - c->page->inuse++; - c->page->freelist = get_freepointer(s, object); - c->node = -1; -@@ -2095,10 +2122,11 @@ static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node) - struct page *page; - struct kmem_cache_node *n; - unsigned long flags; -+ int reserve; - - BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node)); - -- page = new_slab(kmalloc_caches, gfpflags, node); -+ page = new_slab(kmalloc_caches, gfpflags, node, &reserve); - - BUG_ON(!page); - if (page_to_nid(page) != node) { --- -1.7.1.1 - --- -To unsubscribe, send a message with 'unsubscribe linux-mm' in -the body to majordomo@kvack.org. For more info on Linux MM, -see: http://www.linux-mm.org/ . -Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a> diff --git a/a/content_digest b/N2/content_digest index 868835d..32f4818 100644 --- a/a/content_digest +++ b/N2/content_digest @@ -16,445 +16,5 @@ " davem@davemloft.net\0" "\00:1\0" "b\0" - ">From fba0bdebc34d3db41a2c975eb38e9548ea5c2ed1 Mon Sep 17 00:00:00 2001\n" - "From: Xiaotian Feng <dfeng@redhat.com>\n" - "Date: Tue, 13 Jul 2010 10:40:05 +0800\n" - "Subject: [PATCH 05/30] mm: sl[au]b: add knowledge of reserve pages\n" - "\n" - "Restrict objects from reserve slabs (ALLOC_NO_WATERMARKS) to allocation\n" - "contexts that are entitled to it. This is done to ensure reserve pages don't\n" - "leak out and get consumed.\n" - "\n" - "The basic pattern used for all # allocators is the following, for each active\n" - "slab page we store if it came from an emergency allocation. When we find it\n" - "did, make sure the current allocation context would have been able to allocate\n" - "page from the emergency reserves as well. In that case allow the allocation. If\n" - "not, force a new slab allocation. When that works the memory pressure has\n" - "lifted enough to allow this context to get an object, otherwise fail the\n" - "allocation.\n" - "\n" - "[mszeredi@suse.cz: Fix use of uninitialized variable in cache_grow]\n" - "[dfeng@redhat.com: Minor fix related with SLABDEBUG]\n" - "Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>\n" - "Signed-off-by: Miklos Szeredi <mszeredi@suse.cz>\n" - "Signed-off-by: Suresh Jayaraman <sjayaraman@suse.de>\n" - "Signed-off-by: Xiaotian Feng <dfeng@redhat.com>\n" - "---\n" - " include/linux/slub_def.h | 1 +\n" - " mm/slab.c | 62 +++++++++++++++++++++++++++++++++++++++------\n" - " mm/slob.c | 16 +++++++++++-\n" - " mm/slub.c | 42 ++++++++++++++++++++++++++-----\n" - " 4 files changed, 104 insertions(+), 17 deletions(-)\n" - "\n" - "diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h\n" - "index 6447a72..9ef61f4 100644\n" - "--- a/include/linux/slub_def.h\n" - "+++ b/include/linux/slub_def.h\n" - "@@ -39,6 +39,7 @@ struct kmem_cache_cpu {\n" - " \tvoid **freelist;\t/* Pointer to first free per cpu object */\n" - " \tstruct page *page;\t/* The slab from which we are allocating */\n" - " \tint node;\t\t/* The node of the page (or -1 for debug) */\n" - "+\tint reserve;\t\t/* Did the current page come from the reserve */\n" - " #ifdef CONFIG_SLUB_STATS\n" - " \tunsigned stat[NR_SLUB_STAT_ITEMS];\n" - " #endif\n" - "diff --git a/mm/slab.c b/mm/slab.c\n" - "index 4e9c46f..d8cd757 100644\n" - "--- a/mm/slab.c\n" - "+++ b/mm/slab.c\n" - "@@ -120,6 +120,8 @@\n" - " #include\t<asm/tlbflush.h>\n" - " #include\t<asm/page.h>\n" - " \n" - "+#include \t\"internal.h\"\n" - "+\n" - " /*\n" - " * DEBUG\t- 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.\n" - " *\t\t 0 for faster, smaller code (especially in the critical paths).\n" - "@@ -244,7 +246,8 @@ struct array_cache {\n" - " \tunsigned int avail;\n" - " \tunsigned int limit;\n" - " \tunsigned int batchcount;\n" - "-\tunsigned int touched;\n" - "+\tunsigned int touched:1,\n" - "+\t\t reserve:1;\n" - " \tspinlock_t lock;\n" - " \tvoid *entry[];\t/*\n" - " \t\t\t * Must have this definition in here for the proper\n" - "@@ -680,6 +683,27 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)\n" - " \treturn cachep->array[smp_processor_id()];\n" - " }\n" - " \n" - "+/*\n" - "+ * If the last page came from the reserves, and the current allocation context\n" - "+ * does not have access to them, force an allocation to test the watermarks.\n" - "+ */\n" - "+static inline int slab_force_alloc(struct kmem_cache *cachep, gfp_t flags)\n" - "+{\n" - "+\tif (unlikely(cpu_cache_get(cachep)->reserve) &&\n" - "+\t\t\t!(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS))\n" - "+\t\treturn 1;\n" - "+\n" - "+\treturn 0;\n" - "+}\n" - "+\n" - "+static inline void slab_set_reserve(struct kmem_cache *cachep, int reserve)\n" - "+{\n" - "+\tstruct array_cache *ac = cpu_cache_get(cachep);\n" - "+\n" - "+\tif (unlikely(ac->reserve != reserve))\n" - "+\t\tac->reserve = reserve;\n" - "+}\n" - "+\n" - " static inline struct kmem_cache *__find_general_cachep(size_t size,\n" - " \t\t\t\t\t\t\tgfp_t gfpflags)\n" - " {\n" - "@@ -886,6 +910,7 @@ static struct array_cache *alloc_arraycache(int node, int entries,\n" - " \t\tnc->limit = entries;\n" - " \t\tnc->batchcount = batchcount;\n" - " \t\tnc->touched = 0;\n" - "+\t\tnc->reserve = 0;\n" - " \t\tspin_lock_init(&nc->lock);\n" - " \t}\n" - " \treturn nc;\n" - "@@ -1674,7 +1699,8 @@ __initcall(cpucache_init);\n" - " * did not request dmaable memory, we might get it, but that\n" - " * would be relatively rare and ignorable.\n" - " */\n" - "-static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)\n" - "+static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid,\n" - "+\t\tint *reserve)\n" - " {\n" - " \tstruct page *page;\n" - " \tint nr_pages;\n" - "@@ -1696,6 +1722,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)\n" - " \tif (!page)\n" - " \t\treturn NULL;\n" - " \n" - "+\t*reserve = page->reserve;\n" - " \tnr_pages = (1 << cachep->gfporder);\n" - " \tif (cachep->flags & SLAB_RECLAIM_ACCOUNT)\n" - " \t\tadd_zone_page_state(page_zone(page),\n" - "@@ -2128,6 +2155,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)\n" - " \tcpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;\n" - " \tcpu_cache_get(cachep)->batchcount = 1;\n" - " \tcpu_cache_get(cachep)->touched = 0;\n" - "+\tcpu_cache_get(cachep)->reserve = 0;\n" - " \tcachep->batchcount = 1;\n" - " \tcachep->limit = BOOT_CPUCACHE_ENTRIES;\n" - " \treturn 0;\n" - "@@ -2813,6 +2841,7 @@ static int cache_grow(struct kmem_cache *cachep,\n" - " \tsize_t offset;\n" - " \tgfp_t local_flags;\n" - " \tstruct kmem_list3 *l3;\n" - "+\tint reserve = -1;\n" - " \n" - " \t/*\n" - " \t * Be lazy and only check for valid flags here, keeping it out of the\n" - "@@ -2851,7 +2880,7 @@ static int cache_grow(struct kmem_cache *cachep,\n" - " \t * 'nodeid'.\n" - " \t */\n" - " \tif (!objp)\n" - "-\t\tobjp = kmem_getpages(cachep, local_flags, nodeid);\n" - "+\t\tobjp = kmem_getpages(cachep, local_flags, nodeid, &reserve);\n" - " \tif (!objp)\n" - " \t\tgoto failed;\n" - " \n" - "@@ -2868,6 +2897,8 @@ static int cache_grow(struct kmem_cache *cachep,\n" - " \tif (local_flags & __GFP_WAIT)\n" - " \t\tlocal_irq_disable();\n" - " \tcheck_irq_off();\n" - "+\tif (reserve != -1)\n" - "+\t\tslab_set_reserve(cachep, reserve);\n" - " \tspin_lock(&l3->list_lock);\n" - " \n" - " \t/* Make slab active. */\n" - "@@ -3002,7 +3033,8 @@ bad:\n" - " #define check_slabp(x,y) do { } while(0)\n" - " #endif\n" - " \n" - "-static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)\n" - "+static void *cache_alloc_refill(struct kmem_cache *cachep,\n" - "+\t\tgfp_t flags, int must_refill)\n" - " {\n" - " \tint batchcount;\n" - " \tstruct kmem_list3 *l3;\n" - "@@ -3012,6 +3044,8 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)\n" - " retry:\n" - " \tcheck_irq_off();\n" - " \tnode = numa_mem_id();\n" - "+\tif (unlikely(must_refill))\n" - "+\t\tgoto force_grow;\n" - " \tac = cpu_cache_get(cachep);\n" - " \tbatchcount = ac->batchcount;\n" - " \tif (!ac->touched && batchcount > BATCHREFILL_LIMIT) {\n" - "@@ -3081,11 +3115,14 @@ alloc_done:\n" - " \n" - " \tif (unlikely(!ac->avail)) {\n" - " \t\tint x;\n" - "+force_grow:\n" - " \t\tx = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);\n" - " \n" - " \t\t/* cache_grow can reenable interrupts, then ac could change. */\n" - " \t\tac = cpu_cache_get(cachep);\n" - "-\t\tif (!x && ac->avail == 0)\t/* no objects in sight? abort */\n" - "+\n" - "+\t\t/* no objects in sight? abort */\n" - "+\t\tif (!x && (ac->avail == 0 || must_refill))\n" - " \t\t\treturn NULL;\n" - " \n" - " \t\tif (!ac->avail)\t\t/* objects refilled by interrupt? */\n" - "@@ -3175,17 +3212,18 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)\n" - " {\n" - " \tvoid *objp;\n" - " \tstruct array_cache *ac;\n" - "+\tint must_refill = slab_force_alloc(cachep, flags);\n" - " \n" - " \tcheck_irq_off();\n" - " \n" - " \tac = cpu_cache_get(cachep);\n" - "-\tif (likely(ac->avail)) {\n" - "+\tif (likely(ac->avail && !must_refill)) {\n" - " \t\tSTATS_INC_ALLOCHIT(cachep);\n" - " \t\tac->touched = 1;\n" - " \t\tobjp = ac->entry[--ac->avail];\n" - " \t} else {\n" - " \t\tSTATS_INC_ALLOCMISS(cachep);\n" - "-\t\tobjp = cache_alloc_refill(cachep, flags);\n" - "+\t\tobjp = cache_alloc_refill(cachep, flags, must_refill);\n" - " \t\t/*\n" - " \t\t * the 'ac' may be updated by cache_alloc_refill(),\n" - " \t\t * and kmemleak_erase() requires its correct value.\n" - "@@ -3243,7 +3281,7 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)\n" - " \tstruct zone *zone;\n" - " \tenum zone_type high_zoneidx = gfp_zone(flags);\n" - " \tvoid *obj = NULL;\n" - "-\tint nid;\n" - "+\tint nid, reserve;\n" - " \n" - " \tif (flags & __GFP_THISNODE)\n" - " \t\treturn NULL;\n" - "@@ -3280,10 +3318,12 @@ retry:\n" - " \t\tif (local_flags & __GFP_WAIT)\n" - " \t\t\tlocal_irq_enable();\n" - " \t\tkmem_flagcheck(cache, flags);\n" - "-\t\tobj = kmem_getpages(cache, local_flags, numa_mem_id());\n" - "+\t\tobj = kmem_getpages(cache, local_flags, numa_mem_id(),\n" - "+\t\t\t\t &reserve);\n" - " \t\tif (local_flags & __GFP_WAIT)\n" - " \t\t\tlocal_irq_disable();\n" - " \t\tif (obj) {\n" - "+\t\t\tslab_set_reserve(cache, reserve);\n" - " \t\t\t/*\n" - " \t\t\t * Insert into the appropriate per node queues\n" - " \t\t\t */\n" - "@@ -3323,6 +3363,9 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,\n" - " \tl3 = cachep->nodelists[nodeid];\n" - " \tBUG_ON(!l3);\n" - " \n" - "+\tif (unlikely(slab_force_alloc(cachep, flags)))\n" - "+\t\tgoto force_grow;\n" - "+\n" - " retry:\n" - " \tcheck_irq_off();\n" - " \tspin_lock(&l3->list_lock);\n" - "@@ -3360,6 +3403,7 @@ retry:\n" - " \n" - " must_grow:\n" - " \tspin_unlock(&l3->list_lock);\n" - "+force_grow:\n" - " \tx = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL);\n" - " \tif (x)\n" - " \t\tgoto retry;\n" - "diff --git a/mm/slob.c b/mm/slob.c\n" - "index 3f19a34..b84b611 100644\n" - "--- a/mm/slob.c\n" - "+++ b/mm/slob.c\n" - "@@ -71,6 +71,7 @@\n" - " #include <trace/events/kmem.h>\n" - " \n" - " #include <asm/atomic.h>\n" - "+#include \"internal.h\"\n" - " \n" - " /*\n" - " * slob_block has a field 'units', which indicates size of block if +ve,\n" - "@@ -193,6 +194,11 @@ struct slob_rcu {\n" - " static DEFINE_SPINLOCK(slob_lock);\n" - " \n" - " /*\n" - "+ * tracks the reserve state for the allocator.\n" - "+ */\n" - "+static int slob_reserve;\n" - "+\n" - "+/*\n" - " * Encode the given size and next info into a free slob block s.\n" - " */\n" - " static void set_slob(slob_t *s, slobidx_t size, slob_t *next)\n" - "@@ -242,7 +248,7 @@ static int slob_last(slob_t *s)\n" - " \n" - " static void *slob_new_pages(gfp_t gfp, int order, int node)\n" - " {\n" - "-\tvoid *page;\n" - "+\tstruct page *page;\n" - " \n" - " #ifdef CONFIG_NUMA\n" - " \tif (node != -1)\n" - "@@ -254,6 +260,8 @@ static void *slob_new_pages(gfp_t gfp, int order, int node)\n" - " \tif (!page)\n" - " \t\treturn NULL;\n" - " \n" - "+\tslob_reserve = page->reserve;\n" - "+\n" - " \treturn page_address(page);\n" - " }\n" - " \n" - "@@ -326,6 +334,11 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)\n" - " \tslob_t *b = NULL;\n" - " \tunsigned long flags;\n" - " \n" - "+\tif (unlikely(slob_reserve)) {\n" - "+\t\tif (!(gfp_to_alloc_flags(gfp) & ALLOC_NO_WATERMARKS))\n" - "+\t\t\tgoto grow;\n" - "+\t}\n" - "+\n" - " \tif (size < SLOB_BREAK1)\n" - " \t\tslob_list = &free_slob_small;\n" - " \telse if (size < SLOB_BREAK2)\n" - "@@ -364,6 +377,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)\n" - " \t}\n" - " \tspin_unlock_irqrestore(&slob_lock, flags);\n" - " \n" - "+grow:\n" - " \t/* Not enough space: must allocate a new page */\n" - " \tif (!b) {\n" - " \t\tb = slob_new_pages(gfp & ~__GFP_ZERO, 0, node);\n" - "diff --git a/mm/slub.c b/mm/slub.c\n" - "index 7bb7940..7a5d6dc 100644\n" - "--- a/mm/slub.c\n" - "+++ b/mm/slub.c\n" - "@@ -27,6 +27,8 @@\n" - " #include <linux/memory.h>\n" - " #include <linux/math64.h>\n" - " #include <linux/fault-inject.h>\n" - "+#include \"internal.h\"\n" - "+\n" - " \n" - " /*\n" - " * Lock order:\n" - "@@ -1139,7 +1141,8 @@ static void setup_object(struct kmem_cache *s, struct page *page,\n" - " \t\ts->ctor(object);\n" - " }\n" - " \n" - "-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)\n" - "+static\n" - "+struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node, int *reserve)\n" - " {\n" - " \tstruct page *page;\n" - " \tvoid *start;\n" - "@@ -1153,6 +1156,8 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)\n" - " \tif (!page)\n" - " \t\tgoto out;\n" - " \n" - "+\t*reserve = page->reserve;\n" - "+\n" - " \tinc_slabs_node(s, page_to_nid(page), page->objects);\n" - " \tpage->slab = s;\n" - " \tpage->flags |= 1 << PG_slab;\n" - "@@ -1606,10 +1611,20 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,\n" - " {\n" - " \tvoid **object;\n" - " \tstruct page *new;\n" - "+\tint reserve;\n" - " \n" - " \t/* We handle __GFP_ZERO in the caller */\n" - " \tgfpflags &= ~__GFP_ZERO;\n" - " \n" - "+\tif (unlikely(c->reserve)) {\n" - "+\t\t/*\n" - "+\t\t * If the current slab is a reserve slab and the current\n" - "+\t\t * allocation context does not allow access to the reserves we\n" - "+\t\t * must force an allocation to test the current levels.\n" - "+\t\t */\n" - "+\t\tif (!(gfp_to_alloc_flags(gfpflags) & ALLOC_NO_WATERMARKS))\n" - "+\t\t\tgoto grow_slab;\n" - "+\t}\n" - " \tif (!c->page)\n" - " \t\tgoto new_slab;\n" - " \n" - "@@ -1623,8 +1638,8 @@ load_freelist:\n" - " \tobject = c->page->freelist;\n" - " \tif (unlikely(!object))\n" - " \t\tgoto another_slab;\n" - "-\tif (unlikely(SLABDEBUG && PageSlubDebug(c->page)))\n" - "-\t\tgoto debug;\n" - "+\tif (unlikely(SLABDEBUG && PageSlubDebug(c->page) || c->reserve))\n" - "+\t\tgoto slow_path;\n" - " \n" - " \tc->freelist = get_freepointer(s, object);\n" - " \tc->page->inuse = c->page->objects;\n" - "@@ -1646,16 +1661,18 @@ new_slab:\n" - " \t\tgoto load_freelist;\n" - " \t}\n" - " \n" - "+grow_slab:\n" - " \tif (gfpflags & __GFP_WAIT)\n" - " \t\tlocal_irq_enable();\n" - " \n" - "-\tnew = new_slab(s, gfpflags, node);\n" - "+\tnew = new_slab(s, gfpflags, node, &reserve);\n" - " \n" - " \tif (gfpflags & __GFP_WAIT)\n" - " \t\tlocal_irq_disable();\n" - " \n" - " \tif (new) {\n" - " \t\tc = __this_cpu_ptr(s->cpu_slab);\n" - "+\t\tc->reserve = reserve;\n" - " \t\tstat(s, ALLOC_SLAB);\n" - " \t\tif (c->page)\n" - " \t\t\tflush_slab(s, c);\n" - "@@ -1667,10 +1684,20 @@ new_slab:\n" - " \tif (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())\n" - " \t\tslab_out_of_memory(s, gfpflags, node);\n" - " \treturn NULL;\n" - "-debug:\n" - "-\tif (!alloc_debug_processing(s, c->page, object, addr))\n" - "+\n" - "+slow_path:\n" - "+\tif (!c->reserve && !alloc_debug_processing(s, c->page, object, addr))\n" - " \t\tgoto another_slab;\n" - " \n" - "+\t/*\n" - "+\t * Avoid the slub fast path in slab_alloc() by not setting\n" - "+\t * c->freelist and the fast path in slab_free() by making\n" - "+\t * node_match() fail by setting c->node to -1.\n" - "+\t *\n" - "+\t * We use this for for debug and reserve checks which need\n" - "+\t * to be done for each allocation.\n" - "+\t */\n" - "+\n" - " \tc->page->inuse++;\n" - " \tc->page->freelist = get_freepointer(s, object);\n" - " \tc->node = -1;\n" - "@@ -2095,10 +2122,11 @@ static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node)\n" - " \tstruct page *page;\n" - " \tstruct kmem_cache_node *n;\n" - " \tunsigned long flags;\n" - "+\tint reserve;\n" - " \n" - " \tBUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node));\n" - " \n" - "-\tpage = new_slab(kmalloc_caches, gfpflags, node);\n" - "+\tpage = new_slab(kmalloc_caches, gfpflags, node, &reserve);\n" - " \n" - " \tBUG_ON(!page);\n" - " \tif (page_to_nid(page) != node) {\n" - "-- \n" - "1.7.1.1\n" - "\n" - "--\n" - "To unsubscribe, send a message with 'unsubscribe linux-mm' in\n" - "the body to majordomo@kvack.org. For more info on Linux MM,\n" - "see: http://www.linux-mm.org/ .\n" - "Don't email: <a href=mailto:\"dont@kvack.org\"> email@kvack.org </a>" -2c52454c6c1e44a8a2b2c54dda1a500bf94ff40d94903453122918a044bbccae +2946f242e1b0503472a3c1c109ea7732a8c73816eda9e9a16ffa52e4f3a664ad
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.