diff for duplicates of <20131030082800.GA5753@lge.com> diff --git a/a/1.txt b/N1/1.txt index ec7974e..c3fc674 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -21,3 +21,904 @@ Changes from v2 to v2-FIX. -----------------------------8<------------------------------- +>From 6d11304824a3b8c3bf7574323a3e55471cc26937 Mon Sep 17 00:00:00 2001 +From: Joonsoo Kim <iamjoonsoo.kim@lge.com> +Date: Wed, 28 Aug 2013 16:30:27 +0900 +Subject: [PATCH v2-FIX 13/15] slab: use struct page for slab management + +Now, there are a few field in struct slab, so we can overload these +over struct page. This will save some memory and reduce cache footprint. + +After this change, slabp_cache and slab_size no longer related to +a struct slab, so rename them as freelist_cache and freelist_size. + +These changes are just mechanical ones and there is no functional change. + +Acked-by: Andi Kleen <ak@linux.intel.com> +Acked-by: Christoph Lameter <cl@linux.com> +Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com> + +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index 8b85d8c..4e17190 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -42,18 +42,22 @@ struct page { + /* First double word block */ + unsigned long flags; /* Atomic flags, some possibly + * updated asynchronously */ +- struct address_space *mapping; /* If low bit clear, points to +- * inode address_space, or NULL. +- * If page mapped as anonymous +- * memory, low bit is set, and +- * it points to anon_vma object: +- * see PAGE_MAPPING_ANON below. +- */ ++ union { ++ struct address_space *mapping; /* If low bit clear, points to ++ * inode address_space, or NULL. ++ * If page mapped as anonymous ++ * memory, low bit is set, and ++ * it points to anon_vma object: ++ * see PAGE_MAPPING_ANON below. ++ */ ++ void *s_mem; /* slab first object */ ++ }; ++ + /* Second double word */ + struct { + union { + pgoff_t index; /* Our offset within mapping. */ +- void *freelist; /* slub/slob first free object */ ++ void *freelist; /* sl[aou]b first free object */ + bool pfmemalloc; /* If set by the page allocator, + * ALLOC_NO_WATERMARKS was set + * and the low watermark was not +@@ -109,6 +113,7 @@ struct page { + }; + atomic_t _count; /* Usage count, see below. */ + }; ++ unsigned int active; /* SLAB */ + }; + }; + +diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h +index e9346b4..09bfffb 100644 +--- a/include/linux/slab_def.h ++++ b/include/linux/slab_def.h +@@ -27,8 +27,8 @@ struct kmem_cache { + + size_t colour; /* cache colouring range */ + unsigned int colour_off; /* colour offset */ +- struct kmem_cache *slabp_cache; +- unsigned int slab_size; ++ struct kmem_cache *freelist_cache; ++ unsigned int freelist_size; + + /* constructor func */ + void (*ctor)(void *obj); +diff --git a/mm/slab.c b/mm/slab.c +index 2ec2336..da01fc70 100644 +--- a/mm/slab.c ++++ b/mm/slab.c +@@ -164,21 +164,6 @@ + static bool pfmemalloc_active __read_mostly; + + /* +- * struct slab +- * +- * Manages the objs in a slab. Placed either at the beginning of mem allocated +- * for a slab, or allocated from an general cache. +- * Slabs are chained into three list: fully used, partial, fully free slabs. +- */ +-struct slab { +- struct { +- struct list_head list; +- void *s_mem; /* including colour offset */ +- unsigned int active; /* num of objs active in slab */ +- }; +-}; +- +-/* + * struct array_cache + * + * Purpose: +@@ -405,18 +390,10 @@ static inline struct kmem_cache *virt_to_cache(const void *obj) + return page->slab_cache; + } + +-static inline struct slab *virt_to_slab(const void *obj) +-{ +- struct page *page = virt_to_head_page(obj); +- +- VM_BUG_ON(!PageSlab(page)); +- return page->slab_page; +-} +- +-static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, ++static inline void *index_to_obj(struct kmem_cache *cache, struct page *page, + unsigned int idx) + { +- return slab->s_mem + cache->size * idx; ++ return page->s_mem + cache->size * idx; + } + + /* +@@ -426,9 +403,9 @@ static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, + * reciprocal_divide(offset, cache->reciprocal_buffer_size) + */ + static inline unsigned int obj_to_index(const struct kmem_cache *cache, +- const struct slab *slab, void *obj) ++ const struct page *page, void *obj) + { +- u32 offset = (obj - slab->s_mem); ++ u32 offset = (obj - page->s_mem); + return reciprocal_divide(offset, cache->reciprocal_buffer_size); + } + +@@ -590,7 +567,7 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) + + static size_t slab_mgmt_size(size_t nr_objs, size_t align) + { +- return ALIGN(sizeof(struct slab)+nr_objs*sizeof(unsigned int), align); ++ return ALIGN(nr_objs * sizeof(unsigned int), align); + } + + /* +@@ -609,7 +586,6 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size, + * on it. For the latter case, the memory allocated for a + * slab is used for: + * +- * - The struct slab + * - One unsigned int for each object + * - Padding to respect alignment of @align + * - @buffer_size bytes for each object +@@ -632,8 +608,7 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size, + * into the memory allocation when taking the padding + * into account. + */ +- nr_objs = (slab_size - sizeof(struct slab)) / +- (buffer_size + sizeof(unsigned int)); ++ nr_objs = (slab_size) / (buffer_size + sizeof(unsigned int)); + + /* + * This calculated number will be either the right +@@ -773,11 +748,11 @@ static struct array_cache *alloc_arraycache(int node, int entries, + return nc; + } + +-static inline bool is_slab_pfmemalloc(struct slab *slabp) ++static inline bool is_slab_pfmemalloc(struct page *page) + { +- struct page *page = virt_to_page(slabp->s_mem); ++ struct page *mem_page = virt_to_page(page->s_mem); + +- return PageSlabPfmemalloc(page); ++ return PageSlabPfmemalloc(mem_page); + } + + /* Clears pfmemalloc_active if no slabs have pfmalloc set */ +@@ -785,23 +760,23 @@ static void recheck_pfmemalloc_active(struct kmem_cache *cachep, + struct array_cache *ac) + { + struct kmem_cache_node *n = cachep->node[numa_mem_id()]; +- struct slab *slabp; ++ struct page *page; + unsigned long flags; + + if (!pfmemalloc_active) + return; + + spin_lock_irqsave(&n->list_lock, flags); +- list_for_each_entry(slabp, &n->slabs_full, list) +- if (is_slab_pfmemalloc(slabp)) ++ list_for_each_entry(page, &n->slabs_full, lru) ++ if (is_slab_pfmemalloc(page)) + goto out; + +- list_for_each_entry(slabp, &n->slabs_partial, list) +- if (is_slab_pfmemalloc(slabp)) ++ list_for_each_entry(page, &n->slabs_partial, lru) ++ if (is_slab_pfmemalloc(page)) + goto out; + +- list_for_each_entry(slabp, &n->slabs_free, list) +- if (is_slab_pfmemalloc(slabp)) ++ list_for_each_entry(page, &n->slabs_free, lru) ++ if (is_slab_pfmemalloc(page)) + goto out; + + pfmemalloc_active = false; +@@ -841,8 +816,8 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, + */ + n = cachep->node[numa_mem_id()]; + if (!list_empty(&n->slabs_free) && force_refill) { +- struct slab *slabp = virt_to_slab(objp); +- ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem)); ++ struct page *page = virt_to_head_page(objp); ++ ClearPageSlabPfmemalloc(virt_to_head_page(page->s_mem)); + clear_obj_pfmemalloc(&objp); + recheck_pfmemalloc_active(cachep, ac); + return objp; +@@ -874,9 +849,9 @@ static void *__ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac, + { + if (unlikely(pfmemalloc_active)) { + /* Some pfmemalloc slabs exist, check if this is one */ +- struct slab *slabp = virt_to_slab(objp); +- struct page *page = virt_to_head_page(slabp->s_mem); +- if (PageSlabPfmemalloc(page)) ++ struct page *page = virt_to_head_page(objp); ++ struct page *mem_page = virt_to_head_page(page->s_mem); ++ if (PageSlabPfmemalloc(mem_page)) + set_obj_pfmemalloc(&objp); + } + +@@ -1633,7 +1608,7 @@ static noinline void + slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) + { + struct kmem_cache_node *n; +- struct slab *slabp; ++ struct page *page; + unsigned long flags; + int node; + +@@ -1652,15 +1627,15 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) + continue; + + spin_lock_irqsave(&n->list_lock, flags); +- list_for_each_entry(slabp, &n->slabs_full, list) { ++ list_for_each_entry(page, &n->slabs_full, lru) { + active_objs += cachep->num; + active_slabs++; + } +- list_for_each_entry(slabp, &n->slabs_partial, list) { +- active_objs += slabp->active; ++ list_for_each_entry(page, &n->slabs_partial, lru) { ++ active_objs += page->active; + active_slabs++; + } +- list_for_each_entry(slabp, &n->slabs_free, list) ++ list_for_each_entry(page, &n->slabs_free, lru) + num_slabs++; + + free_objects += n->free_objects; +@@ -1746,6 +1721,8 @@ static void kmem_freepages(struct kmem_cache *cachep, struct page *page) + BUG_ON(!PageSlab(page)); + __ClearPageSlabPfmemalloc(page); + __ClearPageSlab(page); ++ page_mapcount_reset(page); ++ page->mapping = NULL; + + memcg_release_pages(cachep, cachep->gfporder); + if (current->reclaim_state) +@@ -1910,19 +1887,19 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) + /* Print some data about the neighboring objects, if they + * exist: + */ +- struct slab *slabp = virt_to_slab(objp); ++ struct page *page = virt_to_head_page(objp); + unsigned int objnr; + +- objnr = obj_to_index(cachep, slabp, objp); ++ objnr = obj_to_index(cachep, page, objp); + if (objnr) { +- objp = index_to_obj(cachep, slabp, objnr - 1); ++ objp = index_to_obj(cachep, page, objnr - 1); + realobj = (char *)objp + obj_offset(cachep); + printk(KERN_ERR "Prev obj: start=%p, len=%d\n", + realobj, size); + print_objinfo(cachep, objp, 2); + } + if (objnr + 1 < cachep->num) { +- objp = index_to_obj(cachep, slabp, objnr + 1); ++ objp = index_to_obj(cachep, page, objnr + 1); + realobj = (char *)objp + obj_offset(cachep); + printk(KERN_ERR "Next obj: start=%p, len=%d\n", + realobj, size); +@@ -1933,11 +1910,12 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) + #endif + + #if DEBUG +-static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp) ++static void slab_destroy_debugcheck(struct kmem_cache *cachep, ++ struct page *page) + { + int i; + for (i = 0; i < cachep->num; i++) { +- void *objp = index_to_obj(cachep, slabp, i); ++ void *objp = index_to_obj(cachep, page, i); + + if (cachep->flags & SLAB_POISON) { + #ifdef CONFIG_DEBUG_PAGEALLOC +@@ -1962,7 +1940,8 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab + } + } + #else +-static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp) ++static void slab_destroy_debugcheck(struct kmem_cache *cachep, ++ struct page *page) + { + } + #endif +@@ -1976,11 +1955,12 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab + * Before calling the slab must have been unlinked from the cache. The + * cache-lock is not held/needed. + */ +-static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp) ++static void slab_destroy(struct kmem_cache *cachep, struct page *page) + { +- struct page *page = virt_to_head_page(slabp->s_mem); ++ void *freelist; + +- slab_destroy_debugcheck(cachep, slabp); ++ freelist = page->freelist; ++ slab_destroy_debugcheck(cachep, page); + if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { + struct rcu_head *head; + +@@ -1998,11 +1978,11 @@ static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp) + } + + /* +- * From now on, we don't use slab management ++ * From now on, we don't use freelist + * although actual page can be freed in rcu context + */ + if (OFF_SLAB(cachep)) +- kmem_cache_free(cachep->slabp_cache, slabp); ++ kmem_cache_free(cachep->freelist_cache, freelist); + } + + /** +@@ -2039,7 +2019,7 @@ static size_t calculate_slab_order(struct kmem_cache *cachep, + * use off-slab slabs. Needed to avoid a possible + * looping condition in cache_grow(). + */ +- offslab_limit = size - sizeof(struct slab); ++ offslab_limit = size; + offslab_limit /= sizeof(unsigned int); + + if (num > offslab_limit) +@@ -2162,7 +2142,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) + int + __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) + { +- size_t left_over, slab_size, ralign; ++ size_t left_over, freelist_size, ralign; + gfp_t gfp; + int err; + size_t size = cachep->size; +@@ -2281,22 +2261,21 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) + if (!cachep->num) + return -E2BIG; + +- slab_size = ALIGN(cachep->num * sizeof(unsigned int) +- + sizeof(struct slab), cachep->align); ++ freelist_size = ++ ALIGN(cachep->num * sizeof(unsigned int), cachep->align); + + /* + * If the slab has been placed off-slab, and we have enough space then + * move it on-slab. This is at the expense of any extra colouring. + */ +- if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) { ++ if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) { + flags &= ~CFLGS_OFF_SLAB; +- left_over -= slab_size; ++ left_over -= freelist_size; + } + + if (flags & CFLGS_OFF_SLAB) { + /* really off slab. No need for manual alignment */ +- slab_size = +- cachep->num * sizeof(unsigned int) + sizeof(struct slab); ++ freelist_size = cachep->num * sizeof(unsigned int); + + #ifdef CONFIG_PAGE_POISONING + /* If we're going to use the generic kernel_map_pages() +@@ -2313,7 +2292,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) + if (cachep->colour_off < cachep->align) + cachep->colour_off = cachep->align; + cachep->colour = left_over / cachep->colour_off; +- cachep->slab_size = slab_size; ++ cachep->freelist_size = freelist_size; + cachep->flags = flags; + cachep->allocflags = __GFP_COMP; + if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA)) +@@ -2322,7 +2301,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) + cachep->reciprocal_buffer_size = reciprocal_value(size); + + if (flags & CFLGS_OFF_SLAB) { +- cachep->slabp_cache = kmalloc_slab(slab_size, 0u); ++ cachep->freelist_cache = kmalloc_slab(freelist_size, 0u); + /* + * This is a possibility for one of the malloc_sizes caches. + * But since we go off slab only for object size greater than +@@ -2330,7 +2309,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) + * this should not happen at all. + * But leave a BUG_ON for some lucky dude. + */ +- BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache)); ++ BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache)); + } + + err = setup_cpu_cache(cachep, gfp); +@@ -2436,7 +2415,7 @@ static int drain_freelist(struct kmem_cache *cache, + { + struct list_head *p; + int nr_freed; +- struct slab *slabp; ++ struct page *page; + + nr_freed = 0; + while (nr_freed < tofree && !list_empty(&n->slabs_free)) { +@@ -2448,18 +2427,18 @@ static int drain_freelist(struct kmem_cache *cache, + goto out; + } + +- slabp = list_entry(p, struct slab, list); ++ page = list_entry(p, struct page, lru); + #if DEBUG +- BUG_ON(slabp->active); ++ BUG_ON(page->active); + #endif +- list_del(&slabp->list); ++ list_del(&page->lru); + /* + * Safe to drop the lock. The slab is no longer linked + * to the cache. + */ + n->free_objects -= cache->num; + spin_unlock_irq(&n->list_lock); +- slab_destroy(cache, slabp); ++ slab_destroy(cache, page); + nr_freed++; + } + out: +@@ -2542,50 +2521,42 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep) + * descriptors in kmem_cache_create, we search through the malloc_sizes array. + * If we are creating a malloc_sizes cache here it would not be visible to + * kmem_find_general_cachep till the initialization is complete. +- * Hence we cannot have slabp_cache same as the original cache. ++ * Hence we cannot have freelist_cache same as the original cache. + */ +-static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, ++static void *alloc_slabmgmt(struct kmem_cache *cachep, + struct page *page, int colour_off, + gfp_t local_flags, int nodeid) + { +- struct slab *slabp; ++ void *freelist; + void *addr = page_address(page); + + if (OFF_SLAB(cachep)) { + /* Slab management obj is off-slab. */ +- slabp = kmem_cache_alloc_node(cachep->slabp_cache, ++ freelist = kmem_cache_alloc_node(cachep->freelist_cache, + local_flags, nodeid); +- /* +- * If the first object in the slab is leaked (it's allocated +- * but no one has a reference to it), we want to make sure +- * kmemleak does not treat the ->s_mem pointer as a reference +- * to the object. Otherwise we will not report the leak. +- */ +- kmemleak_scan_area(&slabp->list, sizeof(struct list_head), +- local_flags); +- if (!slabp) ++ if (!freelist) + return NULL; + } else { +- slabp = addr + colour_off; +- colour_off += cachep->slab_size; ++ freelist = addr + colour_off; ++ colour_off += cachep->freelist_size; + } +- slabp->active = 0; +- slabp->s_mem = addr + colour_off; +- return slabp; ++ page->active = 0; ++ page->s_mem = addr + colour_off; ++ return freelist; + } + +-static inline unsigned int *slab_bufctl(struct slab *slabp) ++static inline unsigned int *slab_bufctl(struct page *page) + { +- return (unsigned int *) (slabp + 1); ++ return (unsigned int *)(page->freelist); + } + + static void cache_init_objs(struct kmem_cache *cachep, +- struct slab *slabp) ++ struct page *page) + { + int i; + + for (i = 0; i < cachep->num; i++) { +- void *objp = index_to_obj(cachep, slabp, i); ++ void *objp = index_to_obj(cachep, page, i); + #if DEBUG + /* need to poison the objs? */ + if (cachep->flags & SLAB_POISON) +@@ -2621,7 +2592,7 @@ static void cache_init_objs(struct kmem_cache *cachep, + if (cachep->ctor) + cachep->ctor(objp); + #endif +- slab_bufctl(slabp)[i] = i; ++ slab_bufctl(page)[i] = i; + } + } + +@@ -2635,13 +2606,13 @@ static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) + } + } + +-static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp, ++static void *slab_get_obj(struct kmem_cache *cachep, struct page *page, + int nodeid) + { + void *objp; + +- objp = index_to_obj(cachep, slabp, slab_bufctl(slabp)[slabp->active]); +- slabp->active++; ++ objp = index_to_obj(cachep, page, slab_bufctl(page)[page->active]); ++ page->active++; + #if DEBUG + WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid); + #endif +@@ -2649,10 +2620,10 @@ static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp, + return objp; + } + +-static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp, ++static void slab_put_obj(struct kmem_cache *cachep, struct page *page, + void *objp, int nodeid) + { +- unsigned int objnr = obj_to_index(cachep, slabp, objp); ++ unsigned int objnr = obj_to_index(cachep, page, objp); + #if DEBUG + unsigned int i; + +@@ -2660,16 +2631,16 @@ static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp, + WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid); + + /* Verify double free bug */ +- for (i = slabp->active; i < cachep->num; i++) { +- if (slab_bufctl(slabp)[i] == objnr) { ++ for (i = page->active; i < cachep->num; i++) { ++ if (slab_bufctl(page)[i] == objnr) { + printk(KERN_ERR "slab: double free detected in cache " + "'%s', objp %p\n", cachep->name, objp); + BUG(); + } + } + #endif +- slabp->active--; +- slab_bufctl(slabp)[slabp->active] = objnr; ++ page->active--; ++ slab_bufctl(page)[page->active] = objnr; + } + + /* +@@ -2677,11 +2648,11 @@ static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp, + * for the slab allocator to be able to lookup the cache and slab of a + * virtual address for kfree, ksize, and slab debugging. + */ +-static void slab_map_pages(struct kmem_cache *cache, struct slab *slab, +- struct page *page) ++static void slab_map_pages(struct kmem_cache *cache, struct page *page, ++ void *freelist) + { + page->slab_cache = cache; +- page->slab_page = slab; ++ page->freelist = freelist; + } + + /* +@@ -2691,7 +2662,7 @@ static void slab_map_pages(struct kmem_cache *cache, struct slab *slab, + static int cache_grow(struct kmem_cache *cachep, + gfp_t flags, int nodeid, struct page *page) + { +- struct slab *slabp; ++ void *freelist; + size_t offset; + gfp_t local_flags; + struct kmem_cache_node *n; +@@ -2738,14 +2709,14 @@ static int cache_grow(struct kmem_cache *cachep, + goto failed; + + /* Get slab management. */ +- slabp = alloc_slabmgmt(cachep, page, offset, ++ freelist = alloc_slabmgmt(cachep, page, offset, + local_flags & ~GFP_CONSTRAINT_MASK, nodeid); +- if (!slabp) ++ if (!freelist) + goto opps1; + +- slab_map_pages(cachep, slabp, page); ++ slab_map_pages(cachep, page, freelist); + +- cache_init_objs(cachep, slabp); ++ cache_init_objs(cachep, page); + + if (local_flags & __GFP_WAIT) + local_irq_disable(); +@@ -2753,7 +2724,7 @@ static int cache_grow(struct kmem_cache *cachep, + spin_lock(&n->list_lock); + + /* Make slab active. */ +- list_add_tail(&slabp->list, &(n->slabs_free)); ++ list_add_tail(&page->lru, &(n->slabs_free)); + STATS_INC_GROWN(cachep); + n->free_objects += cachep->num; + spin_unlock(&n->list_lock); +@@ -2808,13 +2779,13 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, + unsigned long caller) + { + unsigned int objnr; +- struct slab *slabp; ++ struct page *page; + + BUG_ON(virt_to_cache(objp) != cachep); + + objp -= obj_offset(cachep); + kfree_debugcheck(objp); +- slabp = virt_to_slab(objp); ++ page = virt_to_head_page(objp); + + if (cachep->flags & SLAB_RED_ZONE) { + verify_redzone_free(cachep, objp); +@@ -2824,10 +2795,10 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, + if (cachep->flags & SLAB_STORE_USER) + *dbg_userword(cachep, objp) = (void *)caller; + +- objnr = obj_to_index(cachep, slabp, objp); ++ objnr = obj_to_index(cachep, page, objp); + + BUG_ON(objnr >= cachep->num); +- BUG_ON(objp != index_to_obj(cachep, slabp, objnr)); ++ BUG_ON(objp != index_to_obj(cachep, page, objnr)); + + if (cachep->flags & SLAB_POISON) { + #ifdef CONFIG_DEBUG_PAGEALLOC +@@ -2886,7 +2857,7 @@ retry: + + while (batchcount > 0) { + struct list_head *entry; +- struct slab *slabp; ++ struct page *page; + /* Get slab alloc is to come from. */ + entry = n->slabs_partial.next; + if (entry == &n->slabs_partial) { +@@ -2896,7 +2867,7 @@ retry: + goto must_grow; + } + +- slabp = list_entry(entry, struct slab, list); ++ page = list_entry(entry, struct page, lru); + check_spinlock_acquired(cachep); + + /* +@@ -2904,23 +2875,23 @@ retry: + * there must be at least one object available for + * allocation. + */ +- BUG_ON(slabp->active >= cachep->num); ++ BUG_ON(page->active >= cachep->num); + +- while (slabp->active < cachep->num && batchcount--) { ++ while (page->active < cachep->num && batchcount--) { + STATS_INC_ALLOCED(cachep); + STATS_INC_ACTIVE(cachep); + STATS_SET_HIGH(cachep); + +- ac_put_obj(cachep, ac, slab_get_obj(cachep, slabp, ++ ac_put_obj(cachep, ac, slab_get_obj(cachep, page, + node)); + } + + /* move slabp to correct slabp list: */ +- list_del(&slabp->list); +- if (slabp->active == cachep->num) +- list_add(&slabp->list, &n->slabs_full); ++ list_del(&page->lru); ++ if (page->active == cachep->num) ++ list_add(&page->list, &n->slabs_full); + else +- list_add(&slabp->list, &n->slabs_partial); ++ list_add(&page->list, &n->slabs_partial); + } + + must_grow: +@@ -3175,7 +3146,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, + int nodeid) + { + struct list_head *entry; +- struct slab *slabp; ++ struct page *page; + struct kmem_cache_node *n; + void *obj; + int x; +@@ -3195,24 +3166,24 @@ retry: + goto must_grow; + } + +- slabp = list_entry(entry, struct slab, list); ++ page = list_entry(entry, struct page, lru); + check_spinlock_acquired_node(cachep, nodeid); + + STATS_INC_NODEALLOCS(cachep); + STATS_INC_ACTIVE(cachep); + STATS_SET_HIGH(cachep); + +- BUG_ON(slabp->active == cachep->num); ++ BUG_ON(page->active == cachep->num); + +- obj = slab_get_obj(cachep, slabp, nodeid); ++ obj = slab_get_obj(cachep, page, nodeid); + n->free_objects--; + /* move slabp to correct slabp list: */ +- list_del(&slabp->list); ++ list_del(&page->lru); + +- if (slabp->active == cachep->num) +- list_add(&slabp->list, &n->slabs_full); ++ if (page->active == cachep->num) ++ list_add(&page->lru, &n->slabs_full); + else +- list_add(&slabp->list, &n->slabs_partial); ++ list_add(&page->lru, &n->slabs_partial); + + spin_unlock(&n->list_lock); + goto done; +@@ -3362,21 +3333,21 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, + + for (i = 0; i < nr_objects; i++) { + void *objp; +- struct slab *slabp; ++ struct page *page; + + clear_obj_pfmemalloc(&objpp[i]); + objp = objpp[i]; + +- slabp = virt_to_slab(objp); ++ page = virt_to_head_page(objp); + n = cachep->node[node]; +- list_del(&slabp->list); ++ list_del(&page->lru); + check_spinlock_acquired_node(cachep, node); +- slab_put_obj(cachep, slabp, objp, node); ++ slab_put_obj(cachep, page, objp, node); + STATS_DEC_ACTIVE(cachep); + n->free_objects++; + + /* fixup slab chains */ +- if (slabp->active == 0) { ++ if (page->active == 0) { + if (n->free_objects > n->free_limit) { + n->free_objects -= cachep->num; + /* No need to drop any previously held +@@ -3385,16 +3356,16 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, + * a different cache, refer to comments before + * alloc_slabmgmt. + */ +- slab_destroy(cachep, slabp); ++ slab_destroy(cachep, page); + } else { +- list_add(&slabp->list, &n->slabs_free); ++ list_add(&page->lru, &n->slabs_free); + } + } else { + /* Unconditionally move a slab to the end of the + * partial list on free - maximum time for the + * other objects to be freed, too. + */ +- list_add_tail(&slabp->list, &n->slabs_partial); ++ list_add_tail(&page->lru, &n->slabs_partial); + } + } + } +@@ -3434,10 +3405,10 @@ free_done: + + p = n->slabs_free.next; + while (p != &(n->slabs_free)) { +- struct slab *slabp; ++ struct page *page; + +- slabp = list_entry(p, struct slab, list); +- BUG_ON(slabp->active); ++ page = list_entry(p, struct page, lru); ++ BUG_ON(page->active); + + i++; + p = p->next; +@@ -4041,7 +4012,7 @@ out: + #ifdef CONFIG_SLABINFO + void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) + { +- struct slab *slabp; ++ struct page *page; + unsigned long active_objs; + unsigned long num_objs; + unsigned long active_slabs = 0; +@@ -4061,22 +4032,22 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) + check_irq_on(); + spin_lock_irq(&n->list_lock); + +- list_for_each_entry(slabp, &n->slabs_full, list) { +- if (slabp->active != cachep->num && !error) ++ list_for_each_entry(page, &n->slabs_full, lru) { ++ if (page->active != cachep->num && !error) + error = "slabs_full accounting error"; + active_objs += cachep->num; + active_slabs++; + } +- list_for_each_entry(slabp, &n->slabs_partial, list) { +- if (slabp->active == cachep->num && !error) ++ list_for_each_entry(page, &n->slabs_partial, lru) { ++ if (page->active == cachep->num && !error) + error = "slabs_partial accounting error"; +- if (!slabp->active && !error) ++ if (!page->active && !error) + error = "slabs_partial accounting error"; +- active_objs += slabp->active; ++ active_objs += page->active; + active_slabs++; + } +- list_for_each_entry(slabp, &n->slabs_free, list) { +- if (slabp->active && !error) ++ list_for_each_entry(page, &n->slabs_free, lru) { ++ if (page->active && !error) + error = "slabs_free accounting error"; + num_slabs++; + } +@@ -4229,19 +4200,20 @@ static inline int add_caller(unsigned long *n, unsigned long v) + return 1; + } + +-static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s) ++static void handle_slab(unsigned long *n, struct kmem_cache *c, ++ struct page *page) + { + void *p; + int i, j; + + if (n[0] == n[1]) + return; +- for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) { ++ for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) { + bool active = true; + +- for (j = s->active; j < c->num; j++) { ++ for (j = page->active; j < c->num; j++) { + /* Skip freed item */ +- if (slab_bufctl(s)[j] == i) { ++ if (slab_bufctl(page)[j] == i) { + active = false; + break; + } +@@ -4273,7 +4245,7 @@ static void show_symbol(struct seq_file *m, unsigned long address) + static int leaks_show(struct seq_file *m, void *p) + { + struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); +- struct slab *slabp; ++ struct page *page; + struct kmem_cache_node *n; + const char *name; + unsigned long *x = m->private; +@@ -4297,10 +4269,10 @@ static int leaks_show(struct seq_file *m, void *p) + check_irq_on(); + spin_lock_irq(&n->list_lock); + +- list_for_each_entry(slabp, &n->slabs_full, list) +- handle_slab(x, cachep, slabp); +- list_for_each_entry(slabp, &n->slabs_partial, list) +- handle_slab(x, cachep, slabp); ++ list_for_each_entry(page, &n->slabs_full, lru) ++ handle_slab(x, cachep, page); ++ list_for_each_entry(page, &n->slabs_partial, lru) ++ handle_slab(x, cachep, page); + spin_unlock_irq(&n->list_lock); + } + name = cachep->name; +-- +1.7.9.5 diff --git a/a/content_digest b/N1/content_digest index cf0d722..64b0e62 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -34,6 +34,907 @@ "2. %s/'struct freelist *'/'void *'\n" "\n" "\n" - -----------------------------8<------------------------------- + "-----------------------------8<-------------------------------\n" + ">From 6d11304824a3b8c3bf7574323a3e55471cc26937 Mon Sep 17 00:00:00 2001\n" + "From: Joonsoo Kim <iamjoonsoo.kim@lge.com>\n" + "Date: Wed, 28 Aug 2013 16:30:27 +0900\n" + "Subject: [PATCH v2-FIX 13/15] slab: use struct page for slab management\n" + "\n" + "Now, there are a few field in struct slab, so we can overload these\n" + "over struct page. This will save some memory and reduce cache footprint.\n" + "\n" + "After this change, slabp_cache and slab_size no longer related to\n" + "a struct slab, so rename them as freelist_cache and freelist_size.\n" + "\n" + "These changes are just mechanical ones and there is no functional change.\n" + "\n" + "Acked-by: Andi Kleen <ak@linux.intel.com>\n" + "Acked-by: Christoph Lameter <cl@linux.com>\n" + "Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>\n" + "\n" + "diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h\n" + "index 8b85d8c..4e17190 100644\n" + "--- a/include/linux/mm_types.h\n" + "+++ b/include/linux/mm_types.h\n" + "@@ -42,18 +42,22 @@ struct page {\n" + " \t/* First double word block */\n" + " \tunsigned long flags;\t\t/* Atomic flags, some possibly\n" + " \t\t\t\t\t * updated asynchronously */\n" + "-\tstruct address_space *mapping;\t/* If low bit clear, points to\n" + "-\t\t\t\t\t * inode address_space, or NULL.\n" + "-\t\t\t\t\t * If page mapped as anonymous\n" + "-\t\t\t\t\t * memory, low bit is set, and\n" + "-\t\t\t\t\t * it points to anon_vma object:\n" + "-\t\t\t\t\t * see PAGE_MAPPING_ANON below.\n" + "-\t\t\t\t\t */\n" + "+\tunion {\n" + "+\t\tstruct address_space *mapping;\t/* If low bit clear, points to\n" + "+\t\t\t\t\t\t * inode address_space, or NULL.\n" + "+\t\t\t\t\t\t * If page mapped as anonymous\n" + "+\t\t\t\t\t\t * memory, low bit is set, and\n" + "+\t\t\t\t\t\t * it points to anon_vma object:\n" + "+\t\t\t\t\t\t * see PAGE_MAPPING_ANON below.\n" + "+\t\t\t\t\t\t */\n" + "+\t\tvoid *s_mem;\t\t\t/* slab first object */\n" + "+\t};\n" + "+\n" + " \t/* Second double word */\n" + " \tstruct {\n" + " \t\tunion {\n" + " \t\t\tpgoff_t index;\t\t/* Our offset within mapping. */\n" + "-\t\t\tvoid *freelist;\t\t/* slub/slob first free object */\n" + "+\t\t\tvoid *freelist;\t\t/* sl[aou]b first free object */\n" + " \t\t\tbool pfmemalloc;\t/* If set by the page allocator,\n" + " \t\t\t\t\t\t * ALLOC_NO_WATERMARKS was set\n" + " \t\t\t\t\t\t * and the low watermark was not\n" + "@@ -109,6 +113,7 @@ struct page {\n" + " \t\t\t\t};\n" + " \t\t\t\tatomic_t _count;\t\t/* Usage count, see below. */\n" + " \t\t\t};\n" + "+\t\t\tunsigned int active;\t/* SLAB */\n" + " \t\t};\n" + " \t};\n" + " \n" + "diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h\n" + "index e9346b4..09bfffb 100644\n" + "--- a/include/linux/slab_def.h\n" + "+++ b/include/linux/slab_def.h\n" + "@@ -27,8 +27,8 @@ struct kmem_cache {\n" + " \n" + " \tsize_t colour;\t\t\t/* cache colouring range */\n" + " \tunsigned int colour_off;\t/* colour offset */\n" + "-\tstruct kmem_cache *slabp_cache;\n" + "-\tunsigned int slab_size;\n" + "+\tstruct kmem_cache *freelist_cache;\n" + "+\tunsigned int freelist_size;\n" + " \n" + " \t/* constructor func */\n" + " \tvoid (*ctor)(void *obj);\n" + "diff --git a/mm/slab.c b/mm/slab.c\n" + "index 2ec2336..da01fc70 100644\n" + "--- a/mm/slab.c\n" + "+++ b/mm/slab.c\n" + "@@ -164,21 +164,6 @@\n" + " static bool pfmemalloc_active __read_mostly;\n" + " \n" + " /*\n" + "- * struct slab\n" + "- *\n" + "- * Manages the objs in a slab. Placed either at the beginning of mem allocated\n" + "- * for a slab, or allocated from an general cache.\n" + "- * Slabs are chained into three list: fully used, partial, fully free slabs.\n" + "- */\n" + "-struct slab {\n" + "-\tstruct {\n" + "-\t\tstruct list_head list;\n" + "-\t\tvoid *s_mem;\t\t/* including colour offset */\n" + "-\t\tunsigned int active;\t/* num of objs active in slab */\n" + "-\t};\n" + "-};\n" + "-\n" + "-/*\n" + " * struct array_cache\n" + " *\n" + " * Purpose:\n" + "@@ -405,18 +390,10 @@ static inline struct kmem_cache *virt_to_cache(const void *obj)\n" + " \treturn page->slab_cache;\n" + " }\n" + " \n" + "-static inline struct slab *virt_to_slab(const void *obj)\n" + "-{\n" + "-\tstruct page *page = virt_to_head_page(obj);\n" + "-\n" + "-\tVM_BUG_ON(!PageSlab(page));\n" + "-\treturn page->slab_page;\n" + "-}\n" + "-\n" + "-static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,\n" + "+static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,\n" + " \t\t\t\t unsigned int idx)\n" + " {\n" + "-\treturn slab->s_mem + cache->size * idx;\n" + "+\treturn page->s_mem + cache->size * idx;\n" + " }\n" + " \n" + " /*\n" + "@@ -426,9 +403,9 @@ static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,\n" + " * reciprocal_divide(offset, cache->reciprocal_buffer_size)\n" + " */\n" + " static inline unsigned int obj_to_index(const struct kmem_cache *cache,\n" + "-\t\t\t\t\tconst struct slab *slab, void *obj)\n" + "+\t\t\t\t\tconst struct page *page, void *obj)\n" + " {\n" + "-\tu32 offset = (obj - slab->s_mem);\n" + "+\tu32 offset = (obj - page->s_mem);\n" + " \treturn reciprocal_divide(offset, cache->reciprocal_buffer_size);\n" + " }\n" + " \n" + "@@ -590,7 +567,7 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)\n" + " \n" + " static size_t slab_mgmt_size(size_t nr_objs, size_t align)\n" + " {\n" + "-\treturn ALIGN(sizeof(struct slab)+nr_objs*sizeof(unsigned int), align);\n" + "+\treturn ALIGN(nr_objs * sizeof(unsigned int), align);\n" + " }\n" + " \n" + " /*\n" + "@@ -609,7 +586,6 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size,\n" + " \t * on it. For the latter case, the memory allocated for a\n" + " \t * slab is used for:\n" + " \t *\n" + "-\t * - The struct slab\n" + " \t * - One unsigned int for each object\n" + " \t * - Padding to respect alignment of @align\n" + " \t * - @buffer_size bytes for each object\n" + "@@ -632,8 +608,7 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size,\n" + " \t\t * into the memory allocation when taking the padding\n" + " \t\t * into account.\n" + " \t\t */\n" + "-\t\tnr_objs = (slab_size - sizeof(struct slab)) /\n" + "-\t\t\t (buffer_size + sizeof(unsigned int));\n" + "+\t\tnr_objs = (slab_size) / (buffer_size + sizeof(unsigned int));\n" + " \n" + " \t\t/*\n" + " \t\t * This calculated number will be either the right\n" + "@@ -773,11 +748,11 @@ static struct array_cache *alloc_arraycache(int node, int entries,\n" + " \treturn nc;\n" + " }\n" + " \n" + "-static inline bool is_slab_pfmemalloc(struct slab *slabp)\n" + "+static inline bool is_slab_pfmemalloc(struct page *page)\n" + " {\n" + "-\tstruct page *page = virt_to_page(slabp->s_mem);\n" + "+\tstruct page *mem_page = virt_to_page(page->s_mem);\n" + " \n" + "-\treturn PageSlabPfmemalloc(page);\n" + "+\treturn PageSlabPfmemalloc(mem_page);\n" + " }\n" + " \n" + " /* Clears pfmemalloc_active if no slabs have pfmalloc set */\n" + "@@ -785,23 +760,23 @@ static void recheck_pfmemalloc_active(struct kmem_cache *cachep,\n" + " \t\t\t\t\t\tstruct array_cache *ac)\n" + " {\n" + " \tstruct kmem_cache_node *n = cachep->node[numa_mem_id()];\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \tunsigned long flags;\n" + " \n" + " \tif (!pfmemalloc_active)\n" + " \t\treturn;\n" + " \n" + " \tspin_lock_irqsave(&n->list_lock, flags);\n" + "-\tlist_for_each_entry(slabp, &n->slabs_full, list)\n" + "-\t\tif (is_slab_pfmemalloc(slabp))\n" + "+\tlist_for_each_entry(page, &n->slabs_full, lru)\n" + "+\t\tif (is_slab_pfmemalloc(page))\n" + " \t\t\tgoto out;\n" + " \n" + "-\tlist_for_each_entry(slabp, &n->slabs_partial, list)\n" + "-\t\tif (is_slab_pfmemalloc(slabp))\n" + "+\tlist_for_each_entry(page, &n->slabs_partial, lru)\n" + "+\t\tif (is_slab_pfmemalloc(page))\n" + " \t\t\tgoto out;\n" + " \n" + "-\tlist_for_each_entry(slabp, &n->slabs_free, list)\n" + "-\t\tif (is_slab_pfmemalloc(slabp))\n" + "+\tlist_for_each_entry(page, &n->slabs_free, lru)\n" + "+\t\tif (is_slab_pfmemalloc(page))\n" + " \t\t\tgoto out;\n" + " \n" + " \tpfmemalloc_active = false;\n" + "@@ -841,8 +816,8 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,\n" + " \t\t */\n" + " \t\tn = cachep->node[numa_mem_id()];\n" + " \t\tif (!list_empty(&n->slabs_free) && force_refill) {\n" + "-\t\t\tstruct slab *slabp = virt_to_slab(objp);\n" + "-\t\t\tClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem));\n" + "+\t\t\tstruct page *page = virt_to_head_page(objp);\n" + "+\t\t\tClearPageSlabPfmemalloc(virt_to_head_page(page->s_mem));\n" + " \t\t\tclear_obj_pfmemalloc(&objp);\n" + " \t\t\trecheck_pfmemalloc_active(cachep, ac);\n" + " \t\t\treturn objp;\n" + "@@ -874,9 +849,9 @@ static void *__ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,\n" + " {\n" + " \tif (unlikely(pfmemalloc_active)) {\n" + " \t\t/* Some pfmemalloc slabs exist, check if this is one */\n" + "-\t\tstruct slab *slabp = virt_to_slab(objp);\n" + "-\t\tstruct page *page = virt_to_head_page(slabp->s_mem);\n" + "-\t\tif (PageSlabPfmemalloc(page))\n" + "+\t\tstruct page *page = virt_to_head_page(objp);\n" + "+\t\tstruct page *mem_page = virt_to_head_page(page->s_mem);\n" + "+\t\tif (PageSlabPfmemalloc(mem_page))\n" + " \t\t\tset_obj_pfmemalloc(&objp);\n" + " \t}\n" + " \n" + "@@ -1633,7 +1608,7 @@ static noinline void\n" + " slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)\n" + " {\n" + " \tstruct kmem_cache_node *n;\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \tunsigned long flags;\n" + " \tint node;\n" + " \n" + "@@ -1652,15 +1627,15 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)\n" + " \t\t\tcontinue;\n" + " \n" + " \t\tspin_lock_irqsave(&n->list_lock, flags);\n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_full, list) {\n" + "+\t\tlist_for_each_entry(page, &n->slabs_full, lru) {\n" + " \t\t\tactive_objs += cachep->num;\n" + " \t\t\tactive_slabs++;\n" + " \t\t}\n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_partial, list) {\n" + "-\t\t\tactive_objs += slabp->active;\n" + "+\t\tlist_for_each_entry(page, &n->slabs_partial, lru) {\n" + "+\t\t\tactive_objs += page->active;\n" + " \t\t\tactive_slabs++;\n" + " \t\t}\n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_free, list)\n" + "+\t\tlist_for_each_entry(page, &n->slabs_free, lru)\n" + " \t\t\tnum_slabs++;\n" + " \n" + " \t\tfree_objects += n->free_objects;\n" + "@@ -1746,6 +1721,8 @@ static void kmem_freepages(struct kmem_cache *cachep, struct page *page)\n" + " \tBUG_ON(!PageSlab(page));\n" + " \t__ClearPageSlabPfmemalloc(page);\n" + " \t__ClearPageSlab(page);\n" + "+\tpage_mapcount_reset(page);\n" + "+\tpage->mapping = NULL;\n" + " \n" + " \tmemcg_release_pages(cachep, cachep->gfporder);\n" + " \tif (current->reclaim_state)\n" + "@@ -1910,19 +1887,19 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp)\n" + " \t\t/* Print some data about the neighboring objects, if they\n" + " \t\t * exist:\n" + " \t\t */\n" + "-\t\tstruct slab *slabp = virt_to_slab(objp);\n" + "+\t\tstruct page *page = virt_to_head_page(objp);\n" + " \t\tunsigned int objnr;\n" + " \n" + "-\t\tobjnr = obj_to_index(cachep, slabp, objp);\n" + "+\t\tobjnr = obj_to_index(cachep, page, objp);\n" + " \t\tif (objnr) {\n" + "-\t\t\tobjp = index_to_obj(cachep, slabp, objnr - 1);\n" + "+\t\t\tobjp = index_to_obj(cachep, page, objnr - 1);\n" + " \t\t\trealobj = (char *)objp + obj_offset(cachep);\n" + " \t\t\tprintk(KERN_ERR \"Prev obj: start=%p, len=%d\\n\",\n" + " \t\t\t realobj, size);\n" + " \t\t\tprint_objinfo(cachep, objp, 2);\n" + " \t\t}\n" + " \t\tif (objnr + 1 < cachep->num) {\n" + "-\t\t\tobjp = index_to_obj(cachep, slabp, objnr + 1);\n" + "+\t\t\tobjp = index_to_obj(cachep, page, objnr + 1);\n" + " \t\t\trealobj = (char *)objp + obj_offset(cachep);\n" + " \t\t\tprintk(KERN_ERR \"Next obj: start=%p, len=%d\\n\",\n" + " \t\t\t realobj, size);\n" + "@@ -1933,11 +1910,12 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp)\n" + " #endif\n" + " \n" + " #if DEBUG\n" + "-static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp)\n" + "+static void slab_destroy_debugcheck(struct kmem_cache *cachep,\n" + "+\t\t\t\t\t\tstruct page *page)\n" + " {\n" + " \tint i;\n" + " \tfor (i = 0; i < cachep->num; i++) {\n" + "-\t\tvoid *objp = index_to_obj(cachep, slabp, i);\n" + "+\t\tvoid *objp = index_to_obj(cachep, page, i);\n" + " \n" + " \t\tif (cachep->flags & SLAB_POISON) {\n" + " #ifdef CONFIG_DEBUG_PAGEALLOC\n" + "@@ -1962,7 +1940,8 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab\n" + " \t}\n" + " }\n" + " #else\n" + "-static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp)\n" + "+static void slab_destroy_debugcheck(struct kmem_cache *cachep,\n" + "+\t\t\t\t\t\tstruct page *page)\n" + " {\n" + " }\n" + " #endif\n" + "@@ -1976,11 +1955,12 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab\n" + " * Before calling the slab must have been unlinked from the cache. The\n" + " * cache-lock is not held/needed.\n" + " */\n" + "-static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)\n" + "+static void slab_destroy(struct kmem_cache *cachep, struct page *page)\n" + " {\n" + "-\tstruct page *page = virt_to_head_page(slabp->s_mem);\n" + "+\tvoid *freelist;\n" + " \n" + "-\tslab_destroy_debugcheck(cachep, slabp);\n" + "+\tfreelist = page->freelist;\n" + "+\tslab_destroy_debugcheck(cachep, page);\n" + " \tif (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {\n" + " \t\tstruct rcu_head *head;\n" + " \n" + "@@ -1998,11 +1978,11 @@ static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)\n" + " \t}\n" + " \n" + " \t/*\n" + "-\t * From now on, we don't use slab management\n" + "+\t * From now on, we don't use freelist\n" + " \t * although actual page can be freed in rcu context\n" + " \t */\n" + " \tif (OFF_SLAB(cachep))\n" + "-\t\tkmem_cache_free(cachep->slabp_cache, slabp);\n" + "+\t\tkmem_cache_free(cachep->freelist_cache, freelist);\n" + " }\n" + " \n" + " /**\n" + "@@ -2039,7 +2019,7 @@ static size_t calculate_slab_order(struct kmem_cache *cachep,\n" + " \t\t\t * use off-slab slabs. Needed to avoid a possible\n" + " \t\t\t * looping condition in cache_grow().\n" + " \t\t\t */\n" + "-\t\t\toffslab_limit = size - sizeof(struct slab);\n" + "+\t\t\toffslab_limit = size;\n" + " \t\t\toffslab_limit /= sizeof(unsigned int);\n" + " \n" + " \t\t\tif (num > offslab_limit)\n" + "@@ -2162,7 +2142,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)\n" + " int\n" + " __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)\n" + " {\n" + "-\tsize_t left_over, slab_size, ralign;\n" + "+\tsize_t left_over, freelist_size, ralign;\n" + " \tgfp_t gfp;\n" + " \tint err;\n" + " \tsize_t size = cachep->size;\n" + "@@ -2281,22 +2261,21 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)\n" + " \tif (!cachep->num)\n" + " \t\treturn -E2BIG;\n" + " \n" + "-\tslab_size = ALIGN(cachep->num * sizeof(unsigned int)\n" + "-\t\t\t + sizeof(struct slab), cachep->align);\n" + "+\tfreelist_size =\n" + "+\t\tALIGN(cachep->num * sizeof(unsigned int), cachep->align);\n" + " \n" + " \t/*\n" + " \t * If the slab has been placed off-slab, and we have enough space then\n" + " \t * move it on-slab. This is at the expense of any extra colouring.\n" + " \t */\n" + "-\tif (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {\n" + "+\tif (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {\n" + " \t\tflags &= ~CFLGS_OFF_SLAB;\n" + "-\t\tleft_over -= slab_size;\n" + "+\t\tleft_over -= freelist_size;\n" + " \t}\n" + " \n" + " \tif (flags & CFLGS_OFF_SLAB) {\n" + " \t\t/* really off slab. No need for manual alignment */\n" + "-\t\tslab_size =\n" + "-\t\t cachep->num * sizeof(unsigned int) + sizeof(struct slab);\n" + "+\t\tfreelist_size = cachep->num * sizeof(unsigned int);\n" + " \n" + " #ifdef CONFIG_PAGE_POISONING\n" + " \t\t/* If we're going to use the generic kernel_map_pages()\n" + "@@ -2313,7 +2292,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)\n" + " \tif (cachep->colour_off < cachep->align)\n" + " \t\tcachep->colour_off = cachep->align;\n" + " \tcachep->colour = left_over / cachep->colour_off;\n" + "-\tcachep->slab_size = slab_size;\n" + "+\tcachep->freelist_size = freelist_size;\n" + " \tcachep->flags = flags;\n" + " \tcachep->allocflags = __GFP_COMP;\n" + " \tif (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))\n" + "@@ -2322,7 +2301,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)\n" + " \tcachep->reciprocal_buffer_size = reciprocal_value(size);\n" + " \n" + " \tif (flags & CFLGS_OFF_SLAB) {\n" + "-\t\tcachep->slabp_cache = kmalloc_slab(slab_size, 0u);\n" + "+\t\tcachep->freelist_cache = kmalloc_slab(freelist_size, 0u);\n" + " \t\t/*\n" + " \t\t * This is a possibility for one of the malloc_sizes caches.\n" + " \t\t * But since we go off slab only for object size greater than\n" + "@@ -2330,7 +2309,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)\n" + " \t\t * this should not happen at all.\n" + " \t\t * But leave a BUG_ON for some lucky dude.\n" + " \t\t */\n" + "-\t\tBUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));\n" + "+\t\tBUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));\n" + " \t}\n" + " \n" + " \terr = setup_cpu_cache(cachep, gfp);\n" + "@@ -2436,7 +2415,7 @@ static int drain_freelist(struct kmem_cache *cache,\n" + " {\n" + " \tstruct list_head *p;\n" + " \tint nr_freed;\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \n" + " \tnr_freed = 0;\n" + " \twhile (nr_freed < tofree && !list_empty(&n->slabs_free)) {\n" + "@@ -2448,18 +2427,18 @@ static int drain_freelist(struct kmem_cache *cache,\n" + " \t\t\tgoto out;\n" + " \t\t}\n" + " \n" + "-\t\tslabp = list_entry(p, struct slab, list);\n" + "+\t\tpage = list_entry(p, struct page, lru);\n" + " #if DEBUG\n" + "-\t\tBUG_ON(slabp->active);\n" + "+\t\tBUG_ON(page->active);\n" + " #endif\n" + "-\t\tlist_del(&slabp->list);\n" + "+\t\tlist_del(&page->lru);\n" + " \t\t/*\n" + " \t\t * Safe to drop the lock. The slab is no longer linked\n" + " \t\t * to the cache.\n" + " \t\t */\n" + " \t\tn->free_objects -= cache->num;\n" + " \t\tspin_unlock_irq(&n->list_lock);\n" + "-\t\tslab_destroy(cache, slabp);\n" + "+\t\tslab_destroy(cache, page);\n" + " \t\tnr_freed++;\n" + " \t}\n" + " out:\n" + "@@ -2542,50 +2521,42 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep)\n" + " * descriptors in kmem_cache_create, we search through the malloc_sizes array.\n" + " * If we are creating a malloc_sizes cache here it would not be visible to\n" + " * kmem_find_general_cachep till the initialization is complete.\n" + "- * Hence we cannot have slabp_cache same as the original cache.\n" + "+ * Hence we cannot have freelist_cache same as the original cache.\n" + " */\n" + "-static struct slab *alloc_slabmgmt(struct kmem_cache *cachep,\n" + "+static void *alloc_slabmgmt(struct kmem_cache *cachep,\n" + " \t\t\t\t struct page *page, int colour_off,\n" + " \t\t\t\t gfp_t local_flags, int nodeid)\n" + " {\n" + "-\tstruct slab *slabp;\n" + "+\tvoid *freelist;\n" + " \tvoid *addr = page_address(page);\n" + " \n" + " \tif (OFF_SLAB(cachep)) {\n" + " \t\t/* Slab management obj is off-slab. */\n" + "-\t\tslabp = kmem_cache_alloc_node(cachep->slabp_cache,\n" + "+\t\tfreelist = kmem_cache_alloc_node(cachep->freelist_cache,\n" + " \t\t\t\t\t local_flags, nodeid);\n" + "-\t\t/*\n" + "-\t\t * If the first object in the slab is leaked (it's allocated\n" + "-\t\t * but no one has a reference to it), we want to make sure\n" + "-\t\t * kmemleak does not treat the ->s_mem pointer as a reference\n" + "-\t\t * to the object. Otherwise we will not report the leak.\n" + "-\t\t */\n" + "-\t\tkmemleak_scan_area(&slabp->list, sizeof(struct list_head),\n" + "-\t\t\t\t local_flags);\n" + "-\t\tif (!slabp)\n" + "+\t\tif (!freelist)\n" + " \t\t\treturn NULL;\n" + " \t} else {\n" + "-\t\tslabp = addr + colour_off;\n" + "-\t\tcolour_off += cachep->slab_size;\n" + "+\t\tfreelist = addr + colour_off;\n" + "+\t\tcolour_off += cachep->freelist_size;\n" + " \t}\n" + "-\tslabp->active = 0;\n" + "-\tslabp->s_mem = addr + colour_off;\n" + "-\treturn slabp;\n" + "+\tpage->active = 0;\n" + "+\tpage->s_mem = addr + colour_off;\n" + "+\treturn freelist;\n" + " }\n" + " \n" + "-static inline unsigned int *slab_bufctl(struct slab *slabp)\n" + "+static inline unsigned int *slab_bufctl(struct page *page)\n" + " {\n" + "-\treturn (unsigned int *) (slabp + 1);\n" + "+\treturn (unsigned int *)(page->freelist);\n" + " }\n" + " \n" + " static void cache_init_objs(struct kmem_cache *cachep,\n" + "-\t\t\t struct slab *slabp)\n" + "+\t\t\t struct page *page)\n" + " {\n" + " \tint i;\n" + " \n" + " \tfor (i = 0; i < cachep->num; i++) {\n" + "-\t\tvoid *objp = index_to_obj(cachep, slabp, i);\n" + "+\t\tvoid *objp = index_to_obj(cachep, page, i);\n" + " #if DEBUG\n" + " \t\t/* need to poison the objs? */\n" + " \t\tif (cachep->flags & SLAB_POISON)\n" + "@@ -2621,7 +2592,7 @@ static void cache_init_objs(struct kmem_cache *cachep,\n" + " \t\tif (cachep->ctor)\n" + " \t\t\tcachep->ctor(objp);\n" + " #endif\n" + "-\t\tslab_bufctl(slabp)[i] = i;\n" + "+\t\tslab_bufctl(page)[i] = i;\n" + " \t}\n" + " }\n" + " \n" + "@@ -2635,13 +2606,13 @@ static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)\n" + " \t}\n" + " }\n" + " \n" + "-static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp,\n" + "+static void *slab_get_obj(struct kmem_cache *cachep, struct page *page,\n" + " \t\t\t\tint nodeid)\n" + " {\n" + " \tvoid *objp;\n" + " \n" + "-\tobjp = index_to_obj(cachep, slabp, slab_bufctl(slabp)[slabp->active]);\n" + "-\tslabp->active++;\n" + "+\tobjp = index_to_obj(cachep, page, slab_bufctl(page)[page->active]);\n" + "+\tpage->active++;\n" + " #if DEBUG\n" + " \tWARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);\n" + " #endif\n" + "@@ -2649,10 +2620,10 @@ static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp,\n" + " \treturn objp;\n" + " }\n" + " \n" + "-static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,\n" + "+static void slab_put_obj(struct kmem_cache *cachep, struct page *page,\n" + " \t\t\t\tvoid *objp, int nodeid)\n" + " {\n" + "-\tunsigned int objnr = obj_to_index(cachep, slabp, objp);\n" + "+\tunsigned int objnr = obj_to_index(cachep, page, objp);\n" + " #if DEBUG\n" + " \tunsigned int i;\n" + " \n" + "@@ -2660,16 +2631,16 @@ static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,\n" + " \tWARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);\n" + " \n" + " \t/* Verify double free bug */\n" + "-\tfor (i = slabp->active; i < cachep->num; i++) {\n" + "-\t\tif (slab_bufctl(slabp)[i] == objnr) {\n" + "+\tfor (i = page->active; i < cachep->num; i++) {\n" + "+\t\tif (slab_bufctl(page)[i] == objnr) {\n" + " \t\t\tprintk(KERN_ERR \"slab: double free detected in cache \"\n" + " \t\t\t\t\t\"'%s', objp %p\\n\", cachep->name, objp);\n" + " \t\t\tBUG();\n" + " \t\t}\n" + " \t}\n" + " #endif\n" + "-\tslabp->active--;\n" + "-\tslab_bufctl(slabp)[slabp->active] = objnr;\n" + "+\tpage->active--;\n" + "+\tslab_bufctl(page)[page->active] = objnr;\n" + " }\n" + " \n" + " /*\n" + "@@ -2677,11 +2648,11 @@ static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,\n" + " * for the slab allocator to be able to lookup the cache and slab of a\n" + " * virtual address for kfree, ksize, and slab debugging.\n" + " */\n" + "-static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,\n" + "-\t\t\t struct page *page)\n" + "+static void slab_map_pages(struct kmem_cache *cache, struct page *page,\n" + "+\t\t\t void *freelist)\n" + " {\n" + " \tpage->slab_cache = cache;\n" + "-\tpage->slab_page = slab;\n" + "+\tpage->freelist = freelist;\n" + " }\n" + " \n" + " /*\n" + "@@ -2691,7 +2662,7 @@ static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,\n" + " static int cache_grow(struct kmem_cache *cachep,\n" + " \t\tgfp_t flags, int nodeid, struct page *page)\n" + " {\n" + "-\tstruct slab *slabp;\n" + "+\tvoid *freelist;\n" + " \tsize_t offset;\n" + " \tgfp_t local_flags;\n" + " \tstruct kmem_cache_node *n;\n" + "@@ -2738,14 +2709,14 @@ static int cache_grow(struct kmem_cache *cachep,\n" + " \t\tgoto failed;\n" + " \n" + " \t/* Get slab management. */\n" + "-\tslabp = alloc_slabmgmt(cachep, page, offset,\n" + "+\tfreelist = alloc_slabmgmt(cachep, page, offset,\n" + " \t\t\tlocal_flags & ~GFP_CONSTRAINT_MASK, nodeid);\n" + "-\tif (!slabp)\n" + "+\tif (!freelist)\n" + " \t\tgoto opps1;\n" + " \n" + "-\tslab_map_pages(cachep, slabp, page);\n" + "+\tslab_map_pages(cachep, page, freelist);\n" + " \n" + "-\tcache_init_objs(cachep, slabp);\n" + "+\tcache_init_objs(cachep, page);\n" + " \n" + " \tif (local_flags & __GFP_WAIT)\n" + " \t\tlocal_irq_disable();\n" + "@@ -2753,7 +2724,7 @@ static int cache_grow(struct kmem_cache *cachep,\n" + " \tspin_lock(&n->list_lock);\n" + " \n" + " \t/* Make slab active. */\n" + "-\tlist_add_tail(&slabp->list, &(n->slabs_free));\n" + "+\tlist_add_tail(&page->lru, &(n->slabs_free));\n" + " \tSTATS_INC_GROWN(cachep);\n" + " \tn->free_objects += cachep->num;\n" + " \tspin_unlock(&n->list_lock);\n" + "@@ -2808,13 +2779,13 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,\n" + " \t\t\t\t unsigned long caller)\n" + " {\n" + " \tunsigned int objnr;\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \n" + " \tBUG_ON(virt_to_cache(objp) != cachep);\n" + " \n" + " \tobjp -= obj_offset(cachep);\n" + " \tkfree_debugcheck(objp);\n" + "-\tslabp = virt_to_slab(objp);\n" + "+\tpage = virt_to_head_page(objp);\n" + " \n" + " \tif (cachep->flags & SLAB_RED_ZONE) {\n" + " \t\tverify_redzone_free(cachep, objp);\n" + "@@ -2824,10 +2795,10 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,\n" + " \tif (cachep->flags & SLAB_STORE_USER)\n" + " \t\t*dbg_userword(cachep, objp) = (void *)caller;\n" + " \n" + "-\tobjnr = obj_to_index(cachep, slabp, objp);\n" + "+\tobjnr = obj_to_index(cachep, page, objp);\n" + " \n" + " \tBUG_ON(objnr >= cachep->num);\n" + "-\tBUG_ON(objp != index_to_obj(cachep, slabp, objnr));\n" + "+\tBUG_ON(objp != index_to_obj(cachep, page, objnr));\n" + " \n" + " \tif (cachep->flags & SLAB_POISON) {\n" + " #ifdef CONFIG_DEBUG_PAGEALLOC\n" + "@@ -2886,7 +2857,7 @@ retry:\n" + " \n" + " \twhile (batchcount > 0) {\n" + " \t\tstruct list_head *entry;\n" + "-\t\tstruct slab *slabp;\n" + "+\t\tstruct page *page;\n" + " \t\t/* Get slab alloc is to come from. */\n" + " \t\tentry = n->slabs_partial.next;\n" + " \t\tif (entry == &n->slabs_partial) {\n" + "@@ -2896,7 +2867,7 @@ retry:\n" + " \t\t\t\tgoto must_grow;\n" + " \t\t}\n" + " \n" + "-\t\tslabp = list_entry(entry, struct slab, list);\n" + "+\t\tpage = list_entry(entry, struct page, lru);\n" + " \t\tcheck_spinlock_acquired(cachep);\n" + " \n" + " \t\t/*\n" + "@@ -2904,23 +2875,23 @@ retry:\n" + " \t\t * there must be at least one object available for\n" + " \t\t * allocation.\n" + " \t\t */\n" + "-\t\tBUG_ON(slabp->active >= cachep->num);\n" + "+\t\tBUG_ON(page->active >= cachep->num);\n" + " \n" + "-\t\twhile (slabp->active < cachep->num && batchcount--) {\n" + "+\t\twhile (page->active < cachep->num && batchcount--) {\n" + " \t\t\tSTATS_INC_ALLOCED(cachep);\n" + " \t\t\tSTATS_INC_ACTIVE(cachep);\n" + " \t\t\tSTATS_SET_HIGH(cachep);\n" + " \n" + "-\t\t\tac_put_obj(cachep, ac, slab_get_obj(cachep, slabp,\n" + "+\t\t\tac_put_obj(cachep, ac, slab_get_obj(cachep, page,\n" + " \t\t\t\t\t\t\t\t\tnode));\n" + " \t\t}\n" + " \n" + " \t\t/* move slabp to correct slabp list: */\n" + "-\t\tlist_del(&slabp->list);\n" + "-\t\tif (slabp->active == cachep->num)\n" + "-\t\t\tlist_add(&slabp->list, &n->slabs_full);\n" + "+\t\tlist_del(&page->lru);\n" + "+\t\tif (page->active == cachep->num)\n" + "+\t\t\tlist_add(&page->list, &n->slabs_full);\n" + " \t\telse\n" + "-\t\t\tlist_add(&slabp->list, &n->slabs_partial);\n" + "+\t\t\tlist_add(&page->list, &n->slabs_partial);\n" + " \t}\n" + " \n" + " must_grow:\n" + "@@ -3175,7 +3146,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,\n" + " \t\t\t\tint nodeid)\n" + " {\n" + " \tstruct list_head *entry;\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \tstruct kmem_cache_node *n;\n" + " \tvoid *obj;\n" + " \tint x;\n" + "@@ -3195,24 +3166,24 @@ retry:\n" + " \t\t\tgoto must_grow;\n" + " \t}\n" + " \n" + "-\tslabp = list_entry(entry, struct slab, list);\n" + "+\tpage = list_entry(entry, struct page, lru);\n" + " \tcheck_spinlock_acquired_node(cachep, nodeid);\n" + " \n" + " \tSTATS_INC_NODEALLOCS(cachep);\n" + " \tSTATS_INC_ACTIVE(cachep);\n" + " \tSTATS_SET_HIGH(cachep);\n" + " \n" + "-\tBUG_ON(slabp->active == cachep->num);\n" + "+\tBUG_ON(page->active == cachep->num);\n" + " \n" + "-\tobj = slab_get_obj(cachep, slabp, nodeid);\n" + "+\tobj = slab_get_obj(cachep, page, nodeid);\n" + " \tn->free_objects--;\n" + " \t/* move slabp to correct slabp list: */\n" + "-\tlist_del(&slabp->list);\n" + "+\tlist_del(&page->lru);\n" + " \n" + "-\tif (slabp->active == cachep->num)\n" + "-\t\tlist_add(&slabp->list, &n->slabs_full);\n" + "+\tif (page->active == cachep->num)\n" + "+\t\tlist_add(&page->lru, &n->slabs_full);\n" + " \telse\n" + "-\t\tlist_add(&slabp->list, &n->slabs_partial);\n" + "+\t\tlist_add(&page->lru, &n->slabs_partial);\n" + " \n" + " \tspin_unlock(&n->list_lock);\n" + " \tgoto done;\n" + "@@ -3362,21 +3333,21 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,\n" + " \n" + " \tfor (i = 0; i < nr_objects; i++) {\n" + " \t\tvoid *objp;\n" + "-\t\tstruct slab *slabp;\n" + "+\t\tstruct page *page;\n" + " \n" + " \t\tclear_obj_pfmemalloc(&objpp[i]);\n" + " \t\tobjp = objpp[i];\n" + " \n" + "-\t\tslabp = virt_to_slab(objp);\n" + "+\t\tpage = virt_to_head_page(objp);\n" + " \t\tn = cachep->node[node];\n" + "-\t\tlist_del(&slabp->list);\n" + "+\t\tlist_del(&page->lru);\n" + " \t\tcheck_spinlock_acquired_node(cachep, node);\n" + "-\t\tslab_put_obj(cachep, slabp, objp, node);\n" + "+\t\tslab_put_obj(cachep, page, objp, node);\n" + " \t\tSTATS_DEC_ACTIVE(cachep);\n" + " \t\tn->free_objects++;\n" + " \n" + " \t\t/* fixup slab chains */\n" + "-\t\tif (slabp->active == 0) {\n" + "+\t\tif (page->active == 0) {\n" + " \t\t\tif (n->free_objects > n->free_limit) {\n" + " \t\t\t\tn->free_objects -= cachep->num;\n" + " \t\t\t\t/* No need to drop any previously held\n" + "@@ -3385,16 +3356,16 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,\n" + " \t\t\t\t * a different cache, refer to comments before\n" + " \t\t\t\t * alloc_slabmgmt.\n" + " \t\t\t\t */\n" + "-\t\t\t\tslab_destroy(cachep, slabp);\n" + "+\t\t\t\tslab_destroy(cachep, page);\n" + " \t\t\t} else {\n" + "-\t\t\t\tlist_add(&slabp->list, &n->slabs_free);\n" + "+\t\t\t\tlist_add(&page->lru, &n->slabs_free);\n" + " \t\t\t}\n" + " \t\t} else {\n" + " \t\t\t/* Unconditionally move a slab to the end of the\n" + " \t\t\t * partial list on free - maximum time for the\n" + " \t\t\t * other objects to be freed, too.\n" + " \t\t\t */\n" + "-\t\t\tlist_add_tail(&slabp->list, &n->slabs_partial);\n" + "+\t\t\tlist_add_tail(&page->lru, &n->slabs_partial);\n" + " \t\t}\n" + " \t}\n" + " }\n" + "@@ -3434,10 +3405,10 @@ free_done:\n" + " \n" + " \t\tp = n->slabs_free.next;\n" + " \t\twhile (p != &(n->slabs_free)) {\n" + "-\t\t\tstruct slab *slabp;\n" + "+\t\t\tstruct page *page;\n" + " \n" + "-\t\t\tslabp = list_entry(p, struct slab, list);\n" + "-\t\t\tBUG_ON(slabp->active);\n" + "+\t\t\tpage = list_entry(p, struct page, lru);\n" + "+\t\t\tBUG_ON(page->active);\n" + " \n" + " \t\t\ti++;\n" + " \t\t\tp = p->next;\n" + "@@ -4041,7 +4012,7 @@ out:\n" + " #ifdef CONFIG_SLABINFO\n" + " void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)\n" + " {\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \tunsigned long active_objs;\n" + " \tunsigned long num_objs;\n" + " \tunsigned long active_slabs = 0;\n" + "@@ -4061,22 +4032,22 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)\n" + " \t\tcheck_irq_on();\n" + " \t\tspin_lock_irq(&n->list_lock);\n" + " \n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_full, list) {\n" + "-\t\t\tif (slabp->active != cachep->num && !error)\n" + "+\t\tlist_for_each_entry(page, &n->slabs_full, lru) {\n" + "+\t\t\tif (page->active != cachep->num && !error)\n" + " \t\t\t\terror = \"slabs_full accounting error\";\n" + " \t\t\tactive_objs += cachep->num;\n" + " \t\t\tactive_slabs++;\n" + " \t\t}\n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_partial, list) {\n" + "-\t\t\tif (slabp->active == cachep->num && !error)\n" + "+\t\tlist_for_each_entry(page, &n->slabs_partial, lru) {\n" + "+\t\t\tif (page->active == cachep->num && !error)\n" + " \t\t\t\terror = \"slabs_partial accounting error\";\n" + "-\t\t\tif (!slabp->active && !error)\n" + "+\t\t\tif (!page->active && !error)\n" + " \t\t\t\terror = \"slabs_partial accounting error\";\n" + "-\t\t\tactive_objs += slabp->active;\n" + "+\t\t\tactive_objs += page->active;\n" + " \t\t\tactive_slabs++;\n" + " \t\t}\n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_free, list) {\n" + "-\t\t\tif (slabp->active && !error)\n" + "+\t\tlist_for_each_entry(page, &n->slabs_free, lru) {\n" + "+\t\t\tif (page->active && !error)\n" + " \t\t\t\terror = \"slabs_free accounting error\";\n" + " \t\t\tnum_slabs++;\n" + " \t\t}\n" + "@@ -4229,19 +4200,20 @@ static inline int add_caller(unsigned long *n, unsigned long v)\n" + " \treturn 1;\n" + " }\n" + " \n" + "-static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s)\n" + "+static void handle_slab(unsigned long *n, struct kmem_cache *c,\n" + "+\t\t\t\t\t\tstruct page *page)\n" + " {\n" + " \tvoid *p;\n" + " \tint i, j;\n" + " \n" + " \tif (n[0] == n[1])\n" + " \t\treturn;\n" + "-\tfor (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) {\n" + "+\tfor (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {\n" + " \t\tbool active = true;\n" + " \n" + "-\t\tfor (j = s->active; j < c->num; j++) {\n" + "+\t\tfor (j = page->active; j < c->num; j++) {\n" + " \t\t\t/* Skip freed item */\n" + "-\t\t\tif (slab_bufctl(s)[j] == i) {\n" + "+\t\t\tif (slab_bufctl(page)[j] == i) {\n" + " \t\t\t\tactive = false;\n" + " \t\t\t\tbreak;\n" + " \t\t\t}\n" + "@@ -4273,7 +4245,7 @@ static void show_symbol(struct seq_file *m, unsigned long address)\n" + " static int leaks_show(struct seq_file *m, void *p)\n" + " {\n" + " \tstruct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);\n" + "-\tstruct slab *slabp;\n" + "+\tstruct page *page;\n" + " \tstruct kmem_cache_node *n;\n" + " \tconst char *name;\n" + " \tunsigned long *x = m->private;\n" + "@@ -4297,10 +4269,10 @@ static int leaks_show(struct seq_file *m, void *p)\n" + " \t\tcheck_irq_on();\n" + " \t\tspin_lock_irq(&n->list_lock);\n" + " \n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_full, list)\n" + "-\t\t\thandle_slab(x, cachep, slabp);\n" + "-\t\tlist_for_each_entry(slabp, &n->slabs_partial, list)\n" + "-\t\t\thandle_slab(x, cachep, slabp);\n" + "+\t\tlist_for_each_entry(page, &n->slabs_full, lru)\n" + "+\t\t\thandle_slab(x, cachep, page);\n" + "+\t\tlist_for_each_entry(page, &n->slabs_partial, lru)\n" + "+\t\t\thandle_slab(x, cachep, page);\n" + " \t\tspin_unlock_irq(&n->list_lock);\n" + " \t}\n" + " \tname = cachep->name;\n" + "-- \n" + 1.7.9.5 -a7f714a2195077f545db31f2b6321ac0dbf7820d226c0d2fd8c8b90a45e496cc +0dc70af0902a304aa2b260556a222d80f2c8de522f5511cf2b8315948f4aae20
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.