* [PATCH 6.6.y 2/9] mm/zsmalloc: remove unused zspage->isolated
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 3/9] mm/zsmalloc: remove set_zspage_mapping() Sasha Levin
` (6 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Chengming Zhou, Sergey Senozhatsky, Johannes Weiner, Minchan Kim,
Nhat Pham, Yosry Ahmed, Andrew Morton, Sasha Levin
From: Chengming Zhou <zhouchengming@bytedance.com>
[ Upstream commit 4ad63e163264063b8cac8f681c20a303933842f2 ]
The zspage->isolated is not used anywhere, we don't need to maintain it,
which needs to hold the heavy pool lock to update it, so just remove it.
Link: https://lkml.kernel.org/r/20240219-b4-szmalloc-migrate-v1-3-34cd49c6545b@bytedance.com
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 69 ++++++++++++++-------------------------------------
1 file changed, 18 insertions(+), 51 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 874d7d24d2d0f..2f37901ace99a 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -116,7 +116,6 @@
#define HUGE_BITS 1
#define FULLNESS_BITS 4
#define CLASS_BITS 8
-#define ISOLATED_BITS 5
#define MAGIC_VAL_BITS 8
#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL))
@@ -244,7 +243,6 @@ struct zspage {
unsigned int huge:HUGE_BITS;
unsigned int fullness:FULLNESS_BITS;
unsigned int class:CLASS_BITS + 1;
- unsigned int isolated:ISOLATED_BITS;
unsigned int magic:MAGIC_VAL_BITS;
};
unsigned int inuse;
@@ -515,19 +513,19 @@ static int get_size_class_index(int size)
return min_t(int, ZS_SIZE_CLASSES - 1, idx);
}
-static inline void class_stat_inc(struct size_class *class,
- int type, unsigned long cnt)
+static inline void class_stat_add(struct size_class *class, int type,
+ unsigned long cnt)
{
class->stats.objs[type] += cnt;
}
-static inline void class_stat_dec(struct size_class *class,
- int type, unsigned long cnt)
+static inline void class_stat_sub(struct size_class *class, int type,
+ unsigned long cnt)
{
class->stats.objs[type] -= cnt;
}
-static inline unsigned long zs_stat_get(struct size_class *class, int type)
+static inline unsigned long class_stat_read(struct size_class *class, int type)
{
return class->stats.objs[type];
}
@@ -579,12 +577,12 @@ static int zs_stats_size_show(struct seq_file *s, void *v)
seq_printf(s, " %5u %5u ", i, class->size);
for (fg = ZS_INUSE_RATIO_10; fg < NR_FULLNESS_GROUPS; fg++) {
- inuse_totals[fg] += zs_stat_get(class, fg);
- seq_printf(s, "%9lu ", zs_stat_get(class, fg));
+ inuse_totals[fg] += class_stat_read(class, fg);
+ seq_printf(s, "%9lu ", class_stat_read(class, fg));
}
- obj_allocated = zs_stat_get(class, ZS_OBJS_ALLOCATED);
- obj_used = zs_stat_get(class, ZS_OBJS_INUSE);
+ obj_allocated = class_stat_read(class, ZS_OBJS_ALLOCATED);
+ obj_used = class_stat_read(class, ZS_OBJS_INUSE);
freeable = zs_can_compact(class);
spin_unlock(&pool->lock);
@@ -689,7 +687,7 @@ static void insert_zspage(struct size_class *class,
struct zspage *zspage,
int fullness)
{
- class_stat_inc(class, fullness, 1);
+ class_stat_add(class, fullness, 1);
list_add(&zspage->list, &class->fullness_list[fullness]);
}
@@ -704,7 +702,7 @@ static void remove_zspage(struct size_class *class,
VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
list_del_init(&zspage->list);
- class_stat_dec(class, fullness, 1);
+ class_stat_sub(class, fullness, 1);
}
/*
@@ -869,7 +867,7 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class,
cache_free_zspage(pool, zspage);
- class_stat_dec(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage);
+ class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage);
atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated);
}
@@ -1383,7 +1381,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
/* Now move the zspage to another fullness group, if required */
fix_fullness_group(class, zspage);
record_obj(handle, obj);
- class_stat_inc(class, ZS_OBJS_INUSE, 1);
+ class_stat_add(class, ZS_OBJS_INUSE, 1);
goto out;
}
@@ -1403,8 +1401,8 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
set_zspage_mapping(zspage, class->index, newfg);
record_obj(handle, obj);
atomic_long_add(class->pages_per_zspage, &pool->pages_allocated);
- class_stat_inc(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage);
- class_stat_inc(class, ZS_OBJS_INUSE, 1);
+ class_stat_add(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage);
+ class_stat_add(class, ZS_OBJS_INUSE, 1);
/* We completely set up zspage so mark them as movable */
SetZsPageMovable(pool, zspage);
@@ -1463,7 +1461,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
zspage = get_zspage(f_page);
class = zspage_class(pool, zspage);
- class_stat_dec(class, ZS_OBJS_INUSE, 1);
+ class_stat_sub(class, ZS_OBJS_INUSE, 1);
obj_free(class->size, obj);
fullness = fix_fullness_group(class, zspage);
@@ -1733,18 +1731,6 @@ static void migrate_write_unlock(struct zspage *zspage)
write_unlock(&zspage->lock);
}
-/* Number of isolated subpage for *page migration* in this zspage */
-static void inc_zspage_isolation(struct zspage *zspage)
-{
- zspage->isolated++;
-}
-
-static void dec_zspage_isolation(struct zspage *zspage)
-{
- VM_BUG_ON(zspage->isolated == 0);
- zspage->isolated--;
-}
-
static const struct movable_operations zsmalloc_mops;
static void replace_sub_page(struct size_class *class, struct zspage *zspage,
@@ -1772,21 +1758,12 @@ static void replace_sub_page(struct size_class *class, struct zspage *zspage,
static bool zs_page_isolate(struct page *page, isolate_mode_t mode)
{
- struct zs_pool *pool;
- struct zspage *zspage;
-
/*
* Page is locked so zspage couldn't be destroyed. For detail, look at
* lock_zspage in free_zspage.
*/
VM_BUG_ON_PAGE(PageIsolated(page), page);
- zspage = get_zspage(page);
- pool = zspage->pool;
- spin_lock(&pool->lock);
- inc_zspage_isolation(zspage);
- spin_unlock(&pool->lock);
-
return true;
}
@@ -1851,7 +1828,6 @@ static int zs_page_migrate(struct page *newpage, struct page *page,
kunmap_atomic(s_addr);
replace_sub_page(class, zspage, newpage, page);
- dec_zspage_isolation(zspage);
/*
* Since we complete the data copy and set up new zspage structure,
* it's okay to release the pool's lock.
@@ -1873,16 +1849,7 @@ static int zs_page_migrate(struct page *newpage, struct page *page,
static void zs_page_putback(struct page *page)
{
- struct zs_pool *pool;
- struct zspage *zspage;
-
VM_BUG_ON_PAGE(!PageIsolated(page), page);
-
- zspage = get_zspage(page);
- pool = zspage->pool;
- spin_lock(&pool->lock);
- dec_zspage_isolation(zspage);
- spin_unlock(&pool->lock);
}
static const struct movable_operations zsmalloc_mops = {
@@ -1967,8 +1934,8 @@ static inline void zs_flush_migration(struct zs_pool *pool) { }
static unsigned long zs_can_compact(struct size_class *class)
{
unsigned long obj_wasted;
- unsigned long obj_allocated = zs_stat_get(class, ZS_OBJS_ALLOCATED);
- unsigned long obj_used = zs_stat_get(class, ZS_OBJS_INUSE);
+ unsigned long obj_allocated = class_stat_read(class, ZS_OBJS_ALLOCATED);
+ unsigned long obj_used = class_stat_read(class, ZS_OBJS_INUSE);
if (obj_allocated <= obj_used)
return 0;
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 3/9] mm/zsmalloc: remove set_zspage_mapping()
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 2/9] mm/zsmalloc: remove unused zspage->isolated Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 4/9] mm/zsmalloc: remove_zspage() don't need fullness parameter Sasha Levin
` (5 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Chengming Zhou, Sergey Senozhatsky, Johannes Weiner, Minchan Kim,
Nhat Pham, Yosry Ahmed, Andrew Morton, Sasha Levin
From: Chengming Zhou <zhouchengming@bytedance.com>
[ Upstream commit a6a8cdfdde43b6231dd1ed61b8a7a6ee55956376 ]
Patch series "mm/zsmalloc: some cleanup for get/set_zspage_mapping()".
The discussion[1] with Sergey shows there are some cleanup works to do
in get/set_zspage_mapping():
- the fullness returned from get_zspage_mapping() is not stable outside
pool->lock, this usage pattern is confusing, but should be ok in this
free_zspage path.
- we seldom use the class_idx returned from get_zspage_mapping(), only
free_zspage path use to get its class.
- set_zspage_mapping() always set the zspage->class, but it's never
changed after zspage allocated.
[1] https://lore.kernel.org/all/a6c22e30-cf10-4122-91bc-ceb9fb57a5d6@bytedance.com/
This patch (of 3):
We only need to update zspage->fullness when insert_zspage(), since
zspage->class is never changed after allocated.
Link: https://lkml.kernel.org/r/20240220-b4-zsmalloc-cleanup-v1-0-5c5ee4ccdd87@bytedance.com
Link: https://lkml.kernel.org/r/20240220-b4-zsmalloc-cleanup-v1-1-5c5ee4ccdd87@bytedance.com
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 2f37901ace99a..075e2a692acd9 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -487,14 +487,6 @@ static struct size_class *zspage_class(struct zs_pool *pool,
return pool->size_class[zspage->class];
}
-static void set_zspage_mapping(struct zspage *zspage,
- unsigned int class_idx,
- int fullness)
-{
- zspage->class = class_idx;
- zspage->fullness = fullness;
-}
-
/*
* zsmalloc divides the pool into various size classes where each
* class maintains a list of zspages where each zspage is divided
@@ -689,6 +681,7 @@ static void insert_zspage(struct size_class *class,
{
class_stat_add(class, fullness, 1);
list_add(&zspage->list, &class->fullness_list[fullness]);
+ zspage->fullness = fullness;
}
/*
@@ -726,7 +719,6 @@ static int fix_fullness_group(struct size_class *class, struct zspage *zspage)
remove_zspage(class, zspage, currfg);
insert_zspage(class, zspage, newfg);
- set_zspage_mapping(zspage, class_idx, newfg);
out:
return newfg;
}
@@ -1009,6 +1001,7 @@ static struct zspage *alloc_zspage(struct zs_pool *pool,
create_page_chain(class, zspage, pages);
init_zspage(class, zspage);
zspage->pool = pool;
+ zspage->class = class->index;
return zspage;
}
@@ -1398,7 +1391,6 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
obj = obj_malloc(pool, zspage, handle);
newfg = get_fullness_group(class, zspage);
insert_zspage(class, zspage, newfg);
- set_zspage_mapping(zspage, class->index, newfg);
record_obj(handle, obj);
atomic_long_add(class->pages_per_zspage, &pool->pages_allocated);
class_stat_add(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage);
@@ -1656,7 +1648,6 @@ static int putback_zspage(struct size_class *class, struct zspage *zspage)
fullness = get_fullness_group(class, zspage);
insert_zspage(class, zspage, fullness);
- set_zspage_mapping(zspage, class->index, fullness);
return fullness;
}
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 4/9] mm/zsmalloc: remove_zspage() don't need fullness parameter
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 2/9] mm/zsmalloc: remove unused zspage->isolated Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 3/9] mm/zsmalloc: remove set_zspage_mapping() Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 5/9] mm/zsmalloc: remove get_zspage_mapping() Sasha Levin
` (4 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Chengming Zhou, Sergey Senozhatsky, Johannes Weiner, Minchan Kim,
Nhat Pham, Yosry Ahmed, Andrew Morton, Sasha Levin
From: Chengming Zhou <zhouchengming@bytedance.com>
[ Upstream commit 67eaedc1c52f0ac3ea99e4fc0bc3d7940344b94b ]
We must remove_zspage() from its current fullness list, then use
insert_zspage() to update its fullness and insert to new fullness list.
Obviously, remove_zspage() doesn't need the fullness parameter.
Link: https://lkml.kernel.org/r/20240220-b4-zsmalloc-cleanup-v1-2-5c5ee4ccdd87@bytedance.com
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 075e2a692acd9..0cf814a809ec1 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -688,10 +688,10 @@ static void insert_zspage(struct size_class *class,
* This function removes the given zspage from the freelist identified
* by <class, fullness_group>.
*/
-static void remove_zspage(struct size_class *class,
- struct zspage *zspage,
- int fullness)
+static void remove_zspage(struct size_class *class, struct zspage *zspage)
{
+ int fullness = zspage->fullness;
+
VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
list_del_init(&zspage->list);
@@ -717,7 +717,7 @@ static int fix_fullness_group(struct size_class *class, struct zspage *zspage)
if (newfg == currfg)
goto out;
- remove_zspage(class, zspage, currfg);
+ remove_zspage(class, zspage);
insert_zspage(class, zspage, newfg);
out:
return newfg;
@@ -879,7 +879,7 @@ static void free_zspage(struct zs_pool *pool, struct size_class *class,
return;
}
- remove_zspage(class, zspage, ZS_INUSE_RATIO_0);
+ remove_zspage(class, zspage);
__free_zspage(pool, class, zspage);
}
@@ -1610,7 +1610,7 @@ static struct zspage *isolate_src_zspage(struct size_class *class)
zspage = list_first_entry_or_null(&class->fullness_list[fg],
struct zspage, list);
if (zspage) {
- remove_zspage(class, zspage, fg);
+ remove_zspage(class, zspage);
return zspage;
}
}
@@ -1627,7 +1627,7 @@ static struct zspage *isolate_dst_zspage(struct size_class *class)
zspage = list_first_entry_or_null(&class->fullness_list[fg],
struct zspage, list);
if (zspage) {
- remove_zspage(class, zspage, fg);
+ remove_zspage(class, zspage);
return zspage;
}
}
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 5/9] mm/zsmalloc: remove get_zspage_mapping()
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
` (2 preceding siblings ...)
2026-09-04 20:32 ` [PATCH 6.6.y 4/9] mm/zsmalloc: remove_zspage() don't need fullness parameter Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 6/9] mm/zsmalloc: change back to per-size_class lock Sasha Levin
` (3 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Chengming Zhou, Sergey Senozhatsky, Johannes Weiner, Minchan Kim,
Nhat Pham, Yosry Ahmed, Andrew Morton, Sasha Levin
From: Chengming Zhou <zhouchengming@bytedance.com>
[ Upstream commit ce335e0723470203ca3498d70de2eb97e6869a06 ]
Actually we seldom use the class_idx returned from get_zspage_mapping(),
only the zspage->fullness is useful, just use zspage->fullness to remove
this helper.
Note zspage->fullness is not stable outside pool->lock, remove redundant
"VM_BUG_ON(fullness != ZS_INUSE_RATIO_0)" in async_free_zspage() since we
already have the same VM_BUG_ON() in __free_zspage(), which is safe to
access zspage->fullness with pool->lock held.
Link: https://lkml.kernel.org/r/20240220-b4-zsmalloc-cleanup-v1-3-5c5ee4ccdd87@bytedance.com
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 28 ++++------------------------
1 file changed, 4 insertions(+), 24 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 0cf814a809ec1..187936f3ea794 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -471,16 +471,6 @@ static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
zspage->freeobj = obj;
}
-static void get_zspage_mapping(struct zspage *zspage,
- unsigned int *class_idx,
- int *fullness)
-{
- BUG_ON(zspage->magic != ZSPAGE_MAGIC);
-
- *fullness = zspage->fullness;
- *class_idx = zspage->class;
-}
-
static struct size_class *zspage_class(struct zs_pool *pool,
struct zspage *zspage)
{
@@ -709,12 +699,10 @@ static void remove_zspage(struct size_class *class, struct zspage *zspage)
*/
static int fix_fullness_group(struct size_class *class, struct zspage *zspage)
{
- int class_idx;
- int currfg, newfg;
+ int newfg;
- get_zspage_mapping(zspage, &class_idx, &currfg);
newfg = get_fullness_group(class, zspage);
- if (newfg == currfg)
+ if (newfg == zspage->fullness)
goto out;
remove_zspage(class, zspage);
@@ -836,15 +824,11 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class,
struct zspage *zspage)
{
struct page *page, *next;
- int fg;
- unsigned int class_idx;
-
- get_zspage_mapping(zspage, &class_idx, &fg);
assert_spin_locked(&pool->lock);
VM_BUG_ON(get_zspage_inuse(zspage));
- VM_BUG_ON(fg != ZS_INUSE_RATIO_0);
+ VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0);
next = page = get_first_page(zspage);
do {
@@ -1857,8 +1841,6 @@ static void async_free_zspage(struct work_struct *work)
{
int i;
struct size_class *class;
- unsigned int class_idx;
- int fullness;
struct zspage *zspage, *tmp;
LIST_HEAD(free_pages);
struct zs_pool *pool = container_of(work, struct zs_pool,
@@ -1879,10 +1861,8 @@ static void async_free_zspage(struct work_struct *work)
list_del(&zspage->list);
lock_zspage(zspage);
- get_zspage_mapping(zspage, &class_idx, &fullness);
- VM_BUG_ON(fullness != ZS_INUSE_RATIO_0);
- class = pool->size_class[class_idx];
spin_lock(&pool->lock);
+ class = zspage_class(pool, zspage);
__free_zspage(pool, class, zspage);
spin_unlock(&pool->lock);
}
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 6/9] mm/zsmalloc: change back to per-size_class lock
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
` (3 preceding siblings ...)
2026-09-04 20:32 ` [PATCH 6.6.y 5/9] mm/zsmalloc: remove get_zspage_mapping() Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 7/9] mm/zsmalloc: clarify class per-fullness zspage counts Sasha Levin
` (2 subsequent siblings)
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Chengming Zhou, Sergey Senozhatsky, Chengming Zhou,
Johannes Weiner, Minchan Kim, Nhat Pham, Yosry Ahmed,
Andrew Morton, Sasha Levin
From: Chengming Zhou <zhouchengming@bytedance.com>
[ Upstream commit 64bd0197ae0c7d779e21410fe6d1782a3b59ee32 ]
Patch series "mm/zsmalloc: change back to per-size_class lock, v2".
Commit c0547d0b6a4b ("zsmalloc: consolidate zs_pool's migrate_lock and
size_class's locks") changed per-size_class lock to pool spinlock to
prepare reclaim support in zsmalloc. Then reclaim support in zsmalloc had
been dropped in favor of LRU reclaim in zswap, but this locking change had
been left there.
Obviously, the scalability of pool spinlock is worse than per-size_class.
And we have a workaround that using 32 pools in zswap to avoid this
scalability problem, which brings its own problems like memory waste and
more memory fragmentation.
So this series changes back to use per-size_class lock and using testing
data in much stressed situation to verify that we can use only one pool in
zswap. Note we only test and care about the zsmalloc backend, which makes
sense now since zsmalloc became a lot more popular than other backends.
Testing kernel build (make bzImage -j32) on tmpfs with memory.max=1GB, and
zswap shrinker enabled with 10GB swapfile on ext4.
real user sys
6.10.0-rc3 138.18 1241.38 1452.73
6.10.0-rc3-onepool 149.45 1240.45 1844.69
6.10.0-rc3-onepool-perclass 138.23 1242.37 1469.71
We can see from "sys" column that per-size_class locking with only one
pool in zswap can have near performance with the current 32 pools.
This patch (of 2):
This patch is almost the revert of the commit c0547d0b6a4b ("zsmalloc:
consolidate zs_pool's migrate_lock and size_class's locks"), which changed
to use a global pool->lock instead of per-size_class lock and
pool->migrate_lock, was preparation for suppporting reclaim in zsmalloc.
Then reclaim in zsmalloc had been dropped in favor of LRU reclaim in
zswap.
In theory, per-size_class is more fine-grained than the pool->lock, since
a pool can have many size_classes. As for the additional
pool->migrate_lock, only free() and map() need to grab it to access stable
handle to get zspage, and only in read lock mode.
Link: https://lkml.kernel.org/r/20240625-zsmalloc-lock-mm-everything-v3-0-ad941699cb61@linux.dev
Link: https://lkml.kernel.org/r/20240621-zsmalloc-lock-mm-everything-v2-0-d30e9cd2b793@linux.dev
Link: https://lkml.kernel.org/r/20240617-zsmalloc-lock-mm-everything-v1-0-5e5081ea11b3@linux.dev
Link: https://lkml.kernel.org/r/20240617-zsmalloc-lock-mm-everything-v1-1-5e5081ea11b3@linux.dev
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Obviously, the scalability of pool spinlock is worse than per-size_class.
And we have a workaround that using 32 pools in zswap to avoid this
scalability problem, which brings its own problems like memory waste and
more memory fragmentation.
So this series changes back to use per-size_class lock and using testing
data in much stressed situation to verify that we can use only one pool in
zswap. Note we only test and care about the zsmalloc backend, which makes
sense now since zsmalloc became a lot more popular than other backends.
Testing kernel build (make bzImage -j32) on tmpfs with memory.max=1GB, and
zswap shrinker enabled with 10GB swapfile on ext4.
real user sys
6.10.0-rc3 138.18 1241.38 1452.73
6.10.0-rc3-onepool 149.45 1240.45 1844.69
6.10.0-rc3-onepool-perclass 138.23 1242.37 1469.71
We can see from "sys" column that per-size_class locking with only one
pool in zswap can have near performance with the current 32 pools.
This patch (of 2):
This patch is almost the revert of the commit c0547d0b6a4b ("zsmalloc:
consolidate zs_pool's migrate_lock and size_class's locks"), which changed
to use a global pool->lock instead of per-size_class lock and
pool->migrate_lock, was preparation for suppporting reclaim in zsmalloc.
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 85 ++++++++++++++++++++++++++++++---------------------
1 file changed, 50 insertions(+), 35 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 187936f3ea794..6b78584c4183a 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -33,7 +33,8 @@
/*
* lock ordering:
* page_lock
- * pool->lock
+ * pool->migrate_lock
+ * class->lock
* zspage->lock
*/
@@ -180,6 +181,7 @@ static struct dentry *zs_stat_root;
static size_t huge_class_size;
struct size_class {
+ spinlock_t lock;
struct list_head fullness_list[NR_FULLNESS_GROUPS];
/*
* Size of objects stored in this class. Must be multiple
@@ -234,7 +236,8 @@ struct zs_pool {
#ifdef CONFIG_COMPACTION
struct work_struct free_work;
#endif
- spinlock_t lock;
+ /* protect page/zspage migration */
+ rwlock_t migrate_lock;
atomic_t compaction_in_progress;
};
@@ -336,7 +339,7 @@ static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
kmem_cache_free(pool->zspage_cachep, zspage);
}
-/* pool->lock(which owns the handle) synchronizes races */
+/* class->lock(which owns the handle) synchronizes races */
static void record_obj(unsigned long handle, unsigned long obj)
{
*(unsigned long *)handle = obj;
@@ -431,7 +434,7 @@ static __maybe_unused int is_first_page(struct page *page)
return PagePrivate(page);
}
-/* Protected by pool->lock */
+/* Protected by class->lock */
static inline int get_zspage_inuse(struct zspage *zspage)
{
return zspage->inuse;
@@ -555,7 +558,7 @@ static int zs_stats_size_show(struct seq_file *s, void *v)
if (class->index != i)
continue;
- spin_lock(&pool->lock);
+ spin_lock(&class->lock);
seq_printf(s, " %5u %5u ", i, class->size);
for (fg = ZS_INUSE_RATIO_10; fg < NR_FULLNESS_GROUPS; fg++) {
@@ -566,7 +569,7 @@ static int zs_stats_size_show(struct seq_file *s, void *v)
obj_allocated = class_stat_read(class, ZS_OBJS_ALLOCATED);
obj_used = class_stat_read(class, ZS_OBJS_INUSE);
freeable = zs_can_compact(class);
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
objs_per_zspage = class->objs_per_zspage;
pages_used = obj_allocated / objs_per_zspage *
@@ -825,7 +828,7 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class,
{
struct page *page, *next;
- assert_spin_locked(&pool->lock);
+ assert_spin_locked(&class->lock);
VM_BUG_ON(get_zspage_inuse(zspage));
VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0);
@@ -1185,19 +1188,19 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle,
BUG_ON(in_interrupt());
/* It guarantees it can get zspage from handle safely */
- spin_lock(&pool->lock);
+ read_lock(&pool->migrate_lock);
obj = handle_to_obj(handle);
obj_to_location(obj, &page, &obj_idx);
zspage = get_zspage(page);
/*
- * migration cannot move any zpages in this zspage. Here, pool->lock
+ * migration cannot move any zpages in this zspage. Here, class->lock
* is too heavy since callers would take some time until they calls
* zs_unmap_object API so delegate the locking from class to zspage
* which is smaller granularity.
*/
migrate_read_lock(zspage);
- spin_unlock(&pool->lock);
+ read_unlock(&pool->migrate_lock);
class = zspage_class(pool, zspage);
off = offset_in_page(class->size * obj_idx);
@@ -1350,8 +1353,8 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
size += ZS_HANDLE_SIZE;
class = pool->size_class[get_size_class_index(size)];
- /* pool->lock effectively protects the zpage migration */
- spin_lock(&pool->lock);
+ /* class->lock effectively protects the zpage migration */
+ spin_lock(&class->lock);
zspage = find_get_zspage(class);
if (likely(zspage)) {
obj = obj_malloc(pool, zspage, handle);
@@ -1363,7 +1366,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
goto out;
}
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
zspage = alloc_zspage(pool, class, gfp);
if (!zspage) {
@@ -1371,7 +1374,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
return (unsigned long)ERR_PTR(-ENOMEM);
}
- spin_lock(&pool->lock);
+ spin_lock(&class->lock);
obj = obj_malloc(pool, zspage, handle);
newfg = get_fullness_group(class, zspage);
insert_zspage(class, zspage, newfg);
@@ -1383,7 +1386,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
/* We completely set up zspage so mark them as movable */
SetZsPageMovable(pool, zspage);
out:
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
return handle;
}
@@ -1428,14 +1431,16 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
return;
/*
- * The pool->lock protects the race with zpage's migration
+ * The pool->migrate_lock protects the race with zpage's migration
* so it's safe to get the page from handle.
*/
- spin_lock(&pool->lock);
+ read_lock(&pool->migrate_lock);
obj = handle_to_obj(handle);
obj_to_page(obj, &f_page);
zspage = get_zspage(f_page);
class = zspage_class(pool, zspage);
+ spin_lock(&class->lock);
+ read_unlock(&pool->migrate_lock);
class_stat_sub(class, ZS_OBJS_INUSE, 1);
obj_free(class->size, obj);
@@ -1444,7 +1449,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
if (fullness == ZS_INUSE_RATIO_0)
free_zspage(pool, class, zspage);
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
cache_free_handle(pool, handle);
}
EXPORT_SYMBOL_GPL(zs_free);
@@ -1770,12 +1775,16 @@ static int zs_page_migrate(struct page *newpage, struct page *page,
pool = zspage->pool;
/*
- * The pool's lock protects the race between zpage migration
+ * The pool migrate_lock protects the race between zpage migration
* and zs_free.
*/
- spin_lock(&pool->lock);
+ write_lock(&pool->migrate_lock);
class = zspage_class(pool, zspage);
+ /*
+ * the class lock protects zpage alloc/free in the zspage.
+ */
+ spin_lock(&class->lock);
/* the migrate_write_lock protects zpage access via zs_map_object */
migrate_write_lock(zspage);
@@ -1805,9 +1814,10 @@ static int zs_page_migrate(struct page *newpage, struct page *page,
replace_sub_page(class, zspage, newpage, page);
/*
* Since we complete the data copy and set up new zspage structure,
- * it's okay to release the pool's lock.
+ * it's okay to release migration_lock.
*/
- spin_unlock(&pool->lock);
+ write_unlock(&pool->migrate_lock);
+ spin_unlock(&class->lock);
migrate_write_unlock(zspage);
get_page(newpage);
@@ -1851,20 +1861,20 @@ static void async_free_zspage(struct work_struct *work)
if (class->index != i)
continue;
- spin_lock(&pool->lock);
+ spin_lock(&class->lock);
list_splice_init(&class->fullness_list[ZS_INUSE_RATIO_0],
&free_pages);
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
}
list_for_each_entry_safe(zspage, tmp, &free_pages, list) {
list_del(&zspage->list);
lock_zspage(zspage);
- spin_lock(&pool->lock);
class = zspage_class(pool, zspage);
+ spin_lock(&class->lock);
__free_zspage(pool, class, zspage);
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
}
};
@@ -1928,7 +1938,8 @@ static unsigned long __zs_compact(struct zs_pool *pool,
* protect the race between zpage migration and zs_free
* as well as zpage allocation/free
*/
- spin_lock(&pool->lock);
+ write_lock(&pool->migrate_lock);
+ spin_lock(&class->lock);
while (zs_can_compact(class)) {
int fg;
@@ -1954,13 +1965,15 @@ static unsigned long __zs_compact(struct zs_pool *pool,
src_zspage = NULL;
if (get_fullness_group(class, dst_zspage) == ZS_INUSE_RATIO_100
- || spin_is_contended(&pool->lock)) {
+ || rwlock_is_contended(&pool->migrate_lock)) {
putback_zspage(class, dst_zspage);
dst_zspage = NULL;
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
+ write_unlock(&pool->migrate_lock);
cond_resched();
- spin_lock(&pool->lock);
+ write_lock(&pool->migrate_lock);
+ spin_lock(&class->lock);
}
}
@@ -1970,7 +1983,8 @@ static unsigned long __zs_compact(struct zs_pool *pool,
if (dst_zspage)
putback_zspage(class, dst_zspage);
- spin_unlock(&pool->lock);
+ spin_unlock(&class->lock);
+ write_unlock(&pool->migrate_lock);
return pages_freed;
}
@@ -1982,10 +1996,10 @@ unsigned long zs_compact(struct zs_pool *pool)
unsigned long pages_freed = 0;
/*
- * Pool compaction is performed under pool->lock so it is basically
+ * Pool compaction is performed under pool->migrate_lock so it is basically
* single-threaded. Having more than one thread in __zs_compact()
- * will increase pool->lock contention, which will impact other
- * zsmalloc operations that need pool->lock.
+ * will increase pool->migrate_lock contention, which will impact other
+ * zsmalloc operations that need pool->migrate_lock.
*/
if (atomic_xchg(&pool->compaction_in_progress, 1))
return 0;
@@ -2104,7 +2118,7 @@ struct zs_pool *zs_create_pool(const char *name)
return NULL;
init_deferred_free(pool);
- spin_lock_init(&pool->lock);
+ rwlock_init(&pool->migrate_lock);
atomic_set(&pool->compaction_in_progress, 0);
pool->name = kstrdup(name, GFP_KERNEL);
@@ -2176,6 +2190,7 @@ struct zs_pool *zs_create_pool(const char *name)
class->index = i;
class->pages_per_zspage = pages_per_zspage;
class->objs_per_zspage = objs_per_zspage;
+ spin_lock_init(&class->lock);
pool->size_class[i] = class;
fullness = ZS_INUSE_RATIO_0;
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 7/9] mm/zsmalloc: clarify class per-fullness zspage counts
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
` (4 preceding siblings ...)
2026-09-04 20:32 ` [PATCH 6.6.y 6/9] mm/zsmalloc: change back to per-size_class lock Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 8/9] zsmalloc: rename class stat mutators Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 9/9] zsmalloc: account for handle size in class lookup Sasha Levin
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Chengming Zhou, Minchan Kim, Sergey Senozhatsky, Andrew Morton,
Sasha Levin
From: Chengming Zhou <chengming.zhou@linux.dev>
[ Upstream commit 538148f9ba9e3136a877881e72ccbe56733daae2 ]
We always use insert_zspage() and remove_zspage() to update zspage's
fullness location, which will account correctly.
But this special async free path use "splice" instead of remove_zspage(),
so the per-fullness zspage count for ZS_INUSE_RATIO_0 won't decrease.
Clean things up by decreasing when iterate over the zspage free list.
This doesn't actually fix anything. ZS_INUSE_RATIO_0 is just a
"placeholder" which is never used anywhere.
Link: https://lkml.kernel.org/r/20240627075959.611783-1-chengming.zhou@linux.dev
Signed-off-by: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Sergey Senozhatsky <senozhatsky@chromium.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 6b78584c4183a..1249cfa2b67f2 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -1873,6 +1873,7 @@ static void async_free_zspage(struct work_struct *work)
class = zspage_class(pool, zspage);
spin_lock(&class->lock);
+ class_stat_dec(class, ZS_INUSE_RATIO_0, 1);
__free_zspage(pool, class, zspage);
spin_unlock(&class->lock);
}
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 8/9] zsmalloc: rename class stat mutators
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
` (5 preceding siblings ...)
2026-09-04 20:32 ` [PATCH 6.6.y 7/9] mm/zsmalloc: clarify class per-fullness zspage counts Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
2026-09-04 20:32 ` [PATCH 6.6.y 9/9] zsmalloc: account for handle size in class lookup Sasha Levin
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Sergey Senozhatsky, Chengming Zhou, Minchan Kim, Andrew Morton,
Sasha Levin
From: Sergey Senozhatsky <senozhatsky@chromium.org>
[ Upstream commit 791abe1e420c3dad6ddbd0a6c40467e9e24059b7 ]
A cosmetic change.
o Rename class_stat_inc() and class_stat_dec() to class_stat_add()
and class_stat_sub() correspondingly. inc/dec are usually associated
with +1/-1 modifications, while zsmlloc can modify stats by up
to ->objs_per_zspage. Use add/sub (follow atomics naming).
o Rename zs_stat_get() to class_stat_read()
get() is usually associated with ref-counting and is paired with put().
zs_stat_get() simply reads class stat so rename to reflect it.
(This also follows atomics naming).
Link: https://lkml.kernel.org/r/20240701031140.3756345-1-senozhatsky@chromium.org
Signed-off-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Reviewed-by: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7bf5cd5b5f2 ("zsmalloc: account for handle size in class lookup")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 1249cfa2b67f2..b63864f5fa398 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -1873,7 +1873,7 @@ static void async_free_zspage(struct work_struct *work)
class = zspage_class(pool, zspage);
spin_lock(&class->lock);
- class_stat_dec(class, ZS_INUSE_RATIO_0, 1);
+ class_stat_sub(class, ZS_INUSE_RATIO_0, 1);
__free_zspage(pool, class, zspage);
spin_unlock(&class->lock);
}
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6.6.y 9/9] zsmalloc: account for handle size in class lookup
2026-09-04 20:32 ` [PATCH 6.6.y 1/9] mm/zsmalloc: remove migrate_write_lock_nested() Sasha Levin
` (6 preceding siblings ...)
2026-09-04 20:32 ` [PATCH 6.6.y 8/9] zsmalloc: rename class stat mutators Sasha Levin
@ 2026-09-04 20:32 ` Sasha Levin
7 siblings, 0 replies; 10+ messages in thread
From: Sasha Levin @ 2026-09-04 20:32 UTC (permalink / raw)
To: stable
Cc: Longlong Xia, Sergey Senozhatsky, Minchan Kim, Andrew Morton,
Sasha Levin
From: Longlong Xia <xialonglong@kylinos.cn>
[ Upstream commit f7bf5cd5b5f2b13fe2361860880c4e214c08b440 ]
zs_lookup_class_index() lets zram recompression decide whether a newly
compressed object would use a smaller size class. It currently classifies
the payload size directly, while zs_malloc() adds ZS_HANDLE_SIZE before
selecting the class.
This makes lookup disagree with allocation near size-class boundaries.
With 4 KiB pages, CONFIG_ZSMALLOC_CHAIN_SIZE=8, and 64-bit handles, a
1025-to-1024-byte recompression appears to move from class 64 to class 62
although both allocations use class 64. Conversely, a 1049-to-1025-byte
recompression appears to stay in class 64 although the allocations move
from class 65 to class 64.
As a result, zram can accept replacements with no allocation benefit or
reject ones that would save memory, potentially marking the object
incompressible.
Factor size-class selection into lookup_size_class(), account for the
handle there, and use the helper for both lookup and allocation.
Link: https://lore.kernel.org/20260809115518.3791787-1-xialonglong2025@163.com
Fixes: 7c2af309abd2 ("zram: add size class equals check into recompression")
Assisted-by: Codex:gpt-5.6-sol
Signed-off-by: Longlong Xia <xialonglong@kylinos.cn>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/zsmalloc.c | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index b63864f5fa398..dde7162701f2a 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -498,6 +498,11 @@ static int get_size_class_index(int size)
return min_t(int, ZS_SIZE_CLASSES - 1, idx);
}
+static struct size_class *lookup_size_class(struct zs_pool *pool, size_t size)
+{
+ return pool->size_class[get_size_class_index(size + ZS_HANDLE_SIZE)];
+}
+
static inline void class_stat_add(struct size_class *class, int type,
unsigned long cnt)
{
@@ -1140,7 +1145,7 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size)
{
struct size_class *class;
- class = pool->size_class[get_size_class_index(size)];
+ class = lookup_size_class(pool, size);
return class->index;
}
@@ -1349,9 +1354,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
if (!handle)
return (unsigned long)ERR_PTR(-ENOMEM);
- /* extra space in chunk to keep the handle */
- size += ZS_HANDLE_SIZE;
- class = pool->size_class[get_size_class_index(size)];
+ class = lookup_size_class(pool, size);
/* class->lock effectively protects the zpage migration */
spin_lock(&class->lock);
--
2.53.0
^ permalink raw reply related [flat|nested] 10+ messages in thread