From: Sergey Senozhatsky <senozhatsky@chromium.org>
To: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Hongfeng Chen <hongfengchen@xiaomi.com>,
Andrew Morton <akpm@linux-foundation.org>,
Barry Song <baohua@kernel.org>,
Chengming Zhou <chengming.zhou@linux.dev>,
Jens Axboe <axboe@kernel.dk>,
Johannes Weiner <hannes@cmpxchg.org>,
Minchan Kim <minchan@kernel.org>, Nhat Pham <nphamcs@gmail.com>,
Yosry Ahmed <yosry@kernel.org>,
Wenchao Hao <haowenchao@xiaomi.com>,
Wenchao Hao <haowenchao22@gmail.com>,
linux-mm@kvack.org, linux-kernel@vger.kernel.org,
linux-block@vger.kernel.org, chenhongfeng0212@gmail.com
Subject: Re: [RFC PATCH v4 2/2] zram: keep the hot statistics per cpu
Date: Sat, 21 Nov 2026 21:11:33 +0900 [thread overview]
Message-ID: <awGJ4gPFCiAi4Ia8@google.com> (raw)
In-Reply-To: <awFXOyeELHgLfy_D@google.com>
On (26/11/21 17:35), Sergey Senozhatsky wrote:
[..]
> > clear_slot_flag(zram, index, ZRAM_IDLE);
> > - if (test_slot_flag(zram, index, ZRAM_HUGE))
> > - atomic64_dec(&zram->stats.huge_pages);
> > - atomic64_sub(get_slot_size(zram, index), &zram->stats.compr_data_size);
> > + huge = test_slot_flag(zram, index, ZRAM_HUGE) ? -1 : 0;
> > + size = get_slot_size(zram, index);
> > + stats = get_cpu_ptr(zram->pcpu_stats);
> > + u64_stats_update_begin(&stats->syncp);
> > + WRITE_ONCE(stats->huge_pages, stats->huge_pages + huge);
> > + WRITE_ONCE(stats->compr_data_size, stats->compr_data_size - size);
> > + u64_stats_update_end(&stats->syncp);
> > + put_cpu_ptr(zram->pcpu_stats);
>
> Thanks for reminding me that I actually looked into per-CPU stats a while
> ago. And I had much less code, IIRC, let me find and rebase that patch,
> these WRITE_ONCE()-s, etc. are hard to maintain. Let me find my patch.
So this is what I had, basically replicating zsmalloc stats array.
---
drivers/block/zram/zram_drv.c | 108 ++++++++++++++++++++++++++--------
drivers/block/zram/zram_drv.h | 26 ++++++--
2 files changed, 102 insertions(+), 32 deletions(-)
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 54eb691f1409..951a38caeb93 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -95,6 +95,47 @@ static void slot_unlock(struct zram *zram, unsigned long index)
clear_and_wake_up_bit(ZRAM_ENTRY_LOCK_BIT, lock);
}
+static void zram_stat_add(struct zram *zram, enum zram_pcpu_stat item,
+ s64 amount)
+{
+ struct zram_pcpu_stats *stats = get_cpu_ptr(zram->pcpu_stats);
+
+ local64_add(amount, &stats->stat[item]);
+ put_cpu_ptr(zram->pcpu_stats);
+}
+
+static void zram_stat_sub(struct zram *zram, enum zram_pcpu_stat item,
+ s64 amount)
+{
+ struct zram_pcpu_stats *stats = get_cpu_ptr(zram->pcpu_stats);
+
+ local64_sub(amount, &stats->stat[item]);
+ put_cpu_ptr(zram->pcpu_stats);
+}
+
+static u64 zram_stat_read(struct zram *zram, enum zram_pcpu_stat item)
+{
+ struct zram_pcpu_stats *stats;
+ s64 sum;
+ int cpu;
+
+ /*
+ * Other CPUs keep updating while the values are added up: a page
+ * stored on a CPU already summed and freed on one not yet summed
+ * can make the sum negative. Sum them up again then.
+ */
+ for (;;) {
+ sum = 0;
+ for_each_possible_cpu(cpu) {
+ stats = per_cpu_ptr(zram->pcpu_stats, cpu);
+ sum += local64_read(&stats->stat[item]);
+ }
+ if (sum >= 0)
+ return sum;
+ cond_resched();
+ }
+}
+
static inline bool init_done(struct zram *zram)
{
return zram->disksize;
@@ -944,8 +985,9 @@ static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)
clear_slot_flag(zram, index, ZRAM_IDLE);
if (test_slot_flag(zram, index, ZRAM_HUGE))
- atomic64_dec(&zram->stats.huge_pages);
- atomic64_sub(get_slot_size(zram, index), &zram->stats.compr_data_size);
+ zram_stat_sub(zram, ZRAM_STAT_HUGE_PAGES, 1);
+ zram_stat_sub(zram, ZRAM_STAT_COMPR_DATA_SIZE,
+ get_slot_size(zram, index));
zs_free(zram->mem_pool, &zram->table[index].handle);
zram->table[index].blk_idx = req->blk_idx;
set_slot_flag(zram, index, ZRAM_WB);
@@ -1927,7 +1969,7 @@ static ssize_t io_stat_show(struct device *dev, struct device_attribute *attr,
"%8llu %8llu 0 %8llu\n",
(u64)atomic64_read(&zram->stats.failed_reads),
(u64)atomic64_read(&zram->stats.failed_writes),
- (u64)atomic64_read(&zram->stats.notify_free));
+ zram_stat_read(zram, ZRAM_STAT_NOTIFY_FREE));
return ret;
}
@@ -1949,20 +1991,20 @@ static ssize_t mm_stat_show(struct device *dev, struct device_attribute *attr,
zs_pool_stats(zram->mem_pool, &pool_stats);
}
- orig_size = atomic64_read(&zram->stats.pages_stored);
+ orig_size = zram_stat_read(zram, ZRAM_STAT_PAGES_STORED);
max_used = atomic_long_read(&zram->stats.max_used_pages);
ret = sysfs_emit(buf,
"%8llu %8llu %8llu %8lu %8ld %8llu %8lu %8llu %8llu\n",
orig_size << PAGE_SHIFT,
- (u64)atomic64_read(&zram->stats.compr_data_size),
+ zram_stat_read(zram, ZRAM_STAT_COMPR_DATA_SIZE),
mem_used << PAGE_SHIFT,
zram->limit_pages << PAGE_SHIFT,
max_used << PAGE_SHIFT,
- (u64)atomic64_read(&zram->stats.same_pages),
+ zram_stat_read(zram, ZRAM_STAT_SAME_PAGES),
atomic_long_read(&pool_stats.pages_compacted),
- (u64)atomic64_read(&zram->stats.huge_pages),
- (u64)atomic64_read(&zram->stats.huge_pages_since));
+ zram_stat_read(zram, ZRAM_STAT_HUGE_PAGES),
+ zram_stat_read(zram, ZRAM_STAT_HUGE_PAGES_SINCE));
return ret;
}
@@ -2045,7 +2087,7 @@ static void slot_free(struct zram *zram, unsigned long index)
* ZRAM_HUGE flag for deferred decompression path.
*/
if (!test_slot_flag(zram, index, ZRAM_WB))
- atomic64_dec(&zram->stats.huge_pages);
+ zram_stat_sub(zram, ZRAM_STAT_HUGE_PAGES, 1);
clear_slot_flag(zram, index, ZRAM_HUGE);
}
@@ -2061,7 +2103,7 @@ static void slot_free(struct zram *zram, unsigned long index)
*/
if (test_slot_flag(zram, index, ZRAM_SAME)) {
clear_slot_flag(zram, index, ZRAM_SAME);
- atomic64_dec(&zram->stats.same_pages);
+ zram_stat_sub(zram, ZRAM_STAT_SAME_PAGES, 1);
goto out;
}
@@ -2071,10 +2113,10 @@ static void slot_free(struct zram *zram, unsigned long index)
zs_free(zram->mem_pool, handle);
- atomic64_sub(get_slot_size(zram, index),
- &zram->stats.compr_data_size);
+ zram_stat_sub(zram, ZRAM_STAT_COMPR_DATA_SIZE,
+ get_slot_size(zram, index));
out:
- atomic64_dec(&zram->stats.pages_stored);
+ zram_stat_sub(zram, ZRAM_STAT_PAGES_STORED, 1);
zram->table[index].element = 0;
set_slot_size(zram, index, 0);
}
@@ -2246,8 +2288,8 @@ static int write_same_filled_page(struct zram *zram, unsigned long fill,
mark_slot_accessed(zram, index);
slot_unlock(zram, index);
- atomic64_inc(&zram->stats.same_pages);
- atomic64_inc(&zram->stats.pages_stored);
+ zram_stat_add(zram, ZRAM_STAT_SAME_PAGES, 1);
+ zram_stat_add(zram, ZRAM_STAT_PAGES_STORED, 1);
return 0;
}
@@ -2277,10 +2319,10 @@ static int write_incompressible_page(struct zram *zram, struct page *page,
mark_slot_accessed(zram, index);
slot_unlock(zram, index);
- atomic64_add(PAGE_SIZE, &zram->stats.compr_data_size);
- atomic64_inc(&zram->stats.huge_pages);
- atomic64_inc(&zram->stats.huge_pages_since);
- atomic64_inc(&zram->stats.pages_stored);
+ zram_stat_add(zram, ZRAM_STAT_COMPR_DATA_SIZE, PAGE_SIZE);
+ zram_stat_add(zram, ZRAM_STAT_HUGE_PAGES, 1);
+ zram_stat_add(zram, ZRAM_STAT_HUGE_PAGES_SINCE, 1);
+ zram_stat_add(zram, ZRAM_STAT_PAGES_STORED, 1);
return 0;
}
@@ -2340,8 +2382,8 @@ static int zram_write_page(struct zram *zram, struct page *page,
slot_unlock(zram, index);
/* Update stats */
- atomic64_inc(&zram->stats.pages_stored);
- atomic64_add(comp_len, &zram->stats.compr_data_size);
+ zram_stat_add(zram, ZRAM_STAT_PAGES_STORED, 1);
+ zram_stat_add(zram, ZRAM_STAT_COMPR_DATA_SIZE, comp_len);
return ret;
}
@@ -2532,8 +2574,8 @@ static int recompress_slot(struct zram *zram, unsigned long index,
set_slot_size(zram, index, comp_len_new);
set_slot_comp_priority(zram, index, prio);
- atomic64_add(comp_len_new, &zram->stats.compr_data_size);
- atomic64_inc(&zram->stats.pages_stored);
+ zram_stat_add(zram, ZRAM_STAT_COMPR_DATA_SIZE, comp_len_new);
+ zram_stat_add(zram, ZRAM_STAT_PAGES_STORED, 1);
return 0;
}
@@ -2711,7 +2753,7 @@ static void zram_bio_discard(struct zram *zram, struct bio *bio)
slot_lock(zram, index);
slot_free(zram, index);
slot_unlock(zram, index);
- atomic64_inc(&zram->stats.notify_free);
+ zram_stat_add(zram, ZRAM_STAT_NOTIFY_FREE, 1);
index++;
n -= PAGE_SIZE;
}
@@ -2804,7 +2846,7 @@ static void zram_slot_free_notify(struct block_device *bdev,
zram = bdev->bd_disk->private_data;
- atomic64_inc(&zram->stats.notify_free);
+ zram_stat_add(zram, ZRAM_STAT_NOTIFY_FREE, 1);
if (!slot_trylock(zram, index)) {
atomic64_inc(&zram->stats.miss_free);
return;
@@ -2845,6 +2887,8 @@ static void zram_destroy_comps(struct zram *zram)
static void zram_reset_device(struct zram *zram)
{
+ int cpu;
+
guard(rwsem_write)(&zram->dev_lock);
zram->limit_pages = 0;
@@ -2857,6 +2901,9 @@ static void zram_reset_device(struct zram *zram)
zram->disksize = 0;
zram_destroy_comps(zram);
memset(&zram->stats, 0, sizeof(zram->stats));
+ for_each_possible_cpu(cpu)
+ memset(per_cpu_ptr(zram->pcpu_stats, cpu), 0,
+ sizeof(struct zram_pcpu_stats));
reset_bdev(zram);
#ifdef CONFIG_ZRAM_WRITEBACK
zram->wb_limit_enable = false;
@@ -3075,9 +3122,15 @@ static int zram_add(void)
if (!zram)
return -ENOMEM;
+ zram->pcpu_stats = alloc_percpu(struct zram_pcpu_stats);
+ if (!zram->pcpu_stats) {
+ ret = -ENOMEM;
+ goto out_free_dev;
+ }
+
ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
if (ret < 0)
- goto out_free_dev;
+ goto out_free_stats;
device_id = ret;
init_rwsem(&zram->dev_lock);
@@ -3119,6 +3172,8 @@ static int zram_add(void)
put_disk(zram->disk);
out_free_idr:
idr_remove(&zram_index_idr, device_id);
+out_free_stats:
+ free_percpu(zram->pcpu_stats);
out_free_dev:
kfree(zram);
return ret;
@@ -3168,6 +3223,7 @@ static int zram_remove(struct zram *zram)
zram_reset_device(zram);
put_disk(zram->disk);
+ free_percpu(zram->pcpu_stats);
kfree(zram);
return 0;
}
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index 724f8ce6709b..a3fa6e3d007f 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -10,6 +10,7 @@
#define _ZRAM_DRV_H_
#include <asm/byteorder.h>
+#include <asm/local64.h>
#include <linux/rwsem.h>
#include <linux/zsmalloc.h>
@@ -88,15 +89,27 @@ struct zram_table_entry {
};
};
+/*
+ * Updated for every page that is stored or freed, so kept per CPU: with
+ * shared counters, concurrent swap frees bounce their cacheline.
+ */
+enum zram_pcpu_stat {
+ ZRAM_STAT_PAGES_STORED, /* no. of pages currently stored */
+ ZRAM_STAT_COMPR_DATA_SIZE, /* compressed size of pages stored */
+ ZRAM_STAT_SAME_PAGES, /* no. of same element filled pages */
+ ZRAM_STAT_HUGE_PAGES, /* no. of huge pages */
+ ZRAM_STAT_HUGE_PAGES_SINCE, /* no. of huge pages since set up */
+ ZRAM_STAT_NOTIFY_FREE, /* no. of slot free notifications */
+ NR_ZRAM_PCPU_STATS,
+};
+
+struct zram_pcpu_stats {
+ local64_t stat[NR_ZRAM_PCPU_STATS];
+};
+
struct zram_stats {
- atomic64_t compr_data_size; /* compressed size of pages stored */
atomic64_t failed_reads; /* can happen when memory is too low */
atomic64_t failed_writes; /* can happen when memory is too low */
- atomic64_t notify_free; /* no. of swap slot free notifications */
- atomic64_t same_pages; /* no. of same element filled pages */
- atomic64_t huge_pages; /* no. of huge pages */
- atomic64_t huge_pages_since; /* no. of huge pages since zram set up */
- atomic64_t pages_stored; /* no. of pages currently stored */
atomic_long_t max_used_pages; /* no. of maximum pages stored */
atomic64_t miss_free; /* no. of missed free */
#ifdef CONFIG_ZRAM_WRITEBACK
@@ -132,6 +145,7 @@ struct zram {
unsigned long limit_pages;
struct zram_stats stats;
+ struct zram_pcpu_stats __percpu *pcpu_stats;
/*
* This is the limit on amount of *uncompressed* worth of data
* we can store in a disk.
--
2.56.1.750.gf70dc682cf-goog
prev parent reply other threads:[~2026-11-21 12:11 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-16 12:09 [RFC PATCH v4 0/2] mm/zsmalloc: defer object free to per-cpu batches Hongfeng Chen
2026-10-16 12:09 ` [RFC PATCH v4 1/2] mm/zsmalloc: defer object free to per-cpu batches and a drain worker Hongfeng Chen
2026-10-16 16:09 ` [RFC PATCH v4 0/2] mm/zsmalloc: defer object free to per-cpu batches Nhat Pham
2026-10-22 2:55 ` Sergey Senozhatsky
2026-10-23 12:37 ` Hongfeng Chen
2026-10-23 18:19 ` Nhat Pham
2026-11-09 10:59 ` Hongfeng Chen
2026-11-19 5:35 ` Sergey Senozhatsky
2026-11-20 1:44 ` Sergey Senozhatsky
2026-11-20 7:48 ` Sergey Senozhatsky
2026-10-17 3:15 ` [RFC PATCH v4 2/2] zram: keep the hot statistics per cpu Hongfeng Chen
2026-11-21 8:35 ` Sergey Senozhatsky
2026-11-21 12:11 ` Sergey Senozhatsky [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=awGJ4gPFCiAi4Ia8@google.com \
--to=senozhatsky@chromium.org \
--cc=akpm@linux-foundation.org \
--cc=axboe@kernel.dk \
--cc=baohua@kernel.org \
--cc=chengming.zhou@linux.dev \
--cc=chenhongfeng0212@gmail.com \
--cc=hannes@cmpxchg.org \
--cc=haowenchao22@gmail.com \
--cc=haowenchao@xiaomi.com \
--cc=hongfengchen@xiaomi.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=minchan@kernel.org \
--cc=nphamcs@gmail.com \
--cc=yosry@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox