From: Baoquan He <baoquan.he@linux.dev>
To: linux-mm@kvack.org
Cc: akpm@linux-foundation.org, david@kernel.org, kasong@tencent.com,
baohua@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev,
axelrasmussen@google.com, yuanchu@google.com, weixugc@google.com,
baoquan.he@linux.dev, baolin.wang@linux.alibaba.com,
Baoquan He <hebaoquan@kylinos.cn>
Subject: [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers
Date: Fri, 20 Nov 2026 16:58:04 +0800 [thread overview]
Message-ID: <20261120085806.781225-3-baoquan.he@linux.dev> (raw)
In-Reply-To: <20261120085806.781225-1-baoquan.he@linux.dev>
From: Baoquan He <hebaoquan@kylinos.cn>
Generalize the Bloom filter state and helpers so that another filter
level can reuse the same hash, double-buffering and reset logic.
Replace the PMD-level filter array with a two-dimensional filters array
indexed by level, and add BLOOM_FILTER_PUD and BLOOM_FILTER_PMD. The
PUD-level filters operate at PUD granularity and generalize the
suppression one level up: when all PMDs under a PUD fail the PMD-level
filter, the whole PMD iteration can be skipped. Cross-node empty walks
are the most visible case: a foreign mm's PUDs have no young entries for
this lruvec, so the entire PMD iteration is pure waste.
test_bloom_filter(), update_bloom_filter() and reset_bloom_filter() take
the level now and use mm_state->filters[level][gen] directly.
No behavior change yet. No PUD-level bitmap is allocated in this patch;
the next patch fills and uses them for the first time.
Reviewed-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 10 ++++++++--
mm/vmscan.c | 39 ++++++++++++++++++++++-----------------
2 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 902f4877fd0c..a3a13a77ca8e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -589,6 +589,12 @@ enum {
/* double-buffering Bloom filters */
#define NR_BLOOM_FILTERS 2
+enum bloom_filter_level {
+ BLOOM_FILTER_PUD,
+ BLOOM_FILTER_PMD,
+ BLOOM_FILTER_LEVELS,
+};
+
struct lru_gen_mm_state {
/* synced with max_seq after each iteration */
unsigned long seq;
@@ -596,8 +602,8 @@ struct lru_gen_mm_state {
struct list_head *head;
/* where the last iteration ended before */
struct list_head *tail;
- /* Bloom filters flip after each iteration */
- unsigned long *filters[NR_BLOOM_FILTERS];
+ /* Bloom filters flip after each iteration, indexed by level */
+ unsigned long *filters[BLOOM_FILTER_LEVELS][NR_BLOOM_FILTERS];
/* the mm stats for debugging */
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
};
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 36e0946bf2cc..a4c64a190391 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2819,14 +2819,14 @@ static void get_item_key(void *item, int *key)
key[1] = hash >> BLOOM_FILTER_SHIFT;
}
-static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return true;
@@ -2835,14 +2835,14 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s
return test_bit(key[0], filter) && test_bit(key[1], filter);
}
-static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static void update_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return;
@@ -2854,12 +2854,13 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
set_bit(key[1], filter);
}
-static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq)
{
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = mm_state->filters[gen];
+ filter = mm_state->filters[level][gen];
if (filter) {
bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
return;
@@ -2867,7 +2868,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
- WRITE_ONCE(mm_state->filters[gen], filter);
+ WRITE_ONCE(mm_state->filters[level][gen], filter);
}
/******************************************************************************
@@ -3110,7 +3111,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
spin_unlock(&mm_list->lock);
if (mm && first)
- reset_bloom_filter(mm_state, walk->seq + 1);
+ reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1);
if (*iter)
mmdrop(*iter);
@@ -3766,7 +3767,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
}
- if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))
+ if (!walk->force_scan &&
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq, pmd + i))
continue;
walk->mm_stats[MM_NONLEAF_FOUND]++;
@@ -3777,7 +3779,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_NONLEAF_ADDED]++;
/* carry over to the next generation */
- update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
}
walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
@@ -4436,8 +4438,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
/* feedback from rmap walkers to page table walkers */
if (mm_state && suitable_to_scan(i, young))
- update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq),
- pvmw->pmd);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD,
+ READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd);
mem_cgroup_put(memcg);
@@ -6153,6 +6155,7 @@ void lru_gen_init_memcg(struct mem_cgroup *memcg)
void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
int i;
+ int level;
int nid;
struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
@@ -6170,9 +6173,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
if (!mm_state)
continue;
- for (i = 0; i < NR_BLOOM_FILTERS; i++) {
- bitmap_free(mm_state->filters[i]);
- mm_state->filters[i] = NULL;
+ for (level = 0; level < BLOOM_FILTER_LEVELS; level++) {
+ for (i = 0; i < NR_BLOOM_FILTERS; i++) {
+ bitmap_free(mm_state->filters[level][i]);
+ mm_state->filters[level][i] = NULL;
+ }
}
}
}
--
2.52.0
next prev parent reply other threads:[~2026-11-20 8:58 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
2026-11-20 8:58 ` Baoquan He [this message]
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
2026-11-20 8:58 ` [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter Baoquan He
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120085806.781225-3-baoquan.he@linux.dev \
--to=baoquan.he@linux.dev \
--cc=akpm@linux-foundation.org \
--cc=axelrasmussen@google.com \
--cc=baohua@kernel.org \
--cc=baolin.wang@linux.alibaba.com \
--cc=david@kernel.org \
--cc=hebaoquan@kylinos.cn \
--cc=kasong@tencent.com \
--cc=linux-mm@kvack.org \
--cc=qi.zheng@linux.dev \
--cc=shakeel.butt@linux.dev \
--cc=weixugc@google.com \
--cc=yuanchu@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox