From: Baoquan He <baoquan.he@linux.dev>
To: linux-mm@kvack.org
Cc: akpm@linux-foundation.org, david@kernel.org, kasong@tencent.com,
baohua@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev,
axelrasmussen@google.com, yuanchu@google.com, weixugc@google.com,
baoquan.he@linux.dev, baolin.wang@linux.alibaba.com,
Baoquan He <hebaoquan@kylinos.cn>
Subject: [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers
Date: Fri, 20 Nov 2026 16:58:04 +0800 [thread overview]
Message-ID: <20261120085806.781225-3-baoquan.he@linux.dev> (raw)
In-Reply-To: <20261120085806.781225-1-baoquan.he@linux.dev>
From: Baoquan He <hebaoquan@kylinos.cn>
Generalize the Bloom filter state and helpers so that another filter
level can reuse the same hash, double-buffering and reset logic.
Replace the PMD-level filter array with a two-dimensional filters array
indexed by level, and add BLOOM_FILTER_PUD and BLOOM_FILTER_PMD. The
PUD-level filters operate at PUD granularity and generalize the
suppression one level up: when all PMDs under a PUD fail the PMD-level
filter, the whole PMD iteration can be skipped. Cross-node empty walks
are the most visible case: a foreign mm's PUDs have no young entries for
this lruvec, so the entire PMD iteration is pure waste.
test_bloom_filter(), update_bloom_filter() and reset_bloom_filter() take
the level now and use mm_state->filters[level][gen] directly.
No behavior change yet. No PUD-level bitmap is allocated in this patch;
the next patch fills and uses them for the first time.
Reviewed-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 10 ++++++++--
mm/vmscan.c | 39 ++++++++++++++++++++++-----------------
2 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 902f4877fd0c..a3a13a77ca8e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -589,6 +589,12 @@ enum {
/* double-buffering Bloom filters */
#define NR_BLOOM_FILTERS 2
+enum bloom_filter_level {
+ BLOOM_FILTER_PUD,
+ BLOOM_FILTER_PMD,
+ BLOOM_FILTER_LEVELS,
+};
+
struct lru_gen_mm_state {
/* synced with max_seq after each iteration */
unsigned long seq;
@@ -596,8 +602,8 @@ struct lru_gen_mm_state {
struct list_head *head;
/* where the last iteration ended before */
struct list_head *tail;
- /* Bloom filters flip after each iteration */
- unsigned long *filters[NR_BLOOM_FILTERS];
+ /* Bloom filters flip after each iteration, indexed by level */
+ unsigned long *filters[BLOOM_FILTER_LEVELS][NR_BLOOM_FILTERS];
/* the mm stats for debugging */
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
};
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 36e0946bf2cc..a4c64a190391 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2819,14 +2819,14 @@ static void get_item_key(void *item, int *key)
key[1] = hash >> BLOOM_FILTER_SHIFT;
}
-static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return true;
@@ -2835,14 +2835,14 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s
return test_bit(key[0], filter) && test_bit(key[1], filter);
}
-static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static void update_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return;
@@ -2854,12 +2854,13 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
set_bit(key[1], filter);
}
-static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq)
{
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = mm_state->filters[gen];
+ filter = mm_state->filters[level][gen];
if (filter) {
bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
return;
@@ -2867,7 +2868,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
- WRITE_ONCE(mm_state->filters[gen], filter);
+ WRITE_ONCE(mm_state->filters[level][gen], filter);
}
/******************************************************************************
@@ -3110,7 +3111,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
spin_unlock(&mm_list->lock);
if (mm && first)
- reset_bloom_filter(mm_state, walk->seq + 1);
+ reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1);
if (*iter)
mmdrop(*iter);
@@ -3766,7 +3767,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
}
- if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))
+ if (!walk->force_scan &&
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq, pmd + i))
continue;
walk->mm_stats[MM_NONLEAF_FOUND]++;
@@ -3777,7 +3779,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_NONLEAF_ADDED]++;
/* carry over to the next generation */
- update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
}
walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
@@ -4436,8 +4438,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
/* feedback from rmap walkers to page table walkers */
if (mm_state && suitable_to_scan(i, young))
- update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq),
- pvmw->pmd);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD,
+ READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd);
mem_cgroup_put(memcg);
@@ -6153,6 +6155,7 @@ void lru_gen_init_memcg(struct mem_cgroup *memcg)
void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
int i;
+ int level;
int nid;
struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
@@ -6170,9 +6173,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
if (!mm_state)
continue;
- for (i = 0; i < NR_BLOOM_FILTERS; i++) {
- bitmap_free(mm_state->filters[i]);
- mm_state->filters[i] = NULL;
+ for (level = 0; level < BLOOM_FILTER_LEVELS; level++) {
+ for (i = 0; i < NR_BLOOM_FILTERS; i++) {
+ bitmap_free(mm_state->filters[level][i]);
+ mm_state->filters[level][i] = NULL;
+ }
}
}
}
--
2.52.0
next prev parent reply other threads:[~2026-11-20 8:58 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
2026-11-20 8:58 ` Baoquan He [this message]
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
2026-11-20 8:58 ` [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter Baoquan He
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120085806.781225-3-baoquan.he@linux.dev \
--to=baoquan.he@linux.dev \
--cc=akpm@linux-foundation.org \
--cc=axelrasmussen@google.com \
--cc=baohua@kernel.org \
--cc=baolin.wang@linux.alibaba.com \
--cc=david@kernel.org \
--cc=hebaoquan@kylinos.cn \
--cc=kasong@tencent.com \
--cc=linux-mm@kvack.org \
--cc=qi.zheng@linux.dev \
--cc=shakeel.butt@linux.dev \
--cc=weixugc@google.com \
--cc=yuanchu@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.