* [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
2026-11-20 8:58 ` [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers Baoquan He
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He
From: Baoquan He <hebaoquan@kylinos.cn>
Add per-walk counters to measure empty aging walks which traverse
an mm's page tables but find no folio for the current lruvec (node+memcg).
These are common on multi-NUMA node systems because lru_gen_use_mm() marks
an mm for all nodes at every context switch.
New counters (accumulated in mm_state->stats[]):
MM_LEAF_ASSOCIATED - leaf entries whose folio is in this lruvec
MM_WALK_TOTAL - page-table walks completed
MM_WALK_EMPTY - walks that found no folio in this lruvec
MM_LEAF_EMPTY_WALKS - leaf entries scanned during empty walks
A new tracepoint, mm_vmscan_lru_gen_walk(), fires after each walk, and the
debugfs lru_gen output ("TYFALWEE") exposes the new counters.
Reviewed-by: Barry Song <baohua@kernel.org>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 4 ++++
mm/trace_vmscan.h | 28 ++++++++++++++++++++++++++++
mm/vmscan.c | 34 ++++++++++++++++++++++++++++++----
3 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 6a2ad0cff795..902f4877fd0c 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -579,6 +579,10 @@ enum {
MM_LEAF_YOUNG, /* young leaf entries */
MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */
MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */
+ MM_LEAF_ASSOCIATED, /* young leaf entries whose folio is in this lruvec */
+ MM_WALK_TOTAL, /* page-table walks completed */
+ MM_WALK_EMPTY, /* walks that found nothing young for this lruvec */
+ MM_LEAF_EMPTY_WALKS, /* leaf entries scanned during empty walks */
NR_MM_STATS
};
diff --git a/mm/trace_vmscan.h b/mm/trace_vmscan.h
index 9dafdb5bd3ce..9366416faf5a 100644
--- a/mm/trace_vmscan.h
+++ b/mm/trace_vmscan.h
@@ -508,6 +508,34 @@ TRACE_EVENT(mm_mglru_aging,
__print_array(__entry->nr_anon, MAX_NR_GENS, sizeof(unsigned long)),
__print_array(__entry->nr_file, MAX_NR_GENS, sizeof(unsigned long)))
);
+TRACE_EVENT(mm_vmscan_lru_gen_walk,
+
+ TP_PROTO(int nid, unsigned long seq, int leaf_total,
+ int leaf_associated, bool empty),
+
+ TP_ARGS(nid, seq, leaf_total, leaf_associated, empty),
+
+ TP_STRUCT__entry(
+ __field(int, nid)
+ __field(unsigned long, seq)
+ __field(int, leaf_total)
+ __field(int, leaf_associated)
+ __field(bool, empty)
+ ),
+
+ TP_fast_assign(
+ __entry->nid = nid;
+ __entry->seq = seq;
+ __entry->leaf_total = leaf_total;
+ __entry->leaf_associated = leaf_associated;
+ __entry->empty = empty;
+ ),
+
+ TP_printk("nid=%d seq=%lu leaf_total=%d leaf_associated=%d empty=%d",
+ __entry->nid, __entry->seq, __entry->leaf_total,
+ __entry->leaf_associated, __entry->empty)
+);
+
#endif /* CONFIG_LRU_GEN */
TRACE_EVENT(mm_vmscan_write_folio,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bc7579d47f9b..36e0946bf2cc 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3584,6 +3584,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
if (!folio)
continue;
+ walk->mm_stats[MM_LEAF_ASSOCIATED]++;
+
if (folio_test_large(folio)) {
const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
@@ -3591,6 +3593,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
max_nr, FPB_MERGE_YOUNG_DIRTY);
total += nr - 1;
walk->mm_stats[MM_LEAF_TOTAL] += nr - 1;
+ walk->mm_stats[MM_LEAF_ASSOCIATED] += nr - 1;
}
if (!test_and_clear_young_ptes_notify(args->vma, addr, cur_pte, nr))
@@ -3682,6 +3685,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
if (!folio)
goto next;
+ walk->mm_stats[MM_LEAF_ASSOCIATED]++;
+
if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i))
goto next;
@@ -4158,8 +4163,29 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
do {
success = iterate_mm_list(walk, &mm);
- if (mm)
+ if (mm) {
+ bool empty = false;
+
walk_mm(mm, walk);
+ /*
+ * A walk that traversed page tables but found nothing
+ * young for this lruvec (node+memcg) is pure waste:
+ * the PUD-level filter will skip it next generation.
+ */
+ if (walk->mm_stats[MM_LEAF_TOTAL]) {
+ walk->mm_stats[MM_WALK_TOTAL]++;
+ if (walk->mm_stats[MM_LEAF_ASSOCIATED] == 0) {
+ walk->mm_stats[MM_WALK_EMPTY]++;
+ walk->mm_stats[MM_LEAF_EMPTY_WALKS] +=
+ walk->mm_stats[MM_LEAF_TOTAL];
+ empty = true;
+ }
+ }
+ trace_mm_vmscan_lru_gen_walk(
+ lruvec_pgdat(lruvec)->node_id, walk->seq,
+ walk->mm_stats[MM_LEAF_TOTAL],
+ walk->mm_stats[MM_LEAF_ASSOCIATED], empty);
+ }
} while (mm);
done:
if (success) {
@@ -5804,14 +5830,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
seq_puts(m, " ");
for (i = 0; i < NR_MM_STATS; i++) {
- const char *s = "xxxx";
+ const char *s = "xxxxxxxx";
unsigned long n = 0;
if (seq == max_seq && NR_HIST_GENS == 1) {
- s = "TYFA";
+ s = "TYFALWEE";
n = READ_ONCE(mm_state->stats[hist][i]);
} else if (seq != max_seq && NR_HIST_GENS > 1) {
- s = "tyfa";
+ s = "tyfalwee";
n = READ_ONCE(mm_state->stats[hist][i]);
}
--
2.52.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
2026-11-20 8:58 ` [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter Baoquan He
3 siblings, 0 replies; 5+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He
From: Baoquan He <hebaoquan@kylinos.cn>
Generalize the Bloom filter state and helpers so that another filter
level can reuse the same hash, double-buffering and reset logic.
Replace the PMD-level filter array with a two-dimensional filters array
indexed by level, and add BLOOM_FILTER_PUD and BLOOM_FILTER_PMD. The
PUD-level filters operate at PUD granularity and generalize the
suppression one level up: when all PMDs under a PUD fail the PMD-level
filter, the whole PMD iteration can be skipped. Cross-node empty walks
are the most visible case: a foreign mm's PUDs have no young entries for
this lruvec, so the entire PMD iteration is pure waste.
test_bloom_filter(), update_bloom_filter() and reset_bloom_filter() take
the level now and use mm_state->filters[level][gen] directly.
No behavior change yet. No PUD-level bitmap is allocated in this patch;
the next patch fills and uses them for the first time.
Reviewed-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 10 ++++++++--
mm/vmscan.c | 39 ++++++++++++++++++++++-----------------
2 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 902f4877fd0c..a3a13a77ca8e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -589,6 +589,12 @@ enum {
/* double-buffering Bloom filters */
#define NR_BLOOM_FILTERS 2
+enum bloom_filter_level {
+ BLOOM_FILTER_PUD,
+ BLOOM_FILTER_PMD,
+ BLOOM_FILTER_LEVELS,
+};
+
struct lru_gen_mm_state {
/* synced with max_seq after each iteration */
unsigned long seq;
@@ -596,8 +602,8 @@ struct lru_gen_mm_state {
struct list_head *head;
/* where the last iteration ended before */
struct list_head *tail;
- /* Bloom filters flip after each iteration */
- unsigned long *filters[NR_BLOOM_FILTERS];
+ /* Bloom filters flip after each iteration, indexed by level */
+ unsigned long *filters[BLOOM_FILTER_LEVELS][NR_BLOOM_FILTERS];
/* the mm stats for debugging */
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
};
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 36e0946bf2cc..a4c64a190391 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2819,14 +2819,14 @@ static void get_item_key(void *item, int *key)
key[1] = hash >> BLOOM_FILTER_SHIFT;
}
-static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return true;
@@ -2835,14 +2835,14 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s
return test_bit(key[0], filter) && test_bit(key[1], filter);
}
-static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static void update_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return;
@@ -2854,12 +2854,13 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
set_bit(key[1], filter);
}
-static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq)
{
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = mm_state->filters[gen];
+ filter = mm_state->filters[level][gen];
if (filter) {
bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
return;
@@ -2867,7 +2868,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
- WRITE_ONCE(mm_state->filters[gen], filter);
+ WRITE_ONCE(mm_state->filters[level][gen], filter);
}
/******************************************************************************
@@ -3110,7 +3111,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
spin_unlock(&mm_list->lock);
if (mm && first)
- reset_bloom_filter(mm_state, walk->seq + 1);
+ reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1);
if (*iter)
mmdrop(*iter);
@@ -3766,7 +3767,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
}
- if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))
+ if (!walk->force_scan &&
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq, pmd + i))
continue;
walk->mm_stats[MM_NONLEAF_FOUND]++;
@@ -3777,7 +3779,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_NONLEAF_ADDED]++;
/* carry over to the next generation */
- update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
}
walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
@@ -4436,8 +4438,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
/* feedback from rmap walkers to page table walkers */
if (mm_state && suitable_to_scan(i, young))
- update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq),
- pvmw->pmd);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD,
+ READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd);
mem_cgroup_put(memcg);
@@ -6153,6 +6155,7 @@ void lru_gen_init_memcg(struct mem_cgroup *memcg)
void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
int i;
+ int level;
int nid;
struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
@@ -6170,9 +6173,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
if (!mm_state)
continue;
- for (i = 0; i < NR_BLOOM_FILTERS; i++) {
- bitmap_free(mm_state->filters[i]);
- mm_state->filters[i] = NULL;
+ for (level = 0; level < BLOOM_FILTER_LEVELS; level++) {
+ for (i = 0; i < NR_BLOOM_FILTERS; i++) {
+ bitmap_free(mm_state->filters[level][i]);
+ mm_state->filters[level][i] = NULL;
+ }
}
}
}
--
2.52.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
2026-11-20 8:58 ` [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
2026-11-20 8:58 ` [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter Baoquan He
3 siblings, 0 replies; 5+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He, Kairui Song
From: Baoquan He <hebaoquan@kylinos.cn>
The aging walks into every PUD and runs the PMD-level Bloom filter on
each PMD. Add a coarser PUD-level filter one level up:
- walk_pmd_range() reports whether the subtree held anything young for
this lruvec. That is the same condition which already carries a PMD
over into the next generation: a young PMD entry, a huge page owned by
this lruvec, or PTEs with enough young ones to pass suitable_to_scan().
- walk_pud_range() records that in the PUD filter, and
- on subsequent generations, skips the whole PUD subtree when the filter
says it had nothing young last generation.
Use (mm, PUD-aligned address) as the filter key rather than the PUD page
table pointer. It is a stable identity for the range being walked, is
available without walking the page tables in lru_gen_look_around(), and
does not depend on the lifetime of a page table page.
The PUD level keeps the same m and k as the PMD level. It holds far
fewer keys, but a false positive there costs a walk over 512 PMDs, so
the same bitmap size is not an over-allocation.
The double-buffered filter flips with each new iteration, so newly hot or
migrated-in pages are re-checked promptly rather than suppressed
indefinitely. force_scan walks bypass the PUD test, so manual aging and
newly added mms always rescan and re-populate the filter.
To keep hot regions marked, also report the covering PUD from the rmap
feedback path: lru_gen_look_around() for PTE-mapped folios, and
folio_referenced_one() for PMD-mapped ones, which never reaches
lru_gen_look_around(). Without the second path a PUD that holds only
transparent huge pages stays skipped after one cold generation.
Suggested-by: Kairui Song <ryncsn@gmail.com>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 6 +++
mm/rmap.c | 5 ++-
mm/vmscan.c | 99 ++++++++++++++++++++++++++++++++++++------
3 files changed, 96 insertions(+), 14 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a3a13a77ca8e..59963fbee255 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -678,6 +678,7 @@ struct lru_gen_memcg {
void lru_gen_init_pgdat(struct pglist_data *pgdat);
void lru_gen_init_lruvec(struct lruvec *lruvec);
bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
+void lru_gen_mark_pud(struct mm_struct *mm, struct folio *folio, unsigned long addr);
void lru_gen_init_memcg(struct mem_cgroup *memcg);
void lru_gen_exit_memcg(struct mem_cgroup *memcg);
@@ -703,6 +704,11 @@ static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw,
return false;
}
+static inline void lru_gen_mark_pud(struct mm_struct *mm, struct folio *folio,
+ unsigned long addr)
+{
+}
+
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
diff --git a/mm/rmap.c b/mm/rmap.c
index 65646ed6fc45..ea5ada45c09f 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1004,8 +1004,11 @@ static bool folio_referenced_one(struct folio *folio,
referenced++;
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
- pvmw.pmd))
+ pvmw.pmd)) {
referenced++;
+ if (lru_gen_enabled() && !lru_gen_switching())
+ lru_gen_mark_pud(vma->vm_mm, folio, address);
+ }
} else {
/* unexpected pmd-mapped folio? */
WARN_ON_ONCE(1);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index a4c64a190391..2cd3f28f7428 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2794,9 +2794,18 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
* walk_pmd_range(); the eviction also report them when walking the rmap
* in lru_gen_look_around().
*
+ * A second pair of filters works one level above the first pair, over a whole
+ * 1 GiB subtree (a PUD): BLOOM_FILTER_PUD next to BLOOM_FILTER_PMD. A false
+ * positive at the PMD level only costs a PTE walk when the PUD level also
+ * passes. A false positive at the PUD level is rare, because it holds far
+ * fewer keys, but it costs 512 PMD walks. So both levels keep the same m and
+ * k, even though the PUD level holds much fewer keys.
+ *
* For future optimizations:
- * 1. It's not necessary to keep both filters all the time. The spare one can be
- * freed after the RCU grace period and reallocated if needed again.
+ * 1. It's not necessary to keep both filters of a level all the time.
+ * The spare one can be freed after the RCU grace period and reallocated
+ * if needed again. Now: two levels of double buffering take 16 KiB
+ * per lruvec.
* 2. And when reallocating, it's worth scaling its size according to the number
* of inserted entries in the other filter, to reduce the memory overhead on
* small systems and false positives on large systems.
@@ -3110,8 +3119,10 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
spin_unlock(&mm_list->lock);
- if (mm && first)
+ if (mm && first) {
reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1);
+ reset_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1);
+ }
if (*iter)
mmdrop(*iter);
@@ -3626,11 +3637,12 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
return suitable_to_scan(total, young);
}
-static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
+static bool walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
{
int i;
bool dirty;
+ bool young = false;
pmd_t *pmd;
spinlock_t *ptl;
struct folio *last = NULL;
@@ -3644,13 +3656,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
if (*first == -1) {
*first = addr;
bitmap_zero(bitmap, MIN_LRU_BATCH);
- return;
+ return false;
}
i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
if (i && i <= MIN_LRU_BATCH) {
__set_bit(i - 1, bitmap);
- return;
+ return false;
}
pmd = pmd_offset(pud, *first);
@@ -3702,6 +3714,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
dirty = true;
walk->mm_stats[MM_LEAF_YOUNG]++;
+ young = true;
next:
i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
} while (i <= MIN_LRU_BATCH);
@@ -3712,12 +3725,15 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
spin_unlock(ptl);
done:
*first = -1;
+
+ return young;
}
-static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+static bool walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
struct mm_walk *args)
{
int i;
+ bool young = false;
pmd_t *pmd;
unsigned long next;
unsigned long addr;
@@ -3755,7 +3771,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_LEAF_TOTAL]++;
if (pfn != -1)
- walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
+ young |= walk_pmd_range_locked(pud, addr, vma, args,
+ bitmap, &first);
continue;
}
@@ -3777,15 +3794,18 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
continue;
walk->mm_stats[MM_NONLEAF_ADDED]++;
+ young = true;
/* carry over to the next generation */
update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
}
- walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
+ young |= walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
goto restart;
+
+ return young;
}
static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
@@ -3796,6 +3816,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
unsigned long addr;
unsigned long next;
struct lru_gen_mm_walk *walk = args->private;
+ struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
VM_WARN_ON_ONCE(p4d_leaf(*p4d));
@@ -3803,13 +3824,29 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
restart:
for (i = pud_index(start), addr = start; addr != end; i++, addr = next) {
pud_t val = pudp_get(pud + i);
+ void *pud_key;
next = pud_addr_end(addr, end);
if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
continue;
- walk_pmd_range(pud + i, addr, next, args);
+ /*
+ * Use (mm, PUD-aligned address) as the filter key instead of
+ * the page table pointer. It is stable across the aging walk
+ * and the rmap feedback path, and avoids walking the page
+ * tables in lru_gen_look_around().
+ */
+ pud_key = (void *)((unsigned long)args->mm ^ (addr & PUD_MASK));
+
+ /* Skip a subtree that had nothing young for this lruvec last gen */
+ if (!walk->force_scan &&
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key))
+ continue;
+
+ if (walk_pmd_range(pud + i, addr, next, args))
+ update_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1,
+ pud_key);
if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
end = (addr | ~PUD_MASK) + 1;
@@ -4336,6 +4373,39 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
* the PTE table to the Bloom filter. This forms a feedback loop between the
* eviction and the aging.
*/
+/*
+ * Mark the PUD covering addr as young for lruvec, so that the PUD-level filter
+ * walks that 1 GiB subtree in the next generation.
+ */
+static void lru_gen_update_pud_filter(struct lruvec *lruvec, struct mm_struct *mm,
+ unsigned long addr)
+{
+ struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
+ unsigned long max_seq;
+
+ if (!mm_state)
+ return;
+
+ max_seq = READ_ONCE(lruvec->lrugen.max_seq);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PUD, max_seq,
+ (void *)((unsigned long)mm ^ (addr & PUD_MASK)));
+}
+
+/*
+ * Same for a PMD-mapped folio: folio_referenced_one() takes the
+ * pmdp_clear_flush_young_notify() branch for those and never reaches
+ * lru_gen_look_around(), so without this a PUD holding only THPs stays
+ * suppressed once it has been skipped for one generation.
+ */
+void lru_gen_mark_pud(struct mm_struct *mm, struct folio *folio, unsigned long addr)
+{
+ struct mem_cgroup *memcg = get_mem_cgroup_from_folio(folio);
+
+ lru_gen_update_pud_filter(mem_cgroup_lruvec(memcg, folio_pgdat(folio)),
+ mm, addr);
+ mem_cgroup_put(memcg);
+}
+
bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
{
int i;
@@ -4437,9 +4507,12 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
lazy_mmu_mode_disable();
/* feedback from rmap walkers to page table walkers */
- if (mm_state && suitable_to_scan(i, young))
- update_bloom_filter(mm_state, BLOOM_FILTER_PMD,
- READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd);
+ if (mm_state && suitable_to_scan(i, young)) {
+ unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq);
+
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, max_seq, pvmw->pmd);
+ lru_gen_update_pud_filter(lruvec, vma->vm_mm, pvmw->address);
+ }
mem_cgroup_put(memcg);
--
2.52.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
` (2 preceding siblings ...)
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
3 siblings, 0 replies; 5+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He
From: Baoquan He <hebaoquan@kylinos.cn>
Add MM_PUD_EMPTY_SKIPPED, incremented in walk_pud_range() whenever a PUD
subtree is skipped by the filter. Like other counters accumulated
per walk, shown in debugfs lru_gen output (the "S" column of TYFALWEES)
and via the mm_vmscan_lru_gen_walk tracepoint, so the avoided cost
by the filter can be measured directly.
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 1 +
mm/trace_vmscan.h | 10 ++++++----
mm/vmscan.c | 13 ++++++++-----
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 59963fbee255..1b5aef2cbd22 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -583,6 +583,7 @@ enum {
MM_WALK_TOTAL, /* page-table walks completed */
MM_WALK_EMPTY, /* walks that found nothing young for this lruvec */
MM_LEAF_EMPTY_WALKS, /* leaf entries scanned during empty walks */
+ MM_PUD_EMPTY_SKIPPED, /* PUD subtrees skipped by the PUD-level filter */
NR_MM_STATS
};
diff --git a/mm/trace_vmscan.h b/mm/trace_vmscan.h
index 9366416faf5a..a56c7e5f2fd1 100644
--- a/mm/trace_vmscan.h
+++ b/mm/trace_vmscan.h
@@ -511,9 +511,9 @@ TRACE_EVENT(mm_mglru_aging,
TRACE_EVENT(mm_vmscan_lru_gen_walk,
TP_PROTO(int nid, unsigned long seq, int leaf_total,
- int leaf_associated, bool empty),
+ int leaf_associated, bool empty, int pud_skipped),
- TP_ARGS(nid, seq, leaf_total, leaf_associated, empty),
+ TP_ARGS(nid, seq, leaf_total, leaf_associated, empty, pud_skipped),
TP_STRUCT__entry(
__field(int, nid)
@@ -521,6 +521,7 @@ TRACE_EVENT(mm_vmscan_lru_gen_walk,
__field(int, leaf_total)
__field(int, leaf_associated)
__field(bool, empty)
+ __field(int, pud_skipped)
),
TP_fast_assign(
@@ -529,11 +530,12 @@ TRACE_EVENT(mm_vmscan_lru_gen_walk,
__entry->leaf_total = leaf_total;
__entry->leaf_associated = leaf_associated;
__entry->empty = empty;
+ __entry->pud_skipped = pud_skipped;
),
- TP_printk("nid=%d seq=%lu leaf_total=%d leaf_associated=%d empty=%d",
+ TP_printk("nid=%d seq=%lu leaf_total=%d leaf_associated=%d empty=%d pud_skipped=%d",
__entry->nid, __entry->seq, __entry->leaf_total,
- __entry->leaf_associated, __entry->empty)
+ __entry->leaf_associated, __entry->empty, __entry->pud_skipped)
);
#endif /* CONFIG_LRU_GEN */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 2cd3f28f7428..e3616f400014 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3841,8 +3841,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
/* Skip a subtree that had nothing young for this lruvec last gen */
if (!walk->force_scan &&
- !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key))
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key)) {
+ walk->mm_stats[MM_PUD_EMPTY_SKIPPED]++;
continue;
+ }
if (walk_pmd_range(pud + i, addr, next, args))
update_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1,
@@ -4223,7 +4225,8 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
trace_mm_vmscan_lru_gen_walk(
lruvec_pgdat(lruvec)->node_id, walk->seq,
walk->mm_stats[MM_LEAF_TOTAL],
- walk->mm_stats[MM_LEAF_ASSOCIATED], empty);
+ walk->mm_stats[MM_LEAF_ASSOCIATED], empty,
+ walk->mm_stats[MM_PUD_EMPTY_SKIPPED]);
}
} while (mm);
done:
@@ -5905,14 +5908,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
seq_puts(m, " ");
for (i = 0; i < NR_MM_STATS; i++) {
- const char *s = "xxxxxxxx";
+ const char *s = "xxxxxxxxx";
unsigned long n = 0;
if (seq == max_seq && NR_HIST_GENS == 1) {
- s = "TYFALWEE";
+ s = "TYFALWEES";
n = READ_ONCE(mm_state->stats[hist][i]);
} else if (seq != max_seq && NR_HIST_GENS > 1) {
- s = "tyfalwee";
+ s = "tyfalwees";
n = READ_ONCE(mm_state->stats[hist][i]);
}
--
2.52.0
^ permalink raw reply related [flat|nested] 5+ messages in thread