* [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
2026-11-21 8:59 ` sashiko-bot
2026-11-20 8:58 ` [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers Baoquan He
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He
From: Baoquan He <hebaoquan@kylinos.cn>
Add per-walk counters to measure empty aging walks which traverse
an mm's page tables but find no folio for the current lruvec (node+memcg).
These are common on multi-NUMA node systems because lru_gen_use_mm() marks
an mm for all nodes at every context switch.
New counters (accumulated in mm_state->stats[]):
MM_LEAF_ASSOCIATED - leaf entries whose folio is in this lruvec
MM_WALK_TOTAL - page-table walks completed
MM_WALK_EMPTY - walks that found no folio in this lruvec
MM_LEAF_EMPTY_WALKS - leaf entries scanned during empty walks
A new tracepoint, mm_vmscan_lru_gen_walk(), fires after each walk, and the
debugfs lru_gen output ("TYFALWEE") exposes the new counters.
Reviewed-by: Barry Song <baohua@kernel.org>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 4 ++++
mm/trace_vmscan.h | 28 ++++++++++++++++++++++++++++
mm/vmscan.c | 34 ++++++++++++++++++++++++++++++----
3 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 6a2ad0cff795..902f4877fd0c 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -579,6 +579,10 @@ enum {
MM_LEAF_YOUNG, /* young leaf entries */
MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */
MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */
+ MM_LEAF_ASSOCIATED, /* young leaf entries whose folio is in this lruvec */
+ MM_WALK_TOTAL, /* page-table walks completed */
+ MM_WALK_EMPTY, /* walks that found nothing young for this lruvec */
+ MM_LEAF_EMPTY_WALKS, /* leaf entries scanned during empty walks */
NR_MM_STATS
};
diff --git a/mm/trace_vmscan.h b/mm/trace_vmscan.h
index 9dafdb5bd3ce..9366416faf5a 100644
--- a/mm/trace_vmscan.h
+++ b/mm/trace_vmscan.h
@@ -508,6 +508,34 @@ TRACE_EVENT(mm_mglru_aging,
__print_array(__entry->nr_anon, MAX_NR_GENS, sizeof(unsigned long)),
__print_array(__entry->nr_file, MAX_NR_GENS, sizeof(unsigned long)))
);
+TRACE_EVENT(mm_vmscan_lru_gen_walk,
+
+ TP_PROTO(int nid, unsigned long seq, int leaf_total,
+ int leaf_associated, bool empty),
+
+ TP_ARGS(nid, seq, leaf_total, leaf_associated, empty),
+
+ TP_STRUCT__entry(
+ __field(int, nid)
+ __field(unsigned long, seq)
+ __field(int, leaf_total)
+ __field(int, leaf_associated)
+ __field(bool, empty)
+ ),
+
+ TP_fast_assign(
+ __entry->nid = nid;
+ __entry->seq = seq;
+ __entry->leaf_total = leaf_total;
+ __entry->leaf_associated = leaf_associated;
+ __entry->empty = empty;
+ ),
+
+ TP_printk("nid=%d seq=%lu leaf_total=%d leaf_associated=%d empty=%d",
+ __entry->nid, __entry->seq, __entry->leaf_total,
+ __entry->leaf_associated, __entry->empty)
+);
+
#endif /* CONFIG_LRU_GEN */
TRACE_EVENT(mm_vmscan_write_folio,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bc7579d47f9b..36e0946bf2cc 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3584,6 +3584,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
if (!folio)
continue;
+ walk->mm_stats[MM_LEAF_ASSOCIATED]++;
+
if (folio_test_large(folio)) {
const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
@@ -3591,6 +3593,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
max_nr, FPB_MERGE_YOUNG_DIRTY);
total += nr - 1;
walk->mm_stats[MM_LEAF_TOTAL] += nr - 1;
+ walk->mm_stats[MM_LEAF_ASSOCIATED] += nr - 1;
}
if (!test_and_clear_young_ptes_notify(args->vma, addr, cur_pte, nr))
@@ -3682,6 +3685,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
if (!folio)
goto next;
+ walk->mm_stats[MM_LEAF_ASSOCIATED]++;
+
if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i))
goto next;
@@ -4158,8 +4163,29 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
do {
success = iterate_mm_list(walk, &mm);
- if (mm)
+ if (mm) {
+ bool empty = false;
+
walk_mm(mm, walk);
+ /*
+ * A walk that traversed page tables but found nothing
+ * young for this lruvec (node+memcg) is pure waste:
+ * the PUD-level filter will skip it next generation.
+ */
+ if (walk->mm_stats[MM_LEAF_TOTAL]) {
+ walk->mm_stats[MM_WALK_TOTAL]++;
+ if (walk->mm_stats[MM_LEAF_ASSOCIATED] == 0) {
+ walk->mm_stats[MM_WALK_EMPTY]++;
+ walk->mm_stats[MM_LEAF_EMPTY_WALKS] +=
+ walk->mm_stats[MM_LEAF_TOTAL];
+ empty = true;
+ }
+ }
+ trace_mm_vmscan_lru_gen_walk(
+ lruvec_pgdat(lruvec)->node_id, walk->seq,
+ walk->mm_stats[MM_LEAF_TOTAL],
+ walk->mm_stats[MM_LEAF_ASSOCIATED], empty);
+ }
} while (mm);
done:
if (success) {
@@ -5804,14 +5830,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
seq_puts(m, " ");
for (i = 0; i < NR_MM_STATS; i++) {
- const char *s = "xxxx";
+ const char *s = "xxxxxxxx";
unsigned long n = 0;
if (seq == max_seq && NR_HIST_GENS == 1) {
- s = "TYFA";
+ s = "TYFALWEE";
n = READ_ONCE(mm_state->stats[hist][i]);
} else if (seq != max_seq && NR_HIST_GENS > 1) {
- s = "tyfa";
+ s = "tyfalwee";
n = READ_ONCE(mm_state->stats[hist][i]);
}
--
2.52.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
@ 2026-11-21 8:59 ` sashiko-bot
0 siblings, 0 replies; 7+ messages in thread
From: sashiko-bot @ 2026-11-21 8:59 UTC (permalink / raw)
To: Baoquan He, linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baolin.wang, Baoquan He
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] mm/mglru: inconsistent MM_LEAF_ASSOCIATED accounting across
mm_has_notifiers()
--
commit 4f0dadfe0435dfb7f058c5e9c459aef67be5dff2
Author: Baoquan He <hebaoquan@kylinos.cn>
mm/mglru: add MM_WALK_EMPTY stats and tracepoint
This commit adds MGLRU aging statistics (MM_LEAF_ASSOCIATED,
MM_WALK_TOTAL, MM_WALK_EMPTY, and MM_LEAF_EMPTY_WALKS) and the
mm_vmscan_lru_gen_walk tracepoint to track page-table walks that find no
matching folios in the target lruvec.
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 6a2ad0cff7951..902f4877fd0cb 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -579,6 +579,10 @@ enum {
> MM_LEAF_YOUNG, /* young leaf entries */
> MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */
> MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */
> + MM_LEAF_ASSOCIATED, /* young leaf entries whose folio is in this lruvec */
> + MM_WALK_TOTAL, /* page-table walks completed */
> + MM_WALK_EMPTY, /* walks that found nothing young for this lruvec */
> + MM_LEAF_EMPTY_WALKS, /* leaf entries scanned during empty walks */
> NR_MM_STATS
> };
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index bc7579d47f9b4..36e0946bf2cc4 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -3584,6 +3584,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
> if (!folio)
> continue;
>
> + walk->mm_stats[MM_LEAF_ASSOCIATED]++;
> +
> if (folio_test_large(folio)) {
> const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
>
> @@ -3591,6 +3593,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
> max_nr, FPB_MERGE_YOUNG_DIRTY);
> total += nr - 1;
> walk->mm_stats[MM_LEAF_TOTAL] += nr - 1;
> + walk->mm_stats[MM_LEAF_ASSOCIATED] += nr - 1;
> }
>
> if (!test_and_clear_young_ptes_notify(args->vma, addr, cur_pte, nr))
> @@ -3682,6 +3685,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
> if (!folio)
> goto next;
>
> + walk->mm_stats[MM_LEAF_ASSOCIATED]++;
> +
> if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i))
> goto next;
>
[Severity: Medium]
Does MM_LEAF_ASSOCIATED count all leaf entries belonging to the target
lruvec (as described in the commit message), or only young leaf entries
in the target lruvec (as described in the header comments)?
In walk_pte_range() and walk_pmd_range_locked(), get_pte_pfn() and
get_pmd_pfn() filter out non-young entries before get_pfn_folio() is
reached whenever !mm_has_notifiers(vma->vm_mm):
mm/vmscan.c:get_pte_pfn() {
...
if (!pte_young(pte) && !mm_has_notifiers(vma->vm_mm))
return -1;
...
}
When !mm_has_notifiers(vma->vm_mm), every entry that reaches
get_pfn_folio(pfn, memcg, pgdat) is already young, so MM_LEAF_ASSOCIATED
is identical to MM_LEAF_YOUNG (which also only counts young entries after
get_pfn_folio() verifies the folio belongs to this lruvec), and an mm
with old folios in the lruvec is still counted as MM_WALK_EMPTY.
Conversely, when mm_has_notifiers(vma->vm_mm) is true (such as KVM
guests), get_pte_pfn() and get_pmd_pfn() let non-young entries through so
secondary MMU notifiers can be checked by
test_and_clear_young_ptes_notify() and pmdp_test_and_clear_young_notify().
Because MM_LEAF_ASSOCIATED is incremented before those notify calls, it
counts both old and young entries when mm_has_notifiers(vma->vm_mm) is
true, giving MM_LEAF_ASSOCIATED and MM_WALK_EMPTY different meanings
depending on whether the mm has MMU notifiers registered.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120085806.781225-1-baoquan.he@linux.dev?part=1
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
2026-11-20 8:58 ` [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter Baoquan He
3 siblings, 0 replies; 7+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He
From: Baoquan He <hebaoquan@kylinos.cn>
Generalize the Bloom filter state and helpers so that another filter
level can reuse the same hash, double-buffering and reset logic.
Replace the PMD-level filter array with a two-dimensional filters array
indexed by level, and add BLOOM_FILTER_PUD and BLOOM_FILTER_PMD. The
PUD-level filters operate at PUD granularity and generalize the
suppression one level up: when all PMDs under a PUD fail the PMD-level
filter, the whole PMD iteration can be skipped. Cross-node empty walks
are the most visible case: a foreign mm's PUDs have no young entries for
this lruvec, so the entire PMD iteration is pure waste.
test_bloom_filter(), update_bloom_filter() and reset_bloom_filter() take
the level now and use mm_state->filters[level][gen] directly.
No behavior change yet. No PUD-level bitmap is allocated in this patch;
the next patch fills and uses them for the first time.
Reviewed-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 10 ++++++++--
mm/vmscan.c | 39 ++++++++++++++++++++++-----------------
2 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 902f4877fd0c..a3a13a77ca8e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -589,6 +589,12 @@ enum {
/* double-buffering Bloom filters */
#define NR_BLOOM_FILTERS 2
+enum bloom_filter_level {
+ BLOOM_FILTER_PUD,
+ BLOOM_FILTER_PMD,
+ BLOOM_FILTER_LEVELS,
+};
+
struct lru_gen_mm_state {
/* synced with max_seq after each iteration */
unsigned long seq;
@@ -596,8 +602,8 @@ struct lru_gen_mm_state {
struct list_head *head;
/* where the last iteration ended before */
struct list_head *tail;
- /* Bloom filters flip after each iteration */
- unsigned long *filters[NR_BLOOM_FILTERS];
+ /* Bloom filters flip after each iteration, indexed by level */
+ unsigned long *filters[BLOOM_FILTER_LEVELS][NR_BLOOM_FILTERS];
/* the mm stats for debugging */
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
};
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 36e0946bf2cc..a4c64a190391 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2819,14 +2819,14 @@ static void get_item_key(void *item, int *key)
key[1] = hash >> BLOOM_FILTER_SHIFT;
}
-static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return true;
@@ -2835,14 +2835,14 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s
return test_bit(key[0], filter) && test_bit(key[1], filter);
}
-static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
- void *item)
+static void update_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq, void *item)
{
int key[2];
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = READ_ONCE(mm_state->filters[gen]);
+ filter = READ_ONCE(mm_state->filters[level][gen]);
if (!filter)
return;
@@ -2854,12 +2854,13 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
set_bit(key[1], filter);
}
-static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, int level,
+ unsigned long seq)
{
unsigned long *filter;
int gen = filter_gen_from_seq(seq);
- filter = mm_state->filters[gen];
+ filter = mm_state->filters[level][gen];
if (filter) {
bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
return;
@@ -2867,7 +2868,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
- WRITE_ONCE(mm_state->filters[gen], filter);
+ WRITE_ONCE(mm_state->filters[level][gen], filter);
}
/******************************************************************************
@@ -3110,7 +3111,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
spin_unlock(&mm_list->lock);
if (mm && first)
- reset_bloom_filter(mm_state, walk->seq + 1);
+ reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1);
if (*iter)
mmdrop(*iter);
@@ -3766,7 +3767,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
}
- if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))
+ if (!walk->force_scan &&
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq, pmd + i))
continue;
walk->mm_stats[MM_NONLEAF_FOUND]++;
@@ -3777,7 +3779,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_NONLEAF_ADDED]++;
/* carry over to the next generation */
- update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
}
walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
@@ -4436,8 +4438,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
/* feedback from rmap walkers to page table walkers */
if (mm_state && suitable_to_scan(i, young))
- update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq),
- pvmw->pmd);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD,
+ READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd);
mem_cgroup_put(memcg);
@@ -6153,6 +6155,7 @@ void lru_gen_init_memcg(struct mem_cgroup *memcg)
void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
int i;
+ int level;
int nid;
struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
@@ -6170,9 +6173,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
if (!mm_state)
continue;
- for (i = 0; i < NR_BLOOM_FILTERS; i++) {
- bitmap_free(mm_state->filters[i]);
- mm_state->filters[i] = NULL;
+ for (level = 0; level < BLOOM_FILTER_LEVELS; level++) {
+ for (i = 0; i < NR_BLOOM_FILTERS; i++) {
+ bitmap_free(mm_state->filters[level][i]);
+ mm_state->filters[level][i] = NULL;
+ }
}
}
}
--
2.52.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
2026-11-20 8:58 ` [PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint Baoquan He
2026-11-20 8:58 ` [PATCH 2/4] mm/mglru: add PUD-level Bloom filter state and generic helpers Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
2026-11-21 8:59 ` sashiko-bot
2026-11-20 8:58 ` [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter Baoquan He
3 siblings, 1 reply; 7+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He, Kairui Song
From: Baoquan He <hebaoquan@kylinos.cn>
The aging walks into every PUD and runs the PMD-level Bloom filter on
each PMD. Add a coarser PUD-level filter one level up:
- walk_pmd_range() reports whether the subtree held anything young for
this lruvec. That is the same condition which already carries a PMD
over into the next generation: a young PMD entry, a huge page owned by
this lruvec, or PTEs with enough young ones to pass suitable_to_scan().
- walk_pud_range() records that in the PUD filter, and
- on subsequent generations, skips the whole PUD subtree when the filter
says it had nothing young last generation.
Use (mm, PUD-aligned address) as the filter key rather than the PUD page
table pointer. It is a stable identity for the range being walked, is
available without walking the page tables in lru_gen_look_around(), and
does not depend on the lifetime of a page table page.
The PUD level keeps the same m and k as the PMD level. It holds far
fewer keys, but a false positive there costs a walk over 512 PMDs, so
the same bitmap size is not an over-allocation.
The double-buffered filter flips with each new iteration, so newly hot or
migrated-in pages are re-checked promptly rather than suppressed
indefinitely. force_scan walks bypass the PUD test, so manual aging and
newly added mms always rescan and re-populate the filter.
To keep hot regions marked, also report the covering PUD from the rmap
feedback path: lru_gen_look_around() for PTE-mapped folios, and
folio_referenced_one() for PMD-mapped ones, which never reaches
lru_gen_look_around(). Without the second path a PUD that holds only
transparent huge pages stays skipped after one cold generation.
Suggested-by: Kairui Song <ryncsn@gmail.com>
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 6 +++
mm/rmap.c | 5 ++-
mm/vmscan.c | 99 ++++++++++++++++++++++++++++++++++++------
3 files changed, 96 insertions(+), 14 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a3a13a77ca8e..59963fbee255 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -678,6 +678,7 @@ struct lru_gen_memcg {
void lru_gen_init_pgdat(struct pglist_data *pgdat);
void lru_gen_init_lruvec(struct lruvec *lruvec);
bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
+void lru_gen_mark_pud(struct mm_struct *mm, struct folio *folio, unsigned long addr);
void lru_gen_init_memcg(struct mem_cgroup *memcg);
void lru_gen_exit_memcg(struct mem_cgroup *memcg);
@@ -703,6 +704,11 @@ static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw,
return false;
}
+static inline void lru_gen_mark_pud(struct mm_struct *mm, struct folio *folio,
+ unsigned long addr)
+{
+}
+
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
diff --git a/mm/rmap.c b/mm/rmap.c
index 65646ed6fc45..ea5ada45c09f 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1004,8 +1004,11 @@ static bool folio_referenced_one(struct folio *folio,
referenced++;
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
- pvmw.pmd))
+ pvmw.pmd)) {
referenced++;
+ if (lru_gen_enabled() && !lru_gen_switching())
+ lru_gen_mark_pud(vma->vm_mm, folio, address);
+ }
} else {
/* unexpected pmd-mapped folio? */
WARN_ON_ONCE(1);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index a4c64a190391..2cd3f28f7428 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2794,9 +2794,18 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
* walk_pmd_range(); the eviction also report them when walking the rmap
* in lru_gen_look_around().
*
+ * A second pair of filters works one level above the first pair, over a whole
+ * 1 GiB subtree (a PUD): BLOOM_FILTER_PUD next to BLOOM_FILTER_PMD. A false
+ * positive at the PMD level only costs a PTE walk when the PUD level also
+ * passes. A false positive at the PUD level is rare, because it holds far
+ * fewer keys, but it costs 512 PMD walks. So both levels keep the same m and
+ * k, even though the PUD level holds much fewer keys.
+ *
* For future optimizations:
- * 1. It's not necessary to keep both filters all the time. The spare one can be
- * freed after the RCU grace period and reallocated if needed again.
+ * 1. It's not necessary to keep both filters of a level all the time.
+ * The spare one can be freed after the RCU grace period and reallocated
+ * if needed again. Now: two levels of double buffering take 16 KiB
+ * per lruvec.
* 2. And when reallocating, it's worth scaling its size according to the number
* of inserted entries in the other filter, to reduce the memory overhead on
* small systems and false positives on large systems.
@@ -3110,8 +3119,10 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
spin_unlock(&mm_list->lock);
- if (mm && first)
+ if (mm && first) {
reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1);
+ reset_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1);
+ }
if (*iter)
mmdrop(*iter);
@@ -3626,11 +3637,12 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
return suitable_to_scan(total, young);
}
-static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
+static bool walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
{
int i;
bool dirty;
+ bool young = false;
pmd_t *pmd;
spinlock_t *ptl;
struct folio *last = NULL;
@@ -3644,13 +3656,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
if (*first == -1) {
*first = addr;
bitmap_zero(bitmap, MIN_LRU_BATCH);
- return;
+ return false;
}
i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
if (i && i <= MIN_LRU_BATCH) {
__set_bit(i - 1, bitmap);
- return;
+ return false;
}
pmd = pmd_offset(pud, *first);
@@ -3702,6 +3714,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
dirty = true;
walk->mm_stats[MM_LEAF_YOUNG]++;
+ young = true;
next:
i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
} while (i <= MIN_LRU_BATCH);
@@ -3712,12 +3725,15 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
spin_unlock(ptl);
done:
*first = -1;
+
+ return young;
}
-static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+static bool walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
struct mm_walk *args)
{
int i;
+ bool young = false;
pmd_t *pmd;
unsigned long next;
unsigned long addr;
@@ -3755,7 +3771,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_LEAF_TOTAL]++;
if (pfn != -1)
- walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
+ young |= walk_pmd_range_locked(pud, addr, vma, args,
+ bitmap, &first);
continue;
}
@@ -3777,15 +3794,18 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
continue;
walk->mm_stats[MM_NONLEAF_ADDED]++;
+ young = true;
/* carry over to the next generation */
update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
}
- walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
+ young |= walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
goto restart;
+
+ return young;
}
static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
@@ -3796,6 +3816,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
unsigned long addr;
unsigned long next;
struct lru_gen_mm_walk *walk = args->private;
+ struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
VM_WARN_ON_ONCE(p4d_leaf(*p4d));
@@ -3803,13 +3824,29 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
restart:
for (i = pud_index(start), addr = start; addr != end; i++, addr = next) {
pud_t val = pudp_get(pud + i);
+ void *pud_key;
next = pud_addr_end(addr, end);
if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
continue;
- walk_pmd_range(pud + i, addr, next, args);
+ /*
+ * Use (mm, PUD-aligned address) as the filter key instead of
+ * the page table pointer. It is stable across the aging walk
+ * and the rmap feedback path, and avoids walking the page
+ * tables in lru_gen_look_around().
+ */
+ pud_key = (void *)((unsigned long)args->mm ^ (addr & PUD_MASK));
+
+ /* Skip a subtree that had nothing young for this lruvec last gen */
+ if (!walk->force_scan &&
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key))
+ continue;
+
+ if (walk_pmd_range(pud + i, addr, next, args))
+ update_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1,
+ pud_key);
if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
end = (addr | ~PUD_MASK) + 1;
@@ -4336,6 +4373,39 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
* the PTE table to the Bloom filter. This forms a feedback loop between the
* eviction and the aging.
*/
+/*
+ * Mark the PUD covering addr as young for lruvec, so that the PUD-level filter
+ * walks that 1 GiB subtree in the next generation.
+ */
+static void lru_gen_update_pud_filter(struct lruvec *lruvec, struct mm_struct *mm,
+ unsigned long addr)
+{
+ struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
+ unsigned long max_seq;
+
+ if (!mm_state)
+ return;
+
+ max_seq = READ_ONCE(lruvec->lrugen.max_seq);
+ update_bloom_filter(mm_state, BLOOM_FILTER_PUD, max_seq,
+ (void *)((unsigned long)mm ^ (addr & PUD_MASK)));
+}
+
+/*
+ * Same for a PMD-mapped folio: folio_referenced_one() takes the
+ * pmdp_clear_flush_young_notify() branch for those and never reaches
+ * lru_gen_look_around(), so without this a PUD holding only THPs stays
+ * suppressed once it has been skipped for one generation.
+ */
+void lru_gen_mark_pud(struct mm_struct *mm, struct folio *folio, unsigned long addr)
+{
+ struct mem_cgroup *memcg = get_mem_cgroup_from_folio(folio);
+
+ lru_gen_update_pud_filter(mem_cgroup_lruvec(memcg, folio_pgdat(folio)),
+ mm, addr);
+ mem_cgroup_put(memcg);
+}
+
bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
{
int i;
@@ -4437,9 +4507,12 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
lazy_mmu_mode_disable();
/* feedback from rmap walkers to page table walkers */
- if (mm_state && suitable_to_scan(i, young))
- update_bloom_filter(mm_state, BLOOM_FILTER_PMD,
- READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd);
+ if (mm_state && suitable_to_scan(i, young)) {
+ unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq);
+
+ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, max_seq, pvmw->pmd);
+ lru_gen_update_pud_filter(lruvec, vma->vm_mm, pvmw->address);
+ }
mem_cgroup_put(memcg);
--
2.52.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
@ 2026-11-21 8:59 ` sashiko-bot
0 siblings, 0 replies; 7+ messages in thread
From: sashiko-bot @ 2026-11-21 8:59 UTC (permalink / raw)
To: Baoquan He, linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baolin.wang, Baoquan He,
Kairui Song
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] mm/mglru: walk_pmd_range() drops young status of THPs flushed
at non-leaf PMD
--
commit 807f1bc14298f872f3bf880156eb7f9c34a7f570
Author: Baoquan He <hebaoquan@kylinos.cn>
mm/mglru: skip cold PUD subtrees during aging
This commit introduces a PUD-level Bloom filter to MGLRU page table aging
to skip 1 GiB PUD subtrees that had no young pages in the previous
generation, while updating the filter from the page table walk and rmap
feedback paths.
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index a4c64a1903912..2cd3f28f74288 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
[ ... ]
> @@ -3626,11 +3637,12 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
> return suitable_to_scan(total, young);
> }
>
> -static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
> +static bool walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
> struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
> {
> int i;
> bool dirty;
> + bool young = false;
> pmd_t *pmd;
> spinlock_t *ptl;
> struct folio *last = NULL;
> @@ -3644,13 +3656,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
> if (*first == -1) {
> *first = addr;
> bitmap_zero(bitmap, MIN_LRU_BATCH);
> - return;
> + return false;
> }
>
> i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
> if (i && i <= MIN_LRU_BATCH) {
> __set_bit(i - 1, bitmap);
> - return;
> + return false;
> }
[ ... ]
> @@ -3755,7 +3771,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
> walk->mm_stats[MM_LEAF_TOTAL]++;
>
> if (pfn != -1)
> - walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
> + young |= walk_pmd_range_locked(pud, addr, vma, args,
> + bitmap, &first);
> continue;
> }
>
> @@ -3777,15 +3794,18 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
> continue;
>
> walk->mm_stats[MM_NONLEAF_ADDED]++;
> + young = true;
>
> /* carry over to the next generation */
> update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i);
> }
>
> - walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
> + young |= walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
[Severity: Medium]
Can walk_pmd_range() drop the young return value when batched transparent
huge pages are flushed at a non-leaf PMD?
mm/vmscan.c:walk_pmd_range() {
...
if (!walk->force_scan && should_clear_pmd_young() &&
!mm_has_notifiers(args->mm)) {
if (!pmd_young(val))
continue;
walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
}
...
}
walk_pmd_range_locked() buffers up to 1 + MIN_LRU_BATCH entries in first
and bitmap across loop iterations, returning false while buffering. If
earlier iterations queued PMD-mapped THPs and this non-leaf PMD call lies
more than MIN_LRU_BATCH entries past first, it flushes those THPs and
resets first to -1, but its return value is ignored. If no later PMD in
the PUD sets young, walk_pmd_range() returns false and walk_pud_range()
leaves the PUD out of BLOOM_FILTER_PUD, skipping the 1 GiB subtree on the
next aging pass.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120085806.781225-1-baoquan.he@linux.dev?part=3
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH 4/4] mm/mglru: count PUD subtrees skipped by the PUD-level filter
2026-11-20 8:58 [PATCH 0/4] skip empty PUD subtrees during aging with a PUD-level Bloom filter Baoquan He
` (2 preceding siblings ...)
2026-11-20 8:58 ` [PATCH 3/4] mm/mglru: skip cold PUD subtrees during aging Baoquan He
@ 2026-11-20 8:58 ` Baoquan He
3 siblings, 0 replies; 7+ messages in thread
From: Baoquan He @ 2026-11-20 8:58 UTC (permalink / raw)
To: linux-mm
Cc: akpm, david, kasong, baohua, qi.zheng, shakeel.butt,
axelrasmussen, yuanchu, weixugc, baoquan.he, baolin.wang,
Baoquan He
From: Baoquan He <hebaoquan@kylinos.cn>
Add MM_PUD_EMPTY_SKIPPED, incremented in walk_pud_range() whenever a PUD
subtree is skipped by the filter. Like other counters accumulated
per walk, shown in debugfs lru_gen output (the "S" column of TYFALWEES)
and via the mm_vmscan_lru_gen_walk tracepoint, so the avoided cost
by the filter can be measured directly.
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
include/linux/mmzone.h | 1 +
mm/trace_vmscan.h | 10 ++++++----
mm/vmscan.c | 13 ++++++++-----
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 59963fbee255..1b5aef2cbd22 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -583,6 +583,7 @@ enum {
MM_WALK_TOTAL, /* page-table walks completed */
MM_WALK_EMPTY, /* walks that found nothing young for this lruvec */
MM_LEAF_EMPTY_WALKS, /* leaf entries scanned during empty walks */
+ MM_PUD_EMPTY_SKIPPED, /* PUD subtrees skipped by the PUD-level filter */
NR_MM_STATS
};
diff --git a/mm/trace_vmscan.h b/mm/trace_vmscan.h
index 9366416faf5a..a56c7e5f2fd1 100644
--- a/mm/trace_vmscan.h
+++ b/mm/trace_vmscan.h
@@ -511,9 +511,9 @@ TRACE_EVENT(mm_mglru_aging,
TRACE_EVENT(mm_vmscan_lru_gen_walk,
TP_PROTO(int nid, unsigned long seq, int leaf_total,
- int leaf_associated, bool empty),
+ int leaf_associated, bool empty, int pud_skipped),
- TP_ARGS(nid, seq, leaf_total, leaf_associated, empty),
+ TP_ARGS(nid, seq, leaf_total, leaf_associated, empty, pud_skipped),
TP_STRUCT__entry(
__field(int, nid)
@@ -521,6 +521,7 @@ TRACE_EVENT(mm_vmscan_lru_gen_walk,
__field(int, leaf_total)
__field(int, leaf_associated)
__field(bool, empty)
+ __field(int, pud_skipped)
),
TP_fast_assign(
@@ -529,11 +530,12 @@ TRACE_EVENT(mm_vmscan_lru_gen_walk,
__entry->leaf_total = leaf_total;
__entry->leaf_associated = leaf_associated;
__entry->empty = empty;
+ __entry->pud_skipped = pud_skipped;
),
- TP_printk("nid=%d seq=%lu leaf_total=%d leaf_associated=%d empty=%d",
+ TP_printk("nid=%d seq=%lu leaf_total=%d leaf_associated=%d empty=%d pud_skipped=%d",
__entry->nid, __entry->seq, __entry->leaf_total,
- __entry->leaf_associated, __entry->empty)
+ __entry->leaf_associated, __entry->empty, __entry->pud_skipped)
);
#endif /* CONFIG_LRU_GEN */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 2cd3f28f7428..e3616f400014 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3841,8 +3841,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
/* Skip a subtree that had nothing young for this lruvec last gen */
if (!walk->force_scan &&
- !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key))
+ !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key)) {
+ walk->mm_stats[MM_PUD_EMPTY_SKIPPED]++;
continue;
+ }
if (walk_pmd_range(pud + i, addr, next, args))
update_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1,
@@ -4223,7 +4225,8 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
trace_mm_vmscan_lru_gen_walk(
lruvec_pgdat(lruvec)->node_id, walk->seq,
walk->mm_stats[MM_LEAF_TOTAL],
- walk->mm_stats[MM_LEAF_ASSOCIATED], empty);
+ walk->mm_stats[MM_LEAF_ASSOCIATED], empty,
+ walk->mm_stats[MM_PUD_EMPTY_SKIPPED]);
}
} while (mm);
done:
@@ -5905,14 +5908,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
seq_puts(m, " ");
for (i = 0; i < NR_MM_STATS; i++) {
- const char *s = "xxxxxxxx";
+ const char *s = "xxxxxxxxx";
unsigned long n = 0;
if (seq == max_seq && NR_HIST_GENS == 1) {
- s = "TYFALWEE";
+ s = "TYFALWEES";
n = READ_ONCE(mm_state->stats[hist][i]);
} else if (seq != max_seq && NR_HIST_GENS > 1) {
- s = "tyfalwee";
+ s = "tyfalwees";
n = READ_ONCE(mm_state->stats[hist][i]);
}
--
2.52.0
^ permalink raw reply related [flat|nested] 7+ messages in thread