[RFC PATCH 0/6] mm/mglru: skip empty PUD subtrees during aging with PUD-level Bloom filter

[RFC PATCH 0/6] mm/mglru: skip empty PUD subtrees during aging with PUD-level Bloom filter

[Baoquan He]

Problem
=======

MGLRU's aging walks every present PUD of every mm in the mm_list.  When
an mm has no pages on a given NUMA node, all 512 PMDs in each PUD fail
the existing PMD-level Bloom filter test, yet the walker still descends
every PUD and iterates every PMD — pure waste.

These cross-node empty walks are structural: lru_gen_use_mm() sets
mm->lru_gen.bitmap to -1 (all nodes) at each context switch, so every
node's kswapd independently walks the same mm.  On a 2-node KVM guest,
~80% of all aging walks are empty; the worst-case leaf-traversal waste
with a fully resident remote mm is ~73%.

Approach
========

Add a PUD-level Bloom filter (one level above the existing PMD filter).
The walker now:

  1. Checks the PUD filter before descending into a PUD subtree.
  2. If the filter says the PUD had no young entries last generation,
     skips the entire 1GB region — avoiding 512 PMD lookups.
  3. walk_pmd_range() reports upward whether it found any young leaf
     entries; walk_pud_range() records that in the double-buffered
     PUD filter.
  4. The rmap feedback path (lru_gen_look_around()) marks hot PUDs in
     the filter, so newly hot or migrated-in regions are re-examined
     promptly rather than suppressed indefinitely.
  5. force_scan walks bypass the PUD test, so manual aging and newly
     added mm's always populate the filter.

The double-buffered filter flips each aging generation, and the eviction
feedback keeps hot regions marked, so correctness holds naturally without
tracking per-page residency or requiring changes to page fault / rmap
hot paths.

The testing results are from a 2-node KVM guest.  Real-hardware results on
a  ≥4-socket system, would be valuable. I will update the statistics
here if have.

Baoquan He (6):
  mm/mglru: add PUD-level Bloom filter state
  mm/mglru: refactor Bloom filter helpers for two filter levels
  mm/mglru: skip empty PUD subtrees during aging
  mm/mglru: report hot PUDs from the rmap feedback path
  mm/mglru: add MM_WALK_EMPTY stats and tracepoint for cross-node
    measurement
  mm/mglru: count PUD subtrees skipped by the PUD-level filter

 include/linux/mmzone.h        |  15 +++-
 include/trace/events/vmscan.h |  30 ++++++++
 mm/vmscan.c                   | 126 +++++++++++++++++++++++++++++-----
 3 files changed, 153 insertions(+), 18 deletions(-)

--
2.54.0

[RFC PATCH 1/6] mm/mglru: add PUD-level Bloom filter state

[Baoquan He]

Add a second, coarser pair of double-buffered Bloom filters to
struct lru_gen_mm_state. Like the PMD-level filters, they flip
each generation, but they operate at 1GB (PUD) granularity so the
page table walker can skip whole PUD subtrees whose 512 PMDs would
all fail the PMD-level filter. This mainly targets the page tables
of processes whose memory lives only on other NUMA nodes
(cross-node empty walks).

No behavior change yet; the filters are populated and consumed by
follow-up patches.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 include/linux/mmzone.h | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a26c8b855222..2bda24522d9f 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -608,8 +608,16 @@ struct lru_gen_mm_state {
 	struct list_head *head;
 	/* where the last iteration ended before */
 	struct list_head *tail;
-	/* Bloom filters flip after each iteration */
+	/* PMD-level Bloom filters flip after each iteration */
 	unsigned long *filters[NR_BLOOM_FILTERS];
+	/*
+	 * PUD-level Bloom filters flip after each iteration. Same double
+	 * buffering as the PMD-level filters, but coarser: they remember
+	 * which 1GB PUD subtrees had young leaf entries last generation,
+	 * so walk_pud_range() can skip whole subtrees whose PMD iteration
+	 * would find nothing worth scanning (e.g. cross-node empty walks).
+	 */
+	unsigned long *pud_filters[NR_BLOOM_FILTERS];
 	/* the mm stats for debugging */
 	unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
 };
-- 
2.54.0

[RFC PATCH 2/6] mm/mglru: refactor Bloom filter helpers for two filter levels

[Baoquan He]

Split test/update/reset_bloom_filter() into __ prefixed helpers that
operate on a generic filters array, and thin wrappers that pass the
PMD-level mm_state->filters. This lets a second, coarser (PUD-level)
filter pair reuse the exact same hash, double-buffering and reset
machinery without duplicating code. Purely mechanical; no behavior
change.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 mm/vmscan.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 17d2b793cbfc..a397c62b2e5d 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2841,14 +2841,13 @@ static void get_item_key(void *item, int *key)
 	key[1] = hash >> BLOOM_FILTER_SHIFT;
 }
 
-static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
-			      void *item)
+static bool __test_bloom_filter(unsigned long **filters, unsigned long seq, void *item)
 {
 	int key[2];
 	unsigned long *filter;
 	int gen = filter_gen_from_seq(seq);
 
-	filter = READ_ONCE(mm_state->filters[gen]);
+	filter = READ_ONCE(filters[gen]);
 	if (!filter)
 		return true;
 
@@ -2857,14 +2856,13 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s
 	return test_bit(key[0], filter) && test_bit(key[1], filter);
 }
 
-static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
-				void *item)
+static void __update_bloom_filter(unsigned long **filters, unsigned long seq, void *item)
 {
 	int key[2];
 	unsigned long *filter;
 	int gen = filter_gen_from_seq(seq);
 
-	filter = READ_ONCE(mm_state->filters[gen]);
+	filter = READ_ONCE(filters[gen]);
 	if (!filter)
 		return;
 
@@ -2876,12 +2874,12 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
 		set_bit(key[1], filter);
 }
 
-static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+static void __reset_bloom_filter(unsigned long **filters, unsigned long seq)
 {
 	unsigned long *filter;
 	int gen = filter_gen_from_seq(seq);
 
-	filter = mm_state->filters[gen];
+	filter = filters[gen];
 	if (filter) {
 		bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
 		return;
@@ -2889,7 +2887,24 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
 
 	filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
 			       __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
-	WRITE_ONCE(mm_state->filters[gen], filter);
+	WRITE_ONCE(filters[gen], filter);
+}
+
+static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
+			      void *item)
+{
+	return __test_bloom_filter(mm_state->filters, seq, item);
+}
+
+static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
+				void *item)
+{
+	__update_bloom_filter(mm_state->filters, seq, item);
+}
+
+static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+{
+	__reset_bloom_filter(mm_state->filters, seq);
 }
 
 /******************************************************************************
-- 
2.54.0

[RFC PATCH 3/6] mm/mglru: skip empty PUD subtrees during aging

[Baoquan He]

The aging walk descends every present PUD and iterates all 512 of its
PMDs, testing the PMD-level Bloom filter on each. For a process whose
memory lives only on other NUMA nodes, every PUD of this lruvec fails
the PMD test, so the whole PMD iteration is pure waste - and on
multi-socket systems these cross-node walks are common because
lru_gen_use_mm() marks an mm for all nodes at every context switch.

Add a PUD-level Bloom filter (pud_filters) one level up. walk_pmd_range()
now reports whether it found any young leaf entries; walk_pud_range()
records that in the PUD filter and, on subsequent generations, skips the
whole 1GB subtree when the filter says it had none last generation.
The double-buffered filter flips with each new iteration, and the
existing eviction feedback (lru_gen_look_around()) keeps hot regions
marked, so newly hot or migrated-in pages are re-examined promptly
rather than suppressed indefinitely.

force_scan walks bypass the PUD test, so manual aging and newly added
mm's always rescan and re-populate the filter.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 mm/vmscan.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 47 insertions(+), 3 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index a397c62b2e5d..74edfe2a747d 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2816,6 +2816,13 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
  * walk_pmd_range(); the eviction also report them when walking the rmap
  * in lru_gen_look_around().
  *
+ * A second, coarser pair of filters (pud_filters) sits one level up. It
+ * remembers which 1GB PUD subtrees had young leaf entries, so walk_pud_range()
+ * can skip whole subtrees whose 512 PMDs would all fail the PMD-level test —
+ * e.g. the page tables of a process whose memory lives only on other NUMA
+ * nodes (cross-node empty walks). It mirrors the PMD-level filters: populated
+ * by walk_pmd_range()/lru_gen_look_around(), flipped by reset_pud_bloom_filter().
+ *
  * For future optimizations:
  * 1. It's not necessary to keep both filters all the time. The spare one can be
  *    freed after the RCU grace period and reallocated if needed again.
@@ -2907,6 +2914,23 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
 	__reset_bloom_filter(mm_state->filters, seq);
 }
 
+static bool test_pud_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
+				  void *item)
+{
+	return __test_bloom_filter(mm_state->pud_filters, seq, item);
+}
+
+static void update_pud_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
+				    void *item)
+{
+	__update_bloom_filter(mm_state->pud_filters, seq, item);
+}
+
+static void reset_pud_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
+{
+	__reset_bloom_filter(mm_state->pud_filters, seq);
+}
+
 /******************************************************************************
  *                          mm_struct list
  ******************************************************************************/
@@ -3146,8 +3170,10 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
 
 	spin_unlock(&mm_list->lock);
 
-	if (mm && first)
+	if (mm && first) {
 		reset_bloom_filter(mm_state, walk->seq + 1);
+		reset_pud_bloom_filter(mm_state, walk->seq + 1);
+	}
 
 	if (*iter)
 		mmdrop(*iter);
@@ -3728,10 +3754,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 	*first = -1;
 }
 
-static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+static bool walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
 			   struct mm_walk *args)
 {
 	int i;
+	bool young = false;
 	pmd_t *pmd;
 	unsigned long next;
 	unsigned long addr;
@@ -3790,6 +3817,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
 			continue;
 
 		walk->mm_stats[MM_NONLEAF_ADDED]++;
+		young = true;
 
 		/* carry over to the next generation */
 		update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
@@ -3799,6 +3827,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
 
 	if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
 		goto restart;
+
+	return young;
 }
 
 static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
@@ -3809,6 +3839,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
 	unsigned long addr;
 	unsigned long next;
 	struct lru_gen_mm_walk *walk = args->private;
+	struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
 
 	VM_WARN_ON_ONCE(p4d_leaf(*p4d));
 
@@ -3822,7 +3853,20 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
 		if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
 			continue;
 
-		walk_pmd_range(&val, addr, next, args);
+		/*
+		 * Cross-node empty walk suppression. A 1GB PUD subtree whose
+		 * 512 PMDs all failed the PMD-level Bloom filter last generation
+		 * found no young leaf entries for this lruvec, so skip the whole
+		 * PMD iteration instead of re-checking every entry. This mirrors
+		 * the PMD-level filter one level up and mainly cuts the cost of
+		 * walking page tables of processes whose memory lives only on
+		 * other NUMA nodes.
+		 */
+		if (!walk->force_scan && !test_pud_bloom_filter(mm_state, walk->seq, pud + i))
+			continue;
+
+		if (walk_pmd_range(&val, addr, next, args))
+			update_pud_bloom_filter(mm_state, walk->seq + 1, pud + i);
 
 		if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
 			end = (addr | ~PUD_MASK) + 1;
-- 
2.54.0

[RFC PATCH 4/6] mm/mglru: report hot PUDs from the rmap feedback path

[Baoquan He]

lru_gen_look_around() marks the PMD of a young PTE found during the
eviction rmap walk, feeding hot regions back to the aging walker.
With the PUD-level filter in place, that PMD marking alone is not
enough: the next aging walk would test the containing PUD first and
skip the whole 1GB subtree if the PUD is unmarked, never reaching the
PMD. Mark the covering PUD as well, so regions whose hotness is only
observed by eviction keep getting re-scanned by aging.

The PUD entry is re-derived from the mm page tables via pgd_offset()/
p4d_offset()/pud_offset(); it is only hashed, never dereferenced, and
the mmap lock held by the rmap walk keeps the table chain valid.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 mm/vmscan.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 74edfe2a747d..ca0f06641adc 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4416,8 +4416,14 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 	lazy_mmu_mode_disable();
 
 	/* feedback from rmap walkers to page table walkers */
-	if (mm_state && suitable_to_scan(i, young))
+	if (mm_state && suitable_to_scan(i, young)) {
+		/* the PUD entry covering the young PTEs scanned above */
+		pud_t *pud_p = pud_offset(p4d_offset(pgd_offset(vma->vm_mm, pvmw->address),
+						     pvmw->address), pvmw->address);
+
 		update_bloom_filter(mm_state, max_seq, pvmw->pmd);
+		update_pud_bloom_filter(mm_state, max_seq, pud_p);
+	}
 
 	mem_cgroup_put(memcg);
 
-- 
2.54.0

[RFC PATCH 5/6] mm/mglru: add MM_WALK_EMPTY stats and tracepoint for cross-node measurement

[Baoquan He]

Add infrastructure to quantify cross-node empty page table walks in
the MGLRU aging path - walks that traverse an mm's page tables but
find no folio belonging to the current lruvec (node+memcg). These
are the pure waste that the PUD-level Bloom filter (see "skip empty
PUD subtrees during aging") is meant to suppress, and this provides
the counters to verify its effect.

New per-walk counters (accumulated in mm_state->stats[]):

  MM_LEAF_ELIGIBLE    - folios belonging to this lruvec
  MM_WALK_TOTAL       - page-table walks completed
  MM_WALK_EMPTY       - walks that found no eligible folio
  MM_LEAF_TOTAL_EMPTY - leaf entries scanned by empty walks

A new tracepoint, mm_vmscan_lru_gen_walk(nid, seq, leaf_total,
leaf_eligible, empty), fires after each walk for live monitoring.

The debugfs lru_gen output format is updated ("TYFA" -> "TYFALWEE",
"tyfa" -> "tyfalwee") to display the new fields.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 include/linux/mmzone.h        |  4 ++++
 include/trace/events/vmscan.h | 28 ++++++++++++++++++++++++++++
 mm/vmscan.c                   | 32 ++++++++++++++++++++++++++++----
 3 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 2bda24522d9f..508f6fcbbe5a 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -595,6 +595,10 @@ enum {
 	MM_LEAF_YOUNG,		/* young leaf entries */
 	MM_NONLEAF_FOUND,	/* non-leaf entries found in Bloom filters */
 	MM_NONLEAF_ADDED,	/* non-leaf entries added to Bloom filters */
+	MM_LEAF_ELIGIBLE,	/* folios belonging to this lruvec (node+memcg) */
+	MM_WALK_TOTAL,		/* page-table walks completed */
+	MM_WALK_EMPTY,		/* walks that found no eligible folio */
+	MM_LEAF_TOTAL_EMPTY,	/* leaf entries scanned by empty walks */
 	NR_MM_STATS
 };
 
diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
index b4bf7b8def1f..c7c2034715b6 100644
--- a/include/trace/events/vmscan.h
+++ b/include/trace/events/vmscan.h
@@ -659,6 +659,34 @@ TRACE_EVENT(mm_vmscan_kswapd_clear_hopeless,
 		__entry->nid,
 		__print_symbolic(__entry->reason, kswapd_clear_hopeless_reason_ops))
 );
+TRACE_EVENT(mm_vmscan_lru_gen_walk,
+
+	TP_PROTO(int nid, unsigned long seq, int leaf_total,
+		 int leaf_eligible, bool empty),
+
+	TP_ARGS(nid, seq, leaf_total, leaf_eligible, empty),
+
+	TP_STRUCT__entry(
+		__field(int, nid)
+		__field(unsigned long, seq)
+		__field(int, leaf_total)
+		__field(int, leaf_eligible)
+		__field(bool, empty)
+	),
+
+	TP_fast_assign(
+		__entry->nid = nid;
+		__entry->seq = seq;
+		__entry->leaf_total = leaf_total;
+		__entry->leaf_eligible = leaf_eligible;
+		__entry->empty = empty;
+	),
+
+	TP_printk("nid=%d seq=%lu leaf_total=%d leaf_eligible=%d empty=%d",
+		__entry->nid, __entry->seq, __entry->leaf_total,
+		__entry->leaf_eligible, __entry->empty)
+);
+
 #endif /* _TRACE_VMSCAN_H */
 
 /* This part must be outside protection */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index ca0f06641adc..e34179343782 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3628,6 +3628,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 		if (!folio)
 			continue;
 
+		walk->mm_stats[MM_LEAF_ELIGIBLE]++;
+
 		if (folio_test_large(folio)) {
 			const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
 
@@ -3728,6 +3730,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 		if (!folio)
 			goto next;
 
+		walk->mm_stats[MM_LEAF_ELIGIBLE]++;
+
 		if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i))
 			goto next;
 
@@ -4167,8 +4171,28 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
 
 	do {
 		success = iterate_mm_list(walk, &mm);
-		if (mm)
+		if (mm) {
+			bool empty = false;
+
 			walk_mm(mm, walk);
+			/*
+			 * A walk that traversed page tables but found no folio
+			 * belonging to this lruvec (node+memcg) is pure waste.
+			 */
+			if (walk->mm_stats[MM_LEAF_TOTAL]) {
+				walk->mm_stats[MM_WALK_TOTAL]++;
+				if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+					walk->mm_stats[MM_WALK_EMPTY]++;
+					walk->mm_stats[MM_LEAF_TOTAL_EMPTY] +=
+						walk->mm_stats[MM_LEAF_TOTAL];
+					empty = true;
+				}
+			}
+			trace_mm_vmscan_lru_gen_walk(
+					lruvec_pgdat(lruvec)->node_id, walk->seq,
+					walk->mm_stats[MM_LEAF_TOTAL],
+					walk->mm_stats[MM_LEAF_ELIGIBLE], empty);
+		}
 	} while (mm);
 done:
 	if (success) {
@@ -5653,14 +5677,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
 
 	seq_puts(m, "                      ");
 	for (i = 0; i < NR_MM_STATS; i++) {
-		const char *s = "xxxx";
+		const char *s = "xxxxxxxx";
 		unsigned long n = 0;
 
 		if (seq == max_seq && NR_HIST_GENS == 1) {
-			s = "TYFA";
+			s = "TYFALWEE";
 			n = READ_ONCE(mm_state->stats[hist][i]);
 		} else if (seq != max_seq && NR_HIST_GENS > 1) {
-			s = "tyfa";
+			s = "tyfalwee";
 			n = READ_ONCE(mm_state->stats[hist][i]);
 		}
 
-- 
2.54.0

Re: [RFC PATCH 0/6] mm/mglru: skip empty PUD subtrees during aging with PUD-level Bloom filter

[Baoquan He]

On 08/06/26 at 06:29pm, Baoquan He wrote:
> Problem
> =======
> 
> MGLRU's aging walks every present PUD of every mm in the mm_list.  When
> an mm has no pages on a given NUMA node, all 512 PMDs in each PUD fail
> the existing PMD-level Bloom filter test, yet the walker still descends
> every PUD and iterates every PMD ??? pure waste.
> 
> These cross-node empty walks are structural: lru_gen_use_mm() sets
> mm->lru_gen.bitmap to -1 (all nodes) at each context switch, so every
> node's kswapd independently walks the same mm.  On a 2-node KVM guest,
> ~80% of all aging walks are empty; the worst-case leaf-traversal waste
> with a fully resident remote mm is ~73%.

A note on the design background and one limitation.

These patches replace my earlier per-mm empty_map approach, which skipped
an mm for K generations  (default 4) after an empty walk on a node. As
Kairui pointed out, that leaves a window of up to K generations during which
pages migrated into the node are not scanned by aging until the forced  rescan,
and it adds 16 bytes of per-mm state with awkward invalidation semantics.
Following his  suggestion, the series instead adds a PUD-level Bloom filter — a
coarser counterpart of the existing  PMD-level filter — so a 1GB subtree that
had no young entries for this lruvec last generation is skipped, with no per-mm
state and no blind window.

I will paste the empty_map patches to this thread for reviewers'
reference.

[PATCH 1/4] mm/mglru: add MM_WALK_EMPTY stats and tracepoint for cross-node measurement

[Baoquan He]

Add infrastructure to quantify cross-node empty page table walks in the
MGLRU aging path. These are walks that traverse an mm's page tables but
find no folio belonging to the current lruvec (node+memcg) - pure waste
caused by lru_gen_use_mm() setting mm->lru_gen.bitmap to -1 (all nodes)
at context switch.

New per-walk counters (accumulated in mm_state->stats[]):

  MM_LEAF_ELIGIBLE    - folios belonging to this lruvec
  MM_WALK_TOTAL       - page-table walks completed
  MM_WALK_EMPTY       - walks that found no eligible folio
  MM_LEAF_TOTAL_EMPTY - leaf entries scanned by empty walks

A new tracepoint, mm_vmscan_lru_gen_walk(nid, seq, leaf_total,
leaf_eligible, empty), fires after each walk for live monitoring. The
debugfs lru_gen output format is updated ("TYFA" -> "TYFALWEE") to
display the new fields.

This series uses these counters to quantify the cross-node empty walk
cost and to validate the suppression implemented by the follow-up
patches.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 include/linux/mmzone.h        |  4 ++++
 include/trace/events/vmscan.h | 28 ++++++++++++++++++++++++++++
 mm/vmscan.c                   | 30 ++++++++++++++++++++++++++----
 3 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a26c8b855222..34938d4f8e23 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -595,6 +595,10 @@ enum {
 	MM_LEAF_YOUNG,		/* young leaf entries */
 	MM_NONLEAF_FOUND,	/* non-leaf entries found in Bloom filters */
 	MM_NONLEAF_ADDED,	/* non-leaf entries added to Bloom filters */
+	MM_LEAF_ELIGIBLE,	/* folios belonging to this lruvec (node+memcg) */
+	MM_WALK_TOTAL,		/* page-table walks completed */
+	MM_WALK_EMPTY,		/* walks that found no eligible folio */
+	MM_LEAF_TOTAL_EMPTY,	/* leaf entries scanned by empty walks */
 	NR_MM_STATS
 };
 
diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
index b4bf7b8def1f..c7c2034715b6 100644
--- a/include/trace/events/vmscan.h
+++ b/include/trace/events/vmscan.h
@@ -659,6 +659,34 @@ TRACE_EVENT(mm_vmscan_kswapd_clear_hopeless,
 		__entry->nid,
 		__print_symbolic(__entry->reason, kswapd_clear_hopeless_reason_ops))
 );
+TRACE_EVENT(mm_vmscan_lru_gen_walk,
+
+	TP_PROTO(int nid, unsigned long seq, int leaf_total,
+		 int leaf_eligible, bool empty),
+
+	TP_ARGS(nid, seq, leaf_total, leaf_eligible, empty),
+
+	TP_STRUCT__entry(
+		__field(int, nid)
+		__field(unsigned long, seq)
+		__field(int, leaf_total)
+		__field(int, leaf_eligible)
+		__field(bool, empty)
+	),
+
+	TP_fast_assign(
+		__entry->nid = nid;
+		__entry->seq = seq;
+		__entry->leaf_total = leaf_total;
+		__entry->leaf_eligible = leaf_eligible;
+		__entry->empty = empty;
+	),
+
+	TP_printk("nid=%d seq=%lu leaf_total=%d leaf_eligible=%d empty=%d",
+		__entry->nid, __entry->seq, __entry->leaf_total,
+		__entry->leaf_eligible, __entry->empty)
+);
+
 #endif /* _TRACE_VMSCAN_H */
 
 /* This part must be outside protection */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 17d2b793cbfc..83de2b147919 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3587,6 +3587,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 		if (!folio)
 			continue;
 
+		walk->mm_stats[MM_LEAF_ELIGIBLE]++;
+
 		if (folio_test_large(folio)) {
 			const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
 
@@ -3687,6 +3689,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 		if (!folio)
 			goto next;
 
+		walk->mm_stats[MM_LEAF_ELIGIBLE]++;
+
 		if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i))
 			goto next;
 
@@ -4108,8 +4112,26 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
 
 	do {
 		success = iterate_mm_list(walk, &mm);
-		if (mm)
+		if (mm) {
+			bool empty = false;
+
 			walk_mm(mm, walk);
+			/* A walk that traversed page tables but found no folio
+			 * belonging to this lruvec (node+memcg) is pure waste. */
+			if (walk->mm_stats[MM_LEAF_TOTAL]) {
+				walk->mm_stats[MM_WALK_TOTAL]++;
+				if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+					walk->mm_stats[MM_WALK_EMPTY]++;
+					walk->mm_stats[MM_LEAF_TOTAL_EMPTY] +=
+						walk->mm_stats[MM_LEAF_TOTAL];
+					empty = true;
+				}
+			}
+			trace_mm_vmscan_lru_gen_walk(
+					lruvec_pgdat(lruvec)->node_id, walk->seq,
+					walk->mm_stats[MM_LEAF_TOTAL],
+					walk->mm_stats[MM_LEAF_ELIGIBLE], empty);
+		}
 	} while (mm);
 done:
 	if (success) {
@@ -5588,14 +5610,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
 
 	seq_puts(m, "                      ");
 	for (i = 0; i < NR_MM_STATS; i++) {
-		const char *s = "xxxx";
+		const char *s = "xxxxxxxx";
 		unsigned long n = 0;
 
 		if (seq == max_seq && NR_HIST_GENS == 1) {
-			s = "TYFA";
+			s = "TYFALWEE";
 			n = READ_ONCE(mm_state->stats[hist][i]);
 		} else if (seq != max_seq && NR_HIST_GENS > 1) {
-			s = "tyfa";
+			s = "tyfalwee";
 			n = READ_ONCE(mm_state->stats[hist][i]);
 		}
 
-- 
2.54.0

[PATCH 2/4] mm/mglru: suppress cross-node empty page table walks

[Baoquan He]

lru_gen_use_mm() marks an mm used on all nodes at every context switch,
so each node's aging walker independently walks every mm's page tables.
For an mm with memory on only a subset of nodes, the other nodes' walks
find no folio belonging to that lruvec (node+memcg) - pure waste (~80%
of walks on multi-socket systems).

Add per-mm, per-node empty-walk tracking so get_next_mm() skips an mm on
a node for K generations (K=MGLRU_EMPTY_SKIP_GENS=4) after an empty walk,
then force a rescan. empty_map records the node, empty_map_seq the oldest
max_seq among the set bits (via min(), so the rescan never fires late).

A walk is "empty" when it finds no folio for this lruvec, regardless of
MM_LEAF_TOTAL - the PMD-level Bloom filter often keeps leaf_total at 0
for a foreign mm, so gating on it would mean the skip never engages.

The periodic rescan bounds the blind window to K generations, but a page
that appears on a node during the skip (e.g. migration) is not aged until
the rescan. A follow-up patch in this series invalidates the skip on page
fault and migration. mm_struct grows by 16 bytes per process.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 include/linux/mm_types.h | 13 ++++++++++
 mm/vmscan.c              | 54 +++++++++++++++++++++++++++++++++++++---
 2 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index b5d4cd3b067b..68ec8bb2ab71 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1410,6 +1410,17 @@ struct mm_struct {
 			 * page table walkers cleared the corresponding bits.
 			 */
 			unsigned long bitmap;
+			/*
+			 * Cross-node empty-walk suppression: bit N set means
+			 * node N's last aging walk of this mm found no folio
+			 * for this lruvec (pure waste). Skip the mm on node N
+			 * for up to K generations, then force a rescan.
+			 * empty_map_seq is the oldest max_seq among the set
+			 * bits (min(), conservative), so a bit is cleared when
+			 * max_seq >= empty_map_seq + K.
+			 */
+			unsigned long empty_map;
+			unsigned long empty_map_seq;
 #ifdef CONFIG_MEMCG
 			/* points to the memcg of "owner" above */
 			struct mem_cgroup *memcg;
@@ -1503,6 +1514,8 @@ static inline void lru_gen_init_mm(struct mm_struct *mm)
 {
 	INIT_LIST_HEAD(&mm->lru_gen.list);
 	mm->lru_gen.bitmap = 0;
+	mm->lru_gen.empty_map = 0;
+	mm->lru_gen.empty_map_seq = ~0UL;
 #ifdef CONFIG_MEMCG
 	mm->lru_gen.memcg = NULL;
 #endif
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 83de2b147919..c39b392d7b7e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2709,6 +2709,14 @@ static bool should_clear_pmd_young(void)
 	return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG);
 }
 
+/*
+ * Cross-node empty walk suppression. lru_gen_use_mm() marks an mm used on all
+ * nodes, so aging on a node where the mm has no memory wastes a full page table
+ * walk. Skip such an mm for up to MGLRU_EMPTY_SKIP_GENS generations after an
+ * empty walk, then force-rescan to close migration/mlock/NUMA-balancing windows.
+ */
+#define MGLRU_EMPTY_SKIP_GENS 4
+
 /******************************************************************************
  *                          shorthand helpers
  ******************************************************************************/
@@ -2929,9 +2937,27 @@ static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
 	mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
 	key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
 
+	/* skip if this mm hasn't been used on this node since the last walk */
 	if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
 		return NULL;
 
+	/*
+	 * Skip if this node's last walk of this mm was empty and fewer than K
+	 * generations have passed; after K, clear the bit to force a rescan.
+	 * empty_map_seq tracks the oldest marking via min(), so the rescan
+	 * never fires later than K generations on any node.
+	 */
+	if (!walk->force_scan && test_bit(key, &mm->lru_gen.empty_map)) {
+		DEFINE_MAX_SEQ(walk->lruvec);
+		unsigned long empty_seq = READ_ONCE(mm->lru_gen.empty_map_seq);
+
+		if (max_seq < empty_seq + MGLRU_EMPTY_SKIP_GENS)
+			return NULL;		/* skip: < K gens since empty */
+
+		/* K generations passed → force rescan */
+		clear_bit(key, &mm->lru_gen.empty_map);
+	}
+
 	clear_bit(key, &mm->lru_gen.bitmap);
 	mmgrab(mm);
 
@@ -4113,18 +4139,38 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
 	do {
 		success = iterate_mm_list(walk, &mm);
 		if (mm) {
+			int nid = lruvec_pgdat(lruvec)->node_id;
+			int key = nid % BITS_PER_TYPE(mm->lru_gen.bitmap);
 			bool empty = false;
 
 			walk_mm(mm, walk);
-			/* A walk that traversed page tables but found no folio
-			 * belonging to this lruvec (node+memcg) is pure waste. */
+
+			/*
+			 * Any walk that found no folio for this lruvec is empty -
+			 * even if the PMD-level Bloom filter kept MM_LEAF_TOTAL at 0
+			 * (the common case for a foreign mm). Mark it so get_next_mm()
+			 * skips it next time; otherwise the empty-walk skip never
+			 * engages for the walks that matter most.
+			 */
+			if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+				set_bit(key, &mm->lru_gen.empty_map);
+				/* track the oldest marking (conservative) */
+				WRITE_ONCE(mm->lru_gen.empty_map_seq,
+					min(READ_ONCE(mm->lru_gen.empty_map_seq),
+					    walk->seq));
+				empty = true;
+			} else {
+				/* found eligible folios: clear the marking */
+				clear_bit(key, &mm->lru_gen.empty_map);
+			}
+
+			/* measurement stats keep the leaf_total gate */
 			if (walk->mm_stats[MM_LEAF_TOTAL]) {
 				walk->mm_stats[MM_WALK_TOTAL]++;
-				if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+				if (empty) {
 					walk->mm_stats[MM_WALK_EMPTY]++;
 					walk->mm_stats[MM_LEAF_TOTAL_EMPTY] +=
 						walk->mm_stats[MM_LEAF_TOTAL];
-					empty = true;
 				}
 			}
 			trace_mm_vmscan_lru_gen_walk(
-- 
2.54.0

[PATCH 3/4] mm/mglru: add debugfs knob to control cross-node empty walk skip threshold

[Baoquan He]

Convert the hardcoded MGLRU_EMPTY_SKIP_GENS from a #define to a
runtime-tunable variable (mglru_empty_skip_gens, default 4), and
add a debugfs command to control it:

  echo "skip_empty <N>" > /sys/kernel/debug/lru_gen  (0 = disable, default 4)
  echo "skip_empty" > /sys/kernel/debug/lru_gen      (show current value)

Setting N=0 disables the cross-node empty walk suppression entirely,
allowing easy A/B testing without recompiling the kernel.
Default 4 matches MAX_NR_GENS so the skip window covers at most
one full LRU generation slide.

This is intended as a development/tuning aid for RFC. Whether it
should graduate to a permanent ABI (sysctl) can be decided once
field data demonstrates the optimization is worth keeping.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 mm/vmscan.c | 41 ++++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index c39b392d7b7e..631bcd89b196 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2709,14 +2709,6 @@ static bool should_clear_pmd_young(void)
 	return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG);
 }
 
-/*
- * Cross-node empty walk suppression. lru_gen_use_mm() marks an mm used on all
- * nodes, so aging on a node where the mm has no memory wastes a full page table
- * walk. Skip such an mm for up to MGLRU_EMPTY_SKIP_GENS generations after an
- * empty walk, then force-rescan to close migration/mlock/NUMA-balancing windows.
- */
-#define MGLRU_EMPTY_SKIP_GENS 4
-
 /******************************************************************************
  *                          shorthand helpers
  ******************************************************************************/
@@ -2927,6 +2919,9 @@ static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec)
 	return &lruvec->mm_state;
 }
 
+/* tunable empty-walk skip threshold; defined later, get_next_mm() needs it */
+static unsigned long mglru_empty_skip_gens;
+
 static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
 {
 	int key;
@@ -2951,7 +2946,7 @@ static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
 		DEFINE_MAX_SEQ(walk->lruvec);
 		unsigned long empty_seq = READ_ONCE(mm->lru_gen.empty_map_seq);
 
-		if (max_seq < empty_seq + MGLRU_EMPTY_SKIP_GENS)
+		if (max_seq < empty_seq + READ_ONCE(mglru_empty_skip_gens))
 			return NULL;		/* skip: < K gens since empty */
 
 		/* K generations passed → force rescan */
@@ -4273,6 +4268,15 @@ static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc
 /* to protect the working set of the last N jiffies */
 static unsigned long lru_gen_min_ttl __read_mostly;
 
+/*
+ * Cross-node empty-walk skip threshold: skip an mm on node N for up to
+ * @mglru_empty_skip_gens generations after an empty walk, then force-rescan
+ * (closes migration/mlock/NUMA-balancing windows). Default 4 matches
+ * MAX_NR_GENS; 0 disables the suppression. Tunable via:
+ * echo "skip_empty <N>" > /sys/kernel/debug/lru_gen
+ */
+static unsigned long mglru_empty_skip_gens __read_mostly = 4;
+
 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
 {
 	struct mem_cgroup *memcg;
@@ -5870,6 +5874,25 @@ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
 		if (!*cur)
 			continue;
 
+		/*
+		 * set/show the empty-walk skip threshold: "skip_empty <N>"
+		 */
+		if (!strncmp(cur, "skip_empty", 10)) {
+			cur += 10;
+			cur = skip_spaces(cur);
+			if (*cur) {
+				unsigned long val;
+
+				if (sscanf(cur, "%lu", &val) == 1)
+					WRITE_ONCE(mglru_empty_skip_gens, val);
+			} else {
+				pr_info("MGLRU empty skip threshold: %lu generations (0=disabled)\n",
+					READ_ONCE(mglru_empty_skip_gens));
+			}
+			err = 0;
+			continue;
+		}
+
 		n = sscanf(cur, "%c %llu %u %lu %n %4s %n %lu %n", &cmd, &memcg_id, &nid,
 			   &seq, &end, swap_string, &end, &opt, &end);
 		if (n < 4 || cur[end]) {
-- 
2.54.0

[PATCH 4/4] mm/mglru: invalidate empty-walk skip on page fault and migration

[Baoquan He]

empty_map skips an mm on a node for up to K generations after an empty
walk, leaving a window where pages that appear on that node are not aged
until the forced rescan. Notify MGLRU when a page of the mm appears:
set the node's bitmap bit and clear its empty_map bit. Fault and
migration are both software events, so the invalidation is complete - a
node marked empty has no pages there, so a page can only appear via a
fault or a migration.

- mm/memory.c do_anonymous_page()/finish_fault(): mark the folio's node
  after the page is allocated (anon and file/COW faults).
- mm/migrate.c remove_migration_pte(): mark the folio's destination node
  (also covers NUMA-balancing migration).

The bitmap bit is mostly redundant with schedule-time marking, but is
needed when migration targets an idle mm whose bits were already
cleared by a previous walk.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 include/linux/mm_types.h | 18 ++++++++++++++++++
 mm/memory.c              |  7 +++++++
 mm/migrate.c             |  3 +++
 3 files changed, 28 insertions(+)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 68ec8bb2ab71..89b723483675 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1531,6 +1531,20 @@ static inline void lru_gen_use_mm(struct mm_struct *mm)
 	WRITE_ONCE(mm->lru_gen.bitmap, -1);
 }
 
+/*
+ * A page of this mm appeared on (or was accessed on) node @nid — e.g. a page
+ * fault or a migration. Set that node's bitmap bit so the aging walker walks
+ * the mm, and clear the empty-walk skip so a page that just appeared on a node
+ * previously marked empty is not ignored for up to K generations.
+ */
+static inline void lru_gen_mm_accessed(struct mm_struct *mm, int nid)
+{
+	unsigned long key = nid % BITS_PER_TYPE(mm->lru_gen.bitmap);
+
+	set_bit(key, &mm->lru_gen.bitmap);
+	clear_bit(key, &mm->lru_gen.empty_map);
+}
+
 #else /* !CONFIG_LRU_GEN_WALKS_MMU */
 
 static inline void lru_gen_add_mm(struct mm_struct *mm)
@@ -1553,6 +1567,10 @@ static inline void lru_gen_use_mm(struct mm_struct *mm)
 {
 }
 
+static inline void lru_gen_mm_accessed(struct mm_struct *mm, int nid)
+{
+}
+
 #endif /* CONFIG_LRU_GEN_WALKS_MMU */
 
 struct vma_iterator {
diff --git a/mm/memory.c b/mm/memory.c
index 428eb555ecb7..e3d7c8f7ca7d 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5512,6 +5512,9 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 		folio_put(folio);
 		return handle_userfault(vmf, VM_UFFD_MISSING);
 	}
+	/* a new page of this mm lands on this node: invalidate any empty skip */
+	lru_gen_mm_accessed(vma->vm_mm, folio_nid(folio));
+
 	map_anon_folio_pte_pf(folio, vmf->pte, vma, addr,
 			      vmf_orig_pte_uffd_wp(vmf));
 unlock:
@@ -5772,6 +5775,10 @@ vm_fault_t finish_fault(struct vm_fault *vmf)
 		page = vmf->page;
 
 	folio = page_folio(page);
+
+	/* mapping a page of this mm on this node: invalidate any empty skip */
+	lru_gen_mm_accessed(vma->vm_mm, folio_nid(folio));
+
 	/*
 	 * check even for read faults because we might have lost our CoWed
 	 * page
diff --git a/mm/migrate.c b/mm/migrate.c
index b937cbd76480..2e0674e89ba8 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -354,6 +354,9 @@ static bool remove_migration_pte(struct folio *folio,
 	struct rmap_walk_arg *rmap_walk_arg = arg;
 	DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
 
+	/* the folio ends up on folio_nid(): notify MGLRU for this mm */
+	lru_gen_mm_accessed(vma->vm_mm, folio_nid(folio));
+
 	while (page_vma_mapped_walk(&pvmw)) {
 		rmap_t rmap_flags = RMAP_NONE;
 		pte_t old_pte;
-- 
2.54.0