[PATCH 2/4] mm/mglru: suppress cross-node empty page table walks

[Baoquan He <baoquan.he@linux.dev>]

lru_gen_use_mm() marks an mm used on all nodes at every context switch,
so each node's aging walker independently walks every mm's page tables.
For an mm with memory on only a subset of nodes, the other nodes' walks
find no folio belonging to that lruvec (node+memcg) - pure waste (~80%
of walks on multi-socket systems).

Add per-mm, per-node empty-walk tracking so get_next_mm() skips an mm on
a node for K generations (K=MGLRU_EMPTY_SKIP_GENS=4) after an empty walk,
then force a rescan. empty_map records the node, empty_map_seq the oldest
max_seq among the set bits (via min(), so the rescan never fires late).

A walk is "empty" when it finds no folio for this lruvec, regardless of
MM_LEAF_TOTAL - the PMD-level Bloom filter often keeps leaf_total at 0
for a foreign mm, so gating on it would mean the skip never engages.

The periodic rescan bounds the blind window to K generations, but a page
that appears on a node during the skip (e.g. migration) is not aged until
the rescan. A follow-up patch in this series invalidates the skip on page
fault and migration. mm_struct grows by 16 bytes per process.

Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
 include/linux/mm_types.h | 13 ++++++++++
 mm/vmscan.c              | 54 +++++++++++++++++++++++++++++++++++++---
 2 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index b5d4cd3b067b..68ec8bb2ab71 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1410,6 +1410,17 @@ struct mm_struct {
 			 * page table walkers cleared the corresponding bits.
 			 */
 			unsigned long bitmap;
+			/*
+			 * Cross-node empty-walk suppression: bit N set means
+			 * node N's last aging walk of this mm found no folio
+			 * for this lruvec (pure waste). Skip the mm on node N
+			 * for up to K generations, then force a rescan.
+			 * empty_map_seq is the oldest max_seq among the set
+			 * bits (min(), conservative), so a bit is cleared when
+			 * max_seq >= empty_map_seq + K.
+			 */
+			unsigned long empty_map;
+			unsigned long empty_map_seq;
 #ifdef CONFIG_MEMCG
 			/* points to the memcg of "owner" above */
 			struct mem_cgroup *memcg;
@@ -1503,6 +1514,8 @@ static inline void lru_gen_init_mm(struct mm_struct *mm)
 {
 	INIT_LIST_HEAD(&mm->lru_gen.list);
 	mm->lru_gen.bitmap = 0;
+	mm->lru_gen.empty_map = 0;
+	mm->lru_gen.empty_map_seq = ~0UL;
 #ifdef CONFIG_MEMCG
 	mm->lru_gen.memcg = NULL;
 #endif
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 83de2b147919..c39b392d7b7e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2709,6 +2709,14 @@ static bool should_clear_pmd_young(void)
 	return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG);
 }
 
+/*
+ * Cross-node empty walk suppression. lru_gen_use_mm() marks an mm used on all
+ * nodes, so aging on a node where the mm has no memory wastes a full page table
+ * walk. Skip such an mm for up to MGLRU_EMPTY_SKIP_GENS generations after an
+ * empty walk, then force-rescan to close migration/mlock/NUMA-balancing windows.
+ */
+#define MGLRU_EMPTY_SKIP_GENS 4
+
 /******************************************************************************
  *                          shorthand helpers
  ******************************************************************************/
@@ -2929,9 +2937,27 @@ static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
 	mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
 	key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
 
+	/* skip if this mm hasn't been used on this node since the last walk */
 	if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
 		return NULL;
 
+	/*
+	 * Skip if this node's last walk of this mm was empty and fewer than K
+	 * generations have passed; after K, clear the bit to force a rescan.
+	 * empty_map_seq tracks the oldest marking via min(), so the rescan
+	 * never fires later than K generations on any node.
+	 */
+	if (!walk->force_scan && test_bit(key, &mm->lru_gen.empty_map)) {
+		DEFINE_MAX_SEQ(walk->lruvec);
+		unsigned long empty_seq = READ_ONCE(mm->lru_gen.empty_map_seq);
+
+		if (max_seq < empty_seq + MGLRU_EMPTY_SKIP_GENS)
+			return NULL;		/* skip: < K gens since empty */
+
+		/* K generations passed → force rescan */
+		clear_bit(key, &mm->lru_gen.empty_map);
+	}
+
 	clear_bit(key, &mm->lru_gen.bitmap);
 	mmgrab(mm);
 
@@ -4113,18 +4139,38 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
 	do {
 		success = iterate_mm_list(walk, &mm);
 		if (mm) {
+			int nid = lruvec_pgdat(lruvec)->node_id;
+			int key = nid % BITS_PER_TYPE(mm->lru_gen.bitmap);
 			bool empty = false;
 
 			walk_mm(mm, walk);
-			/* A walk that traversed page tables but found no folio
-			 * belonging to this lruvec (node+memcg) is pure waste. */
+
+			/*
+			 * Any walk that found no folio for this lruvec is empty -
+			 * even if the PMD-level Bloom filter kept MM_LEAF_TOTAL at 0
+			 * (the common case for a foreign mm). Mark it so get_next_mm()
+			 * skips it next time; otherwise the empty-walk skip never
+			 * engages for the walks that matter most.
+			 */
+			if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+				set_bit(key, &mm->lru_gen.empty_map);
+				/* track the oldest marking (conservative) */
+				WRITE_ONCE(mm->lru_gen.empty_map_seq,
+					min(READ_ONCE(mm->lru_gen.empty_map_seq),
+					    walk->seq));
+				empty = true;
+			} else {
+				/* found eligible folios: clear the marking */
+				clear_bit(key, &mm->lru_gen.empty_map);
+			}
+
+			/* measurement stats keep the leaf_total gate */
 			if (walk->mm_stats[MM_LEAF_TOTAL]) {
 				walk->mm_stats[MM_WALK_TOTAL]++;
-				if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+				if (empty) {
 					walk->mm_stats[MM_WALK_EMPTY]++;
 					walk->mm_stats[MM_LEAF_TOTAL_EMPTY] +=
 						walk->mm_stats[MM_LEAF_TOTAL];
-					empty = true;
 				}
 			}
 			trace_mm_vmscan_lru_gen_walk(
-- 
2.54.0