* [PATCH v5 01/17] mm/mglru: rename the generation offset in lru_gen_folio_seq()
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 02/17] mm/memcontrol: allow update of LRU statistic without holding LRU lock Kairui Song via B4 Relay
` (15 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Ridong Chen <chenridong@xiaomi.com>
lru_gen_folio_seq() computes how many generations below max_seq a folio
should be added to, but stores it in a variable named "gen". Elsewhere in
MGLRU "gen" means an index into lrugen->folios[], i.e. seq % MAX_NR_GENS,
so reusing the name for a relative offset is confusing.
Rename it to "age", which matches what the value means: the newest
generation is age 1, and a larger age is an older generation.
No functional change.
[kasong: dropped the comment table part, the series rewrites it]
Signed-off-by: Ridong Chen <chenridong@xiaomi.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index f6f4bc9a704f..2395ca8ee305 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -302,7 +302,7 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
const struct folio *folio,
bool reclaiming)
{
- int gen;
+ int age;
int type = folio_is_file_lru(folio);
const struct lru_gen_folio *lrugen = &lruvec->lrugen;
@@ -319,17 +319,17 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
* |<---------------------------- MAX_NR_GENS ---------------------------->|
*/
if (folio_test_active(folio))
- gen = MIN_NR_GENS - folio_test_workingset(folio);
+ age = MIN_NR_GENS - folio_test_workingset(folio);
else if (reclaiming)
- gen = MAX_NR_GENS;
+ age = MAX_NR_GENS;
else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) ||
(folio_test_reclaim(folio) &&
(folio_test_dirty(folio) || folio_test_writeback(folio))))
- gen = MIN_NR_GENS;
+ age = MIN_NR_GENS;
else
- gen = MAX_NR_GENS - (folio_test_workingset(folio) || folio_test_referenced(folio));
+ age = MAX_NR_GENS - (folio_test_workingset(folio) || folio_test_referenced(folio));
- return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type]));
+ return max(READ_ONCE(lrugen->max_seq) - age + 1, READ_ONCE(lrugen->min_seq[type]));
}
static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 02/17] mm/memcontrol: allow update of LRU statistic without holding LRU lock
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 01/17] mm/mglru: rename the generation offset in lru_gen_folio_seq() Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 03/17] mm/mglru: make generation page counters atomic Kairui Song via B4 Relay
` (14 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
To enable moving file pages in folio_mark_accessed directly and lazily
for MGLRU, allow updating the LRU statistic atomically without holding a
lock. It may cause temporary counter underflow, which should be fine as
the counter is eventually consistent and only serves as an estimate.
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 2395ca8ee305..2edb9beef6d3 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -47,12 +47,11 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec,
{
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
- lockdep_assert_held(&lruvec->lru_lock);
WARN_ON_ONCE(nr_pages != (int)nr_pages);
mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
- __mod_zone_page_state(&pgdat->node_zones[zid],
- NR_ZONE_LRU_BASE + lru, nr_pages);
+ mod_zone_page_state(&pgdat->node_zones[zid],
+ NR_ZONE_LRU_BASE + lru, nr_pages);
}
static __always_inline void update_lru_size(struct lruvec *lruvec,
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 03/17] mm/mglru: make generation page counters atomic
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 01/17] mm/mglru: rename the generation offset in lru_gen_folio_seq() Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 02/17] mm/memcontrol: allow update of LRU statistic without holding LRU lock Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 04/17] mm/memcg: add folio-based lruvec live helper Kairui Song via B4 Relay
` (13 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
No feature change, convert them to atomic so we can update them without
holding the LRU lock. A temporary underflow is fine: readers clamp a
negative value to zero, and the counters only need to be eventually
consistent.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 6 ++----
include/linux/mmzone.h | 2 +-
mm/vmscan.c | 26 +++++++++++++-------------
mm/vmscan.h | 2 +-
4 files changed, 17 insertions(+), 19 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 2edb9beef6d3..46beded51c7d 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -265,11 +265,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
if (old_gen >= 0)
- WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
- lrugen->nr_pages[old_gen][type][zone] - delta);
+ atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
if (new_gen >= 0)
- WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
- lrugen->nr_pages[new_gen][type][zone] + delta);
+ atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]);
/* addition */
if (old_gen < 0) {
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index c161d563ce8f..4c4aad95b0b6 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -554,7 +554,7 @@ struct lru_gen_folio {
/* the multi-gen LRU lists, lazily sorted on eviction */
struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the multi-gen LRU sizes, eventually consistent */
- long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+ atomic_long_t nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
/* the exponential moving average of evicted+protected */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index fe1fe1439c7c..8d14831cdf83 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3350,8 +3350,7 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
continue;
walk->nr_pages[gen][type][zone] = 0;
- WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
- lrugen->nr_pages[gen][type][zone] + delta);
+ atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]);
if (lru_gen_is_active(lruvec, gen))
lru += LRU_ACTIVE;
@@ -3971,11 +3970,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
break;
}
flush_lru_batch(head, &batch_end, target_list);
-
- WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
- lrugen->nr_pages[old_gen][type][zone] - delta);
- WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],
- lrugen->nr_pages[target_gen][type][zone] + delta);
+ atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
+ atomic_long_add(delta, &lrugen->nr_pages[target_gen][type][zone]);
if (!remaining)
return false;
}
@@ -4093,8 +4089,8 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness
for (type = 0; type < ANON_AND_FILE; type++) {
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
enum lru_list lru = type * LRU_INACTIVE_FILE;
- long delta = lrugen->nr_pages[prev][type][zone] -
- lrugen->nr_pages[next][type][zone];
+ long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) -
+ atomic_long_read(&lrugen->nr_pages[next][type][zone]);
if (!delta)
continue;
@@ -4625,8 +4621,13 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
child_lrugen = &child_lruvec->lrugen;
parent_lrugen = &parent_lruvec->lrugen;
+ /*
+ * The child is offline and the lockless updaters, which hold
+ * rcu_read_lock() over the update, avoid racing with reparenting,
+ * so the read/set/add operations below are safe.
+ */
for (gen = 0; gen < MAX_NR_GENS; gen++) {
- long nr_pages = child_lrugen->nr_pages[gen][type][zone];
+ long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]);
int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0;
int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0;
@@ -4634,9 +4635,8 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
list_splice_tail_init(&child_lrugen->folios[gen][type][zone],
&parent_lrugen->folios[gen][type][zone]);
- WRITE_ONCE(child_lrugen->nr_pages[gen][type][zone], 0);
- WRITE_ONCE(parent_lrugen->nr_pages[gen][type][zone],
- parent_lrugen->nr_pages[gen][type][zone] + nr_pages);
+ atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0);
+ atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]);
if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) {
__update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages);
diff --git a/mm/vmscan.h b/mm/vmscan.h
index c2cea36289f3..6551f0a6691a 100644
--- a/mm/vmscan.h
+++ b/mm/vmscan.h
@@ -126,7 +126,7 @@ static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
unsigned long size = 0;
for (int zone = 0; zone < max_zone; zone++)
- size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+ size += max(atomic_long_read(&lrugen->nr_pages[gen][type][zone]), 0L);
return size;
}
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 04/17] mm/memcg: add folio-based lruvec live helper
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (2 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 03/17] mm/mglru: make generation page counters atomic Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 05/17] mm/mglru: skip the flags cmpxchg when nothing changes Kairui Song via B4 Relay
` (12 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
Add a helper that resolves a live lruvec for a folio under RCU
without taking the lruvec lock. It takes a folio directly so the
lruvec lookup happens inside the RCU read-side critical section,
which a lruvec-based interface cannot guarantee. The first user is
the lockless folio_inc_lru_refs() path later in the series.
Factor the ancestor walk out of lruvec_live_lock_irq() into
__lruvec_get_live(), so that both entry points share it, and spell
out why the walk is enough while only the RCU read lock is held:
reparenting of a memcg that starts dying waits for a grace period,
so it cannot race with the caller until the lock is released.
Move memcg_is_dying() up in memcontrol.h, next to the other memcg state
predicates, and use it for the walk rather than repeating the open coded
css_is_dying() test a few definitions away from where it is explained.
No functional change for existing callers.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/memcontrol.h | 71 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 56 insertions(+), 15 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 980358bc6f49..441a5d57e723 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -571,6 +571,11 @@ static inline bool mem_cgroup_disabled(void)
return !cgroup_subsys_enabled(memory_cgrp_subsys);
}
+static inline bool memcg_is_dying(struct mem_cgroup *memcg)
+{
+ return memcg ? css_is_dying(&memcg->css) : false;
+}
+
static inline void mem_cgroup_protection(const struct mem_cgroup *root,
const struct mem_cgroup *memcg,
unsigned long *min,
@@ -1349,6 +1354,11 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
return true;
}
+static inline bool memcg_is_dying(struct mem_cgroup *memcg)
+{
+ return false;
+}
+
static inline
unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec,
enum lru_list lru, int zone_idx)
@@ -1510,28 +1520,41 @@ static inline void lruvec_lock_irq(struct lruvec *lruvec)
spin_lock_irq(&lruvec->lru_lock);
}
-static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec)
+/*
+ * Reparenting moves a dying memcg's LRU state to its parent at any time,
+ * so anything added to that LRU afterwards would be lost. This helper
+ * walks up to the nearest live ancestor while the caller must hold the
+ * RCU read lock: reparenting of a memcg that starts dying waits for a
+ * grace period, so it cannot race with the caller until the lock is
+ * released.
+ */
+static inline struct lruvec *__lruvec_get_live(struct lruvec *lruvec)
{
#ifdef CONFIG_MEMCG
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
- rcu_read_lock();
-
/*
* The memcg can be NULL when the memory controller is disabled.
* Otherwise, the caller keeps the memcg owning @lruvec alive.
*/
- while (unlikely(memcg && css_is_dying(&memcg->css))) {
+ while (unlikely(memcg_is_dying(memcg))) {
memcg = parent_mem_cgroup(memcg);
lruvec = mem_cgroup_lruvec(memcg, pgdat);
}
+#endif
+ return lruvec;
+}
+static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec)
+{
+#ifdef CONFIG_MEMCG
+ rcu_read_lock();
+ lruvec = __lruvec_get_live(lruvec);
spin_lock_irq(&lruvec->lru_lock);
#else
lruvec_lock_irq(lruvec);
#endif
-
return lruvec;
}
@@ -1561,6 +1584,34 @@ static inline bool folio_matches_lruvec(const struct folio *folio,
lruvec_memcg(lruvec) == folio_memcg(folio);
}
+/**
+ * folio_lruvec_live_get - get a live lruvec for a folio under RCU
+ * @folio: the folio
+ *
+ * Same as folio_lruvec(), except that a folio whose memcg is dying is
+ * accounted to the nearest live ancestor, which is where reparenting
+ * would put it anyway.
+ *
+ * Return: the live lruvec, with rcu_read_lock() held. Pair it with
+ * folio_lruvec_live_put().
+ */
+static inline struct lruvec *folio_lruvec_live_get(struct folio *folio)
+{
+#ifdef CONFIG_MEMCG
+ rcu_read_lock();
+ return __lruvec_get_live(folio_lruvec(folio));
+#else
+ return folio_lruvec(folio);
+#endif
+}
+
+static inline void folio_lruvec_live_put(struct lruvec *lruvec)
+{
+#ifdef CONFIG_MEMCG
+ rcu_read_unlock();
+#endif
+}
+
/* Don't lock again iff page's lruvec locked */
static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio,
struct lruvec *locked_lruvec)
@@ -1826,11 +1877,6 @@ void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask);
void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg);
-static inline bool memcg_is_dying(struct mem_cgroup *memcg)
-{
- return memcg ? css_is_dying(&memcg->css) : false;
-}
-
#else
static inline bool mem_cgroup_kmem_disabled(void)
{
@@ -1901,11 +1947,6 @@ static inline void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg,
static inline void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg)
{
}
-
-static inline bool memcg_is_dying(struct mem_cgroup *memcg)
-{
- return false;
-}
#endif /* CONFIG_MEMCG */
#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN)
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 05/17] mm/mglru: skip the flags cmpxchg when nothing changes
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (3 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 04/17] mm/memcg: add folio-based lruvec live helper Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 06/17] mm/mglru: frequency guided workingset promotion (MGLRU-FG) Kairui Song via B4 Relay
` (11 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Ridong Chen <chenridong@xiaomi.com>
folio_update_gen() and folio_set_lru_refs() always end with a
try_cmpxchg() on folio->flags, even when the new value equals the old
one. The locked instruction still takes the struct page cacheline in
exclusive state.
That is common in practice: a folio mapped by many processes is visited
once per mm during a page table walk, and after the first visit it is
already in the youngest generation with refs cleared and PG_workingset
set. isolate_folio() and the eviction paths also call
folio_set_lru_refs(folio, 0) on folios whose refs are already zero.
Break out of the loop when the flags would not change. The update is
complete at the point of that read: the flags already hold the target
value, and a concurrent change to them is ordered after it. When the
cmpxchg would have succeeded, the value written is the same, so no new
state becomes reachable.
What the patch drops is the retry. A retry re-reads the flags and
recomputes new_flags from the newer value, so it overwrites a concurrent
update with what this CPU wanted. For folio_update_gen(), a walker that
read an older max_seq can lose the cmpxchg against a promotion done by
another CPU, and then write its older generation back on the retry.
folio_set_lru_refs() can likewise clear a refs increment that raced with
it. Breaking out keeps the newer state.
No caller relies on the full barrier that the locked instruction
implied on the skipped path: the eviction and isolation callers run
under the LRU lock, and the others (page table walks,
folio_mark_accessed(), lru_gen_clear_refs()) only care about the
resulting flags value, which is already in place. Note that
lru_gen_inc_refs() already returns without any atomic when refs is at
LRU_REFS_MAX and the folio is in the workingset.
Tested on an i7-13700HX with turbo disabled and the performance
governor. N processes map the same 2GiB tmpfs file (huge=never) and
read every page, then one aging pass is triggered on their memcg
through /sys/kernel/debug/lru_gen ("+ <memcg_id> 0 <max_seq> 200 1").
Time per aging pass in ms, median of 5 runs of 10 passes each, averaged
over 2 boots of each kernel:
processes base patched
1 32.9 31.7 -3.5%
8 195.1 160.2 -17.9%
32 755.8 592.2 -21.7%
64 1597.2 1260.6 -21.1%
With one process, every visit changes the generation, so there is
nothing to skip. Before the patch, perf annotate attributes 53-58% of
the walk_update_folio() samples to the lock cmpxchg in the inlined
folio_update_gen() in this test.
[kasong: folio_update_gen() is replaced by folio_inc_lru_refs_walk()
later in the series, which keeps the same early exit.]
Reviewed-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Ridong Chen <chenridong@xiaomi.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 2 ++
mm/vmscan.c | 2 ++
2 files changed, 4 insertions(+)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 46beded51c7d..2571b3f01226 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -233,6 +233,8 @@ static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
do {
new_flags = old_flags;
lru_set_refs_flags(&new_flags, refs);
+ if (new_flags == old_flags)
+ break;
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 8d14831cdf83..0f24b6ebc1eb 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3274,6 +3274,8 @@ static int folio_update_gen(struct folio *folio, int new_gen, int *type,
lru_set_gen_flags(&new_flags, new_gen);
lru_set_refs_flags(&new_flags, 0);
new_flags |= BIT(PG_workingset);
+ if (new_flags == old_flags)
+ break;
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
*type = folio_flags_is_file_lru(&old_flags);
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 06/17] mm/mglru: frequency guided workingset promotion (MGLRU-FG)
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (4 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 05/17] mm/mglru: skip the flags cmpxchg when nothing changes Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 07/17] mm/mglru: make folio lru referenced times count a generic API Kairui Song via B4 Relay
` (10 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
Complement MGLRU's eviction-time tier-PID protection with access-time
frequency-guided promotion. Introduce a unified set of helpers based on
the referenced (access) count of a folio.
Like LFU, this uses access frequency to distinguish hotter folios, but it
still reclaims by generation: the generation and the bounded refs together
form a "logical refs" count.
Accesses increase refs; refs crossing a threshold trigger promotion, which
moves the folio one generation newer and caps its refs at a lower
threshold, so a folio in a newer generation with lower refs is hotter than
a folio in an older generation with higher refs. Gen aging then lowers
the logical refs without updating every folio. Accesses from the page
table and from file descriptors are balanced with defined rules.
This limits protection from old hotness at close to no overhead, and
enhances MGLRU's ability to distinguish hotness levels over a much larger
range.
A folio's refs still map to a logarithmic tier just like before, but with
more formal bit definitions and more finely defined thresholds:
LRU_REFS_REFERENCED (1), LRU_REFS_WORKINGSET (2), LRU_REFS_PROTECTED (3),
and LRU_REFS_MAX (7). The PID controller can still operate coherently
with the new FG promotion, but the protection ratio is softer (2:3 -> 1:2)
since proactive promotion is mostly good enough. The setpoint now also
spans the cumulative mass of the tiers below the candidate instead of tier
0 alone, as the new thresholds are fallback tiers that a folio never falls
below.
By making use of the newly defined thresholds, simplify MGLRU's usage of
PG_workingset and PG_referenced: they become the low two bits of the refs
count, with the higher bits provided by LRU_REFS_MASK. This reduces
MGLRU's original refs count bit usage by one, since only one extra bit is
now needed to record a max referenced count of 7.
This doesn't affect classical LRU, and it addresses several shortcomings
of MGLRU's file-descriptor cache protection:
- Long feedback loop: protection is only activated after enough re-faults,
by which time the hot folios are already evicted, or no longer hot.
- Limited tier resolution: once the refs count saturated, MGLRU could no
longer distinguish hotter folios within a tier. PG_workingset also
forces a folio to stay on tier 3.
- Eviction hotness reversion: because PID protection activates upon
eviction and always targets the LRU tail, it tends to protect cold tail
folios at the expense of hotter head folios. Once the tail folios
consume the PID protection budget, head folios lose their protection.
- Additionally, the PID cannot distinguish the access time of folios that
share the same reference count, and there are only 4 tiers.
To achieve a frequency-guided framework, this commit introduces and
reworks the LRU_REFS related helpers and definitions. Most of the work
are done by the helpers below, check their inline comments for details.
They collect references from different access channels, following the
same rule but different context:
- folio_inc_lru_refs(): Called on any cache access (folio_mark_accessed)
or page table access. This is the main helper: it promotes folios
according to their access frequency. This performs lazy promotion.
- folio_inc_lru_refs_walk(): Used by the PTE walk path during aging and by
the batched look-around, both without the LRU lock. This also performs
lazy promotion.
- folio_inc_lru_refs_isolated(): Used by the rmap check before eviction.
The folio is isolated and hence this doesn't perform promotion by
itself; the folio will be added back to the right gen upon return
according to the refs and PG_active placement hint.
Refaults restore the saved refs, capped at LRU_REFS_PROTECTED.
Fault-driven refaults with prior references also set PG_active, keeping
the stronger protection for page table access.
Consequently the placement hint for newly added file folios also uses
LRU_REFS_WORKINGSET instead of PG_referenced, so a folio accessed only
once now starts in the oldest generation rather than the second oldest.
The previous one-generation grace was an artifact of the fault bias which
is well described in the cover letter of this series, and it depended on
LRU drain timing.
PG_workingset and PG_referenced are no longer independent flags under
MGLRU, so existing raw folio_test_*() callers need to be adjusted, which
is covered by the following commits.
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 107 +++++++++------
include/linux/mmzone.h | 126 +++++++++++------
kernel/bounds.c | 2 +-
mm/folio.c | 49 +------
mm/vmscan.c | 341 +++++++++++++++++++++++++++++++---------------
mm/workingset.c | 47 ++++---
6 files changed, 416 insertions(+), 256 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 2571b3f01226..f38dad1198fc 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -144,12 +144,14 @@ static inline int lru_hist_from_seq(unsigned long seq)
return seq % NR_HIST_GENS;
}
-static inline int lru_tier_from_refs(int refs, bool workingset)
+static inline int lru_tier_from_refs(unsigned int refs)
{
- VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));
-
- /* see the comment on MAX_NR_TIERS */
- return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs);
+ /* The refs range must map into the tier index space. */
+ BUILD_BUG_ON(BIT(LRU_TIER_MAX) < LRU_REFS_MAX);
+ VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
+ if (refs < LRU_REFS_WORKINGSET)
+ return 0;
+ return fls(refs - 1);
}
/**
@@ -187,21 +189,24 @@ static inline int lru_get_gen_flags(unsigned long flags)
* @flags: pointer to the folio flags
* @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive.
*
- * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the
- * remaining refs. For classical LRU the extra bits are not used, so it can
- * also be seen as the refs count never exceeds 1. In both cases, refs == 1
- * means PG_referenced is set and the extra bits are zero, and refs == 0 means
- * PG_referenced and the extra bits are all unset.
+ * For MGLRU, PG_referenced, PG_workingset are used as the lower two bits of
+ * refs counter, and extra bits hold the remaining higher bits. For classical
+ * LRU the extra bits are not used, and the two flags have no direct
+ * relationship with each other, but this helper can still be used to sync
+ * them. For both cases, refs == 0 means these two flags and the extra bits
+ * are all unset. And refs == 1 / 2 / 3 means PG_referenced and PG_workingset
+ * encode the low two bits in binary order.
*/
static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs)
{
VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
- BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1);
-
+ BUILD_BUG_ON(LRU_REFS_MASK & (BIT(PG_referenced) | BIT(PG_workingset)));
*flags &= ~LRU_REFS_FLAGS;
- if (!refs)
- return;
- *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF));
+ if (refs & BIT(0))
+ *flags |= BIT(PG_referenced);
+ if (refs & BIT(1))
+ *flags |= BIT(PG_workingset);
+ *flags |= ((unsigned long)refs >> 2) << LRU_REFS_PGOFF;
}
/**
@@ -210,22 +215,31 @@ static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs)
*
* Reads the LRU referenced count set by lru_set_refs_flags().
*/
-static inline int lru_get_refs_flags(unsigned long flags)
+static inline unsigned int lru_get_refs_flags(unsigned long flags)
{
- if (!(flags & BIT(PG_referenced)))
- return 0;
- /*
- * Return the total number of accesses including PG_referenced. Also see
- * the comment on LRU_REFS_FLAGS.
- */
- return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1;
+ unsigned int refs;
+
+ /* Decode the per-folio reference state. */
+ refs = !!(flags & BIT(PG_referenced));
+ refs |= !!(flags & BIT(PG_workingset)) << 1;
+ refs |= ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) << 2;
+ return refs;
}
-static inline int folio_lru_refs(const struct folio *folio)
+static inline unsigned int folio_lru_refs(const struct folio *folio)
{
return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0)));
}
+/*
+ * Referenced at least twice. Unlike lru_refs_is_active() this is not
+ * bounds-checked: folio_inc_lru_refs() evaluates refs unclamped.
+ */
+static inline bool lru_refs_is_workingset(unsigned int refs)
+{
+ return refs >= LRU_REFS_WORKINGSET;
+}
+
static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
@@ -238,14 +252,26 @@ static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
}
+void folio_inc_lru_refs(struct folio *folio, unsigned int flags);
+
static inline int folio_lru_gen(const struct folio *folio)
{
return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
}
+static inline bool lru_gen_is_old(struct lruvec *lruvec, int type, int gen)
+{
+ unsigned long min_seq = READ_ONCE(lruvec->lrugen.min_seq[type]);
+
+ /* The MIN_NR_GENS oldest generations, which eviction scans next */
+ BUILD_BUG_ON(MIN_NR_GENS != 2);
+ return gen == lru_gen_from_seq(min_seq) ||
+ gen == lru_gen_from_seq(min_seq + 1);
+}
+
static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
{
- unsigned long max_seq = lruvec->lrugen.max_seq;
+ unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq);
VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
@@ -302,23 +328,24 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
bool reclaiming)
{
int age;
+ bool workingset = lru_refs_is_workingset(folio_lru_refs(folio));
int type = folio_is_file_lru(folio);
const struct lru_gen_folio *lrugen = &lruvec->lrugen;
/*
- * +-----------------------------------+-----------------------------------+
- * | Accessed through page tables and | Accessed through file descriptors |
- * | promoted by folio_update_gen() | and protected by folio_inc_gen() |
- * +-----------------------------------+-----------------------------------+
- * | PG_active (set while isolated) | |
- * +-----------------+-----------------+-----------------+-----------------+
- * | PG_workingset | PG_referenced | PG_workingset | LRU_REFS_FLAGS |
- * +-----------------------------------+-----------------------------------+
- * |<---------- MIN_NR_GENS ---------->| |
- * |<---------------------------- MAX_NR_GENS ---------------------------->|
+ * +------------------------------------------+------------------------------------------+
+ * | Accessed through page tables and | Accessed through file descriptors |
+ * | promoted by folio_inc_lru_refs_walk() | protected by folio_inc_lru_refs/inc_gen |
+ * +------------------------------------------+------------------------------------------+
+ * | PG_active (set at isolation or refault) | |
+ * +--------------------+---------------------+--------------------+---------------------+
+ * | LRU_REFS_MAX | LRU_REFS_WORKINGSET | LRU_REFS_MAX | LRU_REFS_WORKINGSET |
+ * +------------------------------------------+------------------------------------------+
+ * |<-------------- MIN_NR_GENS ------------->| |
+ * |<----------------------------------- MAX_NR_GENS ----------------------------------->|
*/
if (folio_test_active(folio))
- age = MIN_NR_GENS - folio_test_workingset(folio);
+ age = MIN_NR_GENS - workingset;
else if (reclaiming)
age = MAX_NR_GENS;
else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) ||
@@ -326,7 +353,7 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
(folio_test_dirty(folio) || folio_test_writeback(folio))))
age = MIN_NR_GENS;
else
- age = MAX_NR_GENS - (folio_test_workingset(folio) || folio_test_referenced(folio));
+ age = MAX_NR_GENS - workingset;
return max(READ_ONCE(lrugen->max_seq) - age + 1, READ_ONCE(lrugen->min_seq[type]));
}
@@ -341,6 +368,7 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
int zone = folio_zonenum(folio);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH - 1) != MAX_NR_GENS);
VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
if (folio_test_unevictable(folio) || !lrugen->enabled)
@@ -408,7 +436,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
*/
static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
{
- BUILD_BUG_ON(LRU_REFS_MASK & BIT(PG_referenced));
folio_set_lru_refs(new, folio_lru_refs(old));
}
#else /* !CONFIG_LRU_GEN */
@@ -444,6 +471,10 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
return false;
}
+static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
+{
+}
+
static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
{
if (folio_test_referenced(old))
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 4c4aad95b0b6..5470dd708e44 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -453,57 +453,103 @@ enum lruvec_flags {
#define MAX_NR_GENS 4U
/*
- * Each generation is divided into multiple tiers. A folio accessed N times
- * through file descriptors is in tier order_base_2(N). A folio in the first
- * tier (N=0,1) is marked by PG_referenced unless it was faulted in through page
- * tables or read ahead. A folio in the last tier (MAX_NR_TIERS-1) is marked by
- * PG_workingset. A folio in any other tier (1<N<5) between the first and last
- * is marked by additional bits of LRU_REFS_WIDTH in folio->flags.
+ * Each generation is divided into multiple tiers. A folio's referenced
+ * count maps to a tier as shown below (with one extra refs bit):
*
- * In contrast to moving across generations which requires the LRU lock, moving
- * across tiers only involves atomic operations on folio->flags and therefore
- * has a negligible cost in the buffered access path. In the eviction path,
- * comparisons of refaulted/(evicted+protected) from the first tier and the rest
- * infer whether folios accessed multiple times through file descriptors are
- * statistically hot and thus worth protecting.
+ * MGLRU (frequency guidance)
+ * Refs Tier |- Refs: observed accesses, capped on promotion.
+ * 0 0 |- Mostly cold pages, readahead, etc. [1]
+ * 1 0 |= LRU_REFS_REFERENCED: Used at least once. [2]
+ * -WORKINGSET-+|- Workingset floor, unless explicitly reset. [3]
+ * 2 1<-+|= LRU_REFS_WORKINGSET: Classical workingset, accessed twice, protected. [4]
+ * 3 2 |- LRU_REFS_PROTECTED: Protected workingset, promoted pages capped at here. [5]
+ * 4 2 |
+ * 5 3 |- The tier here is MAX_NR_TIERS - 1
+ * 6 3 |
+ * 7 3 |= LRU_REFS_MAX: Promotion candidate. [6]
+ * -PROMOTION->-/
*
- * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
- * number of categories of the active/inactive LRU when keeping track of
- * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
- * folio->flags, masked by LRU_REFS_MASK.
+ * Ideally each tier holds folios of similar access patterns: lower tiers
+ * are less important and evicted faster. A page's reference count and
+ * tier are capped when it changes generation, preventing it from
+ * dominating the new generation based on old-generation access history.
+ * The generation and refs together represent logical refs: hotness relative
+ * to the aging window. Promotion moves the folio forward and caps refs;
+ * gen aging then lowers logical refs even without updating the stored count.
+ * This avoids a separate full-cache aging walk to reduce old hotness.
+ *
+ * Refs tracks accesses from two sources: page table (lazily collected by
+ * the page table aging walk or rmap eviction lookup) and file descriptors
+ * (by folio_mark_accessed). Page table accesses promote more readily
+ * because the accessed bit is sticky (undercounts repeated accesses),
+ * passively collected, and page faults are generally more important as
+ * userspace does not expect a memory access to block on reclaim. Both
+ * access types increment refs by one; the result is capped at
+ * LRU_REFS_PROTECTED on promotion or deferral, or LRU_REFS_MAX otherwise.
+ *
+ * 1. Tier is fls(N-1) for N > 1, 0 otherwise. Folios with zero
+ * accesses (refs == 0) are generally cold, e.g. readahead folios.
+ *
+ * 2. Folios accessed once stay on tier 0: one-time usage does not
+ * qualify for tier protection. A second access can advance the folio,
+ * aligning with classical LRU's use-twice threshold. A page table
+ * access that brings refs to LRU_REFS_WORKINGSET promotes to the
+ * latest gen. A file access at that refs count only defers eviction
+ * from the tail gens of its type.
+ *
+ * 3. Folios accessed at least twice are considered workingset. This
+ * mostly aligns with classical LRU: at least one I/O is saved by
+ * keeping them in memory. Unless explicitly reset, these folios stay
+ * at or above tier 1 (the workingset floor), so tier 0 stays a clean tier
+ * for cold cache while tier 1 serves as the fallback line for
+ * actually reused or historically hot folios.
+ *
+ * 4. Starting from tier 1, PID protection sacrifices lower tiers to
+ * protect higher tiers by comparing refault rates for long-term
+ * accuracy, and caps higher refs to LRU_REFS_WORKINGSET. Rejected
+ * folios also keep at most that count on putback, allowing further
+ * accesses to distinguish their hotness again.
+ *
+ * 5. Pages beyond the ordinary workingset tier form new tiers for the
+ * PID controller to protect differently. Folios at or above this
+ * level are capped at LRU_REFS_PROTECTED on promotion or deferral,
+ * and at LRU_REFS_WORKINGSET under PID protection in the oldest
+ * generation, where they represent a historical workingset.
+ *
+ * 6. A further cache access at LRU_REFS_MAX advances an older folio by one
+ * generation, with refs capped to LRU_REFS_PROTECTED. Folios already in
+ * the newest generation just saturate their refs.
+ *
+ * Tiering uses PG_referenced and PG_workingset as the lower two bits,
+ * and the bits masked by LRU_REFS_MASK as the higher bits, so the refs
+ * count ranges from 0 to LRU_REFS_MAX.
+ *
+ * Cold hints can reset refs with lru_gen_clear_refs(). Refault restores
+ * its referenced count, capped at LRU_REFS_PROTECTED, which aligns with
+ * promotion. Fault-driven refaults of previous workingset folios set
+ * PG_active, so they are placed in the latest gen.
+ *
+ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice
+ * the number of categories of the active/inactive LRU.
*/
#define MAX_NR_TIERS 4U
#define LRU_TIER_MIN 0U
#define LRU_TIER_MAX (MAX_NR_TIERS - 1)
+/* Access source flags for folio_inc_lru_refs() */
+#define LRU_REF_MAPPED 0x1U
+#define LRU_REF_EXEC 0x2U
+
+#define LRU_REFS_REFERENCED 0x1
+#define LRU_REFS_WORKINGSET 0x2
+#define LRU_REFS_PROTECTED 0x3
+
#ifndef __GENERATING_BOUNDS_H
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
-#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH)
-
-/*
- * For folios accessed multiple times through file descriptors,
- * lru_gen_inc_refs() sets additional bits of LRU_REFS_WIDTH in folio->flags
- * after PG_referenced, then PG_workingset after LRU_REFS_WIDTH. After all its
- * bits are set, i.e., LRU_REFS_FLAGS|BIT(PG_workingset), a folio is lazily
- * promoted into the second oldest generation in the eviction path. And when
- * folio_inc_gen() does that, it clears LRU_REFS_FLAGS so that
- * lru_gen_inc_refs() can start over. Note that for this case, LRU_REFS_MASK is
- * only valid when PG_referenced is set.
- *
- * For folios accessed multiple times through page tables, folio_update_gen()
- * from a page table walk or lru_gen_set_refs() from a rmap walk sets
- * PG_referenced after the accessed bit is cleared for the first time.
- * Thereafter, those two paths set PG_workingset and promote folios to the
- * youngest generation. Like folio_inc_gen(), folio_update_gen() also clears
- * PG_referenced. Note that for this case, LRU_REFS_MASK is not used.
- *
- * For both cases above, after PG_workingset is set on a folio, it remains until
- * this folio is either reclaimed, or "deactivated" by lru_gen_clear_refs(). It
- * can be set again if lru_gen_test_recent() returns true upon a refault.
- */
-#define LRU_REFS_FLAGS (LRU_REFS_MASK | BIT(PG_referenced))
+#define LRU_REFS_FLAGS (LRU_REFS_MASK | BIT(PG_referenced) | BIT(PG_workingset))
+#define LRU_REFS_MAX (BIT(LRU_REFS_WIDTH + 2) - 1)
struct lruvec;
struct page_vma_mapped_walk;
diff --git a/kernel/bounds.c b/kernel/bounds.c
index 02b619eb6106..06a034713b5d 100644
--- a/kernel/bounds.c
+++ b/kernel/bounds.c
@@ -25,7 +25,7 @@ int main(void)
DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
#ifdef CONFIG_LRU_GEN
DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
- DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2);
+ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 3);
#else
DEFINE(LRU_GEN_WIDTH, 0);
DEFINE(__LRU_REFS_WIDTH, 0);
diff --git a/mm/folio.c b/mm/folio.c
index ea3adb6a89d7..06c9cde834f7 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -352,32 +352,6 @@ static void __lru_cache_activate_folio(struct folio *folio)
#ifdef CONFIG_LRU_GEN
-static void lru_gen_inc_refs(struct folio *folio)
-{
- unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
- int refs;
-
- if (folio_test_unevictable(folio))
- return;
-
- /* see the comment on LRU_REFS_FLAGS */
- if (!folio_lru_refs(folio)) {
- folio_set_lru_refs(folio, 1);
- return;
- }
-
- do {
- new_flags = old_flags;
- refs = lru_get_refs_flags(old_flags);
- if (refs == LRU_REFS_MAX) {
- if (!folio_test_workingset(folio))
- folio_set_workingset(folio);
- return;
- }
- lru_set_refs_flags(&new_flags, refs + 1);
- } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
-}
-
static bool lru_gen_clear_refs(struct folio *folio)
{
int gen = folio_lru_gen(folio);
@@ -388,7 +362,6 @@ static bool lru_gen_clear_refs(struct folio *folio)
return true;
folio_set_lru_refs(folio, 0);
- folio_clear_workingset(folio);
rcu_read_lock();
seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
@@ -399,10 +372,6 @@ static bool lru_gen_clear_refs(struct folio *folio)
#else /* !CONFIG_LRU_GEN */
-static void lru_gen_inc_refs(struct folio *folio)
-{
-}
-
static bool lru_gen_clear_refs(struct folio *folio)
{
return false;
@@ -428,7 +397,8 @@ void folio_mark_accessed(struct folio *folio)
if (folio_test_dropbehind(folio))
return;
if (lru_gen_enabled()) {
- lru_gen_inc_refs(folio);
+ if (!folio_test_unevictable(folio))
+ folio_inc_lru_refs(folio, 0);
return;
}
@@ -474,21 +444,6 @@ void folio_add_lru(struct folio *folio)
folio_test_unevictable(folio), folio);
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
- /*
- * For refaulted workingset folios, set PG_active so they
- * can be added to active generations.
- * For prefaulted file folios, folio_mark_accessed() sets
- * PG_referenced so lru_gen_folio_seq() places them into
- * the second oldest generation.
- */
- if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
- lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) {
- if (folio_test_workingset(folio))
- folio_set_active(folio);
- else if (!folio_test_referenced(folio))
- folio_mark_accessed(folio);
- }
-
folio_batch_add_and_move(folio, lru_add);
}
EXPORT_SYMBOL(folio_add_lru);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 0f24b6ebc1eb..a02504c5a7d7 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -818,38 +818,204 @@ enum folio_references {
};
#ifdef CONFIG_LRU_GEN
+/******************************************************************************
+ * Referenced count feedback
+ ******************************************************************************/
+
/*
- * Only used on a mapped folio in the eviction (rmap walk) path, where promotion
- * needs to be done by taking the folio off the LRU list and then adding it back
- * with PG_active set. In contrast, the aging (page table walk) path uses
- * folio_update_gen().
+ * The folio_inc_lru_refs{_*} helpers below collect the referenced info
+ * (hotness) from the page table walker, the rmap walk upon eviction, the
+ * rmap look-around, and file descriptors (folio_mark_accessed).
+ *
+ * Each access adds one to the refs count, capped at LRU_REFS_MAX. A
+ * fresh folio's refs go from 0 to LRU_REFS_REFERENCED to
+ * LRU_REFS_WORKINGSET, and so on.
+ *
+ * Mapped access (PT, LRU_REF_MAPPED) has a higher priority since the
+ * accessed bit is sticky (it under-counts repeated accesses) and a fault
+ * could be more costly than FD I/O. So, a page table access always
+ * promotes a folio on an expiring gen by at least one gen. And starting
+ * from the second access (refs == LRU_REFS_WORKINGSET), it moves the folio
+ * to the newest gen. Executable file folios promote on their first access
+ * to avoid I/O thrashing. The refs are increased at a normal pace though.
+ *
+ * File descriptor accesses (FD) are a precise access method. So, only
+ * since the second access (refs == LRU_REFS_WORKINGSET), every access
+ * will promote a folio if the folio is on an expiring gen (close to
+ * min_seq). If the folio is not on an expiring gen, it will be promoted
+ * when its refs exceed LRU_REFS_MAX; the newest gen just saturates.
+ *
+ * Promotion, regarding the source, always caps the refs at
+ * LRU_REFS_PROTECTED, so folios in a newer gen start at or below the
+ * protected tier, and folios with lower refs in a newer gen are logically
+ * hotter than a folio in an older gen with higher refs.
+ *
+ * A folio with refs at or above LRU_REFS_WORKINGSET is considered a
+ * workingset, and the tiers starting there engage PID protection.
+ * PID-protected folios (folio_inc_gen) are capped at
+ * LRU_REFS_WORKINGSET independently to make full use of tiering.
*/
-static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
-{
- /* see the comment on LRU_REFS_FLAGS */
- if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
- /* Activate file-backed executable folios after first usage. */
- if (is_exec_file_folio(folio, vma_flags)) {
- folio_set_workingset(folio);
- folio_set_lru_refs(folio, 0);
- return true;
+
+/**
+ * folio_inc_lru_refs - Update the folio's LRU refs and lazily promote it.
+ * @folio: The folio being accessed.
+ * @flags: The type of ref: FD access, or mapped access optionally with
+ * executable access.
+ *
+ * Used by both cache access (flags == 0) and page table access (flags ==
+ * LRU_REF_MAPPED, optionally with LRU_REF_EXEC).
+ * Context: The caller must keep the folio alive by a reference or the page
+ * table lock of a mapped folio.
+ */
+void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
+{
+ int max_gen;
+ int type, refs, old_gen, gen;
+ unsigned long new_flags, old_flags, max_seq;
+ struct lru_gen_folio *lrugen;
+ struct lruvec *lruvec = NULL;
+
+ type = folio_is_file_lru(folio);
+ old_flags = READ_ONCE(*folio_flags(folio, 0));
+ do {
+ new_flags = old_flags;
+ old_gen = lru_get_gen_flags(old_flags);
+ refs = lru_get_refs_flags(old_flags) + 1;
+ gen = old_gen;
+ if (old_gen < 0)
+ goto out;
+ /*
+ * Lock the lruvec if the folio is on-list. We are already
+ * doing lazy promotion so in theory we don't need this,
+ * but for now, concurrent aging would still corrupt the
+ * size counters. This is a temporary limitation and
+ * will be lifted very soon, so the lock here is not a
+ * performance concern.
+ */
+ if (!lruvec) {
+ lruvec = lruvec_live_lock_irq(folio_lruvec(folio));
+ lrugen = &lruvec->lrugen;
}
+ max_seq = READ_ONCE(lrugen->max_seq);
+ max_gen = lru_gen_from_seq(max_seq);
+ if (old_gen == max_gen)
+ goto out;
- folio_set_lru_refs(folio, 1);
- return false;
- }
+ if (flags & (LRU_REF_MAPPED | LRU_REF_EXEC)) {
+ /* Promote second page table access or executable */
+ if (refs > LRU_REFS_REFERENCED || flags & LRU_REF_EXEC)
+ gen = max_gen;
+ /* First access only defers eviction of an old generation */
+ else if (lru_gen_is_old(lruvec, type, old_gen))
+ gen = (old_gen + 1) % MAX_NR_GENS;
+ refs = min(refs, LRU_REFS_PROTECTED);
+ } else if (refs > LRU_REFS_MAX) {
+ /* LRU refs counting overflow, bump the gen */
+ gen = (old_gen + 1) % MAX_NR_GENS;
+ refs = LRU_REFS_PROTECTED;
+ } else if (lru_gen_is_old(lruvec, type, old_gen) &&
+ lru_refs_is_workingset(refs)) {
+ /* Defer eviction of just accessed workingset */
+ gen = (old_gen + 1) % MAX_NR_GENS;
+ refs = min(refs, LRU_REFS_PROTECTED);
+ }
+out:
+ refs = min(refs, LRU_REFS_MAX);
+ lru_set_refs_flags(&new_flags, refs);
+ if (gen != old_gen)
+ lru_set_gen_flags(&new_flags, gen);
+ if (new_flags == old_flags)
+ break;
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
- /* Promote on second access */
- if (folio_lru_refs(folio) > 1) {
- folio_set_workingset(folio);
- folio_set_lru_refs(folio, 0);
- } else {
- folio_mark_accessed(folio);
- }
- return true;
+ if (gen != old_gen)
+ lru_gen_update_size(lruvec, folio, old_gen, gen);
+ if (lruvec)
+ lruvec_unlock_irq(lruvec);
+}
+
+/**
+ * folio_inc_lru_refs_walk - Update the folio's LRU refs and lazily promote it.
+ * @folio: The folio.
+ * @lruvec: The lruvec the folio belongs to.
+ * @vma_flags: The flags of the VMA being walked.
+ * @new_gen: The gen the folio is moved to; may be unchanged.
+ * @type: Returns the folio's type: file or anon.
+ *
+ * Similar to folio_inc_lru_refs(), but for the page table walker and the
+ * look-around. The LRU size accounting is batched by the caller for
+ * better performance.
+ *
+ * Return: The old generation the folio belongs to if the folio is still
+ * on the LRU and not in the newest generation, or -1 otherwise. -1 means
+ * the folio should not be promoted and @new_gen is not set.
+ */
+static int folio_inc_lru_refs_walk(struct folio *folio, struct lruvec *lruvec,
+ const vma_flags_t *vma_flags,
+ int *new_gen, int *type)
+{
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+ int refs, gen, max_gen, ret;
+ unsigned long max_seq;
+
+ do {
+ max_seq = READ_ONCE(lruvec->lrugen.max_seq);
+ max_gen = lru_gen_from_seq(max_seq);
+ gen = lru_get_gen_flags(old_flags);
+ refs = lru_get_refs_flags(old_flags) + 1;
+ *type = folio_flags_is_file_lru(&old_flags);
+ new_flags = old_flags;
+
+ if (gen >= 0 && gen != max_gen) {
+ ret = gen;
+ /* Promote second page table access or executable */
+ if (lru_refs_is_workingset(refs) || is_exec_file_folio(folio, vma_flags))
+ *new_gen = max_gen;
+ /* First access only defers eviction of an old generation */
+ else if (lru_gen_is_old(lruvec, *type, gen))
+ *new_gen = (gen + 1) % MAX_NR_GENS;
+ else
+ *new_gen = gen;
+ lru_set_gen_flags(&new_flags, *new_gen);
+ lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_PROTECTED));
+ } else {
+ ret = -1;
+ lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_MAX));
+ }
+ if (new_flags == old_flags)
+ break;
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+
+ return ret;
+}
+
+/**
+ * folio_inc_lru_refs_isolated - Update the folio's LRU refs
+ * @folio: The folio.
+ * @vma_flags: The flags of all the VMAs that referenced the folio.
+ *
+ * Similar to folio_inc_lru_refs(), but the folio is isolated. Only used
+ * on mapped folios upon the final eviction RMAP walk. Since the folio is
+ * isolated, folio promotion needs to be done by the caller.
+ *
+ * Return: true if the caller should activate the folio, false to put it
+ * back in an older gen and wait for a second chance.
+ */
+static bool folio_inc_lru_refs_isolated(struct folio *folio, const vma_flags_t *vma_flags)
+{
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+ unsigned int refs;
+
+ do {
+ new_flags = old_flags;
+ refs = lru_get_refs_flags(old_flags) + 1;
+ lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_PROTECTED));
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+ /* Promote second page table access or executable */
+ return lru_refs_is_workingset(refs) || is_exec_file_folio(folio, vma_flags);
}
#else
-static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
+static bool folio_inc_lru_refs_isolated(struct folio *folio, const vma_flags_t *vma_flags)
{
return false;
}
@@ -884,7 +1050,8 @@ static enum folio_references folio_check_references(struct folio *folio,
if (!referenced_ptes)
return FOLIOREF_RECLAIM;
- return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
+ return folio_inc_lru_refs_isolated(folio, &vma_flags) ?
+ FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
}
referenced_folio = folio_test_clear_referenced(folio);
@@ -3155,9 +3322,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq)
* P term over the generations previously evicted, using the smoothing factor
* 1/2; the D term isn't supported.
*
- * The setpoint (SP) is always the first tier of one type; the process variable
- * (PV) is either any tier of the other type or any other tier of the same
- * type.
+ * To select a tier to protect, the setpoint (SP) is the cumulative tiers below
+ * it and the process variable (PV) is the tier itself. Anon/file scan budgets
+ * are handled separately by lru_gen_balance_scan().
*
* The error is the difference between the SP and the PV; the correction is to
* turn off protection when SP>PV or turn on protection when SP<PV.
@@ -3245,52 +3412,15 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
* the aging
******************************************************************************/
-/* promote pages accessed through page tables */
-static int folio_update_gen(struct folio *folio, int new_gen, int *type,
- const vma_flags_t *vma_flags)
-{
- unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
- int old_gen;
-
- /*
- * See the comment on LRU_REFS_FLAGS, and activate file-backed
- * executable folios after first usage to avoid typical IO
- * thrashing from reclaiming.
- */
- if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
- !is_exec_file_folio(folio, vma_flags)) {
- folio_set_lru_refs(folio, 1);
- return -1;
- }
-
- do {
- old_gen = lru_get_gen_flags(old_flags);
- new_flags = old_flags;
-
- /* lru_gen_del_folio() has isolated this page? */
- if (old_gen < 0)
- break;
-
- lru_set_gen_flags(&new_flags, new_gen);
- lru_set_refs_flags(&new_flags, 0);
- new_flags |= BIT(PG_workingset);
- if (new_flags == old_flags)
- break;
- } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
-
- *type = folio_flags_is_file_lru(&old_flags);
- return old_gen;
-}
-
static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
- int new_gen;
+ int refs, new_gen;
do {
new_gen = lru_get_gen_flags(old_flags);
- /* folio_update_gen() has promoted this page? */
+ /* folio_inc_lru_refs() has promoted this page? */
if (new_gen >= 0 && new_gen != old_gen) {
if (increased)
*increased = false;
@@ -3299,9 +3429,9 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
new_flags = old_flags;
new_gen = (old_gen + 1) % MAX_NR_GENS;
-
+ refs = lru_get_refs_flags(old_flags);
lru_set_gen_flags(&new_flags, new_gen);
- lru_set_refs_flags(&new_flags, 0);
+ lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_WORKINGSET));
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
if (increased)
@@ -3309,7 +3439,7 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
return new_gen;
}
-/* protect pages accessed multiple times through file descriptors */
+/* protect pages selected by the tier PID controller */
static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen)
{
int new_gen;
@@ -3318,6 +3448,7 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen
new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
if (gen_increased)
lru_gen_update_size(lruvec, folio, old_gen, new_gen);
+
return new_gen;
}
@@ -3512,25 +3643,25 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struc
struct lruvec *lruvec, struct folio *folio, bool dirty)
{
int new_gen, old_gen, type;
+ unsigned int flags = LRU_REF_MAPPED;
if (!folio)
return;
- new_gen = lru_gen_from_seq(READ_ONCE(lruvec->lrugen.max_seq));
-
if (dirty && !folio_test_dirty(folio) &&
!(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
!folio_test_swapcache(folio)))
folio_mark_dirty(folio);
if (walk) {
- old_gen = folio_update_gen(folio, new_gen, &type, &vma->flags);
+ old_gen = folio_inc_lru_refs_walk(folio, lruvec, &vma->flags,
+ &new_gen, &type);
if (old_gen >= 0 && old_gen != new_gen)
update_batch_size(walk, folio, old_gen, new_gen, type);
- } else if (lru_gen_set_refs(folio, &vma->flags)) {
- old_gen = folio_lru_gen(folio);
- if (old_gen >= 0 && old_gen != new_gen)
- folio_activate(folio);
+ } else {
+ if (is_exec_file_folio(folio, &vma->flags))
+ flags |= LRU_REF_EXEC;
+ folio_inc_lru_refs(folio, flags);
}
}
@@ -3936,8 +4067,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
while (pos != head) {
struct folio *folio = list_entry(pos, struct folio, lru);
long nr_pages = folio_nr_pages(folio);
- int refs = folio_lru_refs(folio);
- bool workingset = folio_test_workingset(folio);
+ unsigned int refs = folio_lru_refs(folio);
+ int tier = lru_tier_from_refs(refs);
bool gen_increased;
VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
@@ -3957,13 +4088,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
delta += nr_pages;
batch_end = &folio->lru;
- /* don't count the workingset being lazily promoted */
- if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
- int tier = lru_tier_from_refs(refs, workingset);
-
- WRITE_ONCE(lrugen->protected[hist][type][tier],
- lrugen->protected[hist][type][tier] + nr_pages);
- }
+ WRITE_ONCE(lrugen->protected[hist][type][tier],
+ lrugen->protected[hist][type][tier] + nr_pages);
} else {
flush_lru_batch(head, &batch_end, target_list);
list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]);
@@ -4688,9 +4814,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
- int refs = folio_lru_refs(folio);
- bool workingset = folio_test_workingset(folio);
- int tier = lru_tier_from_refs(refs, workingset);
+ unsigned int refs = folio_lru_refs(folio);
+ int tier = lru_tier_from_refs(refs);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
@@ -4715,17 +4840,15 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
return false;
/* protected */
- if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
+ if (tier > tier_idx) {
+ int hist = lru_hist_from_seq(lrugen->min_seq[type]);
+
gen = folio_inc_gen(lruvec, folio, min_gen);
list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
- /* don't count the workingset being lazily promoted */
- if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
- int hist = lru_hist_from_seq(lrugen->min_seq[type]);
+ WRITE_ONCE(lrugen->protected[hist][type][tier],
+ lrugen->protected[hist][type][tier] + delta);
- WRITE_ONCE(lrugen->protected[hist][type][tier],
- lrugen->protected[hist][type][tier] + delta);
- }
return true;
}
@@ -4746,10 +4869,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
return false;
}
- /* see the comment on LRU_REFS_FLAGS */
- if (!folio_test_referenced(folio))
- folio_set_lru_refs(folio, 0);
-
success = lru_gen_del_folio(lruvec, folio, true);
VM_WARN_ON_ONCE_FOLIO(!success, folio);
@@ -4861,13 +4980,14 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
struct ctrl_pos sp, pv;
/*
- * To leave a margin for fluctuations, use a larger gain factor (2:3).
- * This value is chosen because any other tier would have at least twice
- * as many refaults as the first tier.
+ * To leave a margin for fluctuations, use a larger gain factor (1:2).
+ * Stop at the first tier whose refault rate is clearly worse than
+ * that of the cumulative mass of the tiers below it; the PID
+ * protects the tiers above it.
*/
- read_ctrl_pos(lruvec, type, LRU_TIER_MIN, LRU_TIER_MIN, 2, &sp);
for (tier = LRU_TIER_MIN + 1; tier <= LRU_TIER_MAX; tier++) {
- read_ctrl_pos(lruvec, type, tier, tier, 3, &pv);
+ read_ctrl_pos(lruvec, type, LRU_TIER_MIN, tier - 1, 1, &sp);
+ read_ctrl_pos(lruvec, type, tier, tier, 2, &pv);
if (!positive_ctrl_err(&sp, &pv))
break;
}
@@ -5045,12 +5165,11 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
}
/*
- * See the comments on LRU_REFS_FLAGS.
- *
- * The rejected folios are never added to the oldest generation,
- * so this effectively promotes them by at least one generation.
+ * Putback keeps rejected folios beyond scanned_seq when possible.
+ * PG_active and refs are placement hints; cap refs to retain
+ * workingset history without carrying a high tier into the new gen.
*/
- folio_set_lru_refs(folio, 0);
+ folio_set_lru_refs(folio, min(folio_lru_refs(folio), LRU_REFS_WORKINGSET));
}
move_folios_to_lru(&list, scanned_seq, false);
diff --git a/mm/workingset.c b/mm/workingset.c
index 1504f91cdca5..34e60b5354a3 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -188,6 +188,12 @@
#define EVICTION_SHIFT_ANON (EVICTION_SHIFT + SWAP_COUNT_SHIFT)
#define EVICTION_MASK (~0UL >> EVICTION_SHIFT)
#define EVICTION_MASK_ANON (~0UL >> EVICTION_SHIFT_ANON)
+/*
+ * LRU refs uses LRU_REFS_WIDTH + 2 bits, the 2 bits being PG_workingset
+ * and PG_referenced. The lowest bit is recorded in the workingset field
+ * of the shadow entry (to reuse pack_shadow()).
+ */
+#define LRU_REFS_BITS ((LRU_REFS_WIDTH + 2) - 1)
/*
* Eviction timestamps need to be able to cover the full range of
@@ -241,14 +247,13 @@ static void *lru_gen_eviction(struct folio *folio)
struct lru_gen_folio *lrugen;
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
- int refs = folio_lru_refs(folio);
- bool workingset = folio_test_workingset(folio);
- int tier = lru_tier_from_refs(refs, workingset);
+ unsigned int refs = folio_lru_refs(folio);
+ int tier = lru_tier_from_refs(refs);
struct mem_cgroup *memcg;
struct pglist_data *pgdat = folio_pgdat(folio);
unsigned short memcg_id;
- BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH >
+ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_BITS >
BITS_PER_LONG - max(EVICTION_SHIFT, EVICTION_SHIFT_ANON));
rcu_read_lock();
@@ -256,14 +261,14 @@ static void *lru_gen_eviction(struct folio *folio)
lruvec = mem_cgroup_lruvec(memcg, pgdat);
lrugen = &lruvec->lrugen;
min_seq = READ_ONCE(lrugen->min_seq[type]);
- token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
+ token = (min_seq << LRU_REFS_BITS) | refs >> 1;
hist = lru_hist_from_seq(min_seq);
atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
memcg_id = mem_cgroup_private_id(memcg);
rcu_read_unlock();
- return pack_shadow(memcg_id, pgdat, token, workingset, type);
+ return pack_shadow(memcg_id, pgdat, token, refs & 1, type);
}
/*
@@ -284,9 +289,9 @@ static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
*lruvec = mem_cgroup_lruvec(memcg, pgdat);
max_seq = READ_ONCE((*lruvec)->lrugen.max_seq);
- max_seq &= (file ? EVICTION_MASK : EVICTION_MASK_ANON) >> LRU_REFS_WIDTH;
+ max_seq &= (file ? EVICTION_MASK : EVICTION_MASK_ANON) >> LRU_REFS_BITS;
- return abs_diff(max_seq, *token >> LRU_REFS_WIDTH) < MAX_NR_GENS;
+ return abs_diff(max_seq, *token >> LRU_REFS_BITS) < MAX_NR_GENS;
}
static void lru_gen_refault(struct folio *folio, void *shadow)
@@ -314,22 +319,26 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
lrugen = &lruvec->lrugen;
hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
- refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + 1;
- tier = lru_tier_from_refs(refs, workingset);
+ refs = ((token & (BIT(LRU_REFS_BITS) - 1)) << 1) + workingset;
+ tier = lru_tier_from_refs(refs);
atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
- if (workingset) {
- /*
- * see folio_add_lru(), where folio_set_active() is
- * called for workingset folios
- */
- if (lru_gen_in_fault())
+ /*
+ * Activate a fault-driven refault folio, which would have been
+ * promoted had it stayed in memory.
+ */
+ if (refs >= LRU_REFS_REFERENCED) {
+ if (lru_gen_in_fault()) {
+ folio_set_active(folio);
mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
- folio_set_workingset(folio);
+ }
+ /* Refault is also promotion, cap the refs like folio_inc_lru_refs */
+ folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED));
+ }
+
+ if (lru_refs_is_workingset(refs))
mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
- } else
- set_mask_bits(&folio->flags.f, LRU_REFS_MASK, (refs - 1UL) << LRU_REFS_PGOFF);
unlock:
rcu_read_unlock();
}
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 07/17] mm/mglru: make folio lru referenced times count a generic API
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (5 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 06/17] mm/mglru: frequency guided workingset promotion (MGLRU-FG) Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 08/17] mm/mglru: move add/del LRU size accounting out of lru_gen_update_size() Kairui Song via B4 Relay
` (9 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
Pure code shuffle, no behavior change.
Make the folio LRU refs helpers available regardless of CONFIG_LRU_GEN.
Without MGLRU the extra bits are never set, so they are inert. Move the
lru_gen_* helpers into the CONFIG_LRU_GEN section. This prepares for
unifying the API for checking folio referenced and workingset status.
folio_migrate_lru_refs() now migrates the complete refs count,
including PG_workingset (bit 1 of the encoding), bitwise identical to
the direct copy dropped from folio_migrate_flags().
lru_set_refs_flags() skips the nonexistent mask bits when
LRU_REFS_WIDTH collapses to 0, preserving the two independent flags for
!CONFIG_LRU_GEN builds.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 55 ++++++++++++++++++++++++-----------------------
mm/migrate.c | 2 --
2 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index f38dad1198fc..180db43c3a85 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -184,6 +184,12 @@ static inline int lru_get_gen_flags(unsigned long flags)
return gen;
}
+static inline int folio_lru_gen(const struct folio *folio)
+{
+ return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
+}
+#endif
+
/**
* lru_set_refs_flags - Set the LRU referenced count to folio flags.
* @flags: pointer to the folio flags
@@ -206,7 +212,8 @@ static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs)
*flags |= BIT(PG_referenced);
if (refs & BIT(1))
*flags |= BIT(PG_workingset);
- *flags |= ((unsigned long)refs >> 2) << LRU_REFS_PGOFF;
+ if (LRU_REFS_WIDTH)
+ *flags |= ((unsigned long)refs >> 2) << LRU_REFS_PGOFF;
}
/**
@@ -252,13 +259,31 @@ static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
}
+#ifdef CONFIG_LRU_GEN
void folio_inc_lru_refs(struct folio *folio, unsigned int flags);
+#else
+static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
+{
+ /* Should not be called with !CONFIG_LRU_GEN */
+ WARN_ON_ONCE(1);
+}
+#endif
-static inline int folio_lru_gen(const struct folio *folio)
+/**
+ * folio_migrate_lru_refs - copy the reference state to a new folio
+ * @new: the destination folio
+ * @old: the source folio
+ *
+ * Transfer the reference state to @new during migration: the MGLRU
+ * refs count, or PG_referenced and PG_workingset (bits 0-1 of the
+ * encoding) for the active/inactive LRU.
+ */
+static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
{
- return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
+ folio_set_lru_refs(new, folio_lru_refs(old));
}
+#ifdef CONFIG_LRU_GEN
static inline bool lru_gen_is_old(struct lruvec *lruvec, int type, int gen)
{
unsigned long min_seq = READ_ONCE(lruvec->lrugen.min_seq[type]);
@@ -424,20 +449,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
return true;
}
-
-/**
- * folio_migrate_lru_refs - copy the reference state to a new folio
- * @new: the destination folio
- * @old: the source folio
- *
- * Transfer the reference state to @new during migration: the MGLRU
- * refs count, including PG_referenced, or just PG_referenced for the
- * active/inactive LRU.
- */
-static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
-{
- folio_set_lru_refs(new, folio_lru_refs(old));
-}
#else /* !CONFIG_LRU_GEN */
static inline bool lru_gen_enabled(void)
@@ -470,16 +481,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
{
return false;
}
-
-static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
-{
-}
-
-static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
-{
- if (folio_test_referenced(old))
- folio_set_referenced(new);
-}
#endif /* CONFIG_LRU_GEN */
static __always_inline
diff --git a/mm/migrate.c b/mm/migrate.c
index 28a8d7734735..c8c49357df87 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -796,8 +796,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
folio_set_active(newfolio);
} else if (folio_test_clear_unevictable(folio))
folio_set_unevictable(newfolio);
- if (folio_test_workingset(folio))
- folio_set_workingset(newfolio);
if (folio_test_checked(folio))
folio_set_checked(newfolio);
/*
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 08/17] mm/mglru: move add/del LRU size accounting out of lru_gen_update_size()
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (6 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 07/17] mm/mglru: make folio lru referenced times count a generic API Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 09/17] mm/mglru, gup: mark folios referenced via a fast helper Kairui Song via B4 Relay
` (8 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
Pure code shuffle, no behavior change. lru_gen_update_size() now only
updates the per-generation counters plus the promotion move; the
addition and deletion active/inactive accounting move to
lru_gen_add_folio() and lru_gen_del_folio(), which know the generation
and can apply the active-window test directly.
The unevictable case is untouched: unevictable folios never enter
lru_gen_add_folio() and keep being accounted by lruvec_add_folio() as
before.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 180db43c3a85..b15c3618a1f2 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -322,21 +322,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
if (new_gen >= 0)
atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]);
- /* addition */
- if (old_gen < 0) {
- if (lru_gen_is_active(lruvec, new_gen))
- lru += LRU_ACTIVE;
- __update_lru_size(lruvec, lru, zone, delta);
+ /* return now if not a promotion */
+ if (old_gen < 0 || new_gen < 0)
return;
- }
-
- /* deletion */
- if (new_gen < 0) {
- if (lru_gen_is_active(lruvec, old_gen))
- lru += LRU_ACTIVE;
- __update_lru_size(lruvec, lru, zone, -delta);
- return;
- }
/* promotion */
if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
@@ -391,6 +379,8 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
+ int delta = folio_nr_pages(folio);
+ enum lru_list lru = type * LRU_INACTIVE_FILE;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
BUILD_BUG_ON(BIT(LRU_GEN_WIDTH - 1) != MAX_NR_GENS);
@@ -408,6 +398,10 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags);
lru_gen_update_size(lruvec, folio, -1, gen);
+ if (lru_gen_is_active(lruvec, gen))
+ lru += LRU_ACTIVE;
+ __update_lru_size(lruvec, lru, zone, delta);
+
/* for folio_rotate_reclaimable() */
if (reclaiming)
list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
@@ -432,6 +426,9 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
{
unsigned long flags;
int gen = folio_lru_gen(folio);
+ int zone = folio_zonenum(folio);
+ int delta = folio_nr_pages(folio);
+ enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE;
if (gen < 0)
return false;
@@ -445,6 +442,9 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
lru_gen_update_size(lruvec, folio, gen, -1);
+ if (lru_gen_is_active(lruvec, gen))
+ lru += LRU_ACTIVE;
+ __update_lru_size(lruvec, lru, zone, -delta);
list_del(&folio->lru);
return true;
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 09/17] mm/mglru, gup: mark folios referenced via a fast helper
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (7 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 08/17] mm/mglru: move add/del LRU size accounting out of lru_gen_update_size() Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 10/17] mm/mglru, smaps: convert to LRU refs based operations for MGLRU Kairui Song via B4 Relay
` (7 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
Introduce folio_inc_lru_refs_fast(): raise refs from 0 to
LRU_REFS_REFERENCED via cmpxchg, or just set PG_referenced for the
classical LRU as before. No counter update or promotion is needed,
nonzero refs are left unchanged.
This helper only aims to be fast, it exists so the gup fast path can
stay a fast path. The new helper doesn't account refs above
LRU_REFS_REFERENCED but that's fine for now as GUP never used that
or anything similar for CLRU either.
Convert the gup fast-path sites. No behavior change for classical LRU.
There isn't any regression for MGLRU either.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 25 +++++++++++++++++++++++++
mm/gup.c | 6 +++---
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index b15c3618a1f2..3323a3c7336b 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -483,6 +483,31 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
}
#endif /* CONFIG_LRU_GEN */
+/**
+ * folio_inc_lru_refs_fast - Bump folio refs count without promotion.
+ * @folio: the folio
+ *
+ * Raise refs from 0 to LRU_REFS_REFERENCED and leave hotter folios
+ * untouched. For the classical LRU, a plain PG_referenced set.
+ */
+static __always_inline void folio_inc_lru_refs_fast(struct folio *folio)
+{
+ unsigned long new_flags, old_flags;
+
+ if (!lru_gen_enabled()) {
+ folio_set_referenced(folio);
+ return;
+ }
+
+ old_flags = READ_ONCE(*folio_flags(folio, 0));
+ do {
+ if (lru_get_refs_flags(old_flags))
+ break;
+ new_flags = old_flags;
+ lru_set_refs_flags(&new_flags, LRU_REFS_REFERENCED);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+}
+
static __always_inline
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
{
diff --git a/mm/gup.c b/mm/gup.c
index 2b405cce2ca5..596e62954659 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2911,7 +2911,7 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
gup_put_folio(folio, 1, flags);
goto pte_unmap;
}
- folio_set_referenced(folio);
+ folio_inc_lru_refs_fast(folio);
pages[nr_pages] = page;
nr_pages++;
} while (ptep++, addr += PAGE_SIZE, addr != end);
@@ -2980,7 +2980,7 @@ static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp,
for (i = 0; i < nr_pages; i++)
*(pages++) = page++;
- folio_set_referenced(folio);
+ folio_inc_lru_refs_fast(folio);
return nr_pages;
}
@@ -3022,7 +3022,7 @@ static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp,
for (i = 0; i < nr_pages; i++)
*(pages++) = page++;
- folio_set_referenced(folio);
+ folio_inc_lru_refs_fast(folio);
return nr_pages;
}
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 10/17] mm/mglru, smaps: convert to LRU refs based operations for MGLRU
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (8 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 09/17] mm/mglru, gup: mark folios referenced via a fast helper Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 11/17] mm/damon: " Kairui Song via B4 Relay
` (6 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
PG_referenced is only the lowest bit of the folio LRU refs count
(LRU_REFS_FLAGS) now, and MGLRU never kept it stable: promotion used to
clear it, and now it toggles as the count grows. But with the LRU
referenced count as a stable API now, we can use two helpers to provide
in-kernel users a stable and sane reading of the referenced status of
an LRU managed folio.
Add folio_check_lru_refs() and folio_clear_lru_refs(), which use the refs
count when MGLRU is enabled and PG_referenced otherwise, and switch the
smaps paths over to them. The lru_gen_enabled() test stays inside the
helpers, so its static key is not splattered over the few call sites.
Convert smaps as the first user of them. MGLRU now reports every folio
with a nonzero count, so the reading no longer flickers and clearing no
longer results in an ambiguous status.
The classical LRU case is unchanged.
Signed-off-by: Kairui Song <kasong@tencent.com>
---
fs/proc/task_mmu.c | 6 +++---
include/linux/mm_inline.h | 31 +++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 22361607020d..f76a37d052b1 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -889,7 +889,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
mss->resident += size;
/* Accumulate the size in pages that have been accessed. */
- if (young || folio_test_young(folio) || folio_test_referenced(folio))
+ if (young || folio_test_young(folio) || folio_check_lru_refs(folio))
mss->referenced += size;
/*
@@ -1686,7 +1686,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
/* Clear accessed and referenced bits. */
pmdp_test_and_clear_young(vma, addr, pmd);
folio_test_clear_young(folio);
- folio_clear_referenced(folio);
+ folio_clear_lru_refs(folio);
out:
spin_unlock(ptl);
return 0;
@@ -1715,7 +1715,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
/* Clear accessed and referenced bits. */
ptep_test_and_clear_young(vma, addr, pte);
folio_test_clear_young(folio);
- folio_clear_referenced(folio);
+ folio_clear_lru_refs(folio);
}
pte_unmap_unlock(pte - 1, ptl);
cond_resched();
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 3323a3c7336b..14aab2b1ed08 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -508,6 +508,37 @@ static __always_inline void folio_inc_lru_refs_fast(struct folio *folio)
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
}
+/**
+ * folio_check_lru_refs - Test if an LRU managed folio has been referenced.
+ * @folio: The folio to test.
+ *
+ * Returns true if the MGLRU refs count is nonzero, or PG_referenced is set,
+ * which is also correct for the non-MGLRU case. Kept as a separate helper,
+ * to avoid splattering the lru_gen_enabled() static key.
+ */
+static inline bool folio_check_lru_refs(const struct folio *folio)
+{
+ if (lru_gen_enabled())
+ return folio_lru_refs(folio) > 0;
+ return folio_test_referenced(folio);
+}
+
+/**
+ * folio_clear_lru_refs - Clear the referenced state of an LRU managed folio.
+ * @folio: The folio to clear.
+ *
+ * Reset the MGLRU refs count to zero, or clear PG_referenced, leaving
+ * PG_workingset alone for the non-MGLRU case. Kept as a separate helper
+ * just like folio_check_lru_refs().
+ */
+static inline void folio_clear_lru_refs(struct folio *folio)
+{
+ if (lru_gen_enabled())
+ folio_set_lru_refs(folio, 0);
+ else
+ folio_clear_referenced(folio);
+}
+
static __always_inline
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
{
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 11/17] mm/damon: convert to LRU refs based operations for MGLRU
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (9 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 10/17] mm/mglru, smaps: convert to LRU refs based operations for MGLRU Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 12/17] mm/khugepaged: " Kairui Song via B4 Relay
` (5 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
damon_pa_pageout() cleared PG_referenced directly, which under MGLRU
corrupts the folio's refs count. That flag is now
bit 0 of the count (LRU_REFS_FLAGS).
Reset the count instead under MGLRU.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
mm/damon/paddr.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index e89cb14ee17b..411e1d557ba3 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -279,7 +279,8 @@ static unsigned long damon_pa_pageout(struct damon_region *r,
else
*sz_filter_passed += folio_size(folio) / addr_unit;
- folio_clear_referenced(folio);
+ /* The region is cold, better drop the hotness info. */
+ folio_clear_lru_refs(folio);
folio_test_clear_young(folio);
if (!folio_isolate_lru(folio))
goto put_folio;
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 12/17] mm/khugepaged: convert to LRU refs based operations for MGLRU
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (10 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 11/17] mm/damon: " Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 13/17] mm/madvise: adapt for " Kairui Song via B4 Relay
` (4 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
For MGLRU, PG_referenced is only the lowest bit of the folio LRU refs
count, so folios with nonzero even refs no longer have the bit set.
folio_pte_referenced() testing the raw bit misses hot folios at
refs 2, 4 or 6, under-counting the referenced folios of a candidate range
and aborting otherwise good collapses with SCAN_LACK_REFERENCED_PAGE.
Test the refs count directly when MGLRU is enabled, mirroring the
smaps conversion. The classical LRU keeps the plain bit test.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
mm/khugepaged.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 4bbe319ef86f..68f7a30e36be 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -579,7 +579,9 @@ static bool folio_pte_referenced(struct folio *folio,
struct vm_area_struct *vma, unsigned long addr, pte_t pteval)
{
/* The folio was referenced previously ... */
- if (folio_test_young(folio) || folio_test_referenced(folio))
+ if (folio_test_young(folio))
+ return true;
+ if (folio_check_lru_refs(folio))
return true;
/* ... or the PTE mapping was recently used */
return pte_young(pteval) || mmu_notifier_test_young(vma->vm_mm, addr);
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 13/17] mm/madvise: adapt for LRU refs based operations for MGLRU
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (11 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 12/17] mm/khugepaged: " Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 14/17] mm/huge_memory: mark folio as accessed more accurately on split Kairui Song via B4 Relay
` (3 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
For the active/inactive LRU, madvise clears PG_referenced so that one more
access is not enough to reactivate a folio, and keeps PG_workingset on a
folio demoted out of the active list so its refault is still accounted as
a workingset refault (PSI).
MGLRU keeps that history in the folio's refs count instead, which now
lives in PG_referenced, PG_workingset and LRU_REFS_MASK, so writing to
those bits directly corrupts the count. The two hints want different
things:
- MADV_COLD resets the count in folio_deactivate(), which also requests
deactivation.
- MADV_PAGEOUT clears non-workingset refs and caps the rest at
LRU_REFS_WORKINGSET after isolation, before handing the folio to
reclaim_pages(). The hint marks the folio as cold, so the
eviction shadow records the capped count and does not inflate the
eviction/refault counters of the hot tiers that drive aging, while
folios at or above the workingset threshold keep their PSI
classification.
PG_young is cleared for both LRUs as before, so the referenced walk does
not count stale history as a recent access.
Signed-off-by: Kairui Song <kasong@tencent.com>
---
mm/madvise.c | 54 ++++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 40 insertions(+), 14 deletions(-)
diff --git a/mm/madvise.c b/mm/madvise.c
index 2d54cb9863ea..7486c492f437 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -360,6 +360,42 @@ static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end,
FPB_MERGE_YOUNG_DIRTY);
}
+static void madvise_cold_prep_folio(struct folio *folio)
+{
+ /*
+ * VM couldn't reclaim the folio unless we clear PG_young.
+ * As a side effect, idle-page tracking loses recent referenced history.
+ */
+ folio_test_clear_young(folio);
+
+ /*
+ * For the active/inactive LRU, a folio demoted out of the active
+ * list should have PG_workingset so its refault is still accounted
+ * as a workingset refault, and the referenced bit always needs to
+ * be cleared even for inactive folios. For MGLRU, MADV_COLD clears
+ * refs in folio_deactivate(); MADV_PAGEOUT handles refs after isolation.
+ */
+ if (!lru_gen_enabled()) {
+ folio_clear_referenced(folio);
+ if (folio_test_active(folio))
+ folio_set_workingset(folio);
+ }
+}
+
+static void madvise_pageout_prep_folio(struct folio *folio)
+{
+ unsigned int refs = folio_lru_refs(folio);
+
+ /*
+ * For MGLRU, clear refs below LRU_REFS_WORKINGSET and cap
+ * the rest at it: MADV suggests the folio is cold, so the eviction shadow
+ * should not record its hotness in the high tier counters, but
+ * the workingset classification is preserved for PSI.
+ */
+ if (lru_gen_enabled())
+ folio_set_lru_refs(folio, refs >= LRU_REFS_WORKINGSET ? LRU_REFS_WORKINGSET : 0);
+}
+
static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
unsigned long addr, unsigned long end,
struct mm_walk *walk)
@@ -434,12 +470,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
}
- folio_clear_referenced(folio);
- folio_test_clear_young(folio);
- if (folio_test_active(folio))
- folio_set_workingset(folio);
+ madvise_cold_prep_folio(folio);
if (pageout) {
if (folio_isolate_lru(folio)) {
+ madvise_pageout_prep_folio(folio);
if (folio_test_unevictable(folio))
folio_putback_lru(folio);
else
@@ -544,18 +578,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
tlb_remove_tlb_entries(tlb, pte, nr, addr);
}
- /*
- * We are deactivating a folio for accelerating reclaiming.
- * VM couldn't reclaim the folio unless we clear PG_young.
- * As a side effect, it makes confuse idle-page tracking
- * because they will miss recent referenced history.
- */
- folio_clear_referenced(folio);
- folio_test_clear_young(folio);
- if (folio_test_active(folio))
- folio_set_workingset(folio);
+ madvise_cold_prep_folio(folio);
if (pageout) {
if (folio_isolate_lru(folio)) {
+ madvise_pageout_prep_folio(folio);
if (folio_test_unevictable(folio))
folio_putback_lru(folio);
else
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 14/17] mm/huge_memory: mark folio as accessed more accurately on split
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (12 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 13/17] mm/madvise: adapt for " Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 15/17] mm/mglru: folio LRU refs based active/inactive number accounting Kairui Song via B4 Relay
` (2 subsequent siblings)
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
The behavior of updating the folio's access info isn't consistent for
huge mapping splitting or ordinary unmapping. The page table's young
flag has to be translated into folio's access info.
Right now it only checks and sets the folio's referenced flag, which isn't
enough since folio flag updates on access have their own rules. Ordinary
unmapping (zapping) calls folio_mark_accessed(), and it also checks
if the VMA has recency to avoid false updates.
So first just use the right helper here to be more consistent. Keep
accounting the young PTE as a cache access, sticking to the current
behavior.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
mm/huge_memory.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 0d589be136d2..1a415a6704be 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3104,8 +3104,8 @@ static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
if (!folio_test_dirty(folio) && pud_dirty(old_pud))
folio_mark_dirty(folio);
- if (!folio_test_referenced(folio) && pud_young(old_pud))
- folio_set_referenced(folio);
+ if (pud_young(old_pud) && vma_has_recency(vma))
+ folio_mark_accessed(folio);
folio_remove_rmap_pud(folio, page, vma);
add_mm_counter(vma->vm_mm, mm_counter_file(folio),
-HPAGE_PUD_NR);
@@ -3227,8 +3227,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
folio = page_folio(page);
if (!folio_test_dirty(folio) && pmd_dirty(old_pmd))
folio_mark_dirty(folio);
- if (!folio_test_referenced(folio) && pmd_young(old_pmd))
- folio_set_referenced(folio);
+ if (pmd_young(old_pmd) && vma_has_recency(vma))
+ folio_mark_accessed(folio);
folio_remove_rmap_pmd(folio, page, vma);
add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
folio_put(folio);
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 15/17] mm/mglru: folio LRU refs based active/inactive number accounting
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (13 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 14/17] mm/huge_memory: mark folio as accessed more accurately on split Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:29 ` sashiko-bot
2026-11-18 21:03 ` [PATCH v5 16/17] mm/mglru: make folio_inc_lru_refs lruvec lockless Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 17/17] mm/mglru: make folio_test_workingset() work based on folio LRU refs Kairui Song via B4 Relay
16 siblings, 1 reply; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
Currently the active/inactive LRU sizes exported through /proc/vmstat
and memory.stat are derived from the generation window: folios in the
two newest generations are accounted as active, the rest as inactive.
This causes several problems:
- Unmapped file folios can stay in older gens despite repeated use,
so active files can be under-reported, unlike classical LRU.
- The reading is jumpy: on aging, the second-newest gen, previously
active, becomes inactive all at once, causing a huge drop of active
folios without any change in use.
- The reading is reversed on swapless machines. When swapping is
disabled or unavailable, MGLRU ages anon by moving the gens directly
without walking or touching any folio, to reduce the overhead. So
inactive anon folios can suddenly count as active, or vice versa,
despite no change in their hotness.
With frequency guided promotion, hotness is already tracked per folio by
its referenced count, so we can do folio level accounting instead: a
folio is active once its refs reach LRU_REFS_ACTIVATED (currently
LRU_REFS_PROTECTED).
On reaching this threshold, folios are accounted to the active part.
On forced per-folio aging or PID protection, refs are capped at
LRU_REFS_WORKINGSET, dropping active folios to the inactive part.
Note that on swapless machines anon refs are not reset by aging (the
shortcut skips the per-folio walk), but the counts no longer oscillate
as the generation indices wrap.
The accounting is now per folio and lruvec independent, so no size
adjustment is needed when reparenting a dying memcg: the hierarchical
stats keep the counts in the parent automatically.
folio_migrate_lru_refs() also gains off-LRU VM_WARN_ON_ONCE checks and
__folio_set_lru_refs() warns when the folio is still on an LRU list.
The latter uses lru_get_gen_flags(), so that helper now has to move out of
the CONFIG_LRU_GEN block and is built with the option off, where
LRU_GEN_WIDTH is 0 and LRU_GEN_MASK is zero. Keep the decoded value in
int so the subtraction of the one-off encoding cannot fold to ULONG_MAX
before it happens: that is -Woverflow, and fatal with CONFIG_WERROR.
Signed-off-by: Kairui Song <kasong@tencent.com>
---
Documentation/admin-guide/mm/multigen_lru.rst | 4 +-
include/linux/mm_inline.h | 118 +++++++-------
include/linux/mmzone.h | 23 ++-
mm/folio.c | 39 +----
mm/madvise.c | 2 +-
mm/vmscan.c | 218 +++++++++++++++++---------
mm/workingset.c | 2 +-
7 files changed, 230 insertions(+), 176 deletions(-)
diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst
index 9cb54b4ff5d9..808d2e97b3f3 100644
--- a/Documentation/admin-guide/mm/multigen_lru.rst
+++ b/Documentation/admin-guide/mm/multigen_lru.rst
@@ -149,8 +149,8 @@ generations less than or equal to ``min_gen_nr``.
``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]``
``min_gen_nr`` should be less than ``max_gen_nr-1``, since
-``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to
-the active list) and therefore cannot be evicted. ``swappiness``
+``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged and therefore
+cannot be evicted. ``swappiness``
overrides the default value in ``/proc/sys/vm/swappiness`` and the valid
range is [0-200, max], with max being exclusively used for the reclamation
of anonymous memory. ``nr_to_reclaim`` limits the number of pages to evict.
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 14aab2b1ed08..77f514143266 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -105,6 +105,22 @@ static __always_inline enum lru_list folio_lru_list(const struct folio *folio)
return lru;
}
+/**
+ * lru_get_gen_flags - Return the LRU generation number from folio flags.
+ * @flags: folio flags
+ *
+ * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns
+ * -1 if the flags indicate the folio is off the list (e.g., isolated).
+ */
+static inline int lru_get_gen_flags(unsigned long flags)
+{
+ int gen = (int)((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF);
+
+ /* The gen is stored offset by one, so zero means off-list */
+ VM_WARN_ON_ONCE(gen > MAX_NR_GENS);
+ return gen - 1;
+}
+
#ifdef CONFIG_LRU_GEN
static inline bool lru_gen_switching(void)
@@ -168,22 +184,6 @@ static inline void lru_set_gen_flags(unsigned long *flags, int gen)
*flags |= (gen + 1UL) << LRU_GEN_PGOFF;
}
-/**
- * lru_get_gen_flags - Return the LRU generation number from folio flags.
- * @flags: folio flags
- *
- * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns
- * -1 if the flags indicate the folio is off the list (e.g., isolated).
- */
-static inline int lru_get_gen_flags(unsigned long flags)
-{
- int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
-
- /* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */
- VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS);
- return gen;
-}
-
static inline int folio_lru_gen(const struct folio *folio)
{
return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
@@ -247,12 +247,21 @@ static inline bool lru_refs_is_workingset(unsigned int refs)
return refs >= LRU_REFS_WORKINGSET;
}
-static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
+/**
+ * __folio_set_lru_refs - Set a folio's LRU refs.
+ * @folio: the folio
+ * @refs: the new referenced count (0 .. LRU_REFS_MAX)
+ *
+ * Set the folio's LRU refs. The folio must be off the LRU list (e.g.,
+ * isolated), or use folio_inc_lru_refs or folio_reset_lru_refs instead.
+ */
+static inline void __folio_set_lru_refs(struct folio *folio, unsigned int refs)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
do {
new_flags = old_flags;
+ VM_WARN_ON_ONCE(lru_get_gen_flags(old_flags) != -1);
lru_set_refs_flags(&new_flags, refs);
if (new_flags == old_flags)
break;
@@ -261,12 +270,18 @@ static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
#ifdef CONFIG_LRU_GEN
void folio_inc_lru_refs(struct folio *folio, unsigned int flags);
+bool folio_reset_lru_refs(struct folio *folio);
#else
static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
{
/* Should not be called with !CONFIG_LRU_GEN */
WARN_ON_ONCE(1);
}
+
+static inline bool folio_reset_lru_refs(struct folio *folio)
+{
+ return false;
+}
#endif
/**
@@ -280,10 +295,18 @@ static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
*/
static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
{
- folio_set_lru_refs(new, folio_lru_refs(old));
+ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(old), old);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(new), new);
+ __folio_set_lru_refs(new, folio_lru_refs(old));
}
#ifdef CONFIG_LRU_GEN
+static inline bool lru_refs_is_active(unsigned int refs)
+{
+ VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
+ return refs >= LRU_REFS_ACTIVATED;
+}
+
static inline bool lru_gen_is_old(struct lruvec *lruvec, int type, int gen)
{
unsigned long min_seq = READ_ONCE(lruvec->lrugen.min_seq[type]);
@@ -294,23 +317,12 @@ static inline bool lru_gen_is_old(struct lruvec *lruvec, int type, int gen)
gen == lru_gen_from_seq(min_seq + 1);
}
-static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
-{
- unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq);
-
- VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
-
- /* see the comment on MIN_NR_GENS */
- return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
-}
-
static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
int old_gen, int new_gen)
{
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
- enum lru_list lru = type * LRU_INACTIVE_FILE;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
@@ -321,19 +333,6 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
if (new_gen >= 0)
atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]);
-
- /* return now if not a promotion */
- if (old_gen < 0 || new_gen < 0)
- return;
-
- /* promotion */
- if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
- __update_lru_size(lruvec, lru, zone, -delta);
- __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
- }
-
- /* demotion requires isolation, e.g., lru_deactivate_fn() */
- VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
}
static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
@@ -376,7 +375,8 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
{
unsigned long seq;
unsigned long flags;
- int gen = folio_lru_gen(folio);
+ unsigned int refs;
+ int gen;
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
@@ -384,7 +384,7 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
struct lru_gen_folio *lrugen = &lruvec->lrugen;
BUILD_BUG_ON(BIT(LRU_GEN_WIDTH - 1) != MAX_NR_GENS);
- VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio);
if (folio_test_unevictable(folio) || !lrugen->enabled)
return false;
@@ -395,10 +395,12 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
gen = lru_gen_from_seq(seq);
flags = (gen + 1UL) << LRU_GEN_PGOFF;
/* see the comment on MIN_NR_GENS about PG_active */
- set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags);
+ flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags);
+ /* use the refs from the atomic snapshot to avoid raced update */
+ refs = lru_get_refs_flags(flags);
lru_gen_update_size(lruvec, folio, -1, gen);
- if (lru_gen_is_active(lruvec, gen))
+ if (lru_refs_is_active(refs))
lru += LRU_ACTIVE;
__update_lru_size(lruvec, lru, zone, delta);
@@ -425,24 +427,33 @@ static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct foli
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
unsigned long flags;
- int gen = folio_lru_gen(folio);
+ unsigned int refs;
+ int gen;
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE;
+ unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq);
+ /* Keep LRU_REFS_FLAGS: refs carry access history across isolation */
+ flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, 0);
+ gen = lru_get_gen_flags(flags);
+ refs = lru_get_refs_flags(flags);
if (gen < 0)
return false;
VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
- /* for folio_migrate_flags() */
- flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
- flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags);
- gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ /*
+ * See the comment in lru_gen_folio_seq. For migration, compaction,
+ * or any other isolation of a hot folio, try best to retain its gen
+ * info. Ideally we would keep the full gen info.
+ */
+ if (!reclaiming && ((max_seq - gen) % MAX_NR_GENS) < MIN_NR_GENS)
+ folio_set_active(folio);
lru_gen_update_size(lruvec, folio, gen, -1);
- if (lru_gen_is_active(lruvec, gen))
+ if (lru_refs_is_active(refs))
lru += LRU_ACTIVE;
__update_lru_size(lruvec, lru, zone, -delta);
list_del(&folio->lru);
@@ -499,6 +510,7 @@ static __always_inline void folio_inc_lru_refs_fast(struct folio *folio)
return;
}
+ BUILD_BUG_ON(LRU_REFS_REFERENCED >= LRU_REFS_ACTIVATED);
old_flags = READ_ONCE(*folio_flags(folio, 0));
do {
if (lru_get_refs_flags(old_flags))
@@ -534,7 +546,7 @@ static inline bool folio_check_lru_refs(const struct folio *folio)
static inline void folio_clear_lru_refs(struct folio *folio)
{
if (lru_gen_enabled())
- folio_set_lru_refs(folio, 0);
+ folio_reset_lru_refs(folio);
else
folio_clear_referenced(folio);
}
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 5470dd708e44..2680e15b6de9 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -434,15 +434,15 @@ enum lruvec_flags {
* least twice before handing this folio over to the eviction. The first check
* clears the accessed bit from the initial fault; the second check makes sure
* this folio hasn't been used since then. This process, AKA second chance,
- * requires a minimum of two generations, hence MIN_NR_GENS. And to maintain ABI
- * compatibility with the active/inactive LRU, e.g., /proc/vmstat, these two
- * generations are considered active; the rest of generations, if they exist,
- * are considered inactive. See lru_gen_is_active().
+ * requires a minimum of two generations, hence MIN_NR_GENS.
+ *
+ * Active/inactive is per-folio based on the referenced count;
+ * see the comment above MAX_NR_TIERS.
*
* PG_active is always cleared while a folio is on one of lrugen->folios[] so
- * that the sliding window needs not to worry about it. And it's set again when
- * a folio considered active is isolated for non-reclaiming purposes, e.g.,
- * migration. See lru_gen_add_folio() and lru_gen_del_folio().
+ * that the sliding window needs not to worry about it. It is set on a
+ * fault-driven refault (see lru_gen_refault), or isolation (see
+ * lru_gen_del_folio).
*
* MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the
* number of categories of the active/inactive LRU when keeping track of
@@ -524,11 +524,15 @@ enum lruvec_flags {
* and the bits masked by LRU_REFS_MASK as the higher bits, so the refs
* count ranges from 0 to LRU_REFS_MAX.
*
- * Cold hints can reset refs with lru_gen_clear_refs(). Refault restores
+ * Cold hints can reset refs with folio_reset_lru_refs(). Refault restores
* its referenced count, capped at LRU_REFS_PROTECTED, which aligns with
* promotion. Fault-driven refaults of previous workingset folios set
* PG_active, so they are placed in the latest gen.
*
+ * For the active/inactive LRU ABI (/proc/vmstat), a folio is active
+ * if its referenced count reaches LRU_REFS_ACTIVATED. The threshold
+ * is currently set to LRU_REFS_PROTECTED, which is tier 2.
+ *
* MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice
* the number of categories of the active/inactive LRU.
*/
@@ -543,6 +547,7 @@ enum lruvec_flags {
#define LRU_REFS_REFERENCED 0x1
#define LRU_REFS_WORKINGSET 0x2
#define LRU_REFS_PROTECTED 0x3
+#define LRU_REFS_ACTIVATED LRU_REFS_PROTECTED
#ifndef __GENERATING_BOUNDS_H
@@ -651,6 +656,8 @@ struct lru_gen_mm_walk {
unsigned long seq;
/* the next address within an mm to scan */
unsigned long next_addr;
+ /* to batch activated pages */
+ int nr_activated[ANON_AND_FILE][MAX_NR_ZONES];
/* to batch promoted pages */
int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* to batch the mm stats */
diff --git a/mm/folio.c b/mm/folio.c
index 06c9cde834f7..3b5ae5b51650 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -350,35 +350,6 @@ static void __lru_cache_activate_folio(struct folio *folio)
local_unlock(&cpu_fbatches.lock);
}
-#ifdef CONFIG_LRU_GEN
-
-static bool lru_gen_clear_refs(struct folio *folio)
-{
- int gen = folio_lru_gen(folio);
- int type = folio_is_file_lru(folio);
- unsigned long seq;
-
- if (gen < 0)
- return true;
-
- folio_set_lru_refs(folio, 0);
-
- rcu_read_lock();
- seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
- rcu_read_unlock();
- /* whether can do without shuffling under the LRU lock */
- return gen == lru_gen_from_seq(seq);
-}
-
-#else /* !CONFIG_LRU_GEN */
-
-static bool lru_gen_clear_refs(struct folio *folio)
-{
- return false;
-}
-
-#endif /* CONFIG_LRU_GEN */
-
/**
* folio_mark_accessed - Mark a folio as having seen activity.
* @folio: The folio to mark.
@@ -552,10 +523,12 @@ static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)
folio_test_swapcache(folio) || folio_test_unevictable(folio))
return;
- lru_gen_clear_refs(folio);
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
- folio_clear_referenced(folio);
+ if (lru_gen_enabled())
+ __folio_set_lru_refs(folio, 0);
+ else
+ folio_clear_referenced(folio);
/*
* Lazyfree folios are clean anonymous folios. They have
* the swapbacked flag cleared, to distinguish them from normal
@@ -626,7 +599,7 @@ void deactivate_file_folio(struct folio *folio)
if (folio_test_unevictable(folio) || !folio_test_lru(folio))
return;
- if (lru_gen_enabled() && lru_gen_clear_refs(folio))
+ if (lru_gen_enabled() && !folio_reset_lru_refs(folio))
return;
folio_batch_add_and_move(folio, lru_deactivate_file);
@@ -645,7 +618,7 @@ void folio_deactivate(struct folio *folio)
if (folio_test_unevictable(folio) || !folio_test_lru(folio))
return;
- if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
+ if (lru_gen_enabled() ? !folio_reset_lru_refs(folio) : !folio_test_active(folio))
return;
folio_batch_add_and_move(folio, lru_deactivate);
diff --git a/mm/madvise.c b/mm/madvise.c
index 7486c492f437..51e597e38339 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -393,7 +393,7 @@ static void madvise_pageout_prep_folio(struct folio *folio)
* the workingset classification is preserved for PSI.
*/
if (lru_gen_enabled())
- folio_set_lru_refs(folio, refs >= LRU_REFS_WORKINGSET ? LRU_REFS_WORKINGSET : 0);
+ __folio_set_lru_refs(folio, lru_refs_is_workingset(refs) ? LRU_REFS_WORKINGSET : 0);
}
static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index a02504c5a7d7..78a1b4de7076 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -870,17 +870,20 @@ enum folio_references {
void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
{
int max_gen;
- int type, refs, old_gen, gen;
+ unsigned int refs, old_refs;
+ int file, old_gen, gen;
unsigned long new_flags, old_flags, max_seq;
+ long nr_pages = folio_nr_pages(folio);
struct lru_gen_folio *lrugen;
struct lruvec *lruvec = NULL;
- type = folio_is_file_lru(folio);
old_flags = READ_ONCE(*folio_flags(folio, 0));
do {
new_flags = old_flags;
old_gen = lru_get_gen_flags(old_flags);
- refs = lru_get_refs_flags(old_flags) + 1;
+ old_refs = lru_get_refs_flags(old_flags);
+ file = folio_flags_is_file_lru(&old_flags);
+ refs = old_refs + 1;
gen = old_gen;
if (old_gen < 0)
goto out;
@@ -906,14 +909,14 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
if (refs > LRU_REFS_REFERENCED || flags & LRU_REF_EXEC)
gen = max_gen;
/* First access only defers eviction of an old generation */
- else if (lru_gen_is_old(lruvec, type, old_gen))
+ else if (lru_gen_is_old(lruvec, file, old_gen))
gen = (old_gen + 1) % MAX_NR_GENS;
refs = min(refs, LRU_REFS_PROTECTED);
} else if (refs > LRU_REFS_MAX) {
/* LRU refs counting overflow, bump the gen */
gen = (old_gen + 1) % MAX_NR_GENS;
refs = LRU_REFS_PROTECTED;
- } else if (lru_gen_is_old(lruvec, type, old_gen) &&
+ } else if (lru_gen_is_old(lruvec, file, old_gen) &&
lru_refs_is_workingset(refs)) {
/* Defer eviction of just accessed workingset */
gen = (old_gen + 1) % MAX_NR_GENS;
@@ -930,63 +933,120 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
if (gen != old_gen)
lru_gen_update_size(lruvec, folio, old_gen, gen);
+ if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && old_gen >= 0) {
+ enum lru_list lru = file * LRU_INACTIVE_FILE;
+
+ __update_lru_size(lruvec, lru + lru_refs_is_active(old_refs),
+ folio_zonenum(folio), -nr_pages);
+ __update_lru_size(lruvec, lru + lru_refs_is_active(refs),
+ folio_zonenum(folio), nr_pages);
+ }
if (lruvec)
lruvec_unlock_irq(lruvec);
}
+/*
+ * Reset the folio's lru refs indicator. The caller doesn't need to hold
+ * the folio lock, isolate the folio, or hold the lruvec lock.
+ * Return true if the folio is on-list outside the oldest generation,
+ * so the caller may also want to demote it.
+ */
+bool folio_reset_lru_refs(struct folio *folio)
+{
+ unsigned int refs;
+ int type, gen;
+ unsigned long seq, new_flags, old_flags;
+ long nr_pages = folio_nr_pages(folio);
+ struct lruvec *lruvec;
+ bool reset = false;
+
+ old_flags = READ_ONCE(*folio_flags(folio, 0));
+ do {
+ new_flags = old_flags;
+ refs = lru_get_refs_flags(old_flags);
+ type = folio_flags_is_file_lru(&old_flags);
+ gen = lru_get_gen_flags(old_flags);
+ if (!refs)
+ break;
+ lru_set_refs_flags(&new_flags, 0);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+
+ if (gen >= 0) {
+ lruvec = folio_lruvec_live_get(folio);
+
+ /* If the folio is on list, caller might also want to demote it */
+ seq = READ_ONCE(lruvec->lrugen.min_seq[type]);
+ reset = (gen != lru_gen_from_seq(seq));
+ if (lru_refs_is_active(refs)) {
+ __update_lru_size(lruvec, type * LRU_FILE + LRU_ACTIVE,
+ folio_zonenum(folio), -nr_pages);
+ __update_lru_size(lruvec, type * LRU_FILE,
+ folio_zonenum(folio), nr_pages);
+ }
+
+ folio_lruvec_live_put(lruvec);
+ }
+
+ return reset;
+}
+
/**
* folio_inc_lru_refs_walk - Update the folio's LRU refs and lazily promote it.
* @folio: The folio.
* @lruvec: The lruvec the folio belongs to.
* @vma_flags: The flags of the VMA being walked.
* @new_gen: The gen the folio is moved to; may be unchanged.
+ * @old_refs: Returns the refs count before this access.
+ * @new_refs: Returns the refs count after this access.
* @type: Returns the folio's type: file or anon.
*
* Similar to folio_inc_lru_refs(), but for the page table walker and the
* look-around. The LRU size accounting is batched by the caller for
* better performance.
*
- * Return: The old generation the folio belongs to if the folio is still
- * on the LRU and not in the newest generation, or -1 otherwise. -1 means
- * the folio should not be promoted and @new_gen is not set.
+ * Return: The old generation, or -1 if the folio is off the LRU list.
*/
static int folio_inc_lru_refs_walk(struct folio *folio, struct lruvec *lruvec,
- const vma_flags_t *vma_flags,
- int *new_gen, int *type)
+ const vma_flags_t *vma_flags, int *new_gen,
+ int *old_refs, int *new_refs, int *type)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
- int refs, gen, max_gen, ret;
+ unsigned int refs;
+ int old_gen, max_gen;
unsigned long max_seq;
do {
max_seq = READ_ONCE(lruvec->lrugen.max_seq);
max_gen = lru_gen_from_seq(max_seq);
- gen = lru_get_gen_flags(old_flags);
+ old_gen = lru_get_gen_flags(old_flags);
refs = lru_get_refs_flags(old_flags) + 1;
*type = folio_flags_is_file_lru(&old_flags);
new_flags = old_flags;
- if (gen >= 0 && gen != max_gen) {
- ret = gen;
+ if (old_gen >= 0 && old_gen != max_gen) {
+ *new_refs = min(refs, LRU_REFS_PROTECTED);
/* Promote second page table access or executable */
if (lru_refs_is_workingset(refs) || is_exec_file_folio(folio, vma_flags))
*new_gen = max_gen;
/* First access only defers eviction of an old generation */
- else if (lru_gen_is_old(lruvec, *type, gen))
- *new_gen = (gen + 1) % MAX_NR_GENS;
+ else if (lru_gen_is_old(lruvec, *type, old_gen))
+ *new_gen = (old_gen + 1) % MAX_NR_GENS;
else
- *new_gen = gen;
+ *new_gen = old_gen;
lru_set_gen_flags(&new_flags, *new_gen);
- lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_PROTECTED));
+ lru_set_refs_flags(&new_flags, *new_refs);
} else {
- ret = -1;
- lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_MAX));
+ *new_gen = old_gen;
+ *new_refs = min(refs, LRU_REFS_MAX);
+ lru_set_refs_flags(&new_flags, *new_refs);
}
if (new_flags == old_flags)
break;
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
- return ret;
+ *old_refs = lru_get_refs_flags(old_flags);
+
+ return old_gen;
}
/**
@@ -3412,10 +3472,12 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
* the aging
******************************************************************************/
-static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
+static int __folio_inc_gen(struct lruvec *lruvec, struct folio *folio,
+ int old_gen, bool *increased)
{
unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
- int refs, new_gen;
+ unsigned int refs, old_refs;
+ int file, new_gen;
do {
new_gen = lru_get_gen_flags(old_flags);
@@ -3429,11 +3491,24 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
new_flags = old_flags;
new_gen = (old_gen + 1) % MAX_NR_GENS;
- refs = lru_get_refs_flags(old_flags);
+ old_refs = lru_get_refs_flags(old_flags);
+ file = folio_flags_is_file_lru(&old_flags);
+ refs = min(old_refs, LRU_REFS_WORKINGSET);
lru_set_gen_flags(&new_flags, new_gen);
- lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_WORKINGSET));
+ lru_set_refs_flags(&new_flags, refs);
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+ /* Refs capped below PROTECTED: demote if previously active. */
+ if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs)) {
+ enum lru_list lru = file * LRU_INACTIVE_FILE;
+ int nr_pages = folio_nr_pages(folio);
+
+ __update_lru_size(lruvec, lru + lru_refs_is_active(old_refs),
+ folio_zonenum(folio), -nr_pages);
+ __update_lru_size(lruvec, lru + lru_refs_is_active(refs),
+ folio_zonenum(folio), nr_pages);
+ }
+
if (increased)
*increased = true;
return new_gen;
@@ -3445,7 +3520,7 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen
int new_gen;
bool gen_increased;
- new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
+ new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased);
if (gen_increased)
lru_gen_update_size(lruvec, folio, old_gen, new_gen);
@@ -3453,18 +3528,26 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen
}
static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
- int old_gen, int new_gen, int type)
+ int old_gen, int new_gen, int old_refs, int new_refs, int type)
{
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
- VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS);
- VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS);
-
- walk->batched++;
+ /* gen counter update */
+ if (old_gen != new_gen) {
+ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS);
+ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS);
+ walk->batched++;
+ walk->nr_pages[old_gen][type][zone] -= delta;
+ walk->nr_pages[new_gen][type][zone] += delta;
+ }
- walk->nr_pages[old_gen][type][zone] -= delta;
- walk->nr_pages[new_gen][type][zone] += delta;
+ /* active/inactive counter update */
+ if (lru_refs_is_active(old_refs) != lru_refs_is_active(new_refs)) {
+ walk->nr_activated[type][zone] +=
+ lru_refs_is_active(new_refs) ? delta : -delta;
+ walk->batched++;
+ }
}
static void reset_batch_size(struct lru_gen_mm_walk *walk)
@@ -3476,7 +3559,6 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
walk->batched = 0;
for_each_gen_type_zone(gen, type, zone) {
- enum lru_list lru = type * LRU_INACTIVE_FILE;
int delta = walk->nr_pages[gen][type][zone];
if (!delta)
@@ -3484,10 +3566,21 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
walk->nr_pages[gen][type][zone] = 0;
atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]);
+ }
- if (lru_gen_is_active(lruvec, gen))
- lru += LRU_ACTIVE;
- __update_lru_size(lruvec, lru, zone, delta);
+ /* apply batched active/inactive updates */
+ for (type = 0; type < ANON_AND_FILE; type++) {
+ for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+ enum lru_list lru = type * LRU_INACTIVE_FILE;
+ int delta = walk->nr_activated[type][zone];
+
+ if (delta) {
+ __update_lru_size(lruvec, lru + LRU_ACTIVE,
+ zone, delta);
+ __update_lru_size(lruvec, lru, zone, -delta);
+ walk->nr_activated[type][zone] = 0;
+ }
+ }
}
lruvec_unlock_irq(lruvec);
@@ -3642,7 +3735,7 @@ static bool suitable_to_scan(int total, int young)
static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma,
struct lruvec *lruvec, struct folio *folio, bool dirty)
{
- int new_gen, old_gen, type;
+ int new_gen, old_gen, old_refs, new_refs, type;
unsigned int flags = LRU_REF_MAPPED;
if (!folio)
@@ -3655,9 +3748,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struc
if (walk) {
old_gen = folio_inc_lru_refs_walk(folio, lruvec, &vma->flags,
- &new_gen, &type);
- if (old_gen >= 0 && old_gen != new_gen)
- update_batch_size(walk, folio, old_gen, new_gen, type);
+ &new_gen, &old_refs, &new_refs, &type);
+ if (old_gen >= 0)
+ update_batch_size(walk, folio, old_gen, new_gen,
+ old_refs, new_refs, type);
} else {
if (is_exec_file_folio(folio, &vma->flags))
flags |= LRU_REF_EXEC;
@@ -4016,6 +4110,7 @@ static void clear_mm_walk(void)
VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages)));
VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats)));
+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_activated, 0, sizeof(walk->nr_activated)));
current->reclaim_state->mm_walk = NULL;
@@ -4054,8 +4149,6 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
goto done;
VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) != MAX_NR_GENS);
- VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) !=
- lru_gen_is_active(lruvec, target_gen));
/* prevent cold/hot inversion if the type is evictable */
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
struct list_head *target_list = &lrugen->folios[target_gen][type][zone];
@@ -4078,7 +4171,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
prefetchw_next_lru_folio(folio, head);
pos = pos->next;
- new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
+ new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased);
/*
* If gen_increased is false, this is a promotion. Put folios
* at the head of the promoted gen. Otherwise, put them at
@@ -4178,8 +4271,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness)
{
bool success;
- int prev, next;
- int type, zone;
+ int type, next;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
restart:
if (seq < READ_ONCE(lrugen->max_seq))
@@ -4205,32 +4297,10 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness
goto restart;
}
- /*
- * Update the active/inactive LRU sizes for compatibility. Both sides of
- * the current max_seq need to be covered, since max_seq+1 can overlap
- * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do
- * overlap, cold/hot inversion happens.
- */
- prev = lru_gen_from_seq(lrugen->max_seq - 1);
- next = lru_gen_from_seq(lrugen->max_seq + 1);
-
- for (type = 0; type < ANON_AND_FILE; type++) {
- for (zone = 0; zone < MAX_NR_ZONES; zone++) {
- enum lru_list lru = type * LRU_INACTIVE_FILE;
- long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) -
- atomic_long_read(&lrugen->nr_pages[next][type][zone]);
-
- if (!delta)
- continue;
-
- __update_lru_size(lruvec, lru, zone, delta);
- __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
- }
- }
-
for (type = 0; type < ANON_AND_FILE; type++)
reset_ctrl_pos(lruvec, type, false);
+ next = lru_gen_from_seq(lrugen->max_seq + 1);
WRITE_ONCE(lrugen->timestamps[next], jiffies);
/* make sure preceding modifications appear */
smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
@@ -4743,7 +4813,6 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
int zone, int type)
{
struct lru_gen_folio *child_lrugen, *parent_lrugen;
- enum lru_list lru = type * LRU_INACTIVE_FILE;
int gen;
child_lrugen = &child_lruvec->lrugen;
@@ -4756,8 +4825,6 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
*/
for (gen = 0; gen < MAX_NR_GENS; gen++) {
long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]);
- int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0;
- int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0;
/* Assuming that child pages are colder than parent pages */
list_splice_tail_init(&child_lrugen->folios[gen][type][zone],
@@ -4765,11 +4832,6 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0);
atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]);
-
- if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) {
- __update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages);
- __update_lru_size(parent_lruvec, lru + parent_lru_active, zone, nr_pages);
- }
}
}
@@ -5169,7 +5231,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
* PG_active and refs are placement hints; cap refs to retain
* workingset history without carrying a high tier into the new gen.
*/
- folio_set_lru_refs(folio, min(folio_lru_refs(folio), LRU_REFS_WORKINGSET));
+ __folio_set_lru_refs(folio, min(folio_lru_refs(folio), LRU_REFS_WORKINGSET));
}
move_folios_to_lru(&list, scanned_seq, false);
diff --git a/mm/workingset.c b/mm/workingset.c
index 34e60b5354a3..30474aa2cf12 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -334,7 +334,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
}
/* Refault is also promotion, cap the refs like folio_inc_lru_refs */
- folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED));
+ __folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED));
}
if (lru_refs_is_workingset(refs))
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH v5 15/17] mm/mglru: folio LRU refs based active/inactive number accounting
2026-11-18 21:03 ` [PATCH v5 15/17] mm/mglru: folio LRU refs based active/inactive number accounting Kairui Song via B4 Relay
@ 2026-11-18 21:29 ` sashiko-bot
0 siblings, 0 replies; 19+ messages in thread
From: sashiko-bot @ 2026-11-18 21:29 UTC (permalink / raw)
To: Kairui Song, linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, damon, tj, mkoutny
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] mm/mglru: raw folio_clear_referenced() in lru_deactivate()
paths
--
commit b8ed2e873fb1f3b844c7ba1fec4ae00c0c04fb88
Author: Kairui Song <kasong@tencent.com>
mm/mglru: folio LRU refs based active/inactive number accounting
This commit switches MGLRU active/inactive LRU size accounting from
generation-window tracking to per-folio reference count thresholds
(LRU_REFS_ACTIVATED).
> diff --git a/mm/folio.c b/mm/folio.c
> index 06c9cde834f76..3b5ae5b516504 100644
> --- a/mm/folio.c
> +++ b/mm/folio.c
[ ... ]
> @@ -552,10 +523,12 @@ static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)
> folio_test_swapcache(folio) || folio_test_unevictable(folio))
> return;
>
> - lru_gen_clear_refs(folio);
> lruvec_del_folio(lruvec, folio);
> folio_clear_active(folio);
> - folio_clear_referenced(folio);
> + if (lru_gen_enabled())
> + __folio_set_lru_refs(folio, 0);
> + else
> + folio_clear_referenced(folio);
[Severity: Medium]
Should lru_deactivate_file() and lru_deactivate() also call
__folio_set_lru_refs(folio, 0) when lru_gen_enabled() is true, like
lru_lazyfree() does here?
Although deactivate_file_folio() and folio_deactivate() call
folio_reset_lru_refs(folio) before queuing the folio into the per-CPU
batch, the folio stays on the LRU list until the batch drains, so
concurrent accesses can increment its refs count in the meantime. When
the batch drains, lru_deactivate_file() and lru_deactivate() still call
raw folio_clear_referenced(folio):
mm/folio.c:lru_deactivate() {
...
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
folio_clear_referenced(folio);
lruvec_add_folio(lruvec, folio);
...
}
Under MGLRU, folio_clear_referenced() clears only bit 0 (PG_referenced) of
the multi-bit refs counter, leaving PG_workingset and LRU_REFS_MASK set.
If the folio was accessed at least twice while queued in the batch,
lruvec_add_folio() sees the surviving upper refs bits and fails to place
the folio in the oldest generation (or re-adds it as active if refs
reached >= 4).
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261119-mglru-fg-v5-0-9eee2fe97a2d@tencent.com?part=15
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v5 16/17] mm/mglru: make folio_inc_lru_refs lruvec lockless
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (14 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 15/17] mm/mglru: folio LRU refs based active/inactive number accounting Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 17/17] mm/mglru: make folio_test_workingset() work based on folio LRU refs Kairui Song via B4 Relay
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
The lruvec spinlock in folio_inc_lru_refs was only taken to keep
concurrent aging from corrupting the size counters. This is no
longer needed: the per-generation counters are atomic, and the
active/inactive ABI counters are updated per folio and linearized by
the CAS on the folio flags, so concurrent aging can no longer corrupt
them. The same argument is what lets folio_reset_lru_refs() update
the counters without the lruvec lock.
Use folio_lruvec_live_get()/folio_lruvec_live_put() to hold the RCU
read lock around the lruvec lookup, and drop the spinlock entirely.
Without the lock, aging can advance max_seq while a promotion is in
flight. Generations are indexed by seq % MAX_NR_GENS, so the folio
flags can return to the value read earlier and the cmpxchg succeeds
with a target gen computed from a stale max_seq. Recheck max_seq after
a gen-changing cmpxchg: if it moved, account the committed update and
retry with LRU_REF_FORCE, which promotes the folio to the newest gen.
The retry may count the access twice, which is acceptable and avoids
folio_activate(), whose LRU lock causes latency jitter. smp_rmb()
orders the flags read, including the one returned by a failed
cmpxchg(), before the max_seq read.
Also simplify the post-CAS accounting guards: gen is only assigned
non-negative values after the old_gen < 0 early exit, so gen != old_gen
implies gen >= 0, and lru_gen_update_size() needs no explicit guard.
The active/inactive ABI update keeps its gen >= 0 check because an
off-LRU folio has no lruvec to update.
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 10 +++++-----
include/linux/mmzone.h | 3 ++-
mm/vmscan.c | 51 +++++++++++++++++++++++++++++------------------
3 files changed, 39 insertions(+), 25 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 77f514143266..2c6cbd0df686 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -317,10 +317,9 @@ static inline bool lru_gen_is_old(struct lruvec *lruvec, int type, int gen)
gen == lru_gen_from_seq(min_seq + 1);
}
-static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
- int old_gen, int new_gen)
+static inline void lru_gen_update_size(struct lruvec *lruvec, int type,
+ struct folio *folio, int old_gen, int new_gen)
{
- int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
@@ -399,7 +398,7 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli
/* use the refs from the atomic snapshot to avoid raced update */
refs = lru_get_refs_flags(flags);
- lru_gen_update_size(lruvec, folio, -1, gen);
+ lru_gen_update_size(lruvec, type, folio, -1, gen);
if (lru_refs_is_active(refs))
lru += LRU_ACTIVE;
__update_lru_size(lruvec, lru, zone, delta);
@@ -429,6 +428,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
unsigned long flags;
unsigned int refs;
int gen;
+ int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE;
@@ -452,7 +452,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
if (!reclaiming && ((max_seq - gen) % MAX_NR_GENS) < MIN_NR_GENS)
folio_set_active(folio);
- lru_gen_update_size(lruvec, folio, gen, -1);
+ lru_gen_update_size(lruvec, type, folio, gen, -1);
if (lru_refs_is_active(refs))
lru += LRU_ACTIVE;
__update_lru_size(lruvec, lru, zone, -delta);
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 2680e15b6de9..235fb04f8d78 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -540,9 +540,10 @@ enum lruvec_flags {
#define LRU_TIER_MIN 0U
#define LRU_TIER_MAX (MAX_NR_TIERS - 1)
-/* Access source flags for folio_inc_lru_refs() */
+/* Flags for folio_inc_lru_refs() */
#define LRU_REF_MAPPED 0x1U
#define LRU_REF_EXEC 0x2U
+#define LRU_REF_FORCE 0x4U
#define LRU_REFS_REFERENCED 0x1
#define LRU_REFS_WORKINGSET 0x2
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 78a1b4de7076..4576a7a369e3 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -876,37 +876,35 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
long nr_pages = folio_nr_pages(folio);
struct lru_gen_folio *lrugen;
struct lruvec *lruvec = NULL;
+ bool aged = false;
+retry:
old_flags = READ_ONCE(*folio_flags(folio, 0));
do {
new_flags = old_flags;
old_gen = lru_get_gen_flags(old_flags);
old_refs = lru_get_refs_flags(old_flags);
- file = folio_flags_is_file_lru(&old_flags);
refs = old_refs + 1;
gen = old_gen;
+ file = folio_flags_is_file_lru(&old_flags);
if (old_gen < 0)
goto out;
- /*
- * Lock the lruvec if the folio is on-list. We are already
- * doing lazy promotion so in theory we don't need this,
- * but for now, concurrent aging would still corrupt the
- * size counters. This is a temporary limitation and
- * will be lifted very soon, so the lock here is not a
- * performance concern.
- */
if (!lruvec) {
- lruvec = lruvec_live_lock_irq(folio_lruvec(folio));
+ lruvec = folio_lruvec_live_get(folio);
lrugen = &lruvec->lrugen;
}
+ /*
+ * Read the flags before max_seq, paired with the store of a
+ * gen, which samples max_seq first.
+ */
+ smp_rmb();
max_seq = READ_ONCE(lrugen->max_seq);
max_gen = lru_gen_from_seq(max_seq);
if (old_gen == max_gen)
goto out;
-
- if (flags & (LRU_REF_MAPPED | LRU_REF_EXEC)) {
- /* Promote second page table access or executable */
- if (refs > LRU_REFS_REFERENCED || flags & LRU_REF_EXEC)
+ if (flags & (LRU_REF_MAPPED | LRU_REF_EXEC | LRU_REF_FORCE)) {
+ if (lru_refs_is_workingset(refs) ||
+ flags & (LRU_REF_EXEC | LRU_REF_FORCE))
gen = max_gen;
/* First access only defers eviction of an old generation */
else if (lru_gen_is_old(lruvec, file, old_gen))
@@ -931,9 +929,19 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
break;
} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
- if (gen != old_gen)
- lru_gen_update_size(lruvec, folio, old_gen, gen);
- if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && old_gen >= 0) {
+ if (gen != old_gen) {
+ /*
+ * Gen-index reuse can fool cmpxchg(). Retry locklessly, accepting
+ * an extra reference to avoid folio_activate() latency.
+ */
+ if (unlikely(READ_ONCE(lrugen->max_seq) != max_seq)) {
+ flags = LRU_REF_FORCE;
+ aged = true;
+ }
+ lru_gen_update_size(lruvec, file, folio, old_gen, gen);
+ }
+
+ if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && gen >= 0) {
enum lru_list lru = file * LRU_INACTIVE_FILE;
__update_lru_size(lruvec, lru + lru_refs_is_active(old_refs),
@@ -941,8 +949,12 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags)
__update_lru_size(lruvec, lru + lru_refs_is_active(refs),
folio_zonenum(folio), nr_pages);
}
+ if (aged) {
+ aged = false;
+ goto retry;
+ }
if (lruvec)
- lruvec_unlock_irq(lruvec);
+ folio_lruvec_live_put(lruvec);
}
/*
@@ -3517,12 +3529,13 @@ static int __folio_inc_gen(struct lruvec *lruvec, struct folio *folio,
/* protect pages selected by the tier PID controller */
static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen)
{
+ int type = folio_is_file_lru(folio);
int new_gen;
bool gen_increased;
new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased);
if (gen_increased)
- lru_gen_update_size(lruvec, folio, old_gen, new_gen);
+ lru_gen_update_size(lruvec, type, folio, old_gen, new_gen);
return new_gen;
}
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v5 17/17] mm/mglru: make folio_test_workingset() work based on folio LRU refs
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
` (15 preceding siblings ...)
2026-11-18 21:03 ` [PATCH v5 16/17] mm/mglru: make folio_inc_lru_refs lruvec lockless Kairui Song via B4 Relay
@ 2026-11-18 21:03 ` Kairui Song via B4 Relay
16 siblings, 0 replies; 19+ messages in thread
From: Kairui Song via B4 Relay @ 2026-11-18 21:03 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, Johannes Weiner, Muchun Song, Qi Zheng, Ying Huang,
Chris Li, Baoquan He, Nico Pache, Usama Arif, Michal Hocko,
Roman Gushchin, Shakeel Butt, David Hildenbrand, Lorenzo Stoakes,
Liam R. Howlett, Barry Song, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Vlastimil Babka, Suren Baghdasaryan, Nhat Pham, Youngjun Park,
Zi Yan, Gregory Price, Matthew Wilcox (Oracle), Baolin Wang,
Ryan Roberts, Dev Jain, Lance Yang, Hugh Dickins, SJ Park,
David Rientjes, Yu Zhao, Vernon Yang, Zicheng Wang, Chen Ridong,
Tal Zussman, Kairui Song, Xiang Liu, Ehab Ababneh, Zhao Zhenzhuo,
Lian Wang, Joanne Koong, Kunwu Chan, Hui Zhu, linux-kernel,
cgroups, damon, linux-doc, Kairui Song
From: Kairui Song <kasong@tencent.com>
folio_test_workingset() currently tests the PG_workingset bit, which
is only the second lowest access bit of the folio LRU refs count now.
Folios with refs 4 or 5 carry it in the LRU_REFS_MASK field
and no longer have PG_workingset set, so the bit test misses them.
folio_set_workingset() currently sets the PG_workingset bit with a
plain set_bit(), which also over-promotes folios at a higher refs
count: e.g. a folio at refs 4 is bumped to refs 6, advancing it
toward LRU_REFS_MAX and an earlier generation promotion.
Move the test into mm_inline.h as an inline helper based on
folio_lru_refs(), checking refs >= LRU_REFS_WORKINGSET. The set
stays a plain PG_workingset bit operation reserved for the
classical LRU: under MGLRU the refs count is maintained by
folio_inc_lru_refs(), and now triggers a debug WARN when called
while MGLRU is fully on (the switching window is exempt, as the
classical paths legitimately run alongside MGLRU then). The one
PageWorkingset() user in erofs is converted to the folio helper.
Under the classical LRU the refs count is not maintained, so the
test is equivalent to the PG_workingset bit test, which is the old
behavior.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
fs/btrfs/compression.c | 1 +
fs/erofs/zdata.c | 3 ++-
include/linux/mm_inline.h | 29 +++++++++++++++++++++++++++++
include/linux/page-flags.h | 2 --
mm/filemap.c | 1 +
mm/page_io.c | 1 +
6 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index ad90e14032db..127c5ee38979 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -8,6 +8,7 @@
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
+#include <linux/mm_inline.h>
#include <linux/folio_batch.h>
#include <linux/highmem.h>
#include <linux/kthread.h>
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index e981e371d6c2..34ee41e1ccdc 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -6,6 +6,7 @@
*/
#include "compress.h"
#include <linux/psi.h>
+#include <linux/mm_inline.h>
#include <linux/cpuhotplug.h>
#include <trace/events/erofs.h>
@@ -1713,7 +1714,7 @@ static void z_erofs_submit_queue(struct z_erofs_frontend *f,
DBG_BUGON(bvec.bv_len < sb->s_blocksize);
}
- if (unlikely(PageWorkingset(bvec.bv_page)) &&
+ if (unlikely(folio_test_workingset(page_folio(bvec.bv_page))) &&
!memstall) {
psi_memstall_enter(&pflags);
memstall = 1;
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 2c6cbd0df686..a2442e49aca2 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -300,6 +300,23 @@ static inline void folio_migrate_lru_refs(struct folio *new, const struct folio
__folio_set_lru_refs(new, folio_lru_refs(old));
}
+/**
+ * folio_test_workingset - Test if a folio is in the workingset.
+ * @folio: the folio
+ *
+ * A folio is workingset when its LRU refs count reaches
+ * LRU_REFS_WORKINGSET. Under the classical LRU flag encoding,
+ * this is just testing the PG_workingset bit.
+ * NOTE: folio_set_workingset() must not be used under MGLRU, as the
+ * folio refs are tracked by folio_inc_lru_refs() with rules.
+ *
+ * Return: true if the folio is workingset.
+ */
+static __always_inline bool folio_test_workingset(const struct folio *folio)
+{
+ return lru_refs_is_workingset(folio_lru_refs(folio));
+}
+
#ifdef CONFIG_LRU_GEN
static inline bool lru_refs_is_active(unsigned int refs)
{
@@ -551,6 +568,18 @@ static inline void folio_clear_lru_refs(struct folio *folio)
folio_clear_referenced(folio);
}
+/*
+ * For the classical LRU only: with MGLRU the PG_workingset bit is
+ * part of the folio refs count maintained by folio_inc_lru_refs(),
+ * and a raw set would corrupt it. The switching window is exempt
+ * because the classical paths legitimately run alongside MGLRU then.
+ */
+static __always_inline void folio_set_workingset(struct folio *folio)
+{
+ VM_WARN_ON_ONCE(lru_gen_enabled() && !lru_gen_switching());
+ set_bit(PG_workingset, folio_flags(folio, FOLIO_HEAD_PAGE));
+}
+
static __always_inline
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
{
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index b0ddc652e76c..f6625dc042da 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -549,8 +549,6 @@ PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
FOLIO_FLAG(active, FOLIO_HEAD_PAGE)
__FOLIO_CLEAR_FLAG(active, FOLIO_HEAD_PAGE)
FOLIO_TEST_CLEAR_FLAG(active, FOLIO_HEAD_PAGE)
-PAGEFLAG(Workingset, workingset, PF_HEAD)
- TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */
/* Xen */
diff --git a/mm/filemap.c b/mm/filemap.c
index 648f783454c3..2939647940c1 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -21,6 +21,7 @@
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/swap.h>
+#include <linux/mm_inline.h>
#include <linux/leafops.h>
#include <linux/syscalls.h>
#include <linux/mman.h>
diff --git a/mm/page_io.c b/mm/page_io.c
index a57cdcf2892d..12deab296718 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -16,6 +16,7 @@
#include <linux/gfp.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
+#include <linux/mm_inline.h>
#include <linux/bio.h>
#include <linux/swapops.h>
#include <linux/writeback.h>
--
2.56.1
^ permalink raw reply related [flat|nested] 19+ messages in thread