* [PATCH 6.12.y 2/4] mm/mglru: optimize deactivation
2026-09-03 18:40 ` [PATCH 6.12.y 1/4] mm/mglru: reset page lru tier bits when activating Sasha Levin
@ 2026-09-03 18:40 ` Sasha Levin
2026-09-03 18:40 ` [PATCH 6.12.y 3/4] mm/mglru: use the common routine for dirty/writeback reactivation Sasha Levin
2026-09-03 18:40 ` [PATCH 6.12.y 4/4] mm/mglru: fix and remove redundant unevictable folio handling Sasha Levin
2 siblings, 0 replies; 5+ messages in thread
From: Sasha Levin @ 2026-09-03 18:40 UTC (permalink / raw)
To: stable
Cc: Yu Zhao, Bharata B Rao, Kalesh Singh, Barry Song, David Stevens,
Kairui Song, Andrew Morton, Sasha Levin
From: Yu Zhao <yuzhao@google.com>
[ Upstream commit cc8ec7be78ffaa29f6e0781fd12602974970eb53 ]
Do not shuffle a folio in the deactivation paths if it is already in the
oldest generation. This reduces the LRU lock contention.
Before this patch, the contention is reproducible by FIO, e.g.,
fio -filename=/dev/nvme1n1p2 -direct=0 -thread -size=1024G \
-rwmixwrite=30 --norandommap --randrepeat=0 -ioengine=sync \
-bs=4k -numjobs=400 -runtime=25000 --time_based \
-group_reporting -name=mglru
98.96%--_raw_spin_lock_irqsave
folio_lruvec_lock_irqsave
|
--98.78%--folio_batch_move_lru
|
--98.63%--deactivate_file_folio
mapping_try_invalidate
invalidate_mapping_pages
invalidate_bdev
blkdev_common_ioctl
blkdev_ioctl
After this patch, deactivate_file_folio() bails out early without taking
the LRU lock.
A side effect is that a folio can be left at the head of the oldest
generation, rather than the tail. If reclaim happens at the same time, it
cannot reclaim this folio immediately. Since there is no known
correlation between truncation and reclaim, this side effect is considered
insignificant.
Link: https://lkml.kernel.org/r/20241231043538.4075764-3-yuzhao@google.com
Reported-by: Bharata B Rao <bharata@amd.com>
Closes: https://lore.kernel.org/CAOUHufawNerxqLm7L9Yywp3HJFiYVrYO26ePUb1jH-qxNGWzyA@mail.gmail.com/
Signed-off-by: Yu Zhao <yuzhao@google.com>
Tested-by: Kalesh Singh <kaleshsingh@google.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: David Stevens <stevensd@chromium.org>
Cc: Kairui Song <kasong@tencent.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7e698e326b2 ("mm/mglru: fix and remove redundant unevictable folio handling")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/swap.c | 49 +++++++++++++++++++++++++++++++++++++++++--------
mm/vmscan.c | 38 ++++++++++++++++++++++++++++++--------
2 files changed, 71 insertions(+), 16 deletions(-)
diff --git a/mm/swap.c b/mm/swap.c
index ff846915db454..94fa59d230b18 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -411,7 +411,8 @@ static void __lru_cache_activate_folio(struct folio *folio)
}
#ifdef CONFIG_LRU_GEN
-static void folio_inc_refs(struct folio *folio)
+
+static void lru_gen_inc_refs(struct folio *folio)
{
unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
@@ -438,10 +439,34 @@ static void folio_inc_refs(struct folio *folio)
new_flags |= old_flags & ~LRU_REFS_MASK;
} while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
}
-#else
-static void folio_inc_refs(struct folio *folio)
+
+static bool lru_gen_clear_refs(struct folio *folio)
+{
+ struct lru_gen_folio *lrugen;
+ int gen = folio_lru_gen(folio);
+ int type = folio_is_file_lru(folio);
+
+ if (gen < 0)
+ return true;
+
+ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);
+
+ lrugen = &folio_lruvec(folio)->lrugen;
+ /* whether can do without shuffling under the LRU lock */
+ return gen == lru_gen_from_seq(READ_ONCE(lrugen->min_seq[type]));
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static void lru_gen_inc_refs(struct folio *folio)
+{
+}
+
+static bool lru_gen_clear_refs(struct folio *folio)
{
+ return false;
}
+
#endif /* CONFIG_LRU_GEN */
/**
@@ -460,7 +485,7 @@ static void folio_inc_refs(struct folio *folio)
void folio_mark_accessed(struct folio *folio)
{
if (lru_gen_enabled()) {
- folio_inc_refs(folio);
+ lru_gen_inc_refs(folio);
return;
}
@@ -556,7 +581,7 @@ void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)
*/
static void lru_deactivate_file(struct lruvec *lruvec, struct folio *folio)
{
- bool active = folio_test_active(folio);
+ bool active = folio_test_active(folio) || lru_gen_enabled();
long nr_pages = folio_nr_pages(folio);
if (folio_test_unevictable(folio))
@@ -621,7 +646,10 @@ static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
- folio_clear_referenced(folio);
+ if (lru_gen_enabled())
+ lru_gen_clear_refs(folio);
+ else
+ folio_clear_referenced(folio);
/*
* Lazyfree folios are clean anonymous folios. They have
* the swapbacked flag cleared, to distinguish them from normal
@@ -689,6 +717,9 @@ void deactivate_file_folio(struct folio *folio)
if (folio_test_unevictable(folio) || !folio_test_lru(folio))
return;
+ if (lru_gen_enabled() && lru_gen_clear_refs(folio))
+ return;
+
folio_batch_add_and_move(folio, lru_deactivate_file);
}
@@ -702,8 +733,10 @@ void deactivate_file_folio(struct folio *folio)
*/
void folio_deactivate(struct folio *folio)
{
- if (folio_test_unevictable(folio) || !folio_test_lru(folio) ||
- !(folio_test_active(folio) || lru_gen_enabled()))
+ if (folio_test_unevictable(folio) || !folio_test_lru(folio))
+ return;
+
+ if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
return;
folio_batch_add_and_move(folio, lru_deactivate);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index e5cdf114c3ba5..7a112f93a681a 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4313,7 +4313,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
int tier_idx)
{
bool success;
- bool dirty, writeback;
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
@@ -4359,8 +4358,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
return true;
}
- dirty = folio_test_dirty(folio);
- writeback = folio_test_writeback(folio);
+ bool dirty = folio_test_dirty(folio);
+ bool writeback = folio_test_writeback(folio);
+
if (type == LRU_GEN_FILE && dirty) {
sc->nr.file_taken += delta;
if (!writeback)
@@ -4627,12 +4627,23 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
list_for_each_entry_safe_reverse(folio, next, &list, lru) {
+ DEFINE_MIN_SEQ(lruvec);
+
if (!folio_evictable(folio)) {
list_del(&folio->lru);
folio_putback_lru(folio);
continue;
}
+ /* retry folios that may have missed folio_rotate_reclaimable() */
+ if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
+ !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
+ if (!folio_test_referenced(folio) && !folio_test_locked(folio)) {
+ list_move(&folio->lru, &clean);
+ continue;
+ }
+ }
+
if (folio_test_reclaim(folio) &&
(folio_test_dirty(folio) || folio_test_writeback(folio))) {
/* restore LRU_REFS_FLAGS cleared by isolate_folio() */
@@ -4644,14 +4655,25 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
if (skip_retry || folio_test_active(folio) || folio_test_referenced(folio) ||
folio_mapped(folio) || folio_test_locked(folio) ||
folio_test_dirty(folio) || folio_test_writeback(folio)) {
+ unsigned long seq = READ_ONCE(lruvec->lrugen.max_seq);
+
/* don't add rejected folios to the oldest generation */
- set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS,
- BIT(PG_active));
+ if (!folio_test_active(folio)) {
+ if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) ||
+ (folio_test_reclaim(folio) &&
+ (folio_test_dirty(folio) || folio_test_writeback(folio))))
+ seq--;
+ else if (min_seq[type] + MIN_NR_GENS < seq)
+ seq = min_seq[type] + 1;
+ else
+ seq = min_seq[type];
+ }
+
+ if (seq == min_seq[type])
+ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS,
+ BIT(PG_active));
continue;
}
-
- /* retry folios that may have missed folio_rotate_reclaimable() */
- list_move(&folio->lru, &clean);
}
spin_lock_irq(&lruvec->lru_lock);
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 6.12.y 3/4] mm/mglru: use the common routine for dirty/writeback reactivation
2026-09-03 18:40 ` [PATCH 6.12.y 1/4] mm/mglru: reset page lru tier bits when activating Sasha Levin
2026-09-03 18:40 ` [PATCH 6.12.y 2/4] mm/mglru: optimize deactivation Sasha Levin
@ 2026-09-03 18:40 ` Sasha Levin
2026-09-03 18:40 ` [PATCH 6.12.y 4/4] mm/mglru: fix and remove redundant unevictable folio handling Sasha Levin
2 siblings, 0 replies; 5+ messages in thread
From: Sasha Levin @ 2026-09-03 18:40 UTC (permalink / raw)
To: stable
Cc: Kairui Song, Axel Rasmussen, Barry Song, Baolin Wang, Chen Ridong,
Chris Li, David Hildenbrand, David Stevens, Johannes Weiner,
Kalesh Singh, Leno Hou, Lorenzo Stoakes, Michal Hocko,
Shakeel Butt, Suren Baghdasaryan, Vernon Yang, Wei Xu, Yafang,
Yuanchu Xie, Yu Zhao, Andrew Morton, Sasha Levin
From: Kairui Song <kasong@tencent.com>
[ Upstream commit 75d4c3f5fb980de1b620adede47e43dff4d6a5f3 ]
Currently MGLRU will move the dirty writeback folios to the second oldest
gen instead of reactivate them like the classical LRU. This might help to
reduce the LRU contention as it skipped the isolation. But as a result we
will see these folios at the LRU tail more frequently leading to
inefficient reclaim.
Besides, the dirty / writeback check after isolation in shrink_folio_list
is more accurate and covers more cases. So instead, just drop the special
handling for dirty writeback, use the common routine and re-activate it
like the classical LRU.
This should in theory improve the scan efficiency. These folios will be
rotated back to LRU tail once writeback is done so there is no risk of
hotness inversion. And now each reclaim loop will have a higher success
rate. This also prepares for unifying the writeback and throttling
mechanism with classical LRU, we keep these folios far from tail so
detecting the tail batch will have a similar pattern with classical LRU.
The micro optimization that avoids LRU contention by skipping the
isolation is gone, which should be fine. Compared to IO and writeback
cost, the isolation overhead is trivial.
And using the common routine also keeps the folio's referenced bits (tier
bits), which could improve metrics in the long term. Also no more need to
clean reclaim bit as the common routine will make use of it.
Note the common routine updates a few throttling and writeback counters,
which are not used, and never have been for the MGLRU case. We will start
making use of these in later commits.
Link: https://lore.kernel.org/20260428-mglru-reclaim-v7-10-02fabb92dc43@tencent.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Reviewed-by: Axel Rasmussen <axelrasmussen@google.com>
Reviewed-by: Barry Song <baohua@kernel.org>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Chen Ridong <chenridong@huaweicloud.com>
Cc: Chris Li <chrisl@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: David Stevens <stevensd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Leno Hou <lenohou@gmail.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vernon Yang <vernon2gm@gmail.com>
Cc: Wei Xu <weixugc@google.com>
Cc: Yafang <laoar.shao@gmail.com>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Stable-dep-of: f7e698e326b2 ("mm/mglru: fix and remove redundant unevictable folio handling")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/vmscan.c | 21 ---------------------
1 file changed, 21 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 7a112f93a681a..f38fe097a46be 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4358,23 +4358,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
return true;
}
- bool dirty = folio_test_dirty(folio);
- bool writeback = folio_test_writeback(folio);
-
- if (type == LRU_GEN_FILE && dirty) {
- sc->nr.file_taken += delta;
- if (!writeback)
- sc->nr.unqueued_dirty += delta;
- }
-
- /* waiting for writeback */
- if (folio_test_locked(folio) || writeback ||
- (type == LRU_GEN_FILE && dirty)) {
- gen = folio_inc_gen(lruvec, folio, true);
- list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
- return true;
- }
-
return false;
}
@@ -4402,10 +4385,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
if (!folio_test_referenced(folio))
folio_clear_lru_refs(folio);
- /* for shrink_folio_list() */
- folio_clear_reclaim(folio);
- folio_clear_referenced(folio);
-
success = lru_gen_del_folio(lruvec, folio, true);
VM_WARN_ON_ONCE_FOLIO(!success, folio);
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 6.12.y 4/4] mm/mglru: fix and remove redundant unevictable folio handling
2026-09-03 18:40 ` [PATCH 6.12.y 1/4] mm/mglru: reset page lru tier bits when activating Sasha Levin
2026-09-03 18:40 ` [PATCH 6.12.y 2/4] mm/mglru: optimize deactivation Sasha Levin
2026-09-03 18:40 ` [PATCH 6.12.y 3/4] mm/mglru: use the common routine for dirty/writeback reactivation Sasha Levin
@ 2026-09-03 18:40 ` Sasha Levin
2 siblings, 0 replies; 5+ messages in thread
From: Sasha Levin @ 2026-09-03 18:40 UTC (permalink / raw)
To: stable
Cc: Kairui Song, Barry Song, Baolin Wang, Axel Rasmussen,
Brian Geffon, David Hildenbrand, Jan Alexander Steffens (heftig),
Johannes Weiner, Lorenzo Stoakes, Michal Hocko,
Oleksandr Natalenko, Shakeel Butt, Steven Barrett,
Suleiman Souhlal, Wei Xu, Yuanchu Xie, Yu Zhao, Andrew Morton,
Sasha Levin
From: Kairui Song <kasong@tencent.com>
[ Upstream commit f7e698e326b239a91ea15844817551921209e826 ]
sort_folio() has a shortcut for moving folios that are no longer evictable
but are still sitting on a generation list. However, this shortcut is
buggy. It does not follow the PG_lru usage convention, and it has a more
serious issue.
Unevictable folios are not threaded on lists[LRU_UNEVICTABLE], so that
folio->lru can be reused to hold folio->mlock_count (see the comment in
lruvec_init()). Hence lruvec_add_folio() skips the list_add() for them,
and every other place that turns a folio unevictable initialises
mlock_count explicitly: lru_add() sets it to 0, __mlock_folio() and
__mlock_new_folio() set it to !!folio_test_mlocked(folio). sort_folio()
sets nothing, and the lru_gen_del_folio() right above it may have already
poisoned folio->lru via list_del(), so mlock_count ends up aliasing
LIST_POISON2, which reads as 0x122, i.e. 290. The result is user
visible. On munlock, __munlock_folio() decrements that bogus count, finds
it still non-zero and bails out before clearing PG_mlocked, so the folio
remains unevictable and the Mlocked accounting stays inflated until the
folio is freed.
The shortcut also touches the LRU flags in the wrong order. It calls
lru_gen_del_folio() while PG_lru is still set, so a concurrent
folio_test_clear_lru() (e.g. compaction, folio_isolate_lru()) can succeed
on a folio that has already been taken off the generation list, which may
lead to unexpected behavior.
So fix it by isolating them as common folios and letting the generic
shrink path cull them. This matches the classical LRU behavior, and there
should be no visible effect on the generic eviction or isolation behavior.
There is no performance concern either, such a folio goes through this
once, and then it is off the generation lists for good.
Link: https://lore.kernel.org/20260812-mglru-mlock-fix-v2-1-a3fec5853c08@tencent.com
Fixes: ac35a4902374 ("mm: multi-gen LRU: minimal implementation")
Signed-off-by: Kairui Song <kasong@tencent.com>
Reviewed-by: Barry Song <baohua@kernel.org>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Brian Geffon <bgeffon@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Oleksandr Natalenko <oleksandr@natalenko.name>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Steven Barrett <steven@liquorix.net>
Cc: Suleiman Souhlal <suleiman@google.com>
Cc: Wei Xu <weixugc@google.com>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
mm/vmscan.c | 19 +++++--------------
1 file changed, 5 insertions(+), 14 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f38fe097a46be..709e819436754 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4312,7 +4312,6 @@ void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
int tier_idx)
{
- bool success;
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
@@ -4323,15 +4322,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
- /* unevictable */
- if (!folio_evictable(folio)) {
- success = lru_gen_del_folio(lruvec, folio, true);
- VM_WARN_ON_ONCE_FOLIO(!success, folio);
- folio_set_unevictable(folio);
- lruvec_add_folio(lruvec, folio);
- __count_vm_events(UNEVICTABLE_PGCULLED, delta);
- return true;
- }
+ /* unevictable: let it through and the generic path will cull it */
+ if (!folio_evictable(folio))
+ return false;
/* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
@@ -4608,11 +4601,9 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
list_for_each_entry_safe_reverse(folio, next, &list, lru) {
DEFINE_MIN_SEQ(lruvec);
- if (!folio_evictable(folio)) {
- list_del(&folio->lru);
- folio_putback_lru(folio);
+ /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+ if (!folio_evictable(folio))
continue;
- }
/* retry folios that may have missed folio_rotate_reclaimable() */
if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread