* [RFC PATCH v5 01/14] mm/mglru: report evictable sizes per type
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-27 8:51 ` KunWu Chan
2026-10-22 22:01 ` [RFC PATCH v5 02/14] mm/mglru: extract lruvec_populated_min_seq() helper Barry Song (Xiaomi)
` (13 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
Change lruvec_evictable_size() to return the number of evictable
anonymous and file pages separately instead of a single total, so
that callers can weigh the two types independently.
Update lruvec_is_sizable() and get_nr_to_scan() accordingly; the
summed totals are unchanged, so there is no functional change.
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index fc823fe3bd1f..3be0d6c04a82 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4332,32 +4332,35 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control
sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
}
-static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
+static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
+ unsigned long nr[])
{
int gen, type, zone;
- unsigned long seq, total = 0;
+ unsigned long seq;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
+ nr[LRU_GEN_ANON] = 0;
+ nr[LRU_GEN_FILE] = 0;
+
for_each_evictable_type(type, swappiness) {
for (seq = min_seq[type]; seq <= max_seq; seq++) {
gen = lru_gen_from_seq(seq);
for (zone = 0; zone < MAX_NR_ZONES; zone++)
- total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+ nr[type] += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
}
}
-
- return total;
}
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
- unsigned long total;
+ unsigned long total, sizes[ANON_AND_FILE];
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
- total = lruvec_evictable_size(lruvec, swappiness);
+ lruvec_evictable_size(lruvec, swappiness, sizes);
+ total = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
/* whether the size is big enough to be helpful */
return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
@@ -5181,7 +5184,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
struct mem_cgroup *memcg, int swappiness)
{
- unsigned long nr_to_scan, evictable;
+ unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
/*
@@ -5197,7 +5200,8 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
return 0;
}
- evictable = lruvec_evictable_size(lruvec, swappiness);
+ lruvec_evictable_size(lruvec, swappiness, sizes);
+ evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
/* try to scrape all its memory if this memcg was deleted */
if (!mem_cgroup_online(memcg))
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 01/14] mm/mglru: report evictable sizes per type
2026-10-22 22:01 ` [RFC PATCH v5 01/14] mm/mglru: report evictable sizes per type Barry Song (Xiaomi)
@ 2026-10-27 8:51 ` KunWu Chan
0 siblings, 0 replies; 30+ messages in thread
From: KunWu Chan @ 2026-10-27 8:51 UTC (permalink / raw)
To: Barry Song (Xiaomi)
Cc: akpm, linux-mm, hui.zhu, axelrasmussen, baolin.wang, david,
hannes, linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen
On Fri, Oct 23, 2026 at 6:02 AM Barry Song (Xiaomi) <baohua@kernel.org> wrote:
>
> From: Kairui Song <kasong@tencent.com>
>
> Change lruvec_evictable_size() to return the number of evictable
> anonymous and file pages separately instead of a single total, so
> that callers can weigh the two types independently.
>
> Update lruvec_is_sizable() and get_nr_to_scan() accordingly; the
> summed totals are unchanged, so there is no functional change.
>
> Signed-off-by: Kairui Song <kasong@tencent.com>
> Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
> ---
> mm/vmscan.c | 22 +++++++++++++---------
> 1 file changed, 13 insertions(+), 9 deletions(-)
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index fc823fe3bd1f..3be0d6c04a82 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -4332,32 +4332,35 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control
> sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
> }
>
> -static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
> +static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
> + unsigned long nr[])
> {
> int gen, type, zone;
> - unsigned long seq, total = 0;
> + unsigned long seq;
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> DEFINE_MAX_SEQ(lruvec);
> DEFINE_MIN_SEQ(lruvec);
>
> + nr[LRU_GEN_ANON] = 0;
> + nr[LRU_GEN_FILE] = 0;
> +
> for_each_evictable_type(type, swappiness) {
> for (seq = min_seq[type]; seq <= max_seq; seq++) {
> gen = lru_gen_from_seq(seq);
> for (zone = 0; zone < MAX_NR_ZONES; zone++)
> - total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
> + nr[type] += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
> }
> }
> -
> - return total;
> }
>
> static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
> {
> - unsigned long total;
> + unsigned long total, sizes[ANON_AND_FILE];
> int swappiness = get_swappiness(lruvec, sc);
> struct mem_cgroup *memcg = lruvec_memcg(lruvec);
>
> - total = lruvec_evictable_size(lruvec, swappiness);
> + lruvec_evictable_size(lruvec, swappiness, sizes);
> + total = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
>
> /* whether the size is big enough to be helpful */
> return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
> @@ -5181,7 +5184,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
> static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
> struct mem_cgroup *memcg, int swappiness)
> {
> - unsigned long nr_to_scan, evictable;
> + unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
> struct pglist_data *pgdat = lruvec_pgdat(lruvec);
>
> /*
> @@ -5197,7 +5200,8 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
> return 0;
> }
>
> - evictable = lruvec_evictable_size(lruvec, swappiness);
> + lruvec_evictable_size(lruvec, swappiness, sizes);
> + evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
>
> /* try to scrape all its memory if this memcg was deleted */
> if (!mem_cgroup_online(memcg))
> --
> 2.34.1
>
Both call sites preserve the previous total by summing the
anon and file sizes.
Reviewed-by: Kunwu Chan <kunwu.chan@gmail.com>
Thanks,
Kunwu
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 02/14] mm/mglru: extract lruvec_populated_min_seq() helper
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
2026-10-22 22:01 ` [RFC PATCH v5 01/14] mm/mglru: report evictable sizes per type Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-27 9:39 ` KunWu Chan
2026-10-22 22:01 ` [RFC PATCH v5 03/14] mm/mglru: reclaim from the oldest populated gen Barry Song (Xiaomi)
` (12 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
The search for the oldest populated generation of a type, previously
embedded in try_to_inc_min_seq(), is extracted into the new
lruvec_populated_min_seq() helper. It returns the oldest populated seq
of a given type, bounded by the given max_seq so the search never goes
beyond it.
No functional change.
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 43 +++++++++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 14 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 3be0d6c04a82..500ef23949fd 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4131,30 +4131,45 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
return true;
}
+/* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
+static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
+ unsigned long max_seq)
+{
+ int gen, zone;
+ unsigned long min_seq;
+ struct lru_gen_folio *lrugen = &lruvec->lrugen;
+
+ lockdep_assert_held(&lruvec->lru_lock);
+
+ min_seq = READ_ONCE(lrugen->min_seq[type]);
+ while (min_seq <= max_seq) {
+ gen = lru_gen_from_seq(min_seq);
+ for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+ if (!list_empty(&lrugen->folios[gen][type][zone]))
+ return min_seq;
+ }
+ min_seq++;
+ }
+
+ return max_seq;
+}
+
static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
{
- int gen, type, zone;
+ int type;
bool seq_inc_flag = false;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
DEFINE_MIN_SEQ(lruvec);
+ unsigned long seq;
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
- /* find the oldest populated generation */
+ /* Requires at least MIN_NR_GENS gens for basic functionality */
+ seq = lrugen->max_seq - MIN_NR_GENS;
for_each_evictable_type(type, swappiness) {
- while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
- gen = lru_gen_from_seq(min_seq[type]);
-
- for (zone = 0; zone < MAX_NR_ZONES; zone++) {
- if (!list_empty(&lrugen->folios[gen][type][zone]))
- goto next;
- }
-
- min_seq[type]++;
+ min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
+ if (min_seq[type] > lrugen->min_seq[type])
seq_inc_flag = true;
- }
-next:
- ;
}
/*
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 02/14] mm/mglru: extract lruvec_populated_min_seq() helper
2026-10-22 22:01 ` [RFC PATCH v5 02/14] mm/mglru: extract lruvec_populated_min_seq() helper Barry Song (Xiaomi)
@ 2026-10-27 9:39 ` KunWu Chan
0 siblings, 0 replies; 30+ messages in thread
From: KunWu Chan @ 2026-10-27 9:39 UTC (permalink / raw)
To: Barry Song (Xiaomi)
Cc: akpm, linux-mm, hui.zhu, axelrasmussen, baolin.wang, david,
hannes, linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen
On Fri, Oct 23, 2026 at 6:02 AM Barry Song (Xiaomi) <baohua@kernel.org> wrote:
>
> From: Kairui Song <kasong@tencent.com>
>
> The search for the oldest populated generation of a type, previously
> embedded in try_to_inc_min_seq(), is extracted into the new
> lruvec_populated_min_seq() helper. It returns the oldest populated seq
> of a given type, bounded by the given max_seq so the search never goes
> beyond it.
>
> No functional change.
>
> Signed-off-by: Kairui Song <kasong@tencent.com>
> Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
> ---
> mm/vmscan.c | 43 +++++++++++++++++++++++++++++--------------
> 1 file changed, 29 insertions(+), 14 deletions(-)
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index 3be0d6c04a82..500ef23949fd 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -4131,30 +4131,45 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
> return true;
> }
>
> +/* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
> +static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
> + unsigned long max_seq)
> +{
> + int gen, zone;
> + unsigned long min_seq;
> + struct lru_gen_folio *lrugen = &lruvec->lrugen;
> +
> + lockdep_assert_held(&lruvec->lru_lock);
> +
> + min_seq = READ_ONCE(lrugen->min_seq[type]);
> + while (min_seq <= max_seq) {
> + gen = lru_gen_from_seq(min_seq);
> + for (zone = 0; zone < MAX_NR_ZONES; zone++) {
> + if (!list_empty(&lrugen->folios[gen][type][zone]))
> + return min_seq;
> + }
> + min_seq++;
> + }
> +
> + return max_seq;
> +}
> +
> static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
> {
> - int gen, type, zone;
> + int type;
> bool seq_inc_flag = false;
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> DEFINE_MIN_SEQ(lruvec);
> + unsigned long seq;
>
> VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
>
> - /* find the oldest populated generation */
> + /* Requires at least MIN_NR_GENS gens for basic functionality */
> + seq = lrugen->max_seq - MIN_NR_GENS;
> for_each_evictable_type(type, swappiness) {
> - while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
> - gen = lru_gen_from_seq(min_seq[type]);
> -
> - for (zone = 0; zone < MAX_NR_ZONES; zone++) {
> - if (!list_empty(&lrugen->folios[gen][type][zone]))
> - goto next;
> - }
> -
> - min_seq[type]++;
> + min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
> + if (min_seq[type] > lrugen->min_seq[type])
> seq_inc_flag = true;
> - }
> -next:
> - ;
> }
>
> /*
> --
> 2.34.1
>
The helper preserves the original search result, including the
terminal sequence when no populated generation is found.
Reviewed-by: Kunwu Chan <kunwu.chan@gmail.com>
Thanks,
Kunwu
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 03/14] mm/mglru: reclaim from the oldest populated gen
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
2026-10-22 22:01 ` [RFC PATCH v5 01/14] mm/mglru: report evictable sizes per type Barry Song (Xiaomi)
2026-10-22 22:01 ` [RFC PATCH v5 02/14] mm/mglru: extract lruvec_populated_min_seq() helper Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-27 9:56 ` KunWu Chan
2026-10-22 22:01 ` [RFC PATCH v5 04/14] mm/mglru: lift the forced protection of reclaimer Barry Song (Xiaomi)
` (11 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
Currently MGLRU tries to keep the gen numbers of the two types in sync,
avoiding reducing the gen number of one type when the other type has
more gens, as described in the comment in try_to_inc_min_seq().
This makes scan_folios() stuck reclaiming the type with fewer gens: it
always scanned the oldest gen of the type, even when that gen was
empty, so reclaim made no progress on that type unless forced aging
created a new gen and eliminated the dead empty gen.
There are other approaches, like lifting the gen number
synchronization [1]. However, that would break the gen layout used by
lru_gen_folio_seq(): newly added folios would no longer have a
well-defined hotness according to their gen number, which reduces
accuracy. It is also not pressure-aware, since with gentle reclaim we
still want to respect the gen layout (hotness).
Instead, reclaim from whichever populated oldest gen is left, but only
when the default reclaim priority can't satisfy the reclaimer's
requirement. This is typical for swappiness limitations: e.g., low
swappiness asks the reclaimer to reclaim from file, but file gens are
drained, so the reclaim cycle fails and priority is raised.
At the default priority the scan quota is 1/4096 of the evictable
size, and the old behavior is kept there to respect the hotness
distribution. Once priority is raised, the quota is at least 1/2048,
so the hotness bias introduced by reclaiming from a younger gen is
bounded by that quota, which seems fair. Global reclaim for MGLRU may
start with DEF_PRIORITY / 2 (1/64); the ratio is still reasonable at
that point, and when global pressure is more critical, we should
reclaim harder.
Note that scan_folios() still refuses to reclaim a type left with only
MIN_NR_GENS gens, so a completely drained type still requires aging.
This change only avoids scanning empty gens when populated reclaimable
gens remain.
Link: https://lore.kernel.org/linux-mm/20260726122123.7614-4-baohua@kernel.org/ [1]
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song <baohua@kernel.org>
---
mm/vmscan.c | 30 +++++++++++++++++++-----------
1 file changed, 19 insertions(+), 11 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 500ef23949fd..d7eb15533de1 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3451,11 +3451,9 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
}
/* protect pages accessed multiple times through file descriptors */
-static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen)
{
- int type = folio_is_file_lru(folio);
- struct lru_gen_folio *lrugen = &lruvec->lrugen;
- int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+ int new_gen;
bool gen_increased;
new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
@@ -4836,7 +4834,7 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
******************************************************************************/
static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
- int tier_idx)
+ int min_gen, int tier_idx)
{
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
@@ -4854,14 +4852,14 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
return false;
/* promoted */
- if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
+ if (gen != min_gen) {
list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
return true;
}
/* protected */
if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
- gen = folio_inc_gen(lruvec, folio);
+ gen = folio_inc_gen(lruvec, folio, min_gen);
list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
/* don't count the workingset being lazily promoted */
@@ -4876,7 +4874,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
/* ineligible */
if (zone > sc->reclaim_idx) {
- gen = folio_inc_gen(lruvec, folio);
+ gen = folio_inc_gen(lruvec, folio, min_gen);
list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
return true;
}
@@ -4919,17 +4917,27 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
int scanned = 0;
int isolated = 0;
int skipped = 0;
+ unsigned long min_seq;
unsigned long remaining = nr_to_scan;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
VM_WARN_ON_ONCE(!list_empty(list));
+ /*
+ * Respect the gen hotness distribution at low priority to avoid
+ * aging and scan cost, but prefer to satisfy the reclaimers request
+ * when under pressure to respect swappiness more and reduce IO cost.
+ */
+ if (sc->priority == DEF_PRIORITY)
+ min_seq = lrugen->min_seq[type];
+ else
+ min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq - MIN_NR_GENS + 1);
+
if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
return 0;
- gen = lru_gen_from_seq(lrugen->min_seq[type]);
-
+ gen = lru_gen_from_seq(min_seq);
for (i = MAX_NR_ZONES; i > 0; i--) {
LIST_HEAD(moved);
int skipped_zone = 0;
@@ -4947,7 +4955,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
scanned += delta;
- if (sort_folio(lruvec, folio, sc, tier))
+ if (sort_folio(lruvec, folio, sc, gen, tier))
sorted += delta;
else if (isolate_folio(lruvec, folio, sc)) {
list_add(&folio->lru, list);
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 03/14] mm/mglru: reclaim from the oldest populated gen
2026-10-22 22:01 ` [RFC PATCH v5 03/14] mm/mglru: reclaim from the oldest populated gen Barry Song (Xiaomi)
@ 2026-10-27 9:56 ` KunWu Chan
0 siblings, 0 replies; 30+ messages in thread
From: KunWu Chan @ 2026-10-27 9:56 UTC (permalink / raw)
To: Barry Song (Xiaomi)
Cc: akpm, linux-mm, hui.zhu, axelrasmussen, baolin.wang, david,
hannes, linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen
On Fri, Oct 23, 2026 at 6:02 AM Barry Song (Xiaomi) <baohua@kernel.org> wrote:
>
> From: Kairui Song <kasong@tencent.com>
>
> Currently MGLRU tries to keep the gen numbers of the two types in sync,
> avoiding reducing the gen number of one type when the other type has
> more gens, as described in the comment in try_to_inc_min_seq().
>
> This makes scan_folios() stuck reclaiming the type with fewer gens: it
> always scanned the oldest gen of the type, even when that gen was
> empty, so reclaim made no progress on that type unless forced aging
> created a new gen and eliminated the dead empty gen.
>
> There are other approaches, like lifting the gen number
> synchronization [1]. However, that would break the gen layout used by
> lru_gen_folio_seq(): newly added folios would no longer have a
> well-defined hotness according to their gen number, which reduces
> accuracy. It is also not pressure-aware, since with gentle reclaim we
> still want to respect the gen layout (hotness).
>
> Instead, reclaim from whichever populated oldest gen is left, but only
> when the default reclaim priority can't satisfy the reclaimer's
> requirement. This is typical for swappiness limitations: e.g., low
> swappiness asks the reclaimer to reclaim from file, but file gens are
> drained, so the reclaim cycle fails and priority is raised.
>
> At the default priority the scan quota is 1/4096 of the evictable
> size, and the old behavior is kept there to respect the hotness
> distribution. Once priority is raised, the quota is at least 1/2048,
> so the hotness bias introduced by reclaiming from a younger gen is
> bounded by that quota, which seems fair. Global reclaim for MGLRU may
> start with DEF_PRIORITY / 2 (1/64); the ratio is still reasonable at
> that point, and when global pressure is more critical, we should
> reclaim harder.
>
> Note that scan_folios() still refuses to reclaim a type left with only
> MIN_NR_GENS gens, so a completely drained type still requires aging.
> This change only avoids scanning empty gens when populated reclaimable
> gens remain.
>
> Link: https://lore.kernel.org/linux-mm/20260726122123.7614-4-baohua@kernel.org/ [1]
> Signed-off-by: Kairui Song <kasong@tencent.com>
> Signed-off-by: Barry Song <baohua@kernel.org>
> ---
> mm/vmscan.c | 30 +++++++++++++++++++-----------
> 1 file changed, 19 insertions(+), 11 deletions(-)
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index 500ef23949fd..d7eb15533de1 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -3451,11 +3451,9 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
> }
>
> /* protect pages accessed multiple times through file descriptors */
> -static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
> +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen)
> {
> - int type = folio_is_file_lru(folio);
> - struct lru_gen_folio *lrugen = &lruvec->lrugen;
> - int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
> + int new_gen;
> bool gen_increased;
>
> new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
> @@ -4836,7 +4834,7 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
> ******************************************************************************/
>
> static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
> - int tier_idx)
> + int min_gen, int tier_idx)
> {
> int gen = folio_lru_gen(folio);
> int type = folio_is_file_lru(folio);
> @@ -4854,14 +4852,14 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
> return false;
>
> /* promoted */
> - if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
> + if (gen != min_gen) {
> list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
> return true;
> }
>
> /* protected */
> if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
> - gen = folio_inc_gen(lruvec, folio);
> + gen = folio_inc_gen(lruvec, folio, min_gen);
> list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
>
> /* don't count the workingset being lazily promoted */
> @@ -4876,7 +4874,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
>
> /* ineligible */
> if (zone > sc->reclaim_idx) {
> - gen = folio_inc_gen(lruvec, folio);
> + gen = folio_inc_gen(lruvec, folio, min_gen);
> list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
> return true;
> }
> @@ -4919,17 +4917,27 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> int scanned = 0;
> int isolated = 0;
> int skipped = 0;
> + unsigned long min_seq;
> unsigned long remaining = nr_to_scan;
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
>
> VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
> VM_WARN_ON_ONCE(!list_empty(list));
>
> + /*
> + * Respect the gen hotness distribution at low priority to avoid
> + * aging and scan cost, but prefer to satisfy the reclaimers request
> + * when under pressure to respect swappiness more and reduce IO cost.
> + */
> + if (sc->priority == DEF_PRIORITY)
> + min_seq = lrugen->min_seq[type];
> + else
> + min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq - MIN_NR_GENS + 1);
> +
> if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
> return 0;
>
> - gen = lru_gen_from_seq(lrugen->min_seq[type]);
> -
> + gen = lru_gen_from_seq(min_seq);
> for (i = MAX_NR_ZONES; i > 0; i--) {
> LIST_HEAD(moved);
> int skipped_zone = 0;
> @@ -4947,7 +4955,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
>
> scanned += delta;
>
> - if (sort_folio(lruvec, folio, sc, tier))
> + if (sort_folio(lruvec, folio, sc, gen, tier))
> sorted += delta;
> else if (isolate_folio(lruvec, folio, sc)) {
> list_add(&folio->lru, list);
> --
> 2.34.1
>
At raised priority, scanning from the oldest populated gen avoids
wasted work on empty generations. The change to pass min_gen through
sort_folio() and folio_inc_gen() keeps promotion semantics correct
when the scanned gen is above min_seq[type].
Reviewed-by: Kunwu Chan <kunwu.chan@gmail.com>
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 04/14] mm/mglru: lift the forced protection of reclaimer
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (2 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 03/14] mm/mglru: reclaim from the oldest populated gen Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-28 4:14 ` KunWu Chan
2026-10-22 22:01 ` [RFC PATCH v5 05/14] mm/mglru: skip zones above reclaim_idx instead of rotating them Barry Song (Xiaomi)
` (10 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
When a type is drained down to MIN_NR_GENS gens, the reclaimer will
fail to make any progress on that type without aging.
However aging comes with a cost, and it will take time to finish. Even
if we do aging more proactively, when under great pressure, gens will
still be drained very fast so the new gens produced by aging can't catch
up with the consumption. With MIN_NR_GENS setting blocking the reclaim,
we will run into OOM. This OOM can be reproduced by the previous test
program [1] used for fixing another aging problem [2]. When running in
great pressure, certain workloads will keep seeing premature OOM with
MGLRU, and never see any OOM with classical LRU.
This might get worse if we implement async aging, or avoid hotness
inversion in aging, as they will slow down the aging process even more.
To fix this, let's first drop the get_nr_gens() == MIN_NR_GENS early
return and let the raised-priority scan bound cover max_seq itself.
Mapped folios will still survive as the rmap PTE check will keep them.
Unmapped cache folios may thrashing more when under great pressure,
which is really a problem that should be fixed with refault distance
and frequecy aware promotoin instead [3].
Link: https://github.com/ryncsn/emm-test-project/tree/master/file-anon-mix-pressure [1]
Link: https://lore.kernel.org/linux-mm/20260428-mglru-reclaim-v7-0-02fabb92dc43@tencent.com/ [2]
Link: https://lore.kernel.org/linux-mm/20260502-mglru-fg-v1-0-913619b014d9@tencent.com/ [3]
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 34 +++++++++++++++++++++++++---------
1 file changed, 25 insertions(+), 9 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index d7eb15533de1..2cccaa5dd4e7 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4834,7 +4834,7 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
******************************************************************************/
static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
- int min_gen, int tier_idx)
+ int min_gen, int max_gen, int tier_idx)
{
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
@@ -4851,12 +4851,22 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
if (!folio_evictable(folio))
return false;
- /* promoted */
+ /* promoted, or moved due to reparenting. */
if (gen != min_gen) {
list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
return true;
}
+ /*
+ * We are reclaiming the last gen left. Folios in this gen can no
+ * longer be promoted or protected, so just isolate them. The one
+ * exception is lru_gen_reparent_memcg(), which is handled above.
+ */
+ if (min_gen == max_gen) {
+ VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
+ return false;
+ }
+
/* protected */
if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
gen = folio_inc_gen(lruvec, folio, min_gen);
@@ -4911,15 +4921,16 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
struct list_head *list, int *isolatedp)
{
int i;
- int gen;
enum node_stat_item item;
int sorted = 0;
int scanned = 0;
int isolated = 0;
int skipped = 0;
+ int gen, max_gen;
unsigned long min_seq;
unsigned long remaining = nr_to_scan;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ DEFINE_MAX_SEQ(lruvec);
VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
VM_WARN_ON_ONCE(!list_empty(list));
@@ -4932,18 +4943,24 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
if (sc->priority == DEF_PRIORITY)
min_seq = lrugen->min_seq[type];
else
- min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq - MIN_NR_GENS + 1);
-
- if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
- return 0;
+ min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
gen = lru_gen_from_seq(min_seq);
+ max_gen = lru_gen_from_seq(max_seq);
+
for (i = MAX_NR_ZONES; i > 0; i--) {
LIST_HEAD(moved);
int skipped_zone = 0;
int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES;
struct list_head *head = &lrugen->folios[gen][type][zone];
+ /*
+ * If there is only one gen left, rotating the unreclaimable
+ * zone become meaningless, just skip them.
+ */
+ if (gen == max_gen && zone > sc->reclaim_idx)
+ continue;
+
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
int delta = folio_nr_pages(folio);
@@ -4954,8 +4971,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
scanned += delta;
-
- if (sort_folio(lruvec, folio, sc, gen, tier))
+ if (sort_folio(lruvec, folio, sc, gen, max_gen, tier))
sorted += delta;
else if (isolate_folio(lruvec, folio, sc)) {
list_add(&folio->lru, list);
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 04/14] mm/mglru: lift the forced protection of reclaimer
2026-10-22 22:01 ` [RFC PATCH v5 04/14] mm/mglru: lift the forced protection of reclaimer Barry Song (Xiaomi)
@ 2026-10-28 4:14 ` KunWu Chan
0 siblings, 0 replies; 30+ messages in thread
From: KunWu Chan @ 2026-10-28 4:14 UTC (permalink / raw)
To: Barry Song (Xiaomi)
Cc: akpm, linux-mm, hui.zhu, axelrasmussen, baolin.wang, david,
hannes, linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen
On Fri, Oct 23, 2026 at 6:02 AM Barry Song (Xiaomi) <baohua@kernel.org> wrote:
>
> From: Kairui Song <kasong@tencent.com>
>
> When a type is drained down to MIN_NR_GENS gens, the reclaimer will
> fail to make any progress on that type without aging.
>
> However aging comes with a cost, and it will take time to finish. Even
> if we do aging more proactively, when under great pressure, gens will
> still be drained very fast so the new gens produced by aging can't catch
> up with the consumption. With MIN_NR_GENS setting blocking the reclaim,
> we will run into OOM. This OOM can be reproduced by the previous test
> program [1] used for fixing another aging problem [2]. When running in
> great pressure, certain workloads will keep seeing premature OOM with
> MGLRU, and never see any OOM with classical LRU.
>
> This might get worse if we implement async aging, or avoid hotness
> inversion in aging, as they will slow down the aging process even more.
>
> To fix this, let's first drop the get_nr_gens() == MIN_NR_GENS early
> return and let the raised-priority scan bound cover max_seq itself.
>
> Mapped folios will still survive as the rmap PTE check will keep them.
> Unmapped cache folios may thrashing more when under great pressure,
> which is really a problem that should be fixed with refault distance
> and frequecy aware promotoin instead [3].
>
> Link: https://github.com/ryncsn/emm-test-project/tree/master/file-anon-mix-pressure [1]
> Link: https://lore.kernel.org/linux-mm/20260428-mglru-reclaim-v7-0-02fabb92dc43@tencent.com/ [2]
> Link: https://lore.kernel.org/linux-mm/20260502-mglru-fg-v1-0-913619b014d9@tencent.com/ [3]
> Signed-off-by: Kairui Song <kasong@tencent.com>
> Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
> ---
> mm/vmscan.c | 34 +++++++++++++++++++++++++---------
> 1 file changed, 25 insertions(+), 9 deletions(-)
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index d7eb15533de1..2cccaa5dd4e7 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -4834,7 +4834,7 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
> ******************************************************************************/
>
> static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
> - int min_gen, int tier_idx)
> + int min_gen, int max_gen, int tier_idx)
> {
> int gen = folio_lru_gen(folio);
> int type = folio_is_file_lru(folio);
> @@ -4851,12 +4851,22 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
> if (!folio_evictable(folio))
> return false;
>
> - /* promoted */
> + /* promoted, or moved due to reparenting. */
> if (gen != min_gen) {
> list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
> return true;
> }
>
> + /*
> + * We are reclaiming the last gen left. Folios in this gen can no
> + * longer be promoted or protected, so just isolate them. The one
> + * exception is lru_gen_reparent_memcg(), which is handled above.
> + */
> + if (min_gen == max_gen) {
> + VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
> + return false;
> + }
> +
> /* protected */
> if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
> gen = folio_inc_gen(lruvec, folio, min_gen);
> @@ -4911,15 +4921,16 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> struct list_head *list, int *isolatedp)
> {
> int i;
> - int gen;
> enum node_stat_item item;
> int sorted = 0;
> int scanned = 0;
> int isolated = 0;
> int skipped = 0;
> + int gen, max_gen;
> unsigned long min_seq;
> unsigned long remaining = nr_to_scan;
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> + DEFINE_MAX_SEQ(lruvec);
>
> VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
> VM_WARN_ON_ONCE(!list_empty(list));
> @@ -4932,18 +4943,24 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> if (sc->priority == DEF_PRIORITY)
> min_seq = lrugen->min_seq[type];
> else
> - min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq - MIN_NR_GENS + 1);
> -
> - if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
> - return 0;
> + min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
>
> gen = lru_gen_from_seq(min_seq);
> + max_gen = lru_gen_from_seq(max_seq);
> +
> for (i = MAX_NR_ZONES; i > 0; i--) {
> LIST_HEAD(moved);
> int skipped_zone = 0;
> int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES;
> struct list_head *head = &lrugen->folios[gen][type][zone];
>
> + /*
> + * If there is only one gen left, rotating the unreclaimable
> + * zone become meaningless, just skip them.
> + */
> + if (gen == max_gen && zone > sc->reclaim_idx)
> + continue;
> +
> while (!list_empty(head)) {
> struct folio *folio = lru_to_folio(head);
> int delta = folio_nr_pages(folio);
> @@ -4954,8 +4971,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
>
> scanned += delta;
> -
> - if (sort_folio(lruvec, folio, sc, gen, tier))
> + if (sort_folio(lruvec, folio, sc, gen, max_gen, tier))
> sorted += delta;
> else if (isolate_folio(lruvec, folio, sc)) {
> list_add(&folio->lru, list);
> --
> 2.34.1
>
Removing the MIN_NR_GENS early return allows reclaim to make progress
when only one generation remains. Bypassing promotion and protection
in that case lets reclaim attempt to isolate those folios, while
scan_folios() skips zones outside reclaim_idx.
Reviewed-by: Kunwu Chan <kunwu.chan@gmail.com>
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 05/14] mm/mglru: skip zones above reclaim_idx instead of rotating them
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (3 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 04/14] mm/mglru: lift the forced protection of reclaimer Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-23 22:03 ` sashiko-bot
2026-10-22 22:01 ` [RFC PATCH v5 06/14] mm/mglru: return for unavailable anon reclaim earlier Barry Song (Xiaomi)
` (9 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
scan_folios() used to visit every zone. Folios in zones above
sc->reclaim_idx can never be isolated, so sort_folio() rotated them to
a younger generation, the "ineligible" branch, or, when only one gen
was left, skipped them. This wasted scan work on every pass, and the
rotation kept the protected folios from aging.
Restrict both the scan and the gen selection to the eligible zones,
like the legacy LRU, which only isolates folios in zones up to
reclaim_idx:
- lruvec_populated_min_seq() takes a zone_limit, so the oldest
populated gen is searched only among the zones the scan can
reclaim from. A gen populated only in protected zones no longer
stalls the scan.
- scan_folios() iterates the eligible zones in the descending order
of the legacy zonelist walk, draining the zone at reclaim_idx
first.
- The ineligible branch of sort_folio() is now dead and removed,
with a VM_WARN_ON_ONCE_FOLIO() guarding the new invariant.
try_to_inc_min_seq() still considers all zones, so min_seq[type] is
only bumped once the oldest gen is empty everywhere. At DEF_PRIORITY
the search bound stays min_seq[type], so the scan still starts from
the oldest gen regardless of whether it is populated.
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 47 +++++++++++++++++++----------------------------
1 file changed, 19 insertions(+), 28 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 2cccaa5dd4e7..d3caed8aa789 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4131,7 +4131,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
/* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
- unsigned long max_seq)
+ int zone_limit, unsigned long max_seq)
{
int gen, zone;
unsigned long min_seq;
@@ -4142,7 +4142,7 @@ static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
min_seq = READ_ONCE(lrugen->min_seq[type]);
while (min_seq <= max_seq) {
gen = lru_gen_from_seq(min_seq);
- for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+ for (zone = 0; zone < zone_limit; zone++) {
if (!list_empty(&lrugen->folios[gen][type][zone]))
return min_seq;
}
@@ -4165,7 +4165,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
/* Requires at least MIN_NR_GENS gens for basic functionality */
seq = lrugen->max_seq - MIN_NR_GENS;
for_each_evictable_type(type, swappiness) {
- min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
+ min_seq[type] = lruvec_populated_min_seq(lruvec, type, MAX_NR_ZONES, seq + 1);
if (min_seq[type] > lrugen->min_seq[type])
seq_inc_flag = true;
}
@@ -4846,6 +4846,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
struct lru_gen_folio *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
+ VM_WARN_ON_ONCE_FOLIO(zone > sc->reclaim_idx, folio);
/* unevictable: let it through and the generic path will cull it */
if (!folio_evictable(folio))
@@ -4862,10 +4863,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
* longer be promoted or protected, so just isolate them. The one
* exception is lru_gen_reparent_memcg(), which is handled above.
*/
- if (min_gen == max_gen) {
- VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
+ if (min_gen == max_gen)
return false;
- }
/* protected */
if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
@@ -4882,13 +4881,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
return true;
}
- /* ineligible */
- if (zone > sc->reclaim_idx) {
- gen = folio_inc_gen(lruvec, folio, min_gen);
- list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
- return true;
- }
-
return false;
}
@@ -4920,16 +4912,17 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
struct scan_control *sc, int type, int tier,
struct list_head *list, int *isolatedp)
{
- int i;
enum node_stat_item item;
+ int zone_idx;
int sorted = 0;
int scanned = 0;
int isolated = 0;
int skipped = 0;
int gen, max_gen;
- unsigned long min_seq;
+ unsigned long seq;
unsigned long remaining = nr_to_scan;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ struct pglist_data *pgdat = lruvec_pgdat(lruvec);
DEFINE_MAX_SEQ(lruvec);
VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
@@ -4941,24 +4934,22 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
* when under pressure to respect swappiness more and reduce IO cost.
*/
if (sc->priority == DEF_PRIORITY)
- min_seq = lrugen->min_seq[type];
+ seq = lrugen->min_seq[type];
else
- min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
+ seq = lrugen->max_seq;
- gen = lru_gen_from_seq(min_seq);
+ seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
+ gen = lru_gen_from_seq(seq);
max_gen = lru_gen_from_seq(max_seq);
- for (i = MAX_NR_ZONES; i > 0; i--) {
+ /* Scan eligible zones in descending order. */
+ for (zone_idx = sc->reclaim_idx; zone_idx >= 0; zone_idx--) {
LIST_HEAD(moved);
int skipped_zone = 0;
- int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES;
- struct list_head *head = &lrugen->folios[gen][type][zone];
+ struct zone *zone = &pgdat->node_zones[zone_idx];
+ struct list_head *head = &lrugen->folios[gen][type][zone_idx];
- /*
- * If there is only one gen left, rotating the unreclaimable
- * zone become meaningless, just skip them.
- */
- if (gen == max_gen && zone > sc->reclaim_idx)
+ if (!managed_zone(zone))
continue;
while (!list_empty(head)) {
@@ -4968,7 +4959,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
- VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone_idx, folio);
scanned += delta;
if (sort_folio(lruvec, folio, sc, gen, max_gen, tier))
@@ -4987,7 +4978,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
if (skipped_zone) {
list_splice(&moved, head);
- __count_zid_vm_events(PGSCAN_SKIP, zone, skipped_zone);
+ __count_zid_vm_events(PGSCAN_SKIP, zone_idx, skipped_zone);
skipped += skipped_zone;
}
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 05/14] mm/mglru: skip zones above reclaim_idx instead of rotating them
2026-10-22 22:01 ` [RFC PATCH v5 05/14] mm/mglru: skip zones above reclaim_idx instead of rotating them Barry Song (Xiaomi)
@ 2026-10-23 22:03 ` sashiko-bot
2026-10-24 16:19 ` Kairui Song
0 siblings, 1 reply; 30+ messages in thread
From: sashiko-bot @ 2026-10-23 22:03 UTC (permalink / raw)
To: Barry Song (Xiaomi), akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baolin.wang, david, hannes, ljs, mhocko,
qi.zheng, ryncsn, shakeel.butt, weixugc, yuanchu, zhuhui,
baoquan.he, liuxiang.277, wangzicheng, lianux.mm, kasong, liam,
surenb, mhocko, rppt, zhangbo56, lyugaofei, ridong.chen,
kunwu.chan
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] mm/mglru: pinned min_seq in higher zones stalls aging in scan_folios()
--
commit 1b2f683d6b8bd1e171eb9677315ba8a3d9431b9b
Author: Kairui Song <kasong@tencent.com>
mm/mglru: skip zones above reclaim_idx instead of rotating them
This commit restricts scan_folios() and lruvec_populated_min_seq() during
eviction to zones at or below sc->reclaim_idx, and removes the ineligible-zone
rotation branch from sort_folio(). try_to_inc_min_seq() continues to check all
zones up to MAX_NR_ZONES before advancing min_seq.
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index a365a4bc8c988..09ea72b354a56 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -4132,7 +4132,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
>
> /* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
> static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
> - unsigned long max_seq)
> + int zone_limit, unsigned long max_seq)
> {
> int gen, zone;
> unsigned long min_seq;
> @@ -4143,7 +4143,7 @@ static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
> min_seq = READ_ONCE(lrugen->min_seq[type]);
> while (min_seq <= max_seq) {
> gen = lru_gen_from_seq(min_seq);
> - for (zone = 0; zone < MAX_NR_ZONES; zone++) {
> + for (zone = 0; zone < zone_limit; zone++) {
> if (!list_empty(&lrugen->folios[gen][type][zone]))
> return min_seq;
> }
> @@ -4166,7 +4166,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
> /* Requires at least MIN_NR_GENS gens for basic functionality */
> seq = lrugen->max_seq - MIN_NR_GENS;
> for_each_evictable_type(type, swappiness) {
> - min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
> + min_seq[type] = lruvec_populated_min_seq(lruvec, type, MAX_NR_ZONES, seq + 1);
> if (min_seq[type] > lrugen->min_seq[type])
> seq_inc_flag = true;
> }
[ ... ]
> @@ -4863,10 +4864,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
> * longer be promoted or protected, so just isolate them. The one
> * exception is lru_gen_reparent_memcg(), which is handled above.
> */
> - if (min_gen == max_gen) {
> - VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
> + if (min_gen == max_gen)
> return false;
> - }
>
> /* protected */
> if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
> @@ -4883,13 +4882,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
> return true;
> }
>
> - /* ineligible */
> - if (zone > sc->reclaim_idx) {
> - gen = folio_inc_gen(lruvec, folio, min_gen);
> - list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
> - return true;
> - }
> -
> return false;
> }
>
> @@ -4921,16 +4913,17 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> struct scan_control *sc, int type, int tier,
> struct list_head *list, int *isolatedp)
> {
> - int i;
> enum node_stat_item item;
> + int zone_idx;
> int sorted = 0;
> int scanned = 0;
> int isolated = 0;
> int skipped = 0;
> int gen, max_gen;
> - unsigned long min_seq;
> + unsigned long seq;
> unsigned long remaining = nr_to_scan;
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> + struct pglist_data *pgdat = lruvec_pgdat(lruvec);
> DEFINE_MAX_SEQ(lruvec);
>
> VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
> @@ -4942,24 +4935,22 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> * when under pressure to respect swappiness more and reduce IO cost.
> */
> if (sc->priority == DEF_PRIORITY)
> - min_seq = lrugen->min_seq[type];
> + seq = lrugen->min_seq[type];
> else
> - min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
> + seq = lrugen->max_seq;
>
> - gen = lru_gen_from_seq(min_seq);
> + seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
> + gen = lru_gen_from_seq(seq);
> max_gen = lru_gen_from_seq(max_seq);
>
> - for (i = MAX_NR_ZONES; i > 0; i--) {
> + /* Scan eligible zones in descending order. */
> + for (zone_idx = sc->reclaim_idx; zone_idx >= 0; zone_idx--) {
[Severity: High]
During zone-constrained reclaim in mm/vmscan.c (where sc->reclaim_idx is below
MAX_NR_ZONES - 1 and zones above sc->reclaim_idx are populated, such as
GFP_DMA32 reclaim with a populated ZONE_NORMAL or GFP_KERNEL reclaim with a
populated ZONE_MOVABLE), can keeping MAX_NR_ZONES in try_to_inc_min_seq()
while removing the ineligible-zone rotation from sort_folio() stall aging and
memcg rotation?
Previously, scan_folios() visited all zones and sort_folio() rotated folios in
zones above sc->reclaim_idx from min_gen to min_gen + 1. Once the oldest
generation was emptied across all zones, try_to_inc_min_seq() incremented
lrugen->min_seq[type] from max_seq - 3 (when MAX_NR_GENS generations exist) to
max_seq - 2, allowing should_run_aging() to trigger try_to_inc_max_seq().
With scan_folios() restricted to zones 0..sc->reclaim_idx, folios in zones
above sc->reclaim_idx remain in min_seq[type]. Because try_to_inc_min_seq()
still passes MAX_NR_ZONES to lruvec_populated_min_seq(), lrugen->min_seq[type]
stays pinned at max_seq - 3 even after min_seq[type] is completely drained in
eligible zones 0..sc->reclaim_idx.
In this commit, at sc->priority == DEF_PRIORITY, scan_folios() caps the
lruvec_populated_min_seq() search bound at seq = lrugen->min_seq[type]. Once
min_seq[type] is empty in zones 0..sc->reclaim_idx, lruvec_populated_min_seq()
still returns the pinned min_seq[type], so scan_folios() scans empty lists and
returns 0 at DEF_PRIORITY.
A later commit in this series ("mm/mglru: lift youngest gen protection for
higher priority") changes the DEF_PRIORITY search bound in scan_folios() to
lrugen->max_seq - 1, and another commit ("mm/mglru: Run aging when pages are
rare outside the youngest gen") adds lru_gen_imbalanced() to
should_run_aging(). However, both should_run_aging() and shrink_one() still
rely on state across all zones:
mm/vmscan.c:should_run_aging() {
DEFINE_MIN_SEQ(lruvec);
/* have to run aging, since eviction is not possible anymore */
if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq)
return true;
/* try to avoid aging, do gentle reclaim at the default priority */
if (sc->priority == DEF_PRIORITY)
return false;
/* better to run aging even though eviction is still possible */
return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
}
mm/vmscan.c:shrink_one() {
...
need_rotate = try_to_shrink_lruvec(lruvec, sc);
...
if (need_rotate && mem_cgroup_online(memcg))
return MEMCG_LRU_YOUNG;
if (!need_rotate && lruvec_is_sizable(lruvec, sc))
return 0;
}
When lrugen->min_seq[type] remains pinned at max_seq - 3 by folios in zones
above sc->reclaim_idx (and lru_gen_imbalanced() / lruvec_is_sizable() sum
nr_pages across all MAX_NR_ZONES):
1. should_run_aging() sees min_seq + MIN_NR_GENS < max_seq and does not run
try_to_inc_max_seq() (or inc_min_seq() inside inc_max_seq()), stalling
page-table aging.
2. Once eligible zones 0..sc->reclaim_idx drain their older generations and
lruvec_populated_min_seq() advances seq to max_seq in scan_folios(),
sort_folio() hits min_gen == max_gen and isolates folios from the youngest
generation without tier or reference protection.
3. If a memcg has no folios in eligible zones 0..sc->reclaim_idx but has
folios in zones above sc->reclaim_idx, try_to_shrink_lruvec() returns
need_rotate = false while lruvec_is_sizable() returns true, so shrink_one()
returns 0 instead of rotating the memcg.
Should should_run_aging() and the size checks also account for sc->reclaim_idx,
or how should min_seq and max_seq advance when older generations are only
populated in zones above sc->reclaim_idx?
[ ... ]
([PVF1])
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261022220203.91911-1-baohua@kernel.org?part=5
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 05/14] mm/mglru: skip zones above reclaim_idx instead of rotating them
2026-10-23 22:03 ` sashiko-bot
@ 2026-10-24 16:19 ` Kairui Song
0 siblings, 0 replies; 30+ messages in thread
From: Kairui Song @ 2026-10-24 16:19 UTC (permalink / raw)
To: sashiko-reviews
Cc: Barry Song (Xiaomi), akpm, linux-mm, hui.zhu, axelrasmussen,
baolin.wang, david, hannes, ljs, mhocko, qi.zheng, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, liam, surenb, mhocko, rppt, zhangbo56, lyugaofei,
ridong.chen, kunwu.chan
On Sat, Oct 24, 2026 at 6:03 AM <sashiko-bot@kernel.org> wrote:
>
> Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
> - [High] mm/mglru: pinned min_seq in higher zones stalls aging in scan_folios()
> --
>
> commit 1b2f683d6b8bd1e171eb9677315ba8a3d9431b9b
> Author: Kairui Song <kasong@tencent.com>
>
> mm/mglru: skip zones above reclaim_idx instead of rotating them
>
> This commit restricts scan_folios() and lruvec_populated_min_seq() during
> eviction to zones at or below sc->reclaim_idx, and removes the ineligible-zone
> rotation branch from sort_folio(). try_to_inc_min_seq() continues to check all
> zones up to MAX_NR_ZONES before advancing min_seq.
>
> > diff --git a/mm/vmscan.c b/mm/vmscan.c
> > index a365a4bc8c988..09ea72b354a56 100644
> > --- a/mm/vmscan.c
> > +++ b/mm/vmscan.c
> > @@ -4132,7 +4132,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
> >
> > /* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
> > static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
> > - unsigned long max_seq)
> > + int zone_limit, unsigned long max_seq)
> > {
> > int gen, zone;
> > unsigned long min_seq;
> > @@ -4143,7 +4143,7 @@ static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
> > min_seq = READ_ONCE(lrugen->min_seq[type]);
> > while (min_seq <= max_seq) {
> > gen = lru_gen_from_seq(min_seq);
> > - for (zone = 0; zone < MAX_NR_ZONES; zone++) {
> > + for (zone = 0; zone < zone_limit; zone++) {
> > if (!list_empty(&lrugen->folios[gen][type][zone]))
> > return min_seq;
> > }
> > @@ -4166,7 +4166,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
> > /* Requires at least MIN_NR_GENS gens for basic functionality */
> > seq = lrugen->max_seq - MIN_NR_GENS;
> > for_each_evictable_type(type, swappiness) {
> > - min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
> > + min_seq[type] = lruvec_populated_min_seq(lruvec, type, MAX_NR_ZONES, seq + 1);
> > if (min_seq[type] > lrugen->min_seq[type])
> > seq_inc_flag = true;
> > }
>
> [ ... ]
>
> > @@ -4863,10 +4864,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
> > * longer be promoted or protected, so just isolate them. The one
> > * exception is lru_gen_reparent_memcg(), which is handled above.
> > */
> > - if (min_gen == max_gen) {
> > - VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
> > + if (min_gen == max_gen)
> > return false;
> > - }
> >
> > /* protected */
> > if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
> > @@ -4883,13 +4882,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
> > return true;
> > }
> >
> > - /* ineligible */
> > - if (zone > sc->reclaim_idx) {
> > - gen = folio_inc_gen(lruvec, folio, min_gen);
> > - list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
> > - return true;
> > - }
> > -
> > return false;
> > }
> >
> > @@ -4921,16 +4913,17 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> > struct scan_control *sc, int type, int tier,
> > struct list_head *list, int *isolatedp)
> > {
> > - int i;
> > enum node_stat_item item;
> > + int zone_idx;
> > int sorted = 0;
> > int scanned = 0;
> > int isolated = 0;
> > int skipped = 0;
> > int gen, max_gen;
> > - unsigned long min_seq;
> > + unsigned long seq;
> > unsigned long remaining = nr_to_scan;
> > struct lru_gen_folio *lrugen = &lruvec->lrugen;
> > + struct pglist_data *pgdat = lruvec_pgdat(lruvec);
> > DEFINE_MAX_SEQ(lruvec);
> >
> > VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
> > @@ -4942,24 +4935,22 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> > * when under pressure to respect swappiness more and reduce IO cost.
> > */
> > if (sc->priority == DEF_PRIORITY)
> > - min_seq = lrugen->min_seq[type];
> > + seq = lrugen->min_seq[type];
> > else
> > - min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
> > + seq = lrugen->max_seq;
> >
> > - gen = lru_gen_from_seq(min_seq);
> > + seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
> > + gen = lru_gen_from_seq(seq);
> > max_gen = lru_gen_from_seq(max_seq);
> >
> > - for (i = MAX_NR_ZONES; i > 0; i--) {
> > + /* Scan eligible zones in descending order. */
> > + for (zone_idx = sc->reclaim_idx; zone_idx >= 0; zone_idx--) {
>
> [Severity: High]
> During zone-constrained reclaim in mm/vmscan.c (where sc->reclaim_idx is below
> MAX_NR_ZONES - 1 and zones above sc->reclaim_idx are populated, such as
> GFP_DMA32 reclaim with a populated ZONE_NORMAL or GFP_KERNEL reclaim with a
> populated ZONE_MOVABLE), can keeping MAX_NR_ZONES in try_to_inc_min_seq()
> while removing the ineligible-zone rotation from sort_folio() stall aging and
> memcg rotation?
Yeah it can stall aging, but this series lifted the restriction of
MIN_NR_GEN on eviction, and eviction will start the the last populated
gen of a zone instead of min_seq, and we have more proactive aging
when the gens are imbalanced, so this won't be a problem, it won't
cause OOM. I'm not sure if it will cause any performance issue though,
if this is concerning we can keep the old behavior, or trigger aging
when the populated gen that satisfy a reclain is below (MIN_NR_GEN -
1) or imbalanced, this seems a better solution to me, still avoid a
lot of redundant work while ensure there is no reclaim effenciency
issue. We already have the right helper to do so in this series, so
seem not hard or dirty.
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 06/14] mm/mglru: return for unavailable anon reclaim earlier
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (4 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 05/14] mm/mglru: skip zones above reclaim_idx instead of rotating them Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-22 22:01 ` [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate Barry Song (Xiaomi)
` (8 subsequent siblings)
14 siblings, 0 replies; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
No feature change, just shuffle some code, prepare for later commits.
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 27 +++++++++++++--------------
1 file changed, 13 insertions(+), 14 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index d3caed8aa789..0930bee5906e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5215,20 +5215,6 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
struct mem_cgroup *memcg, int swappiness)
{
unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
-
- /*
- * Proactive reclaim initiated by userspace for anonymous memory only.
- * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
- * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
- * no swap), return 0 to skip the scan entirely, avoiding useless scan
- * work when there is nothing eligible to reclaim.
- */
- if (swappiness == SWAPPINESS_ANON_ONLY) {
- WARN_ON_ONCE(!sc->proactive);
- if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc))
- return 0;
- }
lruvec_evictable_size(lruvec, swappiness, sizes);
evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
@@ -5285,6 +5271,19 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ /*
+ * Proactive reclaim initiated by userspace for anonymous memory only.
+ * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
+ * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
+ * no swap), return 0 to skip the scan entirely, avoiding useless scan
+ * work when there is nothing eligible to reclaim.
+ */
+ if (swappiness == SWAPPINESS_ANON_ONLY) {
+ WARN_ON_ONCE(!sc->proactive);
+ if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
+ return false;
+ }
+
nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
while (nr_to_scan > 0) {
int delta;
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (5 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 06/14] mm/mglru: return for unavailable anon reclaim earlier Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-23 22:03 ` sashiko-bot
2026-10-22 22:01 ` [RFC PATCH v5 08/14] mm/mglru: lift youngest gen protection for higher priority Barry Song (Xiaomi)
` (7 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
Replace get_type_to_scan() and isolate_folios(), which selected one
type per batch, with a proportional split of the scan budget between
anon and file, and scan both types each round.
lru_gen_balance_scan() splits the total by
anon : file = swappiness * r_file * w_anon :
(MAX_SWAPPINESS - swappiness) * r_anon * w_file
where r_anon and r_file are the per-type refault rates scaled by
4096, clamped to [1, 4096]; a rate at the 4096 ceiling means the type
refaults at least as often as it is evicted. The w factors bias the
split toward the LRU sizes at mild swappiness and follow the IO cost
ratio at the extremes. Both rate accumulators start at MIN_LRU_BATCH,
so a type without eviction history reads as 100% until evictions
accumulate, and the floor of 1 keeps a type that never refaults from
zeroing out the other type's share. Swappiness 0 and
SWAPPINESS_ANON_ONLY keep their single-type behavior; at swappiness ==
MAX_SWAPPINESS the file factor is zero, so anon gets the whole budget,
as in the classical LRU; swappiness 1 no longer forces file-only
scanning as the old code did.
The proportional protection and priority reduction are applied to the
total once, before the split, and each budget is capped at its type's
evictable size. Regular reclaim derives the total from the evictable
sizes, so the budgets sum to it. The proactive path re-splits the
remaining target each round, so eviction keeps working while either
type makes progress, until the target is met or no progress is
possible. The debugfs scan_control gets .priority = DEF_PRIORITY so
eviction starts from the oldest gen and reports the default priority
in the tracepoints.
Signed-off-by: Kairui Song <kasong@tencent.com>
Co-developed-by: lyugaofei <lyugaofei@xiaomi.com>
Signed-off-by: lyugaofei <lyugaofei@xiaomi.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 223 +++++++++++++++++++++++++++++++++-------------------
1 file changed, 140 insertions(+), 83 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 0930bee5906e..0e37f5c8f355 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5018,79 +5018,110 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
return tier - 1;
}
-static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
+/*
+ * Balance the anon vs file scan budget using MGLRU's per-type refault
+ * rates, where swappiness is the relative IO cost of anon vs file (see
+ * Documentation/admin-guide/sysctl/vm.rst).
+ *
+ * On entry, nr[] holds the evictable size of each type, on return it
+ * holds each type's scan budget, bounded by its evictable size.
+ *
+ * LRU lock not needed: a stale snapshot merely skews the split for one pass,
+ * and MGLRU aging averages the fault rate so we can simply read it, no need
+ * to do any extra work here. If reclaim is blocked by unfair type hotness or
+ * stall aging, the priority will raise and a raised priority drives the
+ * aging.
+ *
+ * Calculation is basically based on following formula with a LRU size bias:
+ *
+ * anon : file = swappiness * r_file * w_anon :
+ * (MAX_SWAPPINESS - swappiness) * r_anon * w_file
+ *
+ * r_file / r_anon: refault rate for file / anon.
+ * w_file / w_anon: LRU size weight of file / anon.
+ *
+ * We don't calculate the scan cost, MGLRU uses lazy promotion, which will
+ * cancel the scan budget natually.
+ */
+static void lru_gen_balance_scan(struct lruvec *lruvec, int swappiness,
+ unsigned long total_scan, unsigned long nr[])
{
- struct ctrl_pos sp, pv;
+ struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ u64 refault_rate[ANON_AND_FILE], anon_factor, file_factor;
+ u64 anon_scan, file_scan, total;
+ u64 anon_weight, file_weight;
+ int size_bias;
- if (swappiness <= MIN_SWAPPINESS + 1)
- return LRU_GEN_FILE;
+ total = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE];
+ if (!total)
+ return;
- if (swappiness >= MAX_SWAPPINESS)
- return LRU_GEN_ANON;
- /*
- * Compare the sum of all tiers of anon with that of file to determine
- * which type to scan.
- */
- read_ctrl_pos(lruvec, LRU_GEN_ANON, LRU_TIER_MIN, LRU_TIER_MAX,
- swappiness, &sp);
- read_ctrl_pos(lruvec, LRU_GEN_FILE, LRU_TIER_MIN, LRU_TIER_MAX,
- MAX_SWAPPINESS - swappiness, &pv);
+ if (swappiness <= MIN_SWAPPINESS) {
+ nr[LRU_GEN_ANON] = 0;
+ nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
+ return;
+ }
- return positive_ctrl_err(&sp, &pv);
-}
+ if (swappiness == SWAPPINESS_ANON_ONLY) {
+ nr[LRU_GEN_FILE] = 0;
+ nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
+ return;
+ }
-static inline bool is_single_type_reclaim(int swappiness)
-{
- return swappiness == MIN_SWAPPINESS ||
- swappiness == SWAPPINESS_ANON_ONLY;
-}
+ for (int type = LRU_GEN_ANON; type <= LRU_GEN_FILE; type++) {
+ int hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
+ /* start at MIN_LRU_BATCH to smooth the rate on gen start */
+ u64 refaulted = MIN_LRU_BATCH, evicted = MIN_LRU_BATCH;
-static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
- struct scan_control *sc, int swappiness,
- struct list_head *list, int *isolated,
- int *isolate_type, int *isolate_scanned)
-{
- bool type_fallback_allowed = !is_single_type_reclaim(swappiness);
- int type = get_type_to_scan(lruvec, swappiness);
- int total_scanned = 0, scanned, tier;
- bool tried = false;
+ for (int tier = 0; tier < MAX_NR_TIERS; tier++) {
+ refaulted += READ_ONCE(lrugen->avg_refaulted[type][tier]);
+ refaulted += atomic_long_read(&lrugen->refaulted[hist][type][tier]);
-retry:
- tier = get_tier_idx(lruvec, type);
- scanned = scan_folios(nr_to_scan, lruvec, sc,
- type, tier, list, isolated);
+ evicted += READ_ONCE(lrugen->avg_total[type][tier]);
+ evicted += atomic_long_read(&lrugen->evicted[hist][type][tier]);
+ }
- total_scanned += scanned;
- if (*isolated) {
- *isolate_type = type;
- *isolate_scanned = scanned;
- return total_scanned;
+ /*
+ * The rate scaled by 4096 to survive integer division, clamped
+ * to [1, 4096] (~0% - 100%), the floor of 1 keeps a type that
+ * never refaults from zeroing out the other type's share.
+ */
+ refault_rate[type] = mul_u64_u64_div_u64(refaulted, 4096, evicted);
+ refault_rate[type] = clamp_t(u64, refault_rate[type], 1, 4096);
}
/*
- * We are running out of the current reclaim type. Fall back to
- * the other type if allowed.
+ * The more extreme the swappiness, the wider the IO cost gap between
+ * anon and file, so the split should follow the IO cost ratio rather
+ * than the sizes. With a mild swappiness, balance the scan by the
+ * actual LRU size more.
*/
- if (!scanned && !tried && type_fallback_allowed) {
- type = !type;
- tried = true;
- type_fallback_allowed = false;
- goto retry;
- }
+ size_bias = max(1, min(swappiness, MAX_SWAPPINESS - swappiness));
+ anon_weight = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE] / size_bias;
+ file_weight = nr[LRU_GEN_FILE] + nr[LRU_GEN_ANON] / size_bias;
+
+ anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
+ file_factor = (u64)(MAX_SWAPPINESS - swappiness) * refault_rate[LRU_GEN_ANON] * file_weight;
+
+ /* Cross-multiply the ratio above. */
+ anon_scan = mul_u64_u64_div_u64(total_scan, anon_factor, anon_factor + file_factor);
+ file_scan = (u64)total_scan - anon_scan;
+
/*
- * We scanned some folios but failed to isolate any due to promotions,
- * protections, or races. Retry once to avoid a larger loop.
+ * Bound each type by its evictable size. Hand the exceeding part to
+ * the other type. Any excess left when total_scan exceeds both sizes
+ * is dropped, since it only stresses the scanner with no gain.
*/
- if (scanned && !tried) {
- tried = true;
- goto retry;
- }
-
- return total_scanned;
+ if (anon_scan > nr[LRU_GEN_ANON])
+ file_scan += anon_scan - nr[LRU_GEN_ANON];
+ if (file_scan > nr[LRU_GEN_FILE])
+ anon_scan += file_scan - nr[LRU_GEN_FILE];
+ nr[LRU_GEN_ANON] = min_t(u64, anon_scan, nr[LRU_GEN_ANON]);
+ nr[LRU_GEN_FILE] = min_t(u64, file_scan, nr[LRU_GEN_FILE]);
}
static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
- struct scan_control *sc, int swappiness)
+ struct scan_control *sc, int type, int swappiness)
{
LIST_HEAD(list);
LIST_HEAD(clean);
@@ -5099,8 +5130,8 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
enum node_stat_item item;
struct reclaim_stat stat;
struct lru_gen_mm_walk *walk;
- int scanned, reclaimed;
- int isolated = 0, nr_isolated = 0, type, type_scanned;
+ int tier, scanned, reclaimed;
+ int isolated = 0, nr_isolated = 0;
unsigned long total_reclaimed = 0;
bool skip_retry = false;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
@@ -5111,8 +5142,9 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
/* In case folio deletion left empty old gens, flush them */
try_to_inc_min_seq(lruvec, swappiness);
- scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,
- &list, &isolated, &type, &type_scanned);
+ tier = get_tier_idx(lruvec, type);
+ scanned = scan_folios(nr_to_scan, lruvec, sc,
+ type, tier, &list, &isolated);
nr_isolated = isolated;
if (nr_isolated)
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + type,
@@ -5134,7 +5166,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
if (isolated)
handle_reclaim_writeback(isolated, pgdat, sc, &stat);
trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
- type_scanned, reclaimed, &stat, sc->priority,
+ scanned, reclaimed, &stat, sc->priority,
type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
list_for_each_entry_safe_reverse(folio, next, &list, lru) {
@@ -5211,22 +5243,22 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
}
-static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
- struct mem_cgroup *memcg, int swappiness)
+static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
+ struct mem_cgroup *memcg, int swappiness,
+ unsigned long nr_to_scan[])
{
- unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
+ unsigned long total;
- lruvec_evictable_size(lruvec, swappiness, sizes);
- evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
+ lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+ total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
- /* try to scrape all its memory if this memcg was deleted */
+ /* Just scrape all pages calculated above if this memcg was deleted */
if (!mem_cgroup_online(memcg))
- return evictable;
-
- nr_to_scan = apply_proportional_protection(memcg, sc, evictable);
- nr_to_scan >>= sc->priority;
+ return;
- return nr_to_scan;
+ total = apply_proportional_protection(memcg, sc, total);
+ total >>= sc->priority;
+ lru_gen_balance_scan(lruvec, swappiness, total, nr_to_scan);
}
static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
@@ -5259,6 +5291,30 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
return true;
}
+static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
+ struct scan_control *sc, int swappiness, unsigned long max_batch)
+{
+ int type;
+ unsigned long batch, scanned, total_scanned = 0;
+
+ /*
+ * Scan both types with the given budgets; the caller checks if
+ * reclaim is satisfied after each pass.
+ */
+ for (type = LRU_GEN_FILE; type >= LRU_GEN_ANON; type--) {
+ if (!nr_to_scan[type])
+ continue;
+
+ batch = min(nr_to_scan[type], max_batch);
+ scanned = evict_folios(batch, lruvec, sc, type, swappiness);
+
+ nr_to_scan[type] -= min(nr_to_scan[type], scanned);
+ total_scanned += scanned;
+ }
+
+ return total_scanned;
+}
+
/*
* For future optimizations:
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
@@ -5266,8 +5322,8 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
*/
static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
+ unsigned long nr[ANON_AND_FILE];
bool need_rotate = false, should_age = false;
- long nr_batch, nr_to_scan;
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
@@ -5284,9 +5340,9 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
return false;
}
- nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
- while (nr_to_scan > 0) {
- int delta;
+ lru_gen_prepare_scan(lruvec, sc, memcg, swappiness, nr);
+
+ while (nr[LRU_GEN_ANON] > 0 || nr[LRU_GEN_FILE] > 0) {
DEFINE_MAX_SEQ(lruvec);
if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) {
@@ -5300,9 +5356,7 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
should_age = true;
}
- nr_batch = min(nr_to_scan, MIN_LRU_BATCH);
- delta = evict_folios(nr_batch, lruvec, sc, swappiness);
- if (!delta)
+ if (!evict_folio_lists(nr, lruvec, sc, swappiness, MIN_LRU_BATCH))
break;
if (should_abort_scan(lruvec, sc))
@@ -5315,7 +5369,6 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
if (root_reclaim(sc) && should_age)
break;
- nr_to_scan -= delta;
cond_resched();
}
@@ -5930,8 +5983,8 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq,
static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
int swappiness, unsigned long nr_to_reclaim)
{
- int nr_batch;
DEFINE_MAX_SEQ(lruvec);
+ unsigned long nr_to_scan[ANON_AND_FILE];
if (seq + MIN_NR_GENS > max_seq)
return -EINVAL;
@@ -5947,8 +6000,11 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
if (sc->nr_reclaimed >= nr_to_reclaim)
return 0;
- nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
- if (!evict_folios(nr_batch, lruvec, sc, swappiness))
+ lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+ lru_gen_balance_scan(lruvec, swappiness,
+ nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
+
+ if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
return 0;
cond_resched();
@@ -6011,6 +6067,7 @@ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
.may_writepage = true,
.may_unmap = true,
.may_swap = true,
+ .priority = DEF_PRIORITY,
.reclaim_idx = MAX_NR_ZONES - 1,
.gfp_mask = GFP_KERNEL,
.proactive = true,
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate
2026-10-22 22:01 ` [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate Barry Song (Xiaomi)
@ 2026-10-23 22:03 ` sashiko-bot
2026-10-24 16:24 ` Kairui Song
0 siblings, 1 reply; 30+ messages in thread
From: sashiko-bot @ 2026-10-23 22:03 UTC (permalink / raw)
To: Barry Song (Xiaomi), akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baolin.wang, david, hannes, ljs, mhocko,
qi.zheng, ryncsn, shakeel.butt, weixugc, yuanchu, zhuhui,
baoquan.he, liuxiang.277, wangzicheng, lianux.mm, kasong, liam,
surenb, mhocko, rppt, zhangbo56, lyugaofei, ridong.chen,
kunwu.chan
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] mm/mglru: lru_gen_balance_scan() budgets un-scannable gens and zones
--
commit ce196f9c0d172466d44f73b512ed408a9ecb6e36
Author: Kairui Song <kasong@tencent.com>
mm/mglru: balance anon and file scan budget by refault rate
This commit replaces get_type_to_scan() and isolate_folios() with
lru_gen_balance_scan() and evict_folio_lists() to split the scan budget
proportionally between anon and file LRUs based on per-type refault rates
and LRU sizes.
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index cc20ba8c6cf2c..fed39b0422e01 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -5019,79 +5019,110 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
[ ... ]
> +static void lru_gen_balance_scan(struct lruvec *lruvec, int swappiness,
> + unsigned long total_scan, unsigned long nr[])
> {
[ ... ]
> - /*
> - * We are running out of the current reclaim type. Fall back to
> - * the other type if allowed.
> + * The more extreme the swappiness, the wider the IO cost gap between
> + * anon and file, so the split should follow the IO cost ratio rather
> + * than the sizes. With a mild swappiness, balance the scan by the
> + * actual LRU size more.
> */
> - if (!scanned && !tried && type_fallback_allowed) {
> - type = !type;
> - tried = true;
> - type_fallback_allowed = false;
> - goto retry;
> - }
> + size_bias = max(1, min(swappiness, MAX_SWAPPINESS - swappiness));
> + anon_weight = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE] / size_bias;
> + file_weight = nr[LRU_GEN_FILE] + nr[LRU_GEN_ANON] / size_bias;
> +
> + anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
> + file_factor = (u64)(MAX_SWAPPINESS - swappiness) * refault_rate[LRU_GEN_ANON] * file_weight;
> +
> + /* Cross-multiply the ratio above. */
> + anon_scan = mul_u64_u64_div_u64(total_scan, anon_factor, anon_factor + file_factor);
> + file_scan = (u64)total_scan - anon_scan;
> +
> /*
> - * We scanned some folios but failed to isolate any due to promotions,
> - * protections, or races. Retry once to avoid a larger loop.
> + * Bound each type by its evictable size. Hand the exceeding part to
> + * the other type. Any excess left when total_scan exceeds both sizes
> + * is dropped, since it only stresses the scanner with no gain.
> */
> - if (scanned && !tried) {
> - tried = true;
> - goto retry;
> - }
> -
> - return total_scanned;
> + if (anon_scan > nr[LRU_GEN_ANON])
> + file_scan += anon_scan - nr[LRU_GEN_ANON];
> + if (file_scan > nr[LRU_GEN_FILE])
> + anon_scan += file_scan - nr[LRU_GEN_FILE];
> + nr[LRU_GEN_ANON] = min_t(u64, anon_scan, nr[LRU_GEN_ANON]);
> + nr[LRU_GEN_FILE] = min_t(u64, file_scan, nr[LRU_GEN_FILE]);
> }
[ ... ]
> @@ -5212,22 +5244,22 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
> return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
> }
>
> -static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
> - struct mem_cgroup *memcg, int swappiness)
> +static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
> + struct mem_cgroup *memcg, int swappiness,
> + unsigned long nr_to_scan[])
> {
> - unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
> + unsigned long total;
>
> - lruvec_evictable_size(lruvec, swappiness, sizes);
> - evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
> + lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> + total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
>
> - /* try to scrape all its memory if this memcg was deleted */
> + /* Just scrape all pages calculated above if this memcg was deleted */
> if (!mem_cgroup_online(memcg))
> - return evictable;
> -
> - nr_to_scan = apply_proportional_protection(memcg, sc, evictable);
> - nr_to_scan >>= sc->priority;
> + return;
>
> - return nr_to_scan;
> + total = apply_proportional_protection(memcg, sc, total);
> + total >>= sc->priority;
> + lru_gen_balance_scan(lruvec, swappiness, total, nr_to_scan);
> }
>
> static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
> @@ -5260,6 +5292,30 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
> return true;
> }
>
> +static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
> + struct scan_control *sc, int swappiness, unsigned long max_batch)
> +{
> + int type;
> + unsigned long batch, scanned, total_scanned = 0;
> +
> + /*
> + * Scan both types with the given budgets; the caller checks if
> + * reclaim is satisfied after each pass.
> + */
> + for (type = LRU_GEN_FILE; type >= LRU_GEN_ANON; type--) {
> + if (!nr_to_scan[type])
> + continue;
> +
> + batch = min(nr_to_scan[type], max_batch);
> + scanned = evict_folios(batch, lruvec, sc, type, swappiness);
> +
> + nr_to_scan[type] -= min(nr_to_scan[type], scanned);
> + total_scanned += scanned;
> + }
> +
> + return total_scanned;
> +}
[ ... ]
> @@ -5948,8 +6001,11 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
> if (sc->nr_reclaimed >= nr_to_reclaim)
> return 0;
>
> - nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
> - if (!evict_folios(nr_batch, lruvec, sc, swappiness))
> + lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> + lru_gen_balance_scan(lruvec, swappiness,
> + nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
> +
> + if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
> return 0;
>
> cond_resched();
[Severity: Medium]
Can budgeting with lruvec_evictable_size() in lru_gen_balance_scan() cause
lost scan budget and premature reclaim termination in try_to_shrink_lruvec()
and run_eviction() when one type has no folios in scannable generations or
eligible zones?
Previously, isolate_folios() in mm/vmscan.c fell back to scanning !type
whenever scan_folios() returned scanned == 0. With isolate_folios() removed,
lru_gen_prepare_scan() and run_eviction() rely on lruvec_evictable_size()
and lru_gen_balance_scan() to cap each type's budget by nr[type] and hand
any excess to the other type.
However, lruvec_evictable_size() sums folios across all generations up to
max_seq (including the protected youngest generations max_seq - 1 and
max_seq) and across all zones:
mm/vmscan.c:lruvec_evictable_size() {
for_each_evictable_type(type, swappiness) {
for (seq = min_seq[type]; seq <= max_seq; seq++) {
gen = lru_gen_from_seq(seq);
for (zone = 0; zone < MAX_NR_ZONES; zone++)
nr[type] += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
}
}
}
In contrast, scan_folios() only scans zones 0..sc->reclaim_idx, and at
sc->priority == DEF_PRIORITY (or after the later commit "mm/mglru: lift
youngest gen protection for higher priority" in this series, which caps
scanning at max_seq - MIN_NR_GENS) it does not scan the youngest generations:
mm/vmscan.c:scan_folios() {
if (sc->priority == DEF_PRIORITY)
seq = lrugen->min_seq[type];
else
seq = lrugen->max_seq;
seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
}
When one type has no scannable folios in eligible generations or zones (for
example, after its cold generations <= max_seq - MIN_NR_GENS are drained or
when its folios reside in zones > sc->reclaim_idx, such as under GFP_DMA32),
lruvec_evictable_size() still reports a large non-zero nr[type] from the
active generations or higher zones. As a result, lru_gen_balance_scan() does
not cap that type's budget or transfer it to the scannable type.
In the proactive reclaim path (lru_gen_seq_write() -> run_cmd() ->
run_eviction()), sc.priority is fixed at DEF_PRIORITY. Once one type drains
its cold generations while retaining folios in max_seq - 1 or max_seq, any
round where the other type's share rounds down to 0 in:
anon_scan = mul_u64_u64_div_u64(total_scan, anon_factor, anon_factor + file_factor);
file_scan = (u64)total_scan - anon_scan;
(for instance, whenever nr_to_reclaim - sc->nr_reclaimed == 1 for swappiness
between 1 and 199, or for larger remaining targets when one factor dominates,
or at swappiness == MAX_SWAPPINESS when anon's cold generations are empty)
assigns the entire remaining budget to the drained type and 0 to the type
that still has cold folios. evict_folio_lists() then scans 0 from the drained
type, skips the scannable type, and returns 0, causing run_eviction() to
abort before meeting nr_to_reclaim.
Similarly, in regular reclaim (try_to_shrink_lruvec() ->
lru_gen_prepare_scan() -> lru_gen_balance_scan() -> evict_folio_lists()),
when evict_folios() returns scanned == 0 for the un-scannable type:
nr_to_scan[type] -= min(nr_to_scan[type], scanned);
leaves nr_to_scan[type] unchanged. While the scannable type still has budget,
evict_folio_lists() repeatedly calls evict_folios() (acquiring
lruvec->lru_lock) on the empty type on every MIN_LRU_BATCH iteration, and
once the scannable type's share is consumed, the empty type's remaining
budget is lost instead of being transferred to the scannable type.
Should lru_gen_balance_scan() bound each type by the size of its scannable
generations and eligible zones, or should evict_folio_lists() transfer the
remaining budget to the other type when evict_folios() returns 0?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261022220203.91911-1-baohua@kernel.org?part=7
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate
2026-10-23 22:03 ` sashiko-bot
@ 2026-10-24 16:24 ` Kairui Song
2026-10-26 7:34 ` Barry Song
0 siblings, 1 reply; 30+ messages in thread
From: Kairui Song @ 2026-10-24 16:24 UTC (permalink / raw)
To: sashiko-reviews
Cc: Barry Song (Xiaomi), akpm, linux-mm, hui.zhu, axelrasmussen,
baolin.wang, david, hannes, ljs, mhocko, qi.zheng, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, liam, surenb, mhocko, rppt, zhangbo56, lyugaofei,
ridong.chen, kunwu.chan
On Sat, Oct 24, 2026 at 6:03 AM <sashiko-bot@kernel.org> wrote:
>
> Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
> - [Medium] mm/mglru: lru_gen_balance_scan() budgets un-scannable gens and zones
> --
>
> commit ce196f9c0d172466d44f73b512ed408a9ecb6e36
> Author: Kairui Song <kasong@tencent.com>
>
> mm/mglru: balance anon and file scan budget by refault rate
>
> This commit replaces get_type_to_scan() and isolate_folios() with
> lru_gen_balance_scan() and evict_folio_lists() to split the scan budget
> proportionally between anon and file LRUs based on per-type refault rates
> and LRU sizes.
>
> > diff --git a/mm/vmscan.c b/mm/vmscan.c
> > index cc20ba8c6cf2c..fed39b0422e01 100644
> > --- a/mm/vmscan.c
> > +++ b/mm/vmscan.c
> > @@ -5019,79 +5019,110 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
>
> [ ... ]
>
> > +static void lru_gen_balance_scan(struct lruvec *lruvec, int swappiness,
> > + unsigned long total_scan, unsigned long nr[])
> > {
>
> [ ... ]
>
> > - /*
> > - * We are running out of the current reclaim type. Fall back to
> > - * the other type if allowed.
> > + * The more extreme the swappiness, the wider the IO cost gap between
> > + * anon and file, so the split should follow the IO cost ratio rather
> > + * than the sizes. With a mild swappiness, balance the scan by the
> > + * actual LRU size more.
> > */
> > - if (!scanned && !tried && type_fallback_allowed) {
> > - type = !type;
> > - tried = true;
> > - type_fallback_allowed = false;
> > - goto retry;
> > - }
> > + size_bias = max(1, min(swappiness, MAX_SWAPPINESS - swappiness));
> > + anon_weight = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE] / size_bias;
> > + file_weight = nr[LRU_GEN_FILE] + nr[LRU_GEN_ANON] / size_bias;
> > +
> > + anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
> > + file_factor = (u64)(MAX_SWAPPINESS - swappiness) * refault_rate[LRU_GEN_ANON] * file_weight;
> > +
> > + /* Cross-multiply the ratio above. */
> > + anon_scan = mul_u64_u64_div_u64(total_scan, anon_factor, anon_factor + file_factor);
> > + file_scan = (u64)total_scan - anon_scan;
> > +
> > /*
> > - * We scanned some folios but failed to isolate any due to promotions,
> > - * protections, or races. Retry once to avoid a larger loop.
> > + * Bound each type by its evictable size. Hand the exceeding part to
> > + * the other type. Any excess left when total_scan exceeds both sizes
> > + * is dropped, since it only stresses the scanner with no gain.
> > */
> > - if (scanned && !tried) {
> > - tried = true;
> > - goto retry;
> > - }
> > -
> > - return total_scanned;
> > + if (anon_scan > nr[LRU_GEN_ANON])
> > + file_scan += anon_scan - nr[LRU_GEN_ANON];
> > + if (file_scan > nr[LRU_GEN_FILE])
> > + anon_scan += file_scan - nr[LRU_GEN_FILE];
> > + nr[LRU_GEN_ANON] = min_t(u64, anon_scan, nr[LRU_GEN_ANON]);
> > + nr[LRU_GEN_FILE] = min_t(u64, file_scan, nr[LRU_GEN_FILE]);
> > }
>
> [ ... ]
>
> > @@ -5212,22 +5244,22 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
> > return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
> > }
> >
> > -static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
> > - struct mem_cgroup *memcg, int swappiness)
> > +static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
> > + struct mem_cgroup *memcg, int swappiness,
> > + unsigned long nr_to_scan[])
> > {
> > - unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
> > + unsigned long total;
> >
> > - lruvec_evictable_size(lruvec, swappiness, sizes);
> > - evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
> > + lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> > + total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
> >
> > - /* try to scrape all its memory if this memcg was deleted */
> > + /* Just scrape all pages calculated above if this memcg was deleted */
> > if (!mem_cgroup_online(memcg))
> > - return evictable;
> > -
> > - nr_to_scan = apply_proportional_protection(memcg, sc, evictable);
> > - nr_to_scan >>= sc->priority;
> > + return;
> >
> > - return nr_to_scan;
> > + total = apply_proportional_protection(memcg, sc, total);
> > + total >>= sc->priority;
> > + lru_gen_balance_scan(lruvec, swappiness, total, nr_to_scan);
> > }
> >
> > static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
> > @@ -5260,6 +5292,30 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
> > return true;
> > }
> >
> > +static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
> > + struct scan_control *sc, int swappiness, unsigned long max_batch)
> > +{
> > + int type;
> > + unsigned long batch, scanned, total_scanned = 0;
> > +
> > + /*
> > + * Scan both types with the given budgets; the caller checks if
> > + * reclaim is satisfied after each pass.
> > + */
> > + for (type = LRU_GEN_FILE; type >= LRU_GEN_ANON; type--) {
> > + if (!nr_to_scan[type])
> > + continue;
> > +
> > + batch = min(nr_to_scan[type], max_batch);
> > + scanned = evict_folios(batch, lruvec, sc, type, swappiness);
> > +
> > + nr_to_scan[type] -= min(nr_to_scan[type], scanned);
> > + total_scanned += scanned;
> > + }
> > +
> > + return total_scanned;
> > +}
>
> [ ... ]
>
> > @@ -5948,8 +6001,11 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
> > if (sc->nr_reclaimed >= nr_to_reclaim)
> > return 0;
> >
> > - nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
> > - if (!evict_folios(nr_batch, lruvec, sc, swappiness))
> > + lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> > + lru_gen_balance_scan(lruvec, swappiness,
> > + nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
> > +
> > + if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
> > return 0;
> >
> > cond_resched();
>
> [Severity: Medium]
> Can budgeting with lruvec_evictable_size() in lru_gen_balance_scan() cause
> lost scan budget and premature reclaim termination in try_to_shrink_lruvec()
> and run_eviction() when one type has no folios in scannable generations or
> eligible zones?
At the moment it's like part of the design, prioiry will raise and in
the end we will have no scannable generation limit, zone might worth
be considered when getting the evictable size though?
Also, maybe we can improve the aging on imbalance to trigger when the
scan budget is much larger than the scannable size (and priority is
raised beyound default)?
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate
2026-10-24 16:24 ` Kairui Song
@ 2026-10-26 7:34 ` Barry Song
0 siblings, 0 replies; 30+ messages in thread
From: Barry Song @ 2026-10-26 7:34 UTC (permalink / raw)
To: Kairui Song
Cc: sashiko-reviews, akpm, linux-mm, hui.zhu, axelrasmussen,
baolin.wang, david, hannes, ljs, mhocko, qi.zheng, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, liam, surenb, mhocko, rppt, zhangbo56, lyugaofei,
ridong.chen, kunwu.chan
On Sun, Oct 25, 2026 at 12:25 AM Kairui Song <ryncsn@gmail.com> wrote:
[...]
> > > @@ -5948,8 +6001,11 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
> > > if (sc->nr_reclaimed >= nr_to_reclaim)
> > > return 0;
> > >
> > > - nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
> > > - if (!evict_folios(nr_batch, lruvec, sc, swappiness))
> > > + lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> > > + lru_gen_balance_scan(lruvec, swappiness,
> > > + nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
> > > +
> > > + if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
> > > return 0;
> > >
> > > cond_resched();
> >
> > [Severity: Medium]
> > Can budgeting with lruvec_evictable_size() in lru_gen_balance_scan() cause
> > lost scan budget and premature reclaim termination in try_to_shrink_lruvec()
> > and run_eviction() when one type has no folios in scannable generations or
> > eligible zones?
>
> At the moment it's like part of the design, prioiry will raise and in
> the end we will have no scannable generation limit, zone might worth
> be considered when getting the evictable size though?
It should be straightforward to exclude non-reclaimable zones based on
Bo's fix to 14/14 which addresses the GFP_NOIO issue[1].
Since sc is already passed to lruvec_evictable_size(),
this should be fairly simple.
If you want to give it a try, I've pushed the code to:
https://git.kernel.org/pub/scm/linux/kernel/git/baohua/linux.git/log/?h=swappiness-rfc-v5-with-kairui-and-fixes
[1] https://lore.kernel.org/linux-mm/20261025013103.2742885-1-zhangbo56@xiaomi.com/
>
> Also, maybe we can improve the aging on imbalance to trigger when the
> scan budget is much larger than the scannable size (and priority is
> raised beyound default)?
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 08/14] mm/mglru: lift youngest gen protection for higher priority
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (6 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 07/14] mm/mglru: balance anon and file scan budget by refault rate Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-23 22:03 ` sashiko-bot
2026-10-22 22:01 ` [RFC PATCH v5 09/14] mm/mglru: Avoid placing rejected folios below populated gen Barry Song (Xiaomi)
` (6 subsequent siblings)
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Kairui Song <kasong@tencent.com>
The forced protection of younger gens is in contradict of the goal of
minimizing the IO cost for reclaim. And under high pressure, MGLRU runs
into premature OOM constantly, where classical LRU never goes OOM.
The problem is when aging can't catch up with reclaimer, reclaimers will
all abort early seeing there is only empty gens, or only two gens left.
Proactive aging won't help eigher because there will be limited threads
doing aging, but there could be unlimited threads doing reclaim, which
is a fundommental limitation of MGLRU. One way to avoid this is to use a
aging lock, like in commit .... However, aging lock caused several
jitter when under huge pressure, and was reported to be causing
obserable lags already on Android.
What's more, the gen model somehow contradicts with the goal of
swappiness, which aim to reduce IO cost intead of following hotness
blindly.
So instead, let's lift the protection based on priority. With default
priority, we still fully respect the MIN_NR_GEN protection. But as the
priority raised, allow it to reclaim younger gens step by step.
OOM is gone, and swappiness is fair after this commit.
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 0e37f5c8f355..12aca97be56b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4929,12 +4929,14 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
VM_WARN_ON_ONCE(!list_empty(list));
/*
- * Respect the gen hotness distribution at low priority to avoid
- * aging and scan cost, but prefer to satisfy the reclaimers request
- * when under pressure to respect swappiness more and reduce IO cost.
+ * Respect hotness at default priority, and only reach into the
+ * newest generation when under severe pressure, so hot folios are
+ * not evicted as soon as priority is raised.
*/
if (sc->priority == DEF_PRIORITY)
- seq = lrugen->min_seq[type];
+ seq = lrugen->max_seq - MIN_NR_GENS;
+ else if (sc->priority > DEF_PRIORITY / 2)
+ seq = lrugen->max_seq - 1;
else
seq = lrugen->max_seq;
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 08/14] mm/mglru: lift youngest gen protection for higher priority
2026-10-22 22:01 ` [RFC PATCH v5 08/14] mm/mglru: lift youngest gen protection for higher priority Barry Song (Xiaomi)
@ 2026-10-23 22:03 ` sashiko-bot
2026-10-24 18:08 ` Kairui Song
0 siblings, 1 reply; 30+ messages in thread
From: sashiko-bot @ 2026-10-23 22:03 UTC (permalink / raw)
To: Barry Song (Xiaomi), akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baolin.wang, david, hannes, ljs, mhocko,
qi.zheng, ryncsn, shakeel.butt, weixugc, yuanchu, zhuhui,
baoquan.he, liuxiang.277, wangzicheng, lianux.mm, kasong, liam,
surenb, mhocko, rppt, zhangbo56, lyugaofei, ridong.chen,
kunwu.chan
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] mm/mglru: scan_folios() scans expired gen < min_seq at DEF_PRIORITY
--
commit 1329290ce5f86b7de9b77d6b22e4b265f16348ae
Author: Kairui Song <kasong@tencent.com>
mm/mglru: lift youngest gen protection for higher priority
This commit adjusts scan_folios() in mm/vmscan.c to choose the target
generation sequence number based on reclaim priority rather than using
lrugen->min_seq[type] at default priority. It aims to protect younger
generations at default priority while allowing higher-priority reclaim to
scan newer generations step by step.
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index fed39b0422e01..d1e14001f9346 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -4930,12 +4930,14 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> VM_WARN_ON_ONCE(!list_empty(list));
>
> /*
> - * Respect the gen hotness distribution at low priority to avoid
> - * aging and scan cost, but prefer to satisfy the reclaimers request
> - * when under pressure to respect swappiness more and reduce IO cost.
> + * Respect hotness at default priority, and only reach into the
> + * newest generation when under severe pressure, so hot folios are
> + * not evicted as soon as priority is raised.
> */
> if (sc->priority == DEF_PRIORITY)
> - seq = lrugen->min_seq[type];
> + seq = lrugen->max_seq - MIN_NR_GENS;
> + else if (sc->priority > DEF_PRIORITY / 2)
> + seq = lrugen->max_seq - 1;
> else
> seq = lrugen->max_seq;
>
[Severity: Medium]
Can this cause scan_folios() in mm/vmscan.c to scan an already-evicted
generation when a type only has the minimum number of generations left?
In try_to_shrink_lruvec() -> evict_folio_lists() -> evict_folios() ->
scan_folios() at default priority, when get_nr_gens(lruvec, type) is at the
minimum (2 generations), lrugen->min_seq[type] is lrugen->max_seq - 1, while
scan_folios() sets seq to lrugen->max_seq - 2 and passes it as max_seq to
lruvec_populated_min_seq():
mm/vmscan.c:scan_folios() {
if (sc->priority == DEF_PRIORITY)
seq = lrugen->max_seq - MIN_NR_GENS;
...
seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
gen = lru_gen_from_seq(seq);
}
In lruvec_populated_min_seq(), because min_seq (lrugen->max_seq - 1) is
greater than max_seq (lrugen->max_seq - 2), the while loop is skipped and
max_seq is returned directly:
mm/vmscan.c:lruvec_populated_min_seq() {
min_seq = READ_ONCE(lrugen->min_seq[type]);
while (min_seq <= max_seq) {
gen = lru_gen_from_seq(min_seq);
for (zone = 0; zone < zone_limit; zone++) {
if (!list_empty(&lrugen->folios[gen][type][zone]))
return min_seq;
}
min_seq++;
}
return max_seq;
}
When scan_folios() previously scanned younger generations (lrugen->max_seq - 1
or lrugen->max_seq) at elevated priority while get_nr_gens(lruvec, type) had
fewer than the maximum number of generations, sort_folio() can move folios
with a stale folio_lru_gen() value (such as after __lru_gen_reparent_memcg()
or when inc_min_seq() skipped draining old_gen) into that expired
lrugen->folios[gen][type][zone] list:
mm/vmscan.c:sort_folio() {
/* promoted, or moved due to reparenting. */
if (gen != min_gen) {
list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
return true;
}
}
A subsequent default-priority call to scan_folios() will then visit that
expired generation (< lrugen->min_seq[type]) and corrupt active/inactive size
accounting.
In addition, in lru_gen_seq_write() -> run_cmd() -> run_eviction(), debugfs
eviction runs at default priority. If a user requests evicting up to
lrugen->max_seq - 3 when four generations exist and one type has already
emptied lrugen->max_seq - 3 while the other type still has min_seq at
lrugen->max_seq - 3, wouldn't setting seq to lrugen->max_seq - 2 in
scan_folios() evict folios from generation lrugen->max_seq - 2, exceeding the
requested seq bound?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261022220203.91911-1-baohua@kernel.org?part=8
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 08/14] mm/mglru: lift youngest gen protection for higher priority
2026-10-23 22:03 ` sashiko-bot
@ 2026-10-24 18:08 ` Kairui Song
0 siblings, 0 replies; 30+ messages in thread
From: Kairui Song @ 2026-10-24 18:08 UTC (permalink / raw)
To: sashiko-reviews
Cc: Barry Song (Xiaomi), akpm, linux-mm, hui.zhu, axelrasmussen,
baolin.wang, david, hannes, ljs, mhocko, qi.zheng, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, liam, surenb, mhocko, rppt, zhangbo56, lyugaofei,
ridong.chen, kunwu.chan
On Sat, Oct 24, 2026 at 6:03 AM <sashiko-bot@kernel.org> wrote:
>
> Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
> - [Medium] mm/mglru: scan_folios() scans expired gen < min_seq at DEF_PRIORITY
> --
>
> commit 1329290ce5f86b7de9b77d6b22e4b265f16348ae
> Author: Kairui Song <kasong@tencent.com>
>
> mm/mglru: lift youngest gen protection for higher priority
>
> This commit adjusts scan_folios() in mm/vmscan.c to choose the target
> generation sequence number based on reclaim priority rather than using
> lrugen->min_seq[type] at default priority. It aims to protect younger
> generations at default priority while allowing higher-priority reclaim to
> scan newer generations step by step.
>
> > diff --git a/mm/vmscan.c b/mm/vmscan.c
> > index fed39b0422e01..d1e14001f9346 100644
> > --- a/mm/vmscan.c
> > +++ b/mm/vmscan.c
> > @@ -4930,12 +4930,14 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> > VM_WARN_ON_ONCE(!list_empty(list));
> >
> > /*
> > - * Respect the gen hotness distribution at low priority to avoid
> > - * aging and scan cost, but prefer to satisfy the reclaimers request
> > - * when under pressure to respect swappiness more and reduce IO cost.
> > + * Respect hotness at default priority, and only reach into the
> > + * newest generation when under severe pressure, so hot folios are
> > + * not evicted as soon as priority is raised.
> > */
> > if (sc->priority == DEF_PRIORITY)
> > - seq = lrugen->min_seq[type];
> > + seq = lrugen->max_seq - MIN_NR_GENS;
> > + else if (sc->priority > DEF_PRIORITY / 2)
> > + seq = lrugen->max_seq - 1;
> > else
> > seq = lrugen->max_seq;
> >
>
> [Severity: Medium]
> Can this cause scan_folios() in mm/vmscan.c to scan an already-evicted
> generation when a type only has the minimum number of generations left?
>
> In try_to_shrink_lruvec() -> evict_folio_lists() -> evict_folios() ->
> scan_folios() at default priority, when get_nr_gens(lruvec, type) is at the
> minimum (2 generations), lrugen->min_seq[type] is lrugen->max_seq - 1, while
> scan_folios() sets seq to lrugen->max_seq - 2 and passes it as max_seq to
> lruvec_populated_min_seq():
>
> mm/vmscan.c:scan_folios() {
> if (sc->priority == DEF_PRIORITY)
> seq = lrugen->max_seq - MIN_NR_GENS;
> ...
> seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
> gen = lru_gen_from_seq(seq);
> }
>
> In lruvec_populated_min_seq(), because min_seq (lrugen->max_seq - 1) is
> greater than max_seq (lrugen->max_seq - 2), the while loop is skipped and
> max_seq is returned directly:
>
> mm/vmscan.c:lruvec_populated_min_seq() {
> min_seq = READ_ONCE(lrugen->min_seq[type]);
> while (min_seq <= max_seq) {
> gen = lru_gen_from_seq(min_seq);
> for (zone = 0; zone < zone_limit; zone++) {
> if (!list_empty(&lrugen->folios[gen][type][zone]))
> return min_seq;
> }
> min_seq++;
> }
>
> return max_seq;
> }
>
> When scan_folios() previously scanned younger generations (lrugen->max_seq - 1
> or lrugen->max_seq) at elevated priority while get_nr_gens(lruvec, type) had
> fewer than the maximum number of generations, sort_folio() can move folios
> with a stale folio_lru_gen() value (such as after __lru_gen_reparent_memcg()
> or when inc_min_seq() skipped draining old_gen) into that expired
> lrugen->folios[gen][type][zone] list:
>
> mm/vmscan.c:sort_folio() {
> /* promoted, or moved due to reparenting. */
> if (gen != min_gen) {
> list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
> return true;
> }
> }
>
> A subsequent default-priority call to scan_folios() will then visit that
> expired generation (< lrugen->min_seq[type]) and corrupt active/inactive size
> accounting.
>
> In addition, in lru_gen_seq_write() -> run_cmd() -> run_eviction(), debugfs
> eviction runs at default priority. If a user requests evicting up to
> lrugen->max_seq - 3 when four generations exist and one type has already
> emptied lrugen->max_seq - 3 while the other type still has min_seq at
> lrugen->max_seq - 3, wouldn't setting seq to lrugen->max_seq - 2 in
> scan_folios() evict folios from generation lrugen->max_seq - 2, exceeding the
> requested seq bound?
>
Looks a false positive, though it is tricky. A folio on a generation
list is only ever promoted, and min_seq[type] is advanced only after
the old generation has been drained by inc_min_seq() or verified empty
by try_to_inc_min_seq(). So when a type is down to MIN_NR_GENS, the
max_seq - MIN_NR_GENS target is empty, not a populated expired
generation. Lazy folio flags and the non evictable skip can create
temporary skew, and reparenting can look odd across generations, but
that skew is transient and cannot survive until a DEF_PRIORITY scan
targets that generation. sort_folio() also refuses to promote from the
last live generation.
For the debugfs case, lruvec_populated_min_seq() starts from
min_seq[type], so the type with min_seq == max_seq - 3 scans max_seq -
3 while that generation is populated. The type already at max_seq - 2
was scanned at max_seq - 2 before this patch too. Not enforcing the
requested seq per type is a separate pre-existing debugfs issue, not
caused by this change.
^ permalink raw reply [flat|nested] 30+ messages in thread
* [RFC PATCH v5 09/14] mm/mglru: Avoid placing rejected folios below populated gen
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (7 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 08/14] mm/mglru: lift youngest gen protection for higher priority Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-22 22:01 ` [RFC PATCH v5 10/14] mm/mglru: Run aging when pages are rare outside the youngest gen Barry Song (Xiaomi)
` (5 subsequent siblings)
14 siblings, 0 replies; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
We used to scan only the oldest generation in scan_folios(). In
evict_folios(), we checked whether
lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]. If so, the
folio was marked active and moved to younger generations.
This was itself buggy, as folios in generation min_seq[type] - 1
would not be marked active by folio_set_active(), potentially causing
a cold/hot inversion.
Besides this issue, the old approach also differs from the current
behavior, where we can scan fewer than two generations.
Now that we can scan younger generations, including the youngest
generation, this no longer prevents rejected folios from being placed
in a generation below or at the populated generation. Such rejected
folios would then be scanned again and again, resulting in extremely
low `pg_steal/pg_scan` ratios. For example:
pgsteal_file: 2312605
pgscan_file: 36465693
reclamation efficiency: 6.34%
We pass `scanned_seq` to `move_folios_to_lru()` and make sure we either
promote the folio or place it in the youngest generation when the
populated generation is already the youngest one.
Also, folio_set_active() is no longer needed.
With this change, the results become:
pgsteal_file: 2683815
pgscan_file: 4224244
reclamation efficiency: 63.53%
The sys time for building the kernel also decreased from 4m30.532s to
3m50.188s while swappiness==1.
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
include/linux/mm_inline.h | 22 +++++++++++++++++++++-
mm/vmscan.c | 24 ++++++++++++------------
2 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index ab69b9930893..f6f4bc9a704f 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -332,7 +332,8 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type]));
}
-static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
+ unsigned long scanned_seq, bool reclaiming)
{
unsigned long seq;
unsigned long flags;
@@ -347,6 +348,8 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
return false;
seq = lru_gen_folio_seq(lruvec, folio, reclaiming);
+ if (scanned_seq != -1UL)
+ seq = max(seq, min(scanned_seq + 1, lrugen->max_seq));
gen = lru_gen_from_seq(seq);
flags = (gen + 1UL) << LRU_GEN_PGOFF;
/* see the comment on MIN_NR_GENS about PG_active */
@@ -362,6 +365,17 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
return true;
}
+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+ return __lru_gen_add_folio(lruvec, folio, -1UL, reclaiming);
+}
+
+static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio,
+ unsigned long scanned_seq)
+{
+ return __lru_gen_add_folio(lruvec, folio, scanned_seq, false);
+}
+
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
unsigned long flags;
@@ -420,6 +434,12 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
return false;
}
+static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio,
+ unsigned long scanned_seq)
+{
+ return false;
+}
+
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 12aca97be56b..39f77a68be37 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1971,7 +1971,8 @@ static bool too_many_isolated(struct pglist_data *pgdat, int file,
*
* Note: The caller must not hold any lruvec lock.
*/
-static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
+static unsigned int move_folios_to_lru(struct list_head *list, unsigned long seq,
+ bool do_rotate)
{
int nr_pages, nr_moved = 0;
struct lruvec *lruvec = NULL;
@@ -2028,7 +2029,7 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
!folio_test_referenced(folio) &&
folio_ref_count(folio) == folio_expected_ref_count(folio))
lruvec_add_folio_tail(lruvec, folio);
- else
+ else if (seq == -1UL || !lru_gen_add_rejected_folio(lruvec, folio, seq))
lruvec_add_folio(lruvec, folio);
nr_pages = folio_nr_pages(folio);
@@ -2147,7 +2148,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
lruvec_memcg(lruvec));
- move_folios_to_lru(&folio_list, true);
+ move_folios_to_lru(&folio_list, -1UL, true);
mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
stat.nr_demoted);
@@ -2258,8 +2259,8 @@ static void shrink_active_list(unsigned long nr_to_scan,
/*
* Move folios back to the lru list.
*/
- nr_activate = move_folios_to_lru(&l_active, false);
- nr_deactivate = move_folios_to_lru(&l_inactive, false);
+ nr_activate = move_folios_to_lru(&l_active, -1UL, false);
+ nr_deactivate = move_folios_to_lru(&l_inactive, -1UL, false);
count_vm_events(PGDEACTIVATE, nr_deactivate);
count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
@@ -4910,7 +4911,8 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
struct scan_control *sc, int type, int tier,
- struct list_head *list, int *isolatedp)
+ struct list_head *list, int *isolatedp,
+ unsigned long *scanned_seq)
{
enum node_stat_item item;
int zone_idx;
@@ -4941,6 +4943,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
seq = lrugen->max_seq;
seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
+ *scanned_seq = seq;
gen = lru_gen_from_seq(seq);
max_gen = lru_gen_from_seq(max_seq);
@@ -5135,6 +5138,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
int tier, scanned, reclaimed;
int isolated = 0, nr_isolated = 0;
unsigned long total_reclaimed = 0;
+ unsigned long scanned_seq;
bool skip_retry = false;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
@@ -5146,7 +5150,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
tier = get_tier_idx(lruvec, type);
scanned = scan_folios(nr_to_scan, lruvec, sc,
- type, tier, &list, &isolated);
+ type, tier, &list, &isolated, &scanned_seq);
nr_isolated = isolated;
if (nr_isolated)
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + type,
@@ -5172,8 +5176,6 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
list_for_each_entry_safe_reverse(folio, next, &list, lru) {
- DEFINE_MIN_SEQ(lruvec);
-
/* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
if (!folio_evictable(folio))
continue;
@@ -5192,11 +5194,9 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
* so this effectively promotes them by at least one generation.
*/
folio_set_lru_refs(folio, 0);
- if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
- folio_set_active(folio);
}
- move_folios_to_lru(&list, false);
+ move_folios_to_lru(&list, scanned_seq, false);
walk = current->reclaim_state->mm_walk;
if (walk && walk->batched) {
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* [RFC PATCH v5 10/14] mm/mglru: Run aging when pages are rare outside the youngest gen
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (8 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 09/14] mm/mglru: Avoid placing rejected folios below populated gen Barry Song (Xiaomi)
@ 2026-10-22 22:01 ` Barry Song (Xiaomi)
2026-10-22 22:02 ` [RFC PATCH v5 11/14] mm/mglru: Reduce LRU lock contention by shrinking folio batches in inc_min_seq Barry Song (Xiaomi)
` (4 subsequent siblings)
14 siblings, 0 replies; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:01 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: lyugaofei <lyugaofei@xiaomi.com>
Run aging when the total number of pages outside the youngest
generation across both types falls below 1/MAX_NR_GENS of the number
of pages in the youngest generation. For extreme swappiness values
such as 1 and 200, also consider the balance of the corresponding
type.
This improves reclaim efficiency, especially at extreme swappiness
values, while reducing system time.
To avoid the overhead of aging, we only allow imbalance-based aging
when the reclaim priority is low enough that we might reclaim the
youngest generation. We also limit it to once per priority.
Signed-off-by: lyugaofei <lyugaofei@xiaomi.com>
Co-developed-by: Barry Song (Xiaomi) <baohua@kernel.org>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 65 +++++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 56 insertions(+), 9 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 39f77a68be37..e8eeeeb76282 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4346,10 +4346,21 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control
sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
}
+static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
+ int type, unsigned long seq)
+{
+ int gen = lru_gen_from_seq(seq);
+ unsigned long size = 0;
+
+ for (int zone = 0; zone < MAX_NR_ZONES; zone++)
+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+ return size;
+}
+
static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
unsigned long nr[])
{
- int gen, type, zone;
+ int type;
unsigned long seq;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
DEFINE_MAX_SEQ(lruvec);
@@ -4359,11 +4370,8 @@ static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
nr[LRU_GEN_FILE] = 0;
for_each_evictable_type(type, swappiness) {
- for (seq = min_seq[type]; seq <= max_seq; seq++) {
- gen = lru_gen_from_seq(seq);
- for (zone = 0; zone < MAX_NR_ZONES; zone++)
- nr[type] += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
- }
+ for (seq = min_seq[type]; seq <= max_seq; seq++)
+ nr[type] += lruvec_gen_size(lrugen, type, seq);
}
}
@@ -5228,8 +5236,41 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
return scanned;
}
+static bool lru_gen_imbalanced(struct lruvec *lruvec, struct scan_control *sc,
+ int swappiness)
+{
+ unsigned long youngest[ANON_AND_FILE] = {0}, old[ANON_AND_FILE] = {0};
+ struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ DEFINE_MAX_SEQ(lruvec);
+ DEFINE_MIN_SEQ(lruvec);
+ int type;
+
+ for_each_evictable_type(type, swappiness) {
+ unsigned long seq;
+
+ youngest[type] = lruvec_gen_size(lrugen, type, max_seq);
+ for (seq = min_seq[type]; seq < max_seq; seq++)
+ old[type] += lruvec_gen_size(lrugen, type, seq);
+ }
+
+ /* the vast majority of pages sit in the youngest gen */
+ if (youngest[LRU_GEN_ANON] + youngest[LRU_GEN_FILE] >
+ (old[LRU_GEN_ANON] + old[LRU_GEN_FILE]) * MAX_NR_GENS)
+ return true;
+
+ /* Swappiness extremes: the corresponding type matters */
+ if (swappiness == MIN_SWAPPINESS + 1)
+ return youngest[LRU_GEN_FILE] > old[LRU_GEN_FILE] * MAX_NR_GENS;
+
+ if (swappiness == MAX_SWAPPINESS)
+ return youngest[LRU_GEN_ANON] > old[LRU_GEN_ANON] * MAX_NR_GENS;
+
+ return false;
+}
+
static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
- struct scan_control *sc, int swappiness)
+ struct scan_control *sc, int swappiness,
+ bool do_balance)
{
DEFINE_MIN_SEQ(lruvec);
@@ -5242,7 +5283,11 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
return false;
/* better to run aging even though eviction is still possible */
- return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
+ if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq)
+ return true;
+
+ /* run aging when folios outside the youngest gen are rare */
+ return do_balance && lru_gen_imbalanced(lruvec, sc, swappiness);
}
static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
@@ -5328,6 +5373,7 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
bool need_rotate = false, should_age = false;
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ bool do_balance = (sc->priority <= DEF_PRIORITY / 2);
/*
* Proactive reclaim initiated by userspace for anonymous memory only.
@@ -5352,11 +5398,12 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
break;
}
- if (should_run_aging(lruvec, max_seq, sc, swappiness)) {
+ if (should_run_aging(lruvec, max_seq, sc, swappiness, do_balance)) {
if (try_to_inc_max_seq(lruvec, max_seq, swappiness, false))
need_rotate = true;
should_age = true;
}
+ do_balance = false;
if (!evict_folio_lists(nr, lruvec, sc, swappiness, MIN_LRU_BATCH))
break;
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* [RFC PATCH v5 11/14] mm/mglru: Reduce LRU lock contention by shrinking folio batches in inc_min_seq
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (9 preceding siblings ...)
2026-10-22 22:01 ` [RFC PATCH v5 10/14] mm/mglru: Run aging when pages are rare outside the youngest gen Barry Song (Xiaomi)
@ 2026-10-22 22:02 ` Barry Song (Xiaomi)
2026-10-22 22:02 ` [RFC PATCH v5 12/14] mm/mglru: Avoid prolonged anon reclaim with swappiness=200 Barry Song (Xiaomi)
` (3 subsequent siblings)
14 siblings, 0 replies; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:02 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
Reduce the number of folios moved from the oldest generation to the
second-oldest generation in each batch. This shortens the time the
lruvec lock is held, allowing concurrent LRU operations to make
progress and significantly reducing LRU lock contention.
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index e8eeeeb76282..555359cc0368 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4051,7 +4051,7 @@ static inline void flush_lru_batch(struct list_head *head, struct list_head **ba
static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
{
int zone;
- int remaining = MAX_LRU_BATCH;
+ int remaining = MIN_LRU_BATCH;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* [RFC PATCH v5 12/14] mm/mglru: Avoid prolonged anon reclaim with swappiness=200
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (10 preceding siblings ...)
2026-10-22 22:02 ` [RFC PATCH v5 11/14] mm/mglru: Reduce LRU lock contention by shrinking folio batches in inc_min_seq Barry Song (Xiaomi)
@ 2026-10-22 22:02 ` Barry Song (Xiaomi)
2026-10-22 22:02 ` [RFC PATCH v5 13/14] mm/mglru: Make swappiness=1 file reclaim more aggressive Barry Song (Xiaomi)
` (2 subsequent siblings)
14 siblings, 0 replies; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:02 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: lyugaofei <lyugaofei@xiaomi.com>
It has been observed that swappiness=200 can significantly slow down
reclaim on some machines, with some workloads running 3x slower
under MGLRU than under CLRU.
This is because swappiness=200 currently results in file_factor=0.
Give file reclaim a small factor at lower priorities, for example,
when sc->priority < DEF_PRIORITY / 2.
Signed-off-by: lyugaofei <lyugaofei@xiaomi.com>
Co-developed-by: Barry Song (Xiaomi) <baohua@kernel.org>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 555359cc0368..81e81d69e67e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5056,8 +5056,8 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
* We don't calculate the scan cost, MGLRU uses lazy promotion, which will
* cancel the scan budget natually.
*/
-static void lru_gen_balance_scan(struct lruvec *lruvec, int swappiness,
- unsigned long total_scan, unsigned long nr[])
+static void lru_gen_balance_scan(struct lruvec *lruvec, struct scan_control *sc,
+ int swappiness, unsigned long total_scan, unsigned long nr[])
{
struct lru_gen_folio *lrugen = &lruvec->lrugen;
u64 refault_rate[ANON_AND_FILE], anon_factor, file_factor;
@@ -5114,7 +5114,12 @@ static void lru_gen_balance_scan(struct lruvec *lruvec, int swappiness,
file_weight = nr[LRU_GEN_FILE] + nr[LRU_GEN_ANON] / size_bias;
anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
- file_factor = (u64)(MAX_SWAPPINESS - swappiness) * refault_rate[LRU_GEN_ANON] * file_weight;
+ if (swappiness == MAX_SWAPPINESS)
+ file_factor = sc->priority < DEF_PRIORITY / 2 ?
+ refault_rate[LRU_GEN_ANON] * file_weight : 0;
+ else
+ file_factor = (u64)(MAX_SWAPPINESS - swappiness) *
+ refault_rate[LRU_GEN_ANON] * file_weight;
/* Cross-multiply the ratio above. */
anon_scan = mul_u64_u64_div_u64(total_scan, anon_factor, anon_factor + file_factor);
@@ -5305,7 +5310,7 @@ static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
total = apply_proportional_protection(memcg, sc, total);
total >>= sc->priority;
- lru_gen_balance_scan(lruvec, swappiness, total, nr_to_scan);
+ lru_gen_balance_scan(lruvec, sc, swappiness, total, nr_to_scan);
}
static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
@@ -6050,7 +6055,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
return 0;
lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
- lru_gen_balance_scan(lruvec, swappiness,
+ lru_gen_balance_scan(lruvec, sc, swappiness,
nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* [RFC PATCH v5 13/14] mm/mglru: Make swappiness=1 file reclaim more aggressive
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (11 preceding siblings ...)
2026-10-22 22:02 ` [RFC PATCH v5 12/14] mm/mglru: Avoid prolonged anon reclaim with swappiness=200 Barry Song (Xiaomi)
@ 2026-10-22 22:02 ` Barry Song (Xiaomi)
2026-10-22 22:02 ` [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation Barry Song (Xiaomi)
2026-10-24 16:10 ` [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Kairui Song
14 siblings, 0 replies; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:02 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
In MGLRU, swappiness=200 favors anon reclaim, while swappiness=1
favors file reclaim. Swappiness=0 allows only file reclaim, while
swappiness=201 allows only anon reclaim. Thus, swappiness=1 and
swappiness=200 are counterparts for file and anon reclaim,
respectively.
Before this patchset, get_type_to_scan() always returned
LRU_GEN_ANON for swappiness=200 and LRU_GEN_FILE for swappiness=1.
For swappiness=200, file_factor is set to 0 at higher reclaim
priorities. Similarly, anon_factor is set to 0 for swappiness=1
at higher reclaim priorities, making the two settings behave
similarly, although the thresholds are slightly different.
This allows file pages to be reclaimed more aggressively and
benefits workloads that prioritize file reclaim by using
swappiness=1.
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 81e81d69e67e..6d5a823667e0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5113,7 +5113,11 @@ static void lru_gen_balance_scan(struct lruvec *lruvec, struct scan_control *sc,
anon_weight = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE] / size_bias;
file_weight = nr[LRU_GEN_FILE] + nr[LRU_GEN_ANON] / size_bias;
- anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
+ if (swappiness == MIN_SWAPPINESS + 1)
+ anon_factor = sc->priority <= DEF_PRIORITY / 2 ?
+ refault_rate[LRU_GEN_FILE] * anon_weight : 0;
+ else
+ anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
if (swappiness == MAX_SWAPPINESS)
file_factor = sc->priority < DEF_PRIORITY / 2 ?
refault_rate[LRU_GEN_ANON] * file_weight : 0;
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (12 preceding siblings ...)
2026-10-22 22:02 ` [RFC PATCH v5 13/14] mm/mglru: Make swappiness=1 file reclaim more aggressive Barry Song (Xiaomi)
@ 2026-10-22 22:02 ` Barry Song (Xiaomi)
2026-10-23 22:03 ` sashiko-bot
2026-10-24 16:10 ` [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Kairui Song
14 siblings, 1 reply; 30+ messages in thread
From: Barry Song (Xiaomi) @ 2026-10-22 22:02 UTC (permalink / raw)
To: akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baohua, baolin.wang, david, hannes,
linux-kernel, ljs, mhocko, qi.zheng, ryncsn, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, kasong, liam, surenb, mhocko, rppt, zhangbo56,
lyugaofei, ridong.chen, kunwu.chan
From: Bo Zhang <zhangbo56@xiaomi.com>
With GFP_NOIO, we cannot reclaim anon folios unless they are in
the swap cache. CLRU skips anon reclaim when the swap cache is
very low by commit 74d9aa0d8e98 ("mm: vmscan: avoid anon scanning
for GFP_NOIO with low swapcache"). Apply the same logic to MGLRU.
The new scan-budget-based mechanism makes this straightforward to
implement. Thanks also to Xiang Liu, who previously attempted to fix
this before we had the scan-budget-based model [1].
[1] https://lore.kernel.org/linux-mm/179248830836.1744016.8424217811286616919.noio-mglru-rfc-v2@bytedance.com/
Cc: Xiang Liu <liuxiang.277@bytedance.com>
Signed-off-by: Bo Zhang <zhangbo56@xiaomi.com>
Co-developed-by: Barry Song (Xiaomi) <baohua@kernel.org>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
mm/vmscan.c | 20 ++++++--------------
1 file changed, 6 insertions(+), 14 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 6d5a823667e0..d72a5aef3a0f 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -379,13 +379,6 @@ static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg,
if (!sc || (sc->gfp_mask & __GFP_IO))
return false;
- /*
- * FIXME: MGLRU doesn't fully respect can_reclaim_anon_pages() for the
- * scanning type, so only apply this to the traditional LRU for now.
- */
- if (lru_gen_enabled())
- return false;
-
if (memcg) {
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
@@ -5061,26 +5054,23 @@ static void lru_gen_balance_scan(struct lruvec *lruvec, struct scan_control *sc,
{
struct lru_gen_folio *lrugen = &lruvec->lrugen;
u64 refault_rate[ANON_AND_FILE], anon_factor, file_factor;
- u64 anon_scan, file_scan, total;
+ u64 anon_scan, file_scan;
u64 anon_weight, file_weight;
int size_bias;
- total = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE];
- if (!total)
- return;
-
if (swappiness <= MIN_SWAPPINESS) {
nr[LRU_GEN_ANON] = 0;
nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
- return;
}
if (swappiness == SWAPPINESS_ANON_ONLY) {
nr[LRU_GEN_FILE] = 0;
nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
- return;
}
+ if (!nr[LRU_GEN_ANON] || !nr[LRU_GEN_FILE])
+ return;
+
for (int type = LRU_GEN_ANON; type <= LRU_GEN_FILE; type++) {
int hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
/* start at MIN_LRU_BATCH to smooth the rate on gen start */
@@ -5306,6 +5296,8 @@ static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
unsigned long total;
lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+ if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
+ nr_to_scan[LRU_GEN_ANON] = 0;
total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
/* Just scrape all pages calculated above if this memcg was deleted */
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation
2026-10-22 22:02 ` [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation Barry Song (Xiaomi)
@ 2026-10-23 22:03 ` sashiko-bot
2026-10-25 1:31 ` Bo Zhang
0 siblings, 1 reply; 30+ messages in thread
From: sashiko-bot @ 2026-10-23 22:03 UTC (permalink / raw)
To: Barry Song (Xiaomi), akpm, linux-mm
Cc: hui.zhu, axelrasmussen, baolin.wang, david, hannes, ljs, mhocko,
qi.zheng, ryncsn, shakeel.butt, weixugc, yuanchu, zhuhui,
baoquan.he, liuxiang.277, wangzicheng, lianux.mm, kasong, liam,
surenb, mhocko, rppt, zhangbo56, lyugaofei, ridong.chen,
kunwu.chan
Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [High] mm/mglru: unclamped scan budget on early return in lru_gen_balance_scan()
- [Medium] mm/mglru: inconsistent swappiness state when anon reclaim is skipped for GFP_NOIO
--
commit 28e2a43107ca8cde0150881a1f428f1c7e18f418
Author: Bo Zhang <zhangbo56@xiaomi.com>
mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation
This commit enables the low-swapcache check in reclaimable_anon_is_low() for
MGLRU under GFP_NOIO and zeroes nr_to_scan[LRU_GEN_ANON] in
lru_gen_prepare_scan() when can_reclaim_anon_pages() returns false. It also
updates lru_gen_balance_scan() to return early when either nr[LRU_GEN_ANON]
or nr[LRU_GEN_FILE] is zero.
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index c47fe07f7290a..71bc7fedf7a9d 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
[ ... ]
> @@ -5062,26 +5055,23 @@ static void lru_gen_balance_scan(struct lruvec *lruvec, struct scan_control *sc,
> {
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> u64 refault_rate[ANON_AND_FILE], anon_factor, file_factor;
> - u64 anon_scan, file_scan, total;
> + u64 anon_scan, file_scan;
> u64 anon_weight, file_weight;
> int size_bias;
>
> - total = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE];
> - if (!total)
> - return;
> -
> if (swappiness <= MIN_SWAPPINESS) {
> nr[LRU_GEN_ANON] = 0;
> nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
> - return;
> }
>
> if (swappiness == SWAPPINESS_ANON_ONLY) {
> nr[LRU_GEN_FILE] = 0;
> nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
> - return;
> }
>
> + if (!nr[LRU_GEN_ANON] || !nr[LRU_GEN_FILE])
> + return;
[Severity: High]
When 1 <= swappiness <= MAX_SWAPPINESS and either nr[LRU_GEN_ANON] or
nr[LRU_GEN_FILE] is 0, does this early return leave the remaining non-zero
nr[] entry unclamped by total_scan?
Both callers of lru_gen_balance_scan() first populate nr_to_scan[] with the
full unscaled evictable LRU sizes via lruvec_evictable_size(). In
lru_gen_prepare_scan():
lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
nr_to_scan[LRU_GEN_ANON] = 0;
total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
/* Just scrape all pages calculated above if this memcg was deleted */
if (!mem_cgroup_online(memcg))
return;
total = apply_proportional_protection(memcg, sc, total);
total >>= sc->priority;
lru_gen_balance_scan(lruvec, sc, swappiness, total, nr_to_scan);
And in run_eviction():
lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
lru_gen_balance_scan(lruvec, sc, swappiness,
nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
Previously, when one type was 0 and swappiness was between 1 and
MAX_SWAPPINESS, execution continued to the end of lru_gen_balance_scan()
where nr[LRU_GEN_ANON] and nr[LRU_GEN_FILE] were bounded to total_scan.
With this change, nr[] is only clamped to total_scan before the early return
when swappiness <= MIN_SWAPPINESS or swappiness == SWAPPINESS_ANON_ONLY. When
swappiness is in the normal range (for example, the default 60) and one LRU
type is 0 (either because !can_reclaim_anon_pages() zeroed
nr_to_scan[LRU_GEN_ANON] under GFP_NOIO, or because an lruvec only has
evictable pages of one type), lru_gen_balance_scan() returns early with the
non-zero nr[type] still set to the entire unscaled evictable size of that
LRU, bypassing sc->priority scaling, memory.low proportional protection in
apply_proportional_protection(), and the nr_to_reclaim target in
run_eviction().
> for (int type = LRU_GEN_ANON; type <= LRU_GEN_FILE; type++) {
> int hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
> /* start at MIN_LRU_BATCH to smooth the rate on gen start */
> @@ -5307,6 +5297,8 @@ static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
> unsigned long total;
>
> lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> + if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
> + nr_to_scan[LRU_GEN_ANON] = 0;
> total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
>
> /* Just scrape all pages calculated above if this memcg was deleted */
[Severity: Medium]
Could zeroing nr_to_scan[LRU_GEN_ANON] in lru_gen_prepare_scan() while
get_swappiness() still returns a non-zero swappiness cause state
inconsistencies across the rest of the MGLRU reclaim and aging paths under
GFP_NOIO when reclaimable_anon_is_low() is true?
In try_to_shrink_lruvec() and shrink_one(), get_swappiness() does not check
can_reclaim_anon_pages() or reclaimable_anon_is_low():
static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
{
...
if (!sc->may_swap)
return 0;
if (!can_demote(pgdat->node_id, sc, memcg) &&
mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
return 0;
return swappiness;
}
When swap space is available during GFP_NOIO reclaim with low swapcache,
swappiness remains non-zero while anon scanning is skipped:
1. In try_to_inc_min_seq() and should_run_aging(), a non-zero swappiness
causes MGLRU to treat LRU_GEN_ANON as evictable:
if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq)
return true;
Because LRU_GEN_ANON is not scanned, min_seq[LRU_GEN_ANON] does not
advance. When min_seq[LRU_GEN_ANON] lags at max_seq - 3,
try_to_inc_min_seq() clamps min_seq[LRU_GEN_FILE] to max_seq - MIN_NR_GENS
and evictable_min_seq() stays below max_seq - MIN_NR_GENS, preventing
should_run_aging() from aging even after all evictable file generations
are exhausted.
2. In shrink_one() -> lruvec_is_sizable(), a non-zero swappiness causes
lruvec_evictable_size() to count unreclaimable anon pages in total:
lruvec_evictable_size(lruvec, swappiness, sizes);
total = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
For an anon-heavy memcg with little or no file cache under GFP_NOIO,
lruvec_is_sizable() still returns true even though nothing can be scanned,
preventing shrink_one() from rotating the memcg.
Would checking can_reclaim_anon_pages() in get_swappiness() (returning 0 when
anon pages cannot be reclaimed) avoid both this mismatch and the unclamped
nr[LRU_GEN_FILE] budget in lru_gen_balance_scan() for this path?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261022220203.91911-1-baohua@kernel.org?part=14
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation
2026-10-23 22:03 ` sashiko-bot
@ 2026-10-25 1:31 ` Bo Zhang
2026-10-26 8:43 ` KunWu Chan
0 siblings, 1 reply; 30+ messages in thread
From: Bo Zhang @ 2026-10-25 1:31 UTC (permalink / raw)
To: sashiko-bot
Cc: akpm, axelrasmussen, baohua, baolin.wang, baoquan.he, david,
hannes, hui.zhu, kasong, kunwu.chan, liam, lianux.mm, linux-mm,
liuxiang.277, ljs, lyugaofei, mhocko, mhocko, qi.zheng,
ridong.chen, rppt, ryncsn, sashiko-reviews, shakeel.butt, surenb,
wangzicheng, weixugc, yuanchu, zhangbo56, zhuhui
I have checked the two sub-issues separately. Both issues are valid and
the root cause is the same: patch 14 skips anon scanning but leaves
swappiness non-zero, so every path that still treats anon as evictable
(scan budget, aging, lruvec_is_sizable()) misbehaves.
> [High] unclamped scan budget on early return in lru_gen_balance_scan()
Confirmed. With 1 <= swappiness <= MAX_SWAPPINESS and if one of the type's
nr is 0, the early return left the other type at its full unscaled evictable
size, bypassing priority scaling, memory.low protection and nr_to_reclaim.
Note this is not specific for GFP_NOIO, an lruvec that naturally has only
one type has the same problem.
> [Medium] inconsistent swappiness state when anon reclaim is skipped
> Would checking can_reclaim_anon_pages() in get_swappiness() ... avoid both?
Confirmed for both sub-issues, stalled aging via evictable_min_seq(), and
lruvec_is_sizable() counting unreclaimable anon.
Rather than forcing swappiness to 0 in get_swappiness() (which, as Barry
noted in [1], wrongly allows anon cold/hot inversion for the temporary
GFP_NOIO case), I keep swappiness not changed and fix the evictable view at
its source, lruvec_evictable_size():
1. It now filters out anon here when it cannot be reclaimed (e.g. GFP_NOIO
with low swapcache), so nr[LRU_GEN_ANON] is 0 for all consumers. This
drops the per-caller zeroing and lets lruvec_is_sizable() ignore
unreclaimable anon.
2. lru_gen_balance_scan() clamps the remaining type to total_scan before
the single-type early return.
3. should_run_aging() forces aging when file gens run low and anon is
unreclaimable, so a stalled min_seq[LRU_GEN_ANON] no longer blocks aging.
Proposed fix on top of patch 14:
[1] https://lore.kernel.org/lkml/20261022102726.56682-1-baohua@kernel.org/
---
mm/vmscan.c | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index d72a5aef3a0f..32bd9c0fef5d 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4350,12 +4350,14 @@ static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
return size;
}
-static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
- unsigned long nr[])
+static void lruvec_evictable_size(struct lruvec *lruvec, struct scan_control *sc,
+ int swappiness, unsigned long nr[])
{
int type;
unsigned long seq;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ int nid = lruvec_pgdat(lruvec)->node_id;
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
@@ -4363,6 +4365,8 @@ static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
nr[LRU_GEN_FILE] = 0;
for_each_evictable_type(type, swappiness) {
+ if (type == LRU_GEN_ANON && !can_reclaim_anon_pages(memcg, nid, sc))
+ continue;
for (seq = min_seq[type]; seq <= max_seq; seq++)
nr[type] += lruvec_gen_size(lrugen, type, seq);
}
@@ -4374,7 +4378,7 @@ static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
- lruvec_evictable_size(lruvec, swappiness, sizes);
+ lruvec_evictable_size(lruvec, sc, swappiness, sizes);
total = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
/* whether the size is big enough to be helpful */
@@ -5058,18 +5062,17 @@ static void lru_gen_balance_scan(struct lruvec *lruvec, struct scan_control *sc,
u64 anon_weight, file_weight;
int size_bias;
- if (swappiness <= MIN_SWAPPINESS) {
+ if (swappiness <= MIN_SWAPPINESS)
nr[LRU_GEN_ANON] = 0;
- nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
- }
- if (swappiness == SWAPPINESS_ANON_ONLY) {
+ if (swappiness == SWAPPINESS_ANON_ONLY)
nr[LRU_GEN_FILE] = 0;
- nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
- }
- if (!nr[LRU_GEN_ANON] || !nr[LRU_GEN_FILE])
+ if (!nr[LRU_GEN_ANON] || !nr[LRU_GEN_FILE]) {
+ nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
+ nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
return;
+ }
for (int type = LRU_GEN_ANON; type <= LRU_GEN_FILE; type++) {
int hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
@@ -5271,12 +5274,19 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
struct scan_control *sc, int swappiness,
bool do_balance)
{
+ struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
/* have to run aging, since eviction is not possible anymore */
if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq)
return true;
+ /* run aging when GFP_NOIO limits anon reclaim with few swapcache folios */
+ if (swappiness && min_seq[LRU_GEN_FILE] + MIN_NR_GENS > max_seq &&
+ reclaimable_anon_is_low(memcg, pgdat->node_id, sc))
+ return true;
+
/* try to avoid aging, do gentle reclaim at the default priority */
if (sc->priority == DEF_PRIORITY)
return false;
@@ -5295,9 +5305,7 @@ static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
{
unsigned long total;
- lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
- if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
- nr_to_scan[LRU_GEN_ANON] = 0;
+ lruvec_evictable_size(lruvec, sc, swappiness, nr_to_scan);
total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
/* Just scrape all pages calculated above if this memcg was deleted */
@@ -6050,7 +6058,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
if (sc->nr_reclaimed >= nr_to_reclaim)
return 0;
- lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+ lruvec_evictable_size(lruvec, sc, swappiness, nr_to_scan);
lru_gen_balance_scan(lruvec, sc, swappiness,
nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
--
2.34.1
^ permalink raw reply related [flat|nested] 30+ messages in thread* Re: [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation
2026-10-25 1:31 ` Bo Zhang
@ 2026-10-26 8:43 ` KunWu Chan
0 siblings, 0 replies; 30+ messages in thread
From: KunWu Chan @ 2026-10-26 8:43 UTC (permalink / raw)
To: Bo Zhang
Cc: sashiko-bot, akpm, axelrasmussen, baohua, baolin.wang, baoquan.he,
david, hannes, hui.zhu, kasong, liam, lianux.mm, linux-mm,
liuxiang.277, ljs, lyugaofei, mhocko, mhocko, qi.zheng,
ridong.chen, rppt, ryncsn, sashiko-reviews, shakeel.butt, surenb,
wangzicheng, weixugc, yuanchu, zhangbo56, zhuhui
On Sun, Oct 25, 2026 at 9:31 AM Bo Zhang <zhangbo0325@gmail.com> wrote:
>
> I have checked the two sub-issues separately. Both issues are valid and
> the root cause is the same: patch 14 skips anon scanning but leaves
> swappiness non-zero, so every path that still treats anon as evictable
> (scan budget, aging, lruvec_is_sizable()) misbehaves.
>
> > [High] unclamped scan budget on early return in lru_gen_balance_scan()
>
> Confirmed. With 1 <= swappiness <= MAX_SWAPPINESS and if one of the type's
> nr is 0, the early return left the other type at its full unscaled evictable
> size, bypassing priority scaling, memory.low protection and nr_to_reclaim.
> Note this is not specific for GFP_NOIO, an lruvec that naturally has only
> one type has the same problem.
>
> > [Medium] inconsistent swappiness state when anon reclaim is skipped
> > Would checking can_reclaim_anon_pages() in get_swappiness() ... avoid both?
>
> Confirmed for both sub-issues, stalled aging via evictable_min_seq(), and
> lruvec_is_sizable() counting unreclaimable anon.
>
> Rather than forcing swappiness to 0 in get_swappiness() (which, as Barry
> noted in [1], wrongly allows anon cold/hot inversion for the temporary
> GFP_NOIO case), I keep swappiness not changed and fix the evictable view at
> its source, lruvec_evictable_size():
>
> 1. It now filters out anon here when it cannot be reclaimed (e.g. GFP_NOIO
> with low swapcache), so nr[LRU_GEN_ANON] is 0 for all consumers. This
> drops the per-caller zeroing and lets lruvec_is_sizable() ignore
> unreclaimable anon.
>
> 2. lru_gen_balance_scan() clamps the remaining type to total_scan before
> the single-type early return.
>
> 3. should_run_aging() forces aging when file gens run low and anon is
> unreclaimable, so a stalled min_seq[LRU_GEN_ANON] no longer blocks aging.
>
> Proposed fix on top of patch 14:
>
> [1] https://lore.kernel.org/lkml/20261022102726.56682-1-baohua@kernel.org/
>
> ---
> mm/vmscan.c | 29 +++++++++++++++++------------
> 1 file changed, 17 insertions(+), 12 deletions(-)
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index d72a5aef3a0f..32bd9c0fef5d 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -4350,12 +4350,14 @@ static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
> return size;
> }
>
> -static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
> - unsigned long nr[])
> +static void lruvec_evictable_size(struct lruvec *lruvec, struct scan_control *sc,
> + int swappiness, unsigned long nr[])
> {
> int type;
> unsigned long seq;
> struct lru_gen_folio *lrugen = &lruvec->lrugen;
> + struct mem_cgroup *memcg = lruvec_memcg(lruvec);
> + int nid = lruvec_pgdat(lruvec)->node_id;
> DEFINE_MAX_SEQ(lruvec);
> DEFINE_MIN_SEQ(lruvec);
>
> @@ -4363,6 +4365,8 @@ static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
> nr[LRU_GEN_FILE] = 0;
>
> for_each_evictable_type(type, swappiness) {
> + if (type == LRU_GEN_ANON && !can_reclaim_anon_pages(memcg, nid, sc))
> + continue;
> for (seq = min_seq[type]; seq <= max_seq; seq++)
> nr[type] += lruvec_gen_size(lrugen, type, seq);
> }
> @@ -4374,7 +4378,7 @@ static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
> int swappiness = get_swappiness(lruvec, sc);
> struct mem_cgroup *memcg = lruvec_memcg(lruvec);
>
> - lruvec_evictable_size(lruvec, swappiness, sizes);
> + lruvec_evictable_size(lruvec, sc, swappiness, sizes);
> total = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
>
> /* whether the size is big enough to be helpful */
> @@ -5058,18 +5062,17 @@ static void lru_gen_balance_scan(struct lruvec *lruvec, struct scan_control *sc,
> u64 anon_weight, file_weight;
> int size_bias;
>
> - if (swappiness <= MIN_SWAPPINESS) {
> + if (swappiness <= MIN_SWAPPINESS)
> nr[LRU_GEN_ANON] = 0;
> - nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
> - }
>
> - if (swappiness == SWAPPINESS_ANON_ONLY) {
> + if (swappiness == SWAPPINESS_ANON_ONLY)
> nr[LRU_GEN_FILE] = 0;
> - nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
> - }
>
> - if (!nr[LRU_GEN_ANON] || !nr[LRU_GEN_FILE])
> + if (!nr[LRU_GEN_ANON] || !nr[LRU_GEN_FILE]) {
> + nr[LRU_GEN_ANON] = min_t(u64, total_scan, nr[LRU_GEN_ANON]);
> + nr[LRU_GEN_FILE] = min_t(u64, total_scan, nr[LRU_GEN_FILE]);
> return;
> + }
>
> for (int type = LRU_GEN_ANON; type <= LRU_GEN_FILE; type++) {
> int hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
> @@ -5271,12 +5274,19 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
> struct scan_control *sc, int swappiness,
> bool do_balance)
> {
> + struct pglist_data *pgdat = lruvec_pgdat(lruvec);
> + struct mem_cgroup *memcg = lruvec_memcg(lruvec);
> DEFINE_MIN_SEQ(lruvec);
>
> /* have to run aging, since eviction is not possible anymore */
> if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq)
> return true;
>
> + /* run aging when GFP_NOIO limits anon reclaim with few swapcache folios */
> + if (swappiness && min_seq[LRU_GEN_FILE] + MIN_NR_GENS > max_seq &&
> + reclaimable_anon_is_low(memcg, pgdat->node_id, sc))
> + return true;
> +
> /* try to avoid aging, do gentle reclaim at the default priority */
> if (sc->priority == DEF_PRIORITY)
> return false;
> @@ -5295,9 +5305,7 @@ static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
> {
> unsigned long total;
>
> - lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> - if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
> - nr_to_scan[LRU_GEN_ANON] = 0;
> + lruvec_evictable_size(lruvec, sc, swappiness, nr_to_scan);
> total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
>
> /* Just scrape all pages calculated above if this memcg was deleted */
> @@ -6050,7 +6058,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
> if (sc->nr_reclaimed >= nr_to_reclaim)
> return 0;
>
> - lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
> + lruvec_evictable_size(lruvec, sc, swappiness, nr_to_scan);
> lru_gen_balance_scan(lruvec, sc, swappiness,
> nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
>
> --
> 2.34.1
Thanks. I checked the fix against the v5 Patch 14 base.
Moving the anon evictability check into lruvec_evictable_size() makes the
size consistent across lruvec_is_sizable(), lru_gen_prepare_scan(), and
run_eviction(), removing the need for the ad-hoc zeroing in
lru_gen_prepare_scan(). The clamp in lru_gen_balance_scan() on the
early-return path prevents an uncapped nr[] from reaching eviction.
The should_run_aging() trigger correctly bypasses evictable_min_seq
when anon is unreclaimable under GFP_NOIO — without it, a lagging
min_seq[LRU_GEN_ANON] can dominate evictable_min_seq and block
aging even after file generations are exhausted.
Reviewed-by: Kunwu Chan <chentao@kylinos.cn>
Thanks,
Kunwu
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior
2026-10-22 22:01 [RFC PATCH v5 00/14] mm: mglru: fix swappiness behavior Barry Song (Xiaomi)
` (13 preceding siblings ...)
2026-10-22 22:02 ` [RFC PATCH v5 14/14] mm/mglru: Avoid scan anon while swapcache is low for GFP_NOIO allocation Barry Song (Xiaomi)
@ 2026-10-24 16:10 ` Kairui Song
14 siblings, 0 replies; 30+ messages in thread
From: Kairui Song @ 2026-10-24 16:10 UTC (permalink / raw)
To: Barry Song (Xiaomi)
Cc: akpm, linux-mm, hui.zhu, axelrasmussen, baolin.wang, david,
hannes, linux-kernel, ljs, mhocko, qi.zheng, shakeel.butt,
weixugc, yuanchu, zhuhui, baoquan.he, liuxiang.277, wangzicheng,
lianux.mm, liam, surenb, mhocko, rppt, zhangbo56, lyugaofei,
ridong.chen, kunwu.chan
On Fri, Oct 23, 2026 at 6:02 AM Barry Song (Xiaomi) <baohua@kernel.org> wrote:
>
> Classic LRU respects swappiness, while MGLRU only partially honors it
> when selecting the scan type, resulting in a much smaller overall
> impact.
>
> For example, when building the kernel in a memcg limited to 1024 MB
> of memory, with swappiness=200, CLRU barely scans or reclaims file
> pages, while MGLRU still reclaims a significant number of them.
> Similarly, with swappiness=1, MGLRU does not reclaim as many file
> pages as expected for such an extreme swappiness value:
>
> pgsteal_file
> swappiness CLRU MGLRU(no patches)
> 1 5,894,538 756,589
> 36 799,998 497,282
> 71 548,331 415,612
> 106 407,012 383,814
> 141 253,737 354,249
> 176 189,003 347,805
> 200 0 346,078
>
> pgsteal_anon
> swappiness CLRU MGLRU(no patches)
> 1 3,962,404 2,768,654
> 36 2,830,250 2,976,260
> 71 3,121,386 3,142,954
> 106 3,521,720 3,382,383
> 141 3,504,471 2,970,859
> 176 3,592,335 3,163,130
> 200 14,095,617 3,209,132
>
> pgscan_file
> swappiness CLRU MGLRU(no patches)
> 1 10,270,808 886,997
> 36 1,010,341 569,036
> 71 656,136 490,972
> 106 460,326 436,042
> 141 295,184 403,132
> 176 235,147 390,358
> 200 0 398,363
>
> pgscan_anon
> swappiness CLRU MGLRU(no patches)
> 1 19,341,957 5,960,884
> 36 17,530,774 6,394,951
> 71 18,624,264 6,673,569
> 106 20,088,558 6,976,364
> 141 19,686,813 6,206,580
> 176 20,936,177 6,344,854
> 200 45,313,873 6,295,581
>
> refault_file
> swappiness CLRU MGLRU(no patches)
> 1 5,795,115 678,254
> 36 720,877 419,659
> 71 469,270 338,518
> 106 331,584 308,983
> 141 174,509 277,375
> 176 113,352 272,390
> 200 0 269,349
>
> refault_anon
> swappiness CLRU MGLRU(no patches)
> 1 738,618 578,308
> 36 451,410 665,723
> 71 514,961 657,870
> 106 633,161 786,893
> 141 648,640 662,673
> 176 668,779 708,969
> 200 4,816,312 672,514
Hi Barry,
Thanks for pushing this series forward!
One nit pick so far is the test data seems redundant with the one
below, maybe we can just keep one of them.
> This model also allows scanning and reclaiming from the two youngest
> generations, including the youngest generation itself. This can help
> resolve OOM situations[1] and reduce the need for aging.
Right, the OOM reproducer is demostrated here previously:
https://lore.kernel.org/all/20260318-mglru-reclaim-v1-0-2c46f9eb0508@tencent.com/
That series is merged at V7:
https://lore.kernel.org/linux-mm/20260428-mglru-reclaim-v7-0-02fabb92dc43@tencent.com/
The problem is largely eased as described, but it isn't 100% fixed at
that time. It will still go OOM with higher pressure. So before this
series, I could still see OOM after hundreds of loops. And problem is
gone after the fixing patch in this series. Unfortunately it's very
hard to reproduce and that reproducer is the best I have so far.
>
> Patch 9 from Barry fixes a cold/hot inversion exacerbated by patches
> 1–8 (which also exists in mainline) and improves reclaim efficiency.
>
> Patches 10-13 from Gaofei and Barry optimize aging based on folio
> distribution imbalance and improve reclaim behavior at extreme
> swappiness values such as 1 and 200.
>
> Patch 14 from Bo Zhang addresses the GFP_NOIO issue in MGLRU.
>
> After the patchset, the results track CLRU across the full swappiness
> range (see table below), with more file scanning at low swappiness
> and more anon scanning at high swappiness.
>
> pgsteal_file
> swappiness CLRU MGLRU MGLRU+patches
> 1 5,894,538 756,589 7,394,183
> 36 799,998 497,282 526,176
> 71 548,331 415,612 433,353
> 106 407,012 383,814 347,647
> 141 253,737 354,249 314,437
> 176 189,003 347,805 261,677
> 200 0 346,078 188
>
> pgsteal_anon
> swappiness CLRU MGLRU MGLRU+patches
> 1 3,962,404 2,768,654 5,362,153
> 36 2,830,250 2,976,260 2,736,749
> 71 3,121,386 3,142,954 3,246,589
> 106 3,521,720 3,382,383 3,074,851
> 141 3,504,471 2,970,859 3,178,409
> 176 3,592,335 3,163,130 3,650,544
> 200 14,095,617 3,209,132 14,538,399
>
> pgscan_file
> swappiness CLRU MGLRU MGLRU+patches
> 1 10,270,808 886,997 16,246,260
> 36 1,010,341 569,036 614,389
> 71 656,136 490,972 484,769
> 106 460,326 436,042 367,969
> 141 295,184 403,132 328,943
> 176 235,147 390,358 268,142
> 200 0 398,363 188
>
> pgscan_anon
> swappiness CLRU MGLRU MGLRU+patches
> 1 19,341,957 5,960,884 7,646,122
> 36 17,530,774 6,394,951 5,914,405
> 71 18,624,264 6,673,569 7,167,762
> 106 20,088,558 6,976,364 7,821,921
> 141 19,686,813 6,206,580 8,495,970
> 176 20,936,177 6,344,854 10,585,375
> 200 45,313,873 6,295,581 30,313,342
>
> refault_file
> swappiness CLRU MGLRU MGLRU+patches
> 1 5,795,115 678,254 7,315,064
> 36 720,877 419,659 448,199
> 71 469,270 338,518 355,269
> 106 331,584 308,983 270,736
> 141 174,509 277,375 237,643
> 176 113,352 272,390 184,908
> 200 0 269,349 102
>
> refault_anon
> swappiness CLRU MGLRU MGLRU+patches
> 1 738,618 578,308 1,383,725
> 36 451,410 665,723 543,071
> 71 514,961 657,870 683,874
> 106 633,161 786,893 669,716
> 141 648,640 662,673 719,231
> 176 668,779 708,969 901,918
> 200 4,816,312 672,514 5,511,787
Right, that's really nice result. Do you have system time or
performance readings? BTW for the performance reading I think we can
focus on the 36 - 176 range. If a user asks for extreme bias towards
anon/file, they strongly favor reclaim priority over throughput, and
the behavior is arguable at that point.
^ permalink raw reply [flat|nested] 30+ messages in thread