[RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[Bo Zhang]

We have observed some cases where memory is allocated with GFP_NOIO, so
we cannot reclaim any anon folios unless they are in swapcache. We can
end up spending more than 150 ms looping in `shrink_folio_list()` scanning
non-swapcache folios without reclaiming a single folio. This is pure
overhead.

This is particularly true on systems using zRAM, where swapcache is
relatively rare. So let's check whether anon reclaim is allowed by
GFP_IO and whether there is enough swapcache to make it worthwhile. If
the swapcache is extremely low, we're essentially searching for a
needle in a haystack, so let's avoid scanning anon in the first place.

Signed-off-by: Bo Zhang <zhangbo56@xiaomi.com>
---
 mm/vmscan.c | 62 ++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 22 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 56708d1d2dfd..192bd0980121 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -339,6 +339,42 @@ static bool can_demote(int nid, struct scan_control *sc,
 	return !nodes_empty(allowed_mask);
 }
 
+static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
+{
+	struct pglist_data *pgdat = NODE_DATA(nid);
+
+#ifdef CONFIG_MEMCG
+	if (memcg) {
+		struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
+
+		/* see the comment in mem_cgroup_lruvec() */
+		if (!lruvec->pgdat)
+			lruvec->pgdat = pgdat;
+
+		return lruvec;
+	}
+#endif
+	VM_WARN_ON_ONCE(!mem_cgroup_disabled());
+
+	return &pgdat->__lruvec;
+}
+
+static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg,
+		int nid, struct scan_control *sc)
+{
+	struct lruvec *lruvec = get_lruvec(memcg, nid);
+	unsigned long anon_pages, swapcache;
+
+	if (!sc || (sc->gfp_mask & __GFP_IO))
+		return false;
+
+	anon_pages = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
+		     lruvec_page_state(lruvec, NR_ACTIVE_ANON);
+	swapcache = lruvec_page_state(lruvec, NR_SWAPCACHE);
+
+	return swapcache < min(anon_pages >> 6, SWAP_CLUSTER_MAX);
+}
+
 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
 					  int nid,
 					  struct scan_control *sc)
@@ -348,11 +384,13 @@ static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
 		 * For non-memcg reclaim, is there
 		 * space in any swap device?
 		 */
-		if (get_nr_swap_pages() > 0)
+		if (get_nr_swap_pages() > 0 &&
+		    !reclaimable_anon_is_low(memcg, nid, sc))
 			return true;
 	} else {
 		/* Is the memcg below its swap limit? */
-		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
+		if (mem_cgroup_get_nr_swap_pages(memcg) > 0 &&
+		    !reclaimable_anon_is_low(memcg, nid, sc))
 			return true;
 	}
 
@@ -2674,26 +2712,6 @@ static bool should_clear_pmd_young(void)
 #define get_memcg_gen(seq)	((seq) % MEMCG_NR_GENS)
 #define get_memcg_bin(bin)	((bin) % MEMCG_NR_BINS)
 
-static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
-{
-	struct pglist_data *pgdat = NODE_DATA(nid);
-
-#ifdef CONFIG_MEMCG
-	if (memcg) {
-		struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
-
-		/* see the comment in mem_cgroup_lruvec() */
-		if (!lruvec->pgdat)
-			lruvec->pgdat = pgdat;
-
-		return lruvec;
-	}
-#endif
-	VM_WARN_ON_ONCE(!mem_cgroup_disabled());
-
-	return &pgdat->__lruvec;
-}
-
 static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
 {
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
-- 
2.34.1

Re: [RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[Barry Song]

On Thu, Sep 3, 2026 at 12:02 PM Bo Zhang <zhangbo0325@gmail.com> wrote:
>
> We have observed some cases where memory is allocated with GFP_NOIO, so
> we cannot reclaim any anon folios unless they are in swapcache. We can
> end up spending more than 150 ms looping in `shrink_folio_list()` scanning
> non-swapcache folios without reclaiming a single folio. This is pure
> overhead.
>

Yes. The coexistence of `GFP_NOIO` and anon rmap scanning seems
nasty. We might also want to do something like the following, but
the side effect is that it might keep a folio while preserving its
PTE young state, indirectly making the folio semantically hotter.
So this may not be a good option. Skipping anon scanning in the first
place seems more sensible.

diff --git a/mm/vmscan.c b/mm/vmscan.c
index ba7adf36e69f..f15aa3573030 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1248,6 +1248,19 @@ static unsigned int shrink_folio_list(struct
list_head *folio_list,
                        }
                }

+               /*
+                * Don't waste time doing rmap and scanning PTE access for
+                * non-reclaimable folios.
+                */
+               if (!do_demote_pass && folio_test_anon(folio) &&
+                   folio_test_swapbacked(folio) &&
+                   !folio_test_swapcache(folio)) {
+                       if (!(sc->gfp_mask & __GFP_IO))
+                               goto keep_locked;
+                       if (folio_maybe_dma_pinned(folio))
+                               goto keep_locked;
+               }
+
                if (!ignore_references)
                        references = folio_check_references(folio, sc);

@@ -1281,10 +1294,6 @@ static unsigned int shrink_folio_list(struct
list_head *folio_list,
                                !folio_test_swapcache(folio)) {
                        int ret;

-                       if (!(sc->gfp_mask & __GFP_IO))
-                               goto keep_locked;
-                       if (folio_maybe_dma_pinned(folio))
-                               goto keep_locked;
                        if (folio_test_large(folio)) {
                                /* cannot split folio, skip it */
                                if (folio_expected_ref_count(folio) !=

> This is particularly true on systems using zRAM, where swapcache is
> relatively rare. So let's check whether anon reclaim is allowed by
> GFP_IO and whether there is enough swapcache to make it worthwhile. If
> the swapcache is extremely low, we're essentially searching for a
> needle in a haystack, so let's avoid scanning anon in the first place.
>
> Signed-off-by: Bo Zhang <zhangbo56@xiaomi.com>
> ---
>  mm/vmscan.c | 62 ++++++++++++++++++++++++++++++++++-------------------
>  1 file changed, 40 insertions(+), 22 deletions(-)
>
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index 56708d1d2dfd..192bd0980121 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -339,6 +339,42 @@ static bool can_demote(int nid, struct scan_control *sc,
>         return !nodes_empty(allowed_mask);
>  }
>
> +static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
> +{
> +       struct pglist_data *pgdat = NODE_DATA(nid);
> +
> +#ifdef CONFIG_MEMCG
> +       if (memcg) {
> +               struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
> +
> +               /* see the comment in mem_cgroup_lruvec() */
> +               if (!lruvec->pgdat)
> +                       lruvec->pgdat = pgdat;
> +
> +               return lruvec;
> +       }
> +#endif
> +       VM_WARN_ON_ONCE(!mem_cgroup_disabled());
> +
> +       return &pgdat->__lruvec;
> +}
> +
> +static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg,
> +               int nid, struct scan_control *sc)
> +{
> +       struct lruvec *lruvec = get_lruvec(memcg, nid);

https://sashiko.dev/#/patchset/20260903040131.4016290-1-zhangbo56%40xiaomi.com
sashiko says:

"Will this cause a kernel panic during global reclaim when memory cgroups are
enabled?
When can_reclaim_anon_pages() is called for global reclaim, memcg is NULL.
Since get_lruvec() is called unconditionally with this NULL memcg, it will
trigger the VM_WARN_ON_ONCE(!mem_cgroup_disabled()) and return the raw
node lruvec (&pgdat->__lruvec).
When lruvec_page_state() is then called with this non-memcg lruvec:
mm/memcontrol.c:lruvec_page_state() {
    ...
    pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
    x = READ_ONCE(pn->lruvec_stats->state[i]);
    ...
}
Because the lruvec is actually embedded in pglist_data rather than
mem_cgroup_per_node, won't container_of() produce a wild pointer, leading
to a fatal out-of-bounds memory access when trying to read the stats?"

it seems we could just use mem_cgroup_lruvec() instead:

 static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg,
                int nid, struct scan_control *sc)
 {
-       struct lruvec *lruvec = get_lruvec(memcg, nid);
+       struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
        unsigned long anon_pages, swapcache;

        if (!sc || (sc->gfp_mask & __GFP_IO))

I applied your patch on my PC with the above change. I didn't see any
panic or run into any problems.

Best Regards
Barry

Re: [RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[Bo Zhang]

On Thu, Sep 3, 2026 at 6:35 PM Barry Song <baohua@kernel.org> wrote:
>
> Yes. The coexistence of `GFP_NOIO` and anon rmap scanning seems
> nasty. We might also want to do something like the following, but
> the side effect is that it might keep a folio while preserving its
> PTE young state, indirectly making the folio semantically hotter.
> So this may not be a good option. Skipping anon scanning in the first
> place seems more sensible.
>
> +               /*
> +                * Don't waste time doing rmap and scanning PTE access for
> +                * non-reclaimable folios.
> +                */
> +               if (!do_demote_pass && folio_test_anon(folio) &&
> +                   folio_test_swapbacked(folio) &&
> +                   !folio_test_swapcache(folio)) {
> +                       if (!(sc->gfp_mask & __GFP_IO))
> +                               goto keep_locked;
> +                       if (folio_maybe_dma_pinned(folio))
> +                               goto keep_locked;
> +               }

Agreed. That matches my thinking: doing the rmap/PTE walk first and
then keeping the folio would leave its young state set and make it look
hotter, so skipping anon selection up front avoids that side effect.
I'll keep the check in can_reclaim_anon_pages().

> it seems we could just use mem_cgroup_lruvec() instead:
>
> -       struct lruvec *lruvec = get_lruvec(memcg, nid);
> +       struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
>
> I applied your patch on my PC with the above change. I didn't see any
> panic or run into any problems.

Thanks, and thanks to the sashiko bot for catching this. I've switched
to mem_cgroup_lruvec() in v2, which also lets me drop the get_lruvec()
move entirely. Thanks for testing it on your side.

I'll send v2 shortly.

Thanks,
Bo

Re: [RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[Johannes Weiner]

On Thu, Sep 03, 2026 at 12:01:31PM +0800, Bo Zhang wrote:
> We have observed some cases where memory is allocated with GFP_NOIO, so
> we cannot reclaim any anon folios unless they are in swapcache. We can
> end up spending more than 150 ms looping in `shrink_folio_list()` scanning
> non-swapcache folios without reclaiming a single folio. This is pure
> overhead.

Not entirely. There is some value in aging anon alongside file, so
that the next __GFP_IO reclaimer doesn't look at a stale list.

Can you describe a bit more about what you observed? What workload is
running, maybe you have a stack trace of which NOIO requests are
routinely getting stuck in reclaim?

150ms sounds awful indeed. Is this cumulative for a whole reclaim
cycle or single shrink_folio_list() runs?

Re: [RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[Bo Zhang]

On Thu, Sep 03, 2026 at 09:03:04AM -0400, Johannes Weiner wrote:
> On Thu, Sep 03, 2026 at 12:01:31PM +0800, Bo Zhang wrote:
> > We have observed some cases where memory is allocated with GFP_NOIO, so
> > we cannot reclaim any anon folios unless they are in swapcache. We can
> > end up spending more than 150 ms looping in `shrink_folio_list()` scanning
> > non-swapcache folios without reclaiming a single folio. This is pure
> > overhead.
>
> Not entirely. There is some value in aging anon alongside file, so
> that the next __GFP_IO reclaimer doesn't look at a stale list.

You're right, "pure overhead" was too strong - aging anon does have
value for a later __GFP_IO reclaimer, and I don't intend to skip it in
general. Let me describe the case in full, because the reclaim cycle
itself already provides that aging on a later pass, which is what makes
me think the trade-off here leans the other way.

> Can you describe a bit more about what you observed? What workload is
> running, maybe you have a stack trace of which NOIO requests are
> routinely getting stuck in reclaim?

The workload is app launching on Android. The NOIO allocations come from
dm-verity hash-block reads via dm-bufio, which legitimately use GFP_NOIO
because they run underneath the IO path:

  worker_thread
   process_scheduled_works
    verity_work
     verity_verify_io
      verity_hash_for_block
       verity_verify_level
        dm_bufio_read_with_ioprio
         new_read
          __bufio_new
           alloc_buffer
            gfp_mask: GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN

So the NOIO use itself is correct; the problem is on the reclaim side.

Here is the full picture of one such direct reclaim. It runs two rounds
of do_try_to_free_pages(); the target is 32 folios.

Round 1 - partial (shared) memcg walk, 169.20 ms, 0 folios reclaimed
--------------------------------------------------------------------
  prio 12->1 (~1.3 ms):
    cache_trim_mode is on, so get_scan_count() picks SCAN_FILE. Only the
    file side is scanned. Because this is a shared/partial walk, each
    priority only visits a handful of memcgs before the iterator is
    handed off, so very few memcgs are looked at on the way down:
    428 file folios scanned, 0 reclaimed.

  prio 0 (~167.9 ms):
    priority hits 0 without meeting the target, so get_scan_count()
    forces SCAN_EQUAL. The walk lands on a single memcg with a large,
    unswapped anon LRU and a tiny file LRU:

      inactive_anon ~335 MB, inactive_file ~4 MB   (~84:1)
      memcg swap usage ~3.6 MB, so swapcache is negligible

    shrink_lruvec() now keeps feeding that huge anon list into
    shrink_folio_list() - ~2400 shrink_folio_list() calls, ~93,000 anon
    folios scanned - and every folio hits the !__GFP_IO keep_locked path
    (not in swapcache, needs a swap slot). This single shrink_lruvec()
    pass alone is ~168 ms with 0 folios reclaimed.

  Round 1 ends with nr_reclaimed = 0 < target, so reclaim retries with
  sc->memcg_full_walk = 1.

Round 2 - full memcg walk, 2.37 ms, 68 folios reclaimed
-------------------------------------------------------
  With memcg_full_walk = 1, priority resets to 12 and every priority
  now visits the complete subtree (~78 memcgs). Progress is made
  entirely from the file side:

    prio 12: ...  0 reclaimed
    prio 11: ...  0
    prio 10: ...  3
    prio  9: ...  8
    prio  8: ... 18
    prio  7: ... 38   -> cumulative 68 >= target 32, done

  All 68 reclaimed folios come from file LRUs; anon contributes 0.

So the whole 168 ms is spent scanning an anon list that cannot yield a
single folio under GFP_NOIO, and the actual progress comes from file in
a fast full-walk round that follows.

> 150ms sounds awful indeed. Is this cumulative for a whole reclaim
> cycle or single shrink_folio_list() runs?

It is a single shrink_lruvec() invocation on that one memcg, as above -
not accumulated across the cycle. Each shrink_folio_list() only handles
SWAP_CLUSTER_MAX folios and is fast on its own; it's the prio-0 while
loop over the huge anon list that adds up to ~168 ms.

On the aging trade-off
----------------------
I take your point that this pass would otherwise have aged anon for the
next __GFP_IO reclaimer. But in this cycle that benefit is small and the
cost is large:

- The aging is not lost so much as deferred. Round 2 (and any later
  __GFP_IO reclaimer) still walks the full subtree; anon that genuinely
  needs IO to be reclaimed gets aged/reclaimed then, once IO is allowed.
- The 168 ms is spent scanning ~93k anon folios that, by construction of
  GFP_NOIO + negligible swapcache, cannot be reclaimed on this pass at
  all - the aging is the only product, and it comes at the price of a
  ~168 ms stall in a latency-sensitive path.

That's why I'd argue the balance tips towards skipping anon here rather
than aging it. To keep the change narrow, the check only triggers at
priority 0 (where SCAN_EQUAL is forced) and only when swapcache is far
below the anon LRU (below min(anon >> 6, SWAP_CLUSTER_MAX)), i.e. when
essentially no anon on the list is reclaimable without IO. Outside that
corner anon is scanned and aged exactly as before.

And skipping anon in this corner doesn't cost us the aging in practice:
with no reclaimable anon left to scan, Round 1 simply finishes quickly
and reclaim proceeds to the full-walk retry (Round 2), which resets to
priority 12 and walks the whole subtree - anon included - so anon still
gets aged there, just without the ~168 ms detour first.

Thanks,
Bo

Re: [RFC PATCH] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache

[Johannes Weiner]

On Fri, Sep 04, 2026 at 10:07:56AM +0800, Bo Zhang wrote:
> On Thu, Sep 03, 2026 at 09:03:04AM -0400, Johannes Weiner wrote:
> > On Thu, Sep 03, 2026 at 12:01:31PM +0800, Bo Zhang wrote:
> > > We have observed some cases where memory is allocated with GFP_NOIO, so
> > > we cannot reclaim any anon folios unless they are in swapcache. We can
> > > end up spending more than 150 ms looping in `shrink_folio_list()` scanning
> > > non-swapcache folios without reclaiming a single folio. This is pure
> > > overhead.
> >
> > Not entirely. There is some value in aging anon alongside file, so
> > that the next __GFP_IO reclaimer doesn't look at a stale list.
> 
> You're right, "pure overhead" was too strong - aging anon does have
> value for a later __GFP_IO reclaimer, and I don't intend to skip it in
> general. Let me describe the case in full, because the reclaim cycle
> itself already provides that aging on a later pass, which is what makes
> me think the trade-off here leans the other way.
> 
> > Can you describe a bit more about what you observed? What workload is
> > running, maybe you have a stack trace of which NOIO requests are
> > routinely getting stuck in reclaim?
> 
> The workload is app launching on Android. The NOIO allocations come from
> dm-verity hash-block reads via dm-bufio, which legitimately use GFP_NOIO
> because they run underneath the IO path:
> 
>   worker_thread
>    process_scheduled_works
>     verity_work
>      verity_verify_io
>       verity_hash_for_block
>        verity_verify_level
>         dm_bufio_read_with_ioprio
>          new_read
>           __bufio_new
>            alloc_buffer
>             gfp_mask: GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN
> 
> So the NOIO use itself is correct; the problem is on the reclaim side.

Ack.

> Here is the full picture of one such direct reclaim. It runs two rounds
> of do_try_to_free_pages(); the target is 32 folios.
> 
> Round 1 - partial (shared) memcg walk, 169.20 ms, 0 folios reclaimed
> --------------------------------------------------------------------
>   prio 12->1 (~1.3 ms):
>     cache_trim_mode is on, so get_scan_count() picks SCAN_FILE. Only the
>     file side is scanned. Because this is a shared/partial walk, each
>     priority only visits a handful of memcgs before the iterator is
>     handed off, so very few memcgs are looked at on the way down:
>     428 file folios scanned, 0 reclaimed.
> 
>   prio 0 (~167.9 ms):
>     priority hits 0 without meeting the target, so get_scan_count()
>     forces SCAN_EQUAL. The walk lands on a single memcg with a large,
>     unswapped anon LRU and a tiny file LRU:
> 
>       inactive_anon ~335 MB, inactive_file ~4 MB   (~84:1)
>       memcg swap usage ~3.6 MB, so swapcache is negligible
> 
>     shrink_lruvec() now keeps feeding that huge anon list into
>     shrink_folio_list() - ~2400 shrink_folio_list() calls, ~93,000 anon
>     folios scanned - and every folio hits the !__GFP_IO keep_locked path
>     (not in swapcache, needs a swap slot). This single shrink_lruvec()
>     pass alone is ~168 ms with 0 folios reclaimed.

Ack. Thanks for the rich explanation, this is illuminating.

Agree with your fix. This GFP_NOIO just has to get through the day,
and aging 90% of memory it cannot reclaim is an unreasonable side
quest. Leave it to kswapd and the other reclaimers.