* [RFC PATCH v2 01/18] mm: free PG_rcu_free folios only after an RCU grace period
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
@ 2026-11-04 13:00 ` Kiryl Shutsemau (Meta)
2026-11-04 13:00 ` [RFC PATCH v2 02/18] splice: don't steal page cache folios Kiryl Shutsemau (Meta)
` (16 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:00 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
Reading a page cache folio without taking a reference would keep CPUs
that read the same file from bouncing its refcount. But without a
reference the folio can be freed, and its memory reused, while the
reader copies from it.
Add PG_rcu_free: the free of a folio with the flag waits for an RCU
grace period. A reader that saw the flag under rcu_read_lock(), and
then still found the folio where it looked, can read it until
rcu_read_unlock(). A split copies the flag, as such a reader may still
be reading the new folios.
The folios wait on per-CPU lists, one grace period in flight per CPU:
call_rcu() per folio would queue a callback for every folio of a
truncated file. A work, which can reschedule, frees each batch.
Reclaim depends on these frees, so the workqueue is WQ_MEM_RECLAIM.
The flag is 64-bit only (CONFIG_PAGE_RCU_FREE): page flags are scarce
on 32-bit, and the contention this avoids is a problem of large
machines. It is off on PREEMPT_RT, where expedited grace periods are
normal ones, so the waiting pages cannot be freed quickly when memory
runs short.
pgfree_rcu in /proc/vmstat counts the deferred pages. Nothing sets the
flag yet.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: rework for flagging at read time, 64-bit only and not PREEMPT_RT,
shorter comments and changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/page-flags.h | 21 +++++++
include/linux/vm_event_item.h | 1 +
include/trace/events/mmflags.h | 7 +++
mm/Kconfig | 15 +++++
mm/huge_memory.c | 1 +
mm/memory-failure.c | 4 ++
mm/page_alloc.c | 105 +++++++++++++++++++++++++++++++++
mm/vmstat.c | 1 +
8 files changed, 155 insertions(+)
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 7a863572adce..a1e52a1f8a52 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -111,6 +111,9 @@ enum pageflags {
PG_swapbacked, /* Page is backed by RAM/swap */
PG_unevictable, /* Page is "unevictable" */
PG_dropbehind, /* drop pages on IO completion */
+#ifdef CONFIG_PAGE_RCU_FREE
+ PG_rcu_free, /* free only after an RCU grace period */
+#endif
#ifdef CONFIG_MMU
PG_mlocked, /* Page is vma mlocked */
#endif
@@ -602,6 +605,24 @@ FOLIO_FLAG(dropbehind, FOLIO_HEAD_PAGE)
FOLIO_TEST_CLEAR_FLAG(dropbehind, FOLIO_HEAD_PAGE)
__FOLIO_SET_FLAG(dropbehind, FOLIO_HEAD_PAGE)
+/*
+ * A lockless lookup that sees PG_rcu_free and then still finds the folio
+ * where it looked may read it until rcu_read_unlock(). Set under a
+ * reference, cleared only by the free. Not for hugetlb or
+ * free_pages_nolock().
+ */
+#ifdef CONFIG_PAGE_RCU_FREE
+FOLIO_TEST_FLAG(rcu_free, FOLIO_HEAD_PAGE)
+ FOLIO_TEST_SET_FLAG(rcu_free, FOLIO_HEAD_PAGE)
+ __FOLIO_CLEAR_FLAG(rcu_free, FOLIO_HEAD_PAGE)
+#define __PG_RCU_FREE (1UL << PG_rcu_free)
+#else
+FOLIO_TEST_FLAG_FALSE(rcu_free)
+ FOLIO_TEST_SET_FLAG_FALSE(rcu_free)
+ __FOLIO_CLEAR_FLAG_NOOP(rcu_free)
+#define __PG_RCU_FREE 0
+#endif
+
#ifdef CONFIG_HIGHMEM
/*
* Must use a macro here due to header dependency issues. page_zone() is not
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 2628ccda076a..cbe849701653 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -36,6 +36,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
FOR_ALL_ZONES(ALLOCSTALL)
FOR_ALL_ZONES(PGSCAN_SKIP)
PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
+ PGFREE_RCU,
PGFAULT, PGMAJFAULT,
PGLAZYFREED,
PGREUSE,
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index 935893e5ea53..47c3411a75e8 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -117,6 +117,12 @@ TRACE_DEFINE_ENUM(___GFP_LAST_BIT);
#define IF_HAVE_PG_IDLE(_name)
#endif
+#ifdef CONFIG_PAGE_RCU_FREE
+#define IF_HAVE_PG_RCU_FREE(_name) ,{1UL << PG_##_name, __stringify(_name)}
+#else
+#define IF_HAVE_PG_RCU_FREE(_name)
+#endif
+
#ifdef CONFIG_ARCH_USES_PG_ARCH_2
#define IF_HAVE_PG_ARCH_2(_name) ,{1UL << PG_##_name, __stringify(_name)}
#else
@@ -152,6 +158,7 @@ TRACE_DEFINE_ENUM(___GFP_LAST_BIT);
DEF_PAGEFLAG_NAME(swapbacked), \
DEF_PAGEFLAG_NAME(unevictable), \
DEF_PAGEFLAG_NAME(dropbehind) \
+IF_HAVE_PG_RCU_FREE(rcu_free) \
IF_HAVE_PG_MLOCK(mlocked) \
IF_HAVE_PG_HWPOISON(hwpoison) \
IF_HAVE_PG_IDLE(idle) \
diff --git a/mm/Kconfig b/mm/Kconfig
index 604c58199acb..6e5341b0bf64 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -718,6 +718,21 @@ config PCP_BATCH_SCALE_MAX
latency. This option sets the upper limit of scale factor to limit
the maximum latency.
+config PAGE_RCU_FREE
+ def_bool y
+ depends on 64BIT
+ depends on !PREEMPT_RT
+ help
+ Reads from the page cache can copy from folios that are read
+ again and again without taking a reference on them, so small
+ reads of one file from many CPUs do not contend on the folio's
+ reference count. The cost is that memory freed from such
+ folios becomes reusable only after an RCU grace period.
+
+ 64-bit only: it takes a page flag, and those are scarce on
+ 32-bit. Not on PREEMPT_RT, where expedited grace periods are
+ normal ones, so freeing the waiting pages on demand would stall.
+
config PHYS_ADDR_T_64BIT
def_bool 64BIT
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 1e5d68acf62a..25082949a4ca 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3743,6 +3743,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
#endif
(1L << PG_dirty) |
(1L << PG_dropbehind) |
+ __PG_RCU_FREE |
LRU_GEN_MASK | LRU_REFS_MASK));
new_folio->mapping = folio->mapping;
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index a8b03e2920ba..7dde7c3c9a0c 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -211,6 +211,10 @@ static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, boo
return false;
}
+ /* The free of a hwpoisoned page is not deferred: wait for readers */
+ if (release && folio_test_rcu_free(page_folio(page)))
+ synchronize_rcu();
+
SetPageHWPoison(page);
if (release)
put_page(page);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 12fac9084c48..a371e92da273 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1313,6 +1313,96 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
#endif /* CONFIG_MEM_ALLOC_PROFILING */
+/*
+ * Each CPU has at most one grace period in flight (@queued), covering
+ * @batch. Folios freed meanwhile wait on @pending for the next one.
+ */
+struct rcu_free_pcpu {
+ spinlock_t lock;
+ bool queued;
+ struct list_head pending;
+ struct list_head batch;
+ struct rcu_work rwork;
+};
+
+static DEFINE_PER_CPU(struct rcu_free_pcpu, rcu_free_pcpu);
+
+static struct workqueue_struct *rcu_free_wq __ro_after_init;
+
+static void rcu_free_start(struct rcu_free_pcpu *r)
+{
+ lockdep_assert_held(&r->lock);
+
+ if (r->queued || list_empty(&r->pending))
+ return;
+
+ list_splice_init(&r->pending, &r->batch);
+ r->queued = true;
+ WARN_ON_ONCE(!queue_rcu_work(rcu_free_wq, &r->rwork));
+}
+
+static void rcu_free_pages(struct list_head *list)
+{
+ struct folio_batch fbatch;
+ struct folio *folio, *next;
+
+ folio_batch_init(&fbatch);
+ list_for_each_entry_safe(folio, next, list, lru) {
+ if (!folio_batch_add(&fbatch, folio)) {
+ free_unref_folios(&fbatch);
+ cond_resched_tasks_rcu_qs();
+ }
+ }
+ if (folio_batch_count(&fbatch))
+ free_unref_folios(&fbatch);
+}
+
+static void rcu_free_workfn(struct work_struct *work)
+{
+ struct rcu_free_pcpu *r = container_of(to_rcu_work(work),
+ struct rcu_free_pcpu, rwork);
+ LIST_HEAD(done);
+
+ spin_lock_irq(&r->lock);
+ list_splice_init(&r->batch, &done);
+ r->queued = false;
+ rcu_free_start(r);
+ spin_unlock_irq(&r->lock);
+
+ rcu_free_pages(&done);
+}
+
+static void rcu_free_defer(struct folio *folio)
+{
+ struct rcu_free_pcpu *r = raw_cpu_ptr(&rcu_free_pcpu);
+ unsigned long flags;
+
+ spin_lock_irqsave(&r->lock, flags);
+ list_add_tail(&folio->lru, &r->pending);
+ rcu_free_start(r);
+ spin_unlock_irqrestore(&r->lock, flags);
+}
+
+static void __init rcu_free_init(void)
+{
+ int cpu;
+
+ if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
+ return;
+
+ rcu_free_wq = alloc_workqueue("rcu_free",
+ WQ_MEM_RECLAIM | WQ_PERCPU, 0);
+
+ for_each_possible_cpu(cpu) {
+ struct rcu_free_pcpu *r = per_cpu_ptr(&rcu_free_pcpu, cpu);
+
+ spin_lock_init(&r->lock);
+ INIT_LIST_HEAD(&r->pending);
+ INIT_LIST_HEAD(&r->batch);
+ INIT_RCU_WORK(&r->rwork, rcu_free_workfn);
+ }
+}
+
static __always_inline bool __free_pages_prepare(struct page *page,
unsigned int order, fpi_t fpi_flags)
{
@@ -1327,6 +1417,19 @@ static __always_inline bool __free_pages_prepare(struct page *page,
VM_BUG_ON_PAGE(PageTail(page), page);
+ /*
+ * Not for a hwpoisoned order-0 page: it never reaches the allocator,
+ * and page_handle_poison() takes a reference right after this free.
+ */
+ if (unlikely(folio_test_rcu_free(folio)) &&
+ !(PageHWPoison(page) && !order)) {
+ VM_WARN_ON_ONCE(fpi_flags & FPI_NOLOCK);
+ __folio_clear_rcu_free(folio);
+ count_vm_events(PGFREE_RCU, 1 << order);
+ rcu_free_defer(folio);
+ return false;
+ }
+
trace_mm_page_free(page, order);
kmsan_free_page(page, order);
@@ -6401,6 +6504,8 @@ void __init setup_per_cpu_pageset(void)
for_each_online_pgdat(pgdat)
pgdat->per_cpu_nodestats =
alloc_percpu(struct per_cpu_nodestat);
+
+ rcu_free_init();
}
__meminit void zone_pcp_init(struct zone *zone)
diff --git a/mm/vmstat.c b/mm/vmstat.c
index cb57714539fb..66ed017bde46 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1343,6 +1343,7 @@ const char * const vmstat_text[] = {
[I(PGACTIVATE)] = "pgactivate",
[I(PGDEACTIVATE)] = "pgdeactivate",
[I(PGLAZYFREE)] = "pglazyfree",
+ [I(PGFREE_RCU)] = "pgfree_rcu",
[I(PGFAULT)] = "pgfault",
[I(PGMAJFAULT)] = "pgmajfault",
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 02/18] splice: don't steal page cache folios
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
2026-11-04 13:00 ` [RFC PATCH v2 01/18] mm: free PG_rcu_free folios only after an RCU grace period Kiryl Shutsemau (Meta)
@ 2026-11-04 13:00 ` Kiryl Shutsemau (Meta)
2026-11-04 16:05 ` David Hildenbrand (Arm)
2026-11-04 13:00 ` [RFC PATCH v2 03/18] fuse: don't move a folio with PG_rcu_free into a fuse file Kiryl Shutsemau (Meta)
` (15 subsequent siblings)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:00 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
PG_rcu_free holds back the reuse of a folio's memory until a grace
period after the folio is freed. A stolen page cache pipe buffer is
reused without being freed: remove_mapping() takes the folio out of the
page cache and the caller keeps it. fuse_try_move_folio() puts it into
a fuse file, where writes change it under a reader that found it before
the steal.
Drop ->try_steal from page_cache_pipe_buf_ops. fuse_try_move_folio()
and virtio_console's pipe_to_sg() then copy, as they already do when
stealing fails. SPLICE_F_MOVE from the page cache into a fuse file or
a virtio console now copies.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: rewrote the changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
fs/splice.c | 51 ---------------------------------------------------
1 file changed, 51 deletions(-)
diff --git a/fs/splice.c b/fs/splice.c
index 9d8f63e2fd1a..0133c2348da0 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -55,56 +55,6 @@ static noinline void pipe_clear_nowait(struct file *file)
} while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));
}
-/*
- * Attempt to steal a page from a pipe buffer. This should perhaps go into
- * a vm helper function, it's already simplified quite a bit by the
- * addition of remove_mapping(). If success is returned, the caller may
- * attempt to reuse this page for another destination.
- */
-static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
-{
- struct folio *folio = page_folio(buf->page);
- struct address_space *mapping;
-
- folio_lock(folio);
-
- mapping = folio_mapping(folio);
- if (mapping) {
- WARN_ON(!folio_test_uptodate(folio));
-
- /*
- * At least for ext2 with nobh option, we need to wait on
- * writeback completing on this folio, since we'll remove it
- * from the pagecache. Otherwise truncate wont wait on the
- * folio, allowing the disk blocks to be reused by someone else
- * before we actually wrote our data to them. fs corruption
- * ensues.
- */
- folio_wait_writeback(folio);
-
- if (!filemap_release_folio(folio, GFP_KERNEL))
- goto out_unlock;
-
- /*
- * If we succeeded in removing the mapping, set LRU flag
- * and return good.
- */
- if (remove_mapping(mapping, folio)) {
- buf->flags |= PIPE_BUF_FLAG_LRU;
- return true;
- }
- }
-
- /*
- * Raced with truncate or failed to remove folio from current
- * address space, unlock and return failure.
- */
-out_unlock:
- folio_unlock(folio);
- return false;
-}
-
static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
struct pipe_buffer *buf)
{
@@ -155,7 +105,6 @@ static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
const struct pipe_buf_operations page_cache_pipe_buf_ops = {
.confirm = page_cache_pipe_buf_confirm,
.release = page_cache_pipe_buf_release,
- .try_steal = page_cache_pipe_buf_try_steal,
.get = generic_pipe_buf_get,
};
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 02/18] splice: don't steal page cache folios
2026-11-04 13:00 ` [RFC PATCH v2 02/18] splice: don't steal page cache folios Kiryl Shutsemau (Meta)
@ 2026-11-04 16:05 ` David Hildenbrand (Arm)
2026-11-05 14:55 ` Kiryl Shutsemau
0 siblings, 1 reply; 27+ messages in thread
From: David Hildenbrand (Arm) @ 2026-11-04 16:05 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
Matthew Wilcox, Vlastimil Babka, Lorenzo Stoakes, Jan Kara,
Hugh Dickins, Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team
On 11/4/26 14:00, Kiryl Shutsemau (Meta) wrote:
> From: Linus Torvalds <torvalds@linux-foundation.org>
>
> PG_rcu_free holds back the reuse of a folio's memory until a grace
> period after the folio is freed. A stolen page cache pipe buffer is
> reused without being freed: remove_mapping() takes the folio out of the
> page cache and the caller keeps it. fuse_try_move_folio() puts it into
> a fuse file, where writes change it under a reader that found it before
> the steal.
>
> Drop ->try_steal from page_cache_pipe_buf_ops. fuse_try_move_folio()
> and virtio_console's pipe_to_sg() then copy, as they already do when
> stealing fails. SPLICE_F_MOVE from the page cache into a fuse file or
> a virtio console now copies.
>
> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
> [kas: rewrote the changelog]
> Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
> ---
> fs/splice.c | 51 ---------------------------------------------------
Removing code is always nice, even independent of the other changes :)
The description doesn't really describe the impact of this change. I assume a
slowdown of certain workloads (does anybody really care?), and temporary
increased memory consumption (as we keep the pagecache page).
No idea how often stealing actually succeeds ... a bit of history why it was
added in the first place and why we no longer care now would be nice.
--
Cheers,
David
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [RFC PATCH v2 02/18] splice: don't steal page cache folios
2026-11-04 16:05 ` David Hildenbrand (Arm)
@ 2026-11-05 14:55 ` Kiryl Shutsemau
0 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau @ 2026-11-05 14:55 UTC (permalink / raw)
To: David Hildenbrand (Arm)
Cc: Andrew Morton, Linus Torvalds, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo, Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
On Wed, Nov 04, 2026 at 05:05:29PM +0100, David Hildenbrand (Arm) wrote:
> On 11/4/26 14:00, Kiryl Shutsemau (Meta) wrote:
> > From: Linus Torvalds <torvalds@linux-foundation.org>
> >
> > PG_rcu_free holds back the reuse of a folio's memory until a grace
> > period after the folio is freed. A stolen page cache pipe buffer is
> > reused without being freed: remove_mapping() takes the folio out of the
> > page cache and the caller keeps it. fuse_try_move_folio() puts it into
> > a fuse file, where writes change it under a reader that found it before
> > the steal.
> >
> > Drop ->try_steal from page_cache_pipe_buf_ops. fuse_try_move_folio()
> > and virtio_console's pipe_to_sg() then copy, as they already do when
> > stealing fails. SPLICE_F_MOVE from the page cache into a fuse file or
> > a virtio console now copies.
> >
> > Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
> > [kas: rewrote the changelog]
> > Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
> > ---
> > fs/splice.c | 51 ---------------------------------------------------
>
> Removing code is always nice, even independent of the other changes :)
>
> The description doesn't really describe the impact of this change. I assume a
> slowdown of certain workloads (does anybody really care?), and temporary
> increased memory consumption (as we keep the pagecache page).
>
> No idea how often stealing actually succeeds ... a bit of history why it was
> added in the first place and why we no longer care now would be nice.
SPLICE_F_MOVE has not moved page cache folios between files since 2007.
See 485ddb4b9741 ("1/2 splice: dont steal"). The only users that still
steal from a pipe are fuse, when a server splices into /dev/fuse with
SPLICE_F_MOVE, and virtio_console's splice_write(). With this patch
both copy, and the source file keeps its folio cached.
Miklos, Bernd, Amit: do you know of users that depend on the move? Would
the copy hurt them?
--
Kiryl Shutsemau / Kirill A. Shutemov
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 03/18] fuse: don't move a folio with PG_rcu_free into a fuse file
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
2026-11-04 13:00 ` [RFC PATCH v2 01/18] mm: free PG_rcu_free folios only after an RCU grace period Kiryl Shutsemau (Meta)
2026-11-04 13:00 ` [RFC PATCH v2 02/18] splice: don't steal page cache folios Kiryl Shutsemau (Meta)
@ 2026-11-04 13:00 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 04/18] mm: count the pages waiting for a grace period Kiryl Shutsemau (Meta)
` (14 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:00 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
fuse_try_move_folio() takes the folio of a pipe buffer and puts it into
a fuse file, where writes change it. A folio with PG_rcu_free may still
be copied by a lockless page cache read that found it before it left
the page cache.
Such a folio reaches fuse when a mapped file page is gifted with
vmsplice(SPLICE_F_GIFT) and its file is then truncated.
fuse_check_folio() refuses it only because the flag is not among those
it allows, and with a dump_page().
Copy such a folio instead, as fuse does whenever it cannot take one.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
fs/fuse/dev.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 4fec31fc0b84..526929c7b3f1 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -1152,6 +1152,10 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop
newfolio = page_folio(buf->page);
+ /* A lockless page cache read may still be copying from it */
+ if (folio_test_rcu_free(newfolio))
+ goto out_fallback_unlock;
+
folio_clear_uptodate(newfolio);
folio_clear_mappedtodisk(newfolio);
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 04/18] mm: count the pages waiting for a grace period
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (2 preceding siblings ...)
2026-11-04 13:00 ` [RFC PATCH v2 03/18] fuse: don't move a folio with PG_rcu_free into a fuse file Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 05/18] mm: expedite the grace period when a CPU has many deferred frees Kiryl Shutsemau (Meta)
` (13 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
Deferred folios have refcount 0 and are not on the free lists, so no
counter shows them. Memory that waits for a grace period is missing
from the free count and from every LRU count.
Count it per zone in NR_FREE_RCU_PAGES: nr_free_rcu in /proc/vmstat,
free_rcu in show_free_areas(). Preparation for freeing the waiting
pages sooner when they hold back memory that is needed.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/mmzone.h | 1 +
mm/page_alloc.c | 4 ++++
mm/show_mem.c | 9 ++++++---
mm/vmstat.c | 2 ++
4 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 94f9c3ff5416..3336b9e5ee99 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -228,6 +228,7 @@ enum zone_stat_item {
NR_ZSPAGES, /* allocated in zsmalloc */
#endif
NR_FREE_CMA_PAGES,
+ NR_FREE_RCU_PAGES, /* freed, waiting for an RCU grace period */
#ifdef CONFIG_UNACCEPTED_MEMORY
NR_UNACCEPTED,
#endif
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a371e92da273..15bb66e36304 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1348,6 +1348,8 @@ static void rcu_free_pages(struct list_head *list)
folio_batch_init(&fbatch);
list_for_each_entry_safe(folio, next, list, lru) {
+ mod_zone_page_state(folio_zone(folio), NR_FREE_RCU_PAGES,
+ -folio_nr_pages(folio));
if (!folio_batch_add(&fbatch, folio)) {
free_unref_folios(&fbatch);
cond_resched_tasks_rcu_qs();
@@ -1375,9 +1377,11 @@ static void rcu_free_workfn(struct work_struct *work)
static void rcu_free_defer(struct folio *folio)
{
struct rcu_free_pcpu *r = raw_cpu_ptr(&rcu_free_pcpu);
+ struct zone *zone = folio_zone(folio);
unsigned long flags;
spin_lock_irqsave(&r->lock, flags);
+ __mod_zone_page_state(zone, NR_FREE_RCU_PAGES, folio_nr_pages(folio));
list_add_tail(&folio->lru, &r->pending);
rcu_free_start(r);
spin_unlock_irqrestore(&r->lock, flags);
diff --git a/mm/show_mem.c b/mm/show_mem.c
index b938cbcd774a..0d3bddbda108 100644
--- a/mm/show_mem.c
+++ b/mm/show_mem.c
@@ -204,7 +204,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask,
" mapped:%lu shmem:%lu pagetables:%lu\n"
" sec_pagetables:%lu bounce:%lu\n"
" kernel_misc_reclaimable:%lu\n"
- " free:%lu free_pcp:%lu free_cma:%lu\n",
+ " free:%lu free_pcp:%lu free_cma:%lu free_rcu:%lu\n",
global_node_page_state(NR_ACTIVE_ANON),
global_node_page_state(NR_INACTIVE_ANON),
global_node_page_state(NR_ISOLATED_ANON),
@@ -224,7 +224,8 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask,
global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE),
global_zone_page_state(NR_FREE_PAGES),
free_pcp,
- global_zone_page_state(NR_FREE_CMA_PAGES));
+ global_zone_page_state(NR_FREE_CMA_PAGES),
+ global_zone_page_state(NR_FREE_RCU_PAGES));
for_each_online_pgdat(pgdat) {
if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
@@ -325,6 +326,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask,
" free_pcp:%lukB"
" local_pcp:%lukB"
" free_cma:%lukB"
+ " free_rcu:%lukB"
"\n",
zone->name,
K(zone_page_state(zone, NR_FREE_PAGES)),
@@ -351,7 +353,8 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask,
0UL,
K(free_pcp),
K((unsigned long)this_cpu_read(zone->per_cpu_pageset->count)),
- K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
+ K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
+ K(zone_page_state(zone, NR_FREE_RCU_PAGES)));
printk("lowmem_reserve[]:");
for (i = 0; i < MAX_NR_ZONES; i++)
printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 66ed017bde46..64068d73d6d2 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1212,6 +1212,7 @@ const char * const vmstat_text[] = {
[I(NR_ZSPAGES)] = "nr_zspages",
#endif
[I(NR_FREE_CMA_PAGES)] = "nr_free_cma",
+ [I(NR_FREE_RCU_PAGES)] = "nr_free_rcu",
#ifdef CONFIG_UNACCEPTED_MEMORY
[I(NR_UNACCEPTED)] = "nr_unaccepted",
#endif
@@ -2023,6 +2024,7 @@ static int vmstat_refresh(const struct ctl_table *table, int write,
switch (i) {
case NR_ZONE_WRITE_PENDING:
case NR_FREE_CMA_PAGES:
+ case NR_FREE_RCU_PAGES:
continue;
}
val = atomic_long_read(&vm_zone_stat[i]);
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 05/18] mm: expedite the grace period when a CPU has many deferred frees
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (3 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 04/18] mm: count the pages waiting for a grace period Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 06/18] mm: free the pages waiting for a grace period before OOM Kiryl Shutsemau (Meta)
` (12 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
A deferred folio waits one to two grace periods on its CPU's list, where
the page allocator cannot use it. A CPU that frees many flagged folios
at once, for example when a file is truncated, can hold a lot of memory
there.
When more than 4 MiB is pending on a CPU, a work takes both of the CPU's
lists and frees them after an expedited grace period. That grace period
starts after the take, so it covers every reader that found one of the
folios. The task that frees them does not wait.
An expedited grace period interrupts every CPU, so keep it for when the
memory matters. That is when the zone is below its high watermark, or
when the pages waiting in it are more than an eighth of its free pages.
The watermark alone comes too late for a bulk free. A truncate or
fadvise(POSIX_FADV_DONTNEED) of a hot range defers all of it while the
zone still has memory. When the readers that refill the range then run
short, nothing is freed that would check again.
With rcu_normal, the expedited grace period is a normal one.
pgfree_rcu_expedite in /proc/vmstat counts the expedite runs that freed
something.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: take both lists, expedite only when the memory matters, rewrote
the changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/vm_event_item.h | 2 +-
mm/page_alloc.c | 61 ++++++++++++++++++++++++++++++++++-
mm/vmstat.c | 1 +
3 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index cbe849701653..45527fa7d0c1 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -36,7 +36,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
FOR_ALL_ZONES(ALLOCSTALL)
FOR_ALL_ZONES(PGSCAN_SKIP)
PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
- PGFREE_RCU,
+ PGFREE_RCU, PGFREE_RCU_EXPEDITE,
PGFAULT, PGMAJFAULT,
PGLAZYFREED,
PGREUSE,
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 15bb66e36304..a9058d414f92 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1313,6 +1313,8 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
#endif /* CONFIG_MEM_ALLOC_PROFILING */
+#define RCU_FREE_EXPEDITE (SZ_4M >> PAGE_SHIFT)
+
/*
* Each CPU has at most one grace period in flight (@queued), covering
* @batch. Folios freed meanwhile wait on @pending for the next one.
@@ -1322,7 +1324,9 @@ struct rcu_free_pcpu {
bool queued;
struct list_head pending;
struct list_head batch;
+ unsigned long nr_pending;
struct rcu_work rwork;
+ struct work_struct expedite;
};
static DEFINE_PER_CPU(struct rcu_free_pcpu, rcu_free_pcpu);
@@ -1337,17 +1341,29 @@ static void rcu_free_start(struct rcu_free_pcpu *r)
return;
list_splice_init(&r->pending, &r->batch);
+ r->nr_pending = 0;
r->queued = true;
WARN_ON_ONCE(!queue_rcu_work(rcu_free_wq, &r->rwork));
}
-static void rcu_free_pages(struct list_head *list)
+static void rcu_free_take(struct rcu_free_pcpu *r, struct list_head *list)
+{
+ lockdep_assert_held(&r->lock);
+
+ list_splice_tail_init(&r->batch, list);
+ list_splice_tail_init(&r->pending, list);
+ r->nr_pending = 0;
+}
+
+static unsigned long rcu_free_pages(struct list_head *list)
{
struct folio_batch fbatch;
struct folio *folio, *next;
+ unsigned long nr = 0;
folio_batch_init(&fbatch);
list_for_each_entry_safe(folio, next, list, lru) {
+ nr += folio_nr_pages(folio);
mod_zone_page_state(folio_zone(folio), NR_FREE_RCU_PAGES,
-folio_nr_pages(folio));
if (!folio_batch_add(&fbatch, folio)) {
@@ -1357,6 +1373,17 @@ static void rcu_free_pages(struct list_head *list)
}
if (folio_batch_count(&fbatch))
free_unref_folios(&fbatch);
+ return nr;
+}
+
+/* Free what rcu_free_take() took: the grace period starts after that */
+static unsigned long rcu_free_flush(struct list_head *list)
+{
+ if (list_empty(list))
+ return 0;
+
+ synchronize_rcu_expedited();
+ return rcu_free_pages(list);
}
static void rcu_free_workfn(struct work_struct *work)
@@ -1374,17 +1401,48 @@ static void rcu_free_workfn(struct work_struct *work)
rcu_free_pages(&done);
}
+static void rcu_free_expedite_workfn(struct work_struct *work)
+{
+ struct rcu_free_pcpu *r = container_of(work, struct rcu_free_pcpu,
+ expedite);
+ LIST_HEAD(list);
+
+ spin_lock_irq(&r->lock);
+ rcu_free_take(r, &list);
+ spin_unlock_irq(&r->lock);
+
+ if (rcu_free_flush(&list))
+ count_vm_event(PGFREE_RCU_EXPEDITE);
+}
+
static void rcu_free_defer(struct folio *folio)
{
struct rcu_free_pcpu *r = raw_cpu_ptr(&rcu_free_pcpu);
struct zone *zone = folio_zone(folio);
+ unsigned long free, waiting;
unsigned long flags;
spin_lock_irqsave(&r->lock, flags);
__mod_zone_page_state(zone, NR_FREE_RCU_PAGES, folio_nr_pages(folio));
list_add_tail(&folio->lru, &r->pending);
+ r->nr_pending += folio_nr_pages(folio);
rcu_free_start(r);
spin_unlock_irqrestore(&r->lock, flags);
+
+ /* Unlocked: a stale count delays the expedite by one free */
+ if (data_race(r->nr_pending) <= RCU_FREE_EXPEDITE ||
+ work_pending(&r->expedite))
+ return;
+
+ /*
+ * An expedited grace period interrupts every CPU: only while the
+ * zone is short of memory, or while more than an eighth of what it
+ * has free waits here.
+ */
+ free = zone_page_state(zone, NR_FREE_PAGES);
+ waiting = zone_page_state(zone, NR_FREE_RCU_PAGES);
+ if (free < high_wmark_pages(zone) || waiting > free / 8)
+ queue_work(rcu_free_wq, &r->expedite);
}
static void __init rcu_free_init(void)
@@ -1404,6 +1462,7 @@ static void __init rcu_free_init(void)
INIT_LIST_HEAD(&r->pending);
INIT_LIST_HEAD(&r->batch);
INIT_RCU_WORK(&r->rwork, rcu_free_workfn);
+ INIT_WORK(&r->expedite, rcu_free_expedite_workfn);
}
}
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 64068d73d6d2..aa7280b5e69c 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1345,6 +1345,7 @@ const char * const vmstat_text[] = {
[I(PGDEACTIVATE)] = "pgdeactivate",
[I(PGLAZYFREE)] = "pglazyfree",
[I(PGFREE_RCU)] = "pgfree_rcu",
+ [I(PGFREE_RCU_EXPEDITE)] = "pgfree_rcu_expedite",
[I(PGFAULT)] = "pgfault",
[I(PGMAJFAULT)] = "pgmajfault",
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 06/18] mm: free the pages waiting for a grace period before OOM
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (4 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 05/18] mm: expedite the grace period when a CPU has many deferred frees Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-05 13:03 ` sashiko-bot
2026-11-04 13:01 ` [RFC PATCH v2 07/18] mm: free the pages waiting for a grace period before isolation fails Kiryl Shutsemau (Meta)
` (11 subsequent siblings)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
When reclaim fails, should_reclaim_retry() looks only at free and
reclaimable pages. It can hand the allocation to the OOM killer while
enough memory is about to come back from the pages that wait for a
grace period.
Before should_reclaim_retry() gives up, free the waiting pages if they
and what reclaim can still free would bring an eligible zone above its
min watermark, and retry. The drain frees them straight to the buddy
lists, where the retry's watermark check sees them, and waits for the
expedites already in flight. That costs an expedited grace period,
which is cheap next to the OOM killer. The retries are bounded per
no-progress streak, so a drain that does not help still ends in an OOM
kill.
pgfree_rcu_drain in /proc/vmstat counts the drains that freed something.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/vm_event_item.h | 2 +-
mm/page_alloc.c | 119 ++++++++++++++++++++++++++++++----
mm/vmstat.c | 1 +
3 files changed, 110 insertions(+), 12 deletions(-)
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 45527fa7d0c1..7626a3e3a07a 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -36,7 +36,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
FOR_ALL_ZONES(ALLOCSTALL)
FOR_ALL_ZONES(PGSCAN_SKIP)
PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
- PGFREE_RCU, PGFREE_RCU_EXPEDITE,
+ PGFREE_RCU, PGFREE_RCU_EXPEDITE, PGFREE_RCU_DRAIN,
PGFAULT, PGMAJFAULT,
PGLAZYFREED,
PGREUSE,
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a9058d414f92..9caaa57bf51d 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1355,7 +1355,7 @@ static void rcu_free_take(struct rcu_free_pcpu *r, struct list_head *list)
r->nr_pending = 0;
}
-static unsigned long rcu_free_pages(struct list_head *list)
+static unsigned long rcu_free_pages(struct list_head *list, bool direct)
{
struct folio_batch fbatch;
struct folio *folio, *next;
@@ -1366,7 +1366,11 @@ static unsigned long rcu_free_pages(struct list_head *list)
nr += folio_nr_pages(folio);
mod_zone_page_state(folio_zone(folio), NR_FREE_RCU_PAGES,
-folio_nr_pages(folio));
- if (!folio_batch_add(&fbatch, folio)) {
+ if (direct) {
+ __free_pages_ok(&folio->page, folio_order(folio),
+ FPI_NONE);
+ cond_resched();
+ } else if (!folio_batch_add(&fbatch, folio)) {
free_unref_folios(&fbatch);
cond_resched_tasks_rcu_qs();
}
@@ -1376,14 +1380,17 @@ static unsigned long rcu_free_pages(struct list_head *list)
return nr;
}
-/* Free what rcu_free_take() took: the grace period starts after that */
-static unsigned long rcu_free_flush(struct list_head *list)
+/*
+ * Free what rcu_free_take() took: the grace period starts after that.
+ * @direct frees to the buddy lists instead of this CPU's pcp lists.
+ */
+static unsigned long rcu_free_flush(struct list_head *list, bool direct)
{
if (list_empty(list))
return 0;
synchronize_rcu_expedited();
- return rcu_free_pages(list);
+ return rcu_free_pages(list, direct);
}
static void rcu_free_workfn(struct work_struct *work)
@@ -1398,7 +1405,7 @@ static void rcu_free_workfn(struct work_struct *work)
rcu_free_start(r);
spin_unlock_irq(&r->lock);
- rcu_free_pages(&done);
+ rcu_free_pages(&done, false);
}
static void rcu_free_expedite_workfn(struct work_struct *work)
@@ -1411,7 +1418,7 @@ static void rcu_free_expedite_workfn(struct work_struct *work)
rcu_free_take(r, &list);
spin_unlock_irq(&r->lock);
- if (rcu_free_flush(&list))
+ if (rcu_free_flush(&list, false))
count_vm_event(PGFREE_RCU_EXPEDITE);
}
@@ -1445,6 +1452,33 @@ static void rcu_free_defer(struct folio *folio)
queue_work(rcu_free_wq, &r->expedite);
}
+/*
+ * Free what every CPU's lists hold, straight to the buddy lists: the
+ * callers want to allocate the pages next. A concurrent drain holds the
+ * lock until it has freed what it took.
+ */
+static void rcu_free_drain(void)
+{
+ static DEFINE_MUTEX(lock);
+ LIST_HEAD(list);
+ int cpu;
+
+ mutex_lock(&lock);
+ for_each_possible_cpu(cpu) {
+ struct rcu_free_pcpu *r = per_cpu_ptr(&rcu_free_pcpu, cpu);
+
+ spin_lock_irq(&r->lock);
+ rcu_free_take(r, &list);
+ spin_unlock_irq(&r->lock);
+ }
+ if (rcu_free_flush(&list, true))
+ count_vm_event(PGFREE_RCU_DRAIN);
+ /* An expedite in flight holds pages it took off the lists */
+ for_each_possible_cpu(cpu)
+ flush_work(&per_cpu_ptr(&rcu_free_pcpu, cpu)->expedite);
+ mutex_unlock(&lock);
+}
+
static void __init rcu_free_init(void)
{
int cpu;
@@ -4792,6 +4826,58 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
return !!__gfp_pfmemalloc_flags(gfp_mask);
}
+/*
+ * Reclaim is about to give up: if the pages waiting for a grace period,
+ * with what reclaim can still free, would bring an eligible zone above its
+ * min watermark, free them and retry. Not for a dying task, which frees
+ * memory by exiting.
+ */
+static bool rcu_free_wait(gfp_t gfp_mask, unsigned int order,
+ struct alloc_context *ac, int alloc_flags,
+ int *rcu_free_waits)
+{
+ struct zone *zone;
+ struct zoneref *z;
+
+ if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
+ return false;
+ if (tsk_is_oom_victim(current) || fatal_signal_pending(current))
+ return false;
+ if (*rcu_free_waits >= MAX_RECLAIM_RETRIES ||
+ order > PAGE_ALLOC_COSTLY_ORDER)
+ return false;
+
+ for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
+ ac->highest_zoneidx, ac->nodemask) {
+ unsigned long waiting, available;
+
+ if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
+ !__cpuset_zone_allowed(zone, gfp_mask))
+ continue;
+
+ waiting = zone_page_state_snapshot(zone, NR_FREE_RCU_PAGES);
+ if (!waiting)
+ continue;
+
+ available = waiting + zone_reclaimable_pages(zone) +
+ zone_page_state_snapshot(zone, NR_FREE_PAGES);
+ /*
+ * Count pages only: for order > 0 __zone_watermark_ok() also
+ * wants a free block of that order, and the waiting pages are
+ * not free yet.
+ */
+ if (__zone_watermark_ok(zone, 0, min_wmark_pages(zone),
+ ac->highest_zoneidx, alloc_flags,
+ available)) {
+ (*rcu_free_waits)++;
+ rcu_free_drain();
+ return true;
+ }
+ }
+
+ return false;
+}
+
/*
* Checks whether it makes sense to retry the reclaim to make a forward progress
* for the given allocation request.
@@ -4805,7 +4891,8 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
static inline bool
should_reclaim_retry(gfp_t gfp_mask, unsigned order,
struct alloc_context *ac, int alloc_flags,
- bool did_some_progress, int *no_progress_loops)
+ bool did_some_progress, int *no_progress_loops,
+ int *rcu_free_waits)
{
struct zone *zone;
struct zoneref *z;
@@ -4816,10 +4903,12 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
* their order will become available due to high fragmentation so
* always increment the no progress counter for them
*/
- if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
+ if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) {
*no_progress_loops = 0;
- else
+ *rcu_free_waits = 0;
+ } else {
(*no_progress_loops)++;
+ }
if (*no_progress_loops > MAX_RECLAIM_RETRIES)
goto out;
@@ -4872,6 +4961,10 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
else
cond_resched();
out:
+ if (!ret && rcu_free_wait(gfp_mask, order, ac, alloc_flags,
+ rcu_free_waits))
+ return true;
+
/* Before OOM, exhaust highatomic_reserve */
if (!ret)
return unreserve_highatomic_pageblock(ac, true);
@@ -4961,6 +5054,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
enum compact_result compact_result;
int compaction_retries;
int no_progress_loops;
+ int rcu_free_waits;
unsigned int cpuset_mems_cookie;
unsigned int zonelist_iter_cookie;
int reserve_flags;
@@ -4985,6 +5079,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
restart:
compaction_retries = 0;
no_progress_loops = 0;
+ rcu_free_waits = 0;
compact_result = COMPACT_SKIPPED;
compact_priority = DEF_COMPACT_PRIORITY;
cpuset_mems_cookie = read_mems_allowed_begin();
@@ -5163,7 +5258,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
goto restart;
if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
- did_some_progress > 0, &no_progress_loops))
+ did_some_progress > 0, &no_progress_loops,
+ &rcu_free_waits))
goto retry;
/*
@@ -5206,6 +5302,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
/* Retry as long as the OOM killer is making progress */
if (did_some_progress) {
no_progress_loops = 0;
+ rcu_free_waits = 0;
goto retry;
}
diff --git a/mm/vmstat.c b/mm/vmstat.c
index aa7280b5e69c..d994de1fd739 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1346,6 +1346,7 @@ const char * const vmstat_text[] = {
[I(PGLAZYFREE)] = "pglazyfree",
[I(PGFREE_RCU)] = "pgfree_rcu",
[I(PGFREE_RCU_EXPEDITE)] = "pgfree_rcu_expedite",
+ [I(PGFREE_RCU_DRAIN)] = "pgfree_rcu_drain",
[I(PGFAULT)] = "pgfault",
[I(PGMAJFAULT)] = "pgmajfault",
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 06/18] mm: free the pages waiting for a grace period before OOM
2026-11-04 13:01 ` [RFC PATCH v2 06/18] mm: free the pages waiting for a grace period before OOM Kiryl Shutsemau (Meta)
@ 2026-11-05 13:03 ` sashiko-bot
0 siblings, 0 replies; 27+ messages in thread
From: sashiko-bot @ 2026-11-05 13:03 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
David Hildenbrand, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
kernel-team
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
- [High] mm: order-0 check in rcu_free_wait() starves high-order compaction
- [High] mm: rcu_free_drain() misses rcu_free_workfn() and PCP-freed pages
- [Medium] mm: rcu_free_wait() counts unreclaimable pages after retry limit
--
commit 4b538833b03e67798c223181a1b410cee9126502
Author: Kiryl Shutsemau (Meta) <kas@kernel.org>
mm: free the pages waiting for a grace period before OOM
Before should_reclaim_retry() gives up on reclaim, this commit drains
RCU-deferred pages directly to the buddy lists and retries the allocation
if those pages along with reclaimable memory would bring an eligible zone
above its minimum watermark.
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index a9058d414f92d..9caaa57bf51d7 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
[ ... ]
> @@ -1398,7 +1405,7 @@ static void rcu_free_workfn(struct work_struct *work)
> rcu_free_start(r);
> spin_unlock_irq(&r->lock);
>
> - rcu_free_pages(&done);
> + rcu_free_pages(&done, false);
> }
>
> static void rcu_free_expedite_workfn(struct work_struct *work)
> @@ -1411,7 +1418,7 @@ static void rcu_free_expedite_workfn(struct work_struct *work)
> rcu_free_take(r, &list);
> spin_unlock_irq(&r->lock);
>
> - if (rcu_free_flush(&list))
> + if (rcu_free_flush(&list, false))
> count_vm_event(PGFREE_RCU_EXPEDITE);
> }
>
> @@ -1445,6 +1452,33 @@ static void rcu_free_defer(struct folio *folio)
> queue_work(rcu_free_wq, &r->expedite);
> }
>
> +/*
> + * Free what every CPU's lists hold, straight to the buddy lists: the
> + * callers want to allocate the pages next. A concurrent drain holds the
> + * lock until it has freed what it took.
> + */
> +static void rcu_free_drain(void)
> +{
> + static DEFINE_MUTEX(lock);
> + LIST_HEAD(list);
> + int cpu;
> +
> + mutex_lock(&lock);
> + for_each_possible_cpu(cpu) {
> + struct rcu_free_pcpu *r = per_cpu_ptr(&rcu_free_pcpu, cpu);
> +
> + spin_lock_irq(&r->lock);
> + rcu_free_take(r, &list);
> + spin_unlock_irq(&r->lock);
> + }
> + if (rcu_free_flush(&list, true))
> + count_vm_event(PGFREE_RCU_DRAIN);
> + /* An expedite in flight holds pages it took off the lists */
> + for_each_possible_cpu(cpu)
> + flush_work(&per_cpu_ptr(&rcu_free_pcpu, cpu)->expedite);
> + mutex_unlock(&lock);
> +}
[Severity: High]
As reported by Sashiko on RFC [1], what happens if the pages counted in
NR_FREE_RCU_PAGES were already spliced off r->batch or r->pending by an
in-flight rcu_free_workfn() or rcu_free_expedite_workfn() when
rcu_free_drain() runs?
First, rcu_free_workfn() splices r->batch onto its stack-local done list
under r->lock and then drops the lock before calling
rcu_free_pages(&done, false), which decrements NR_FREE_RCU_PAGES one folio
at a time and can sleep at cond_resched_tasks_rcu_qs(). Because
rcu_free_drain() only calls flush_work(&r->expedite) and does not flush
r->rwork.work, rcu_free_take() can find the per-CPU lists empty and
rcu_free_drain() can return before rcu_free_workfn() finishes freeing its
done list.
Second, both rcu_free_workfn() and rcu_free_expedite_workfn() call
rcu_free_pages() with direct set to false, which passes pcp_allowed_order()
folios to free_unref_folios() and places them onto the worker CPU's per-CPU
pageset lists rather than the buddy lists while decrementing
NR_FREE_RCU_PAGES to 0.
On the subsequent retry in __alloc_pages_slowpath(), pages sitting on PCP
lists are not counted in NR_FREE_PAGES, and __alloc_pages_direct_reclaim()
skips drain_all_pages(NULL) when __perform_reclaim() makes no progress. On
the next should_reclaim_retry() call, NR_FREE_RCU_PAGES is 0 and
NR_FREE_PAGES is still below min_wmark_pages(zone). Could this cause
__alloc_pages_slowpath() to proceed to __alloc_pages_may_oom() while those
freed pages remain stranded on PCP lists?
[ ... ]
> @@ -4792,6 +4826,58 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
> return !!__gfp_pfmemalloc_flags(gfp_mask);
> }
>
> +/*
> + * Reclaim is about to give up: if the pages waiting for a grace period,
> + * with what reclaim can still free, would bring an eligible zone above its
> + * min watermark, free them and retry. Not for a dying task, which frees
> + * memory by exiting.
> + */
> +static bool rcu_free_wait(gfp_t gfp_mask, unsigned int order,
> + struct alloc_context *ac, int alloc_flags,
> + int *rcu_free_waits)
> +{
> + struct zone *zone;
> + struct zoneref *z;
> +
> + if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
> + return false;
> + if (tsk_is_oom_victim(current) || fatal_signal_pending(current))
> + return false;
> + if (*rcu_free_waits >= MAX_RECLAIM_RETRIES ||
> + order > PAGE_ALLOC_COSTLY_ORDER)
> + return false;
> +
> + for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
> + ac->highest_zoneidx, ac->nodemask) {
> + unsigned long waiting, available;
> +
> + if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
> + !__cpuset_zone_allowed(zone, gfp_mask))
> + continue;
> +
> + waiting = zone_page_state_snapshot(zone, NR_FREE_RCU_PAGES);
> + if (!waiting)
> + continue;
> +
> + available = waiting + zone_reclaimable_pages(zone) +
> + zone_page_state_snapshot(zone, NR_FREE_PAGES);
> + /*
> + * Count pages only: for order > 0 __zone_watermark_ok() also
> + * wants a free block of that order, and the waiting pages are
> + * not free yet.
> + */
> + if (__zone_watermark_ok(zone, 0, min_wmark_pages(zone),
> + ac->highest_zoneidx, alloc_flags,
> + available)) {
[Severity: Medium]
When should_reclaim_retry() reaches out: via the
*no_progress_loops > MAX_RECLAIM_RETRIES check, direct reclaim has failed
to make progress 16 times in a row even though
zone_reclaimable_pages(zone) + NR_FREE_PAGES was above min_wmark_pages(zone)
(for instance under GFP_NOFS or when remaining LRU pages are dirty, under
writeback, or pinned).
In that situation, since rcu_free_wait() still includes
zone_reclaimable_pages(zone) in available, won't
zone_reclaimable_pages(zone) + NR_FREE_PAGES already exceed
min_wmark_pages(zone) on its own, causing even a single waiting page
(waiting == 1) to trigger up to MAX_RECLAIM_RETRIES additional
rcu_free_drain() calls and no-progress reclaim loops while also skipping
the PF_WQ_WORKER sleep in should_reclaim_retry()?
> + (*rcu_free_waits)++;
> + rcu_free_drain();
> + return true;
> + }
> + }
> +
> + return false;
> +}
> +
> /*
> * Checks whether it makes sense to retry the reclaim to make a forward progress
> * for the given allocation request.
> @@ -4805,7 +4891,8 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
> static inline bool
> should_reclaim_retry(gfp_t gfp_mask, unsigned order,
> struct alloc_context *ac, int alloc_flags,
> - bool did_some_progress, int *no_progress_loops)
> + bool did_some_progress, int *no_progress_loops,
> + int *rcu_free_waits)
> {
> struct zone *zone;
> struct zoneref *z;
> @@ -4816,10 +4903,12 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
> * their order will become available due to high fragmentation so
> * always increment the no progress counter for them
> */
> - if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
> + if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) {
> *no_progress_loops = 0;
> - else
> + *rcu_free_waits = 0;
> + } else {
> (*no_progress_loops)++;
> + }
>
> if (*no_progress_loops > MAX_RECLAIM_RETRIES)
> goto out;
> @@ -4872,6 +4961,10 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
> else
> cond_resched();
> out:
> + if (!ret && rcu_free_wait(gfp_mask, order, ac, alloc_flags,
> + rcu_free_waits))
> + return true;
[Severity: High]
Can this starve compaction for non-costly high-order allocations
(1 <= order <= PAGE_ALLOC_COSTLY_ORDER)?
In should_reclaim_retry(), the zone loop calls:
wmark = __zone_watermark_ok(zone, order, min_wmark,
ac->highest_zoneidx, alloc_flags, available);
For order > 0, __zone_watermark_ok() requires a free block of at least that
order in zone->free_area[]. When a high-order allocation fails due to
fragmentation despite having plenty of order-0 memory
(reclaimable + NR_FREE_PAGES > min_wmark), should_reclaim_retry() previously
returned false so __alloc_pages_slowpath() could reach
should_compact_retry() (to escalate compact_priority toward
COMPACT_PRIO_SYNC_FULL) and relax ALLOC_NOFRAGMENT:
if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
did_some_progress > 0, &no_progress_loops,
&rcu_free_waits))
goto retry;
if (did_some_progress > 0 && can_compact &&
should_compact_retry(gfp_mask, ac, order, alloc_flags,
compact_result, &compact_priority,
&compaction_retries))
goto retry;
With rcu_free_wait() checking __zone_watermark_ok() at order 0, any non-zero
NR_FREE_RCU_PAGES count (including pages just evicted by direct reclaim
itself) satisfies the order-0 watermark check whenever order-0 memory is
already above min_wmark.
Furthermore, because __alloc_pages_direct_reclaim() sets did_some_progress
whenever it reclaims pages or sees sc->compaction_ready,
should_reclaim_retry() resets *rcu_free_waits = 0 on every iteration.
Doesn't this cause __alloc_pages_slowpath() to loop on goto retry via
should_reclaim_retry(), repeatedly firing synchronize_rcu_expedited() in
rcu_free_drain() and reclaiming page cache without ever reaching
should_compact_retry()?
[1] https://lore.kernel.org/r/20261019154913.1765974-7-kas@kernel.org
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104130118.2507513-1-kas@kernel.org?part=6
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 07/18] mm: free the pages waiting for a grace period before isolation fails
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (5 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 06/18] mm: free the pages waiting for a grace period before OOM Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-05 13:03 ` sashiko-bot
2026-11-04 13:01 ` [RFC PATCH v2 08/18] mm/compaction: free the migrated folios that wait for a grace period Kiryl Shutsemau (Meta)
` (10 subsequent siblings)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
alloc_contig_range() and offline_pages() migrate the folios out of a
range and then check with test_pages_isolated() that every page is free.
A migrated folio that had PG_rcu_free waits for a grace period after its
free, so the check fails. alloc_contig_range() returns -EBUSY, and a
CMA or gigantic page allocation can fail; offline_pages() retries until
the grace period ends.
When the check fails and the zone has pages waiting, free them first.
alloc_contig_range() then finds the start of the free range again, as
the freed folio can merge with a free buddy below it.
offline_pages() does so only after a pass that migrated folios: an
offline stuck on a busy page would otherwise start an expedited grace
period on every retry.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/memory_hotplug.c | 6 ++++++
mm/page_alloc.c | 24 +++++++++++++++++++++++-
mm/page_alloc.h | 8 ++++++++
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 226ab9cb078a..c23fab652957 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1967,6 +1967,7 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
.nid = NUMA_NO_NODE,
};
unsigned long flags;
+ bool migrated;
char *reason;
int ret;
unsigned long normal_pages = 0;
@@ -2050,6 +2051,7 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
do {
pfn = start_pfn;
+ migrated = false;
do {
/*
* Historically we always checked for any signal and
@@ -2071,6 +2073,7 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
* out
*/
do_migrate_range(pfn, end_pfn);
+ migrated = true;
}
} while (!ret);
@@ -2092,6 +2095,9 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
ret = test_pages_isolated(start_pfn, end_pfn,
PB_ISOLATE_MODE_MEM_OFFLINE);
+ /* What this pass migrated can wait for a grace period */
+ if (ret && migrated && zone_rcu_free_waiting(zone))
+ rcu_free_sync();
} while (ret);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 9caaa57bf51d..eea287ab2a01 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1479,6 +1479,19 @@ static void rcu_free_drain(void)
mutex_unlock(&lock);
}
+/* Free every waiting folio, including those a work took off the lists */
+void rcu_free_sync(void)
+{
+ int cpu;
+
+ if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
+ return;
+
+ rcu_free_drain();
+ for_each_possible_cpu(cpu)
+ flush_work(&per_cpu_ptr(&rcu_free_pcpu, cpu)->rwork.work);
+}
+
static void __init rcu_free_init(void)
{
int cpu;
@@ -7649,7 +7662,16 @@ int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end,
outer_start = find_large_buddy(start);
/* Make sure the range is really isolated. */
- if (test_pages_isolated(outer_start, end, mode)) {
+ ret = test_pages_isolated(outer_start, end, mode);
+ if (ret && zone_rcu_free_waiting(cc.zone)) {
+ rcu_free_sync();
+
+ /* A freed folio can merge with a free buddy below start */
+ outer_start = find_large_buddy(start);
+ ret = test_pages_isolated(outer_start, end, mode);
+ }
+
+ if (ret) {
ret = -EBUSY;
goto done;
}
diff --git a/mm/page_alloc.h b/mm/page_alloc.h
index b9259deddb59..764e3f43b3b9 100644
--- a/mm/page_alloc.h
+++ b/mm/page_alloc.h
@@ -235,6 +235,14 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid,
alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
void free_frozen_pages(struct page *page, unsigned int order);
void free_unref_folios(struct folio_batch *fbatch);
+void rcu_free_sync(void);
+
+/* Whether @zone has pages waiting for a grace period */
+static inline bool zone_rcu_free_waiting(struct zone *zone)
+{
+ return IS_ENABLED(CONFIG_PAGE_RCU_FREE) &&
+ zone_page_state_snapshot(zone, NR_FREE_RCU_PAGES);
+}
#ifdef CONFIG_NUMA
struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 07/18] mm: free the pages waiting for a grace period before isolation fails
2026-11-04 13:01 ` [RFC PATCH v2 07/18] mm: free the pages waiting for a grace period before isolation fails Kiryl Shutsemau (Meta)
@ 2026-11-05 13:03 ` sashiko-bot
0 siblings, 0 replies; 27+ messages in thread
From: sashiko-bot @ 2026-11-05 13:03 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
David Hildenbrand, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
kernel-team
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
- [Medium] mm: premature NR_FREE_RCU_PAGES decrement in rcu_free_pages() skips sync
- [Medium] mm: stale outer_start in alloc_contig_frozen_range_noprof() after RCU free merge
- [Medium] mm: page_is_unmovable() treats freed compound PG_rcu_free folios as unmovable
--
commit 01209427ff9390747e1e7bf6dbe80b5fa2e38b2b
Author: Kiryl Shutsemau (Meta) <kas@kernel.org>
mm: free the pages waiting for a grace period before isolation fails
This commit adds rcu_free_sync() and zone_rcu_free_waiting() so that
alloc_contig_frozen_range_noprof() and offline_pages() can drain and flush
folios waiting for an RCU grace period when test_pages_isolated() fails.
> diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
> index 226ab9cb078ad..c23fab6529571 100644
> --- a/mm/memory_hotplug.c
> +++ b/mm/memory_hotplug.c
> @@ -1967,6 +1967,7 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
> .nid = NUMA_NO_NODE,
> };
> unsigned long flags;
> + bool migrated;
> char *reason;
> int ret;
> unsigned long normal_pages = 0;
> @@ -2050,6 +2051,7 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
>
> do {
> pfn = start_pfn;
> + migrated = false;
> do {
> /*
> * Historically we always checked for any signal and
> @@ -2071,6 +2073,7 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
> * out
> */
> do_migrate_range(pfn, end_pfn);
> + migrated = true;
> }
> } while (!ret);
>
> @@ -2092,6 +2095,9 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
>
> ret = test_pages_isolated(start_pfn, end_pfn,
> PB_ISOLATE_MODE_MEM_OFFLINE);
> + /* What this pass migrated can wait for a grace period */
> + if (ret && migrated && zone_rcu_free_waiting(zone))
> + rcu_free_sync();
>
> } while (ret);
>
[Severity: Medium]
What happens if a large compound folio with PG_rcu_free was already freed
before offline_pages() or alloc_contig_frozen_range_noprof() calls
start_isolate_page_range()?
When __free_pages_prepare() intercepts a folio with PG_rcu_free and defers
it via rcu_free_defer(), the folio has a reference count of zero and
PageLRU cleared, while PageCompound remains set until rcu_free_pages()
later completes the free:
mm/page_alloc.c:__free_pages_prepare() {
...
if (unlikely(folio_test_rcu_free(folio)) &&
!(PageHWPoison(page) && !order)) {
VM_WARN_ON_ONCE(fpi_flags & FPI_NOLOCK);
__folio_clear_rcu_free(folio);
count_vm_events(PGFREE_RCU, 1 << order);
rcu_free_defer(folio);
return false;
}
...
}
Before reaching test_pages_isolated() and rcu_free_sync(), both
offline_pages() and alloc_contig_frozen_range_noprof() (as well as
alloc_contig_frozen_pages() via pfn_range_valid_contig()) check for
unmovable pages through:
start_isolate_page_range()
-> set_migratetype_isolate()
-> has_unmovable_pages()
-> page_is_unmovable()
In page_is_unmovable() (outside ZONE_MOVABLE), order-0 pages with a zero
reference count fall through to the !page_ref_count(page) check and are
treated as movable, but compound pages are checked first and rejected when
!PageLRU(page) without checking !folio_ref_count(folio):
mm/page_isolation.c:page_is_unmovable() {
...
if (PageCompound(page)) {
struct folio *folio = page_folio(page);
...
if (folio_test_hugetlb(folio)) {
...
} else if (!PageLRU(page)) {
return true;
}
...
}
Won't start_isolate_page_range() or pfn_range_valid_contig() treat a freed
compound folio waiting for an RCU grace period as unmovable and fail with
-EBUSY before rcu_free_sync() is ever reached?
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index 9caaa57bf51d7..eea287ab2a011 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -1479,6 +1479,19 @@ static void rcu_free_drain(void)
> mutex_unlock(&lock);
> }
>
> +/* Free every waiting folio, including those a work took off the lists */
> +void rcu_free_sync(void)
> +{
> + int cpu;
> +
> + if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
> + return;
> +
> + rcu_free_drain();
> + for_each_possible_cpu(cpu)
> + flush_work(&per_cpu_ptr(&rcu_free_pcpu, cpu)->rwork.work);
> +}
> +
> static void __init rcu_free_init(void)
> {
> int cpu;
> @@ -7649,7 +7662,16 @@ int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end,
> outer_start = find_large_buddy(start);
>
> /* Make sure the range is really isolated. */
> - if (test_pages_isolated(outer_start, end, mode)) {
> + ret = test_pages_isolated(outer_start, end, mode);
> + if (ret && zone_rcu_free_waiting(cc.zone)) {
> + rcu_free_sync();
> +
> + /* A freed folio can merge with a free buddy below start */
> + outer_start = find_large_buddy(start);
> + ret = test_pages_isolated(outer_start, end, mode);
> + }
> +
> + if (ret) {
> ret = -EBUSY;
> goto done;
> }
[Severity: Medium]
Can outer_start in alloc_contig_frozen_range_noprof() still become stale if
a background RCU free completes and merges across start or outer_start?
First, suppose a migrated folio at start had PG_rcu_free and a free buddy
exists immediately below start in the same isolated pageblock. When the
initial find_large_buddy(start) runs, the folio at start is not yet freed,
so find_large_buddy(start) returns start:
mm/page_alloc.c:find_large_buddy() {
...
if (pfn + (1 << buddy_order(page)) > start_pfn)
return pfn;
/* Nothing found */
return start_pfn;
}
If rcu_free_workfn() or rcu_free_expedite_workfn() on another CPU finishes
freeing that folio before zone_rcu_free_waiting(cc.zone) is checked,
__free_one_page() merges the folio with the preceding buddy below start,
clearing PageBuddy on start. Then test_pages_isolated(outer_start, end,
mode) fails because outer_start (still equal to start) is now a tail of the
merged buddy, and zone_rcu_free_waiting(cc.zone) returns false because
NR_FREE_RCU_PAGES was already decremented to 0, so outer_start is never
recomputed and alloc_contig_frozen_range_noprof() returns -EBUSY.
Second, suppose all pages in [start, end) are already free in the buddy
allocator when the initial find_large_buddy(start) and
test_pages_isolated(outer_start, end, mode) run, so test_pages_isolated()
returns 0 and rcu_free_sync() is not called. If a folio below outer_start
within the same isolated pageblock was waiting for an RCU grace period and
is freed by rcu_free_workfn() before isolate_freepages_range(&cc,
outer_start, end) runs:
mm/page_alloc.c:alloc_contig_frozen_range_noprof() {
...
/* Grab isolated pages from freelists. */
outer_end = isolate_freepages_range(&cc, outer_start, end);
if (!outer_end) {
ret = -EBUSY;
goto done;
}
...
}
__free_one_page() can merge that newly freed folio with the buddy at
outer_start, moving the buddy head to a lower PFN and causing
isolate_freepages_range() to fail with -EBUSY.
> diff --git a/mm/page_alloc.h b/mm/page_alloc.h
> index b9259deddb59d..764e3f43b3b98 100644
> --- a/mm/page_alloc.h
> +++ b/mm/page_alloc.h
> @@ -235,6 +235,14 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid,
> alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
> void free_frozen_pages(struct page *page, unsigned int order);
> void free_unref_folios(struct folio_batch *fbatch);
> +void rcu_free_sync(void);
> +
> +/* Whether @zone has pages waiting for a grace period */
> +static inline bool zone_rcu_free_waiting(struct zone *zone)
> +{
> + return IS_ENABLED(CONFIG_PAGE_RCU_FREE) &&
> + zone_page_state_snapshot(zone, NR_FREE_RCU_PAGES);
> +}
[Severity: Medium]
As reported by Sashiko on RFC [1], doesn't gating rcu_free_sync() on
zone_rcu_free_waiting() miss folios that are actively being freed inside
rcu_free_pages() after NR_FREE_RCU_PAGES has already been decremented?
In rcu_free_pages(), NR_FREE_RCU_PAGES is decremented for each folio before
__free_pages_ok() or free_unref_folios(&fbatch) prepares the pages and
returns them to the buddy freelists under zone->lock:
mm/page_alloc.c:rcu_free_pages() {
...
list_for_each_entry_safe(folio, next, list, lru) {
nr += folio_nr_pages(folio);
mod_zone_page_state(folio_zone(folio), NR_FREE_RCU_PAGES,
-folio_nr_pages(folio));
if (direct) {
__free_pages_ok(&folio->page, folio_order(folio),
FPI_NONE);
cond_resched();
} else if (!folio_batch_add(&fbatch, folio)) {
free_unref_folios(&fbatch);
cond_resched_tasks_rcu_qs();
}
}
if (folio_batch_count(&fbatch))
free_unref_folios(&fbatch);
...
}
When rcu_free_workfn(), rcu_free_expedite_workfn(), or a concurrent
rcu_free_drain() decrements NR_FREE_RCU_PAGES to 0 for the last waiting
folios in the zone, there is a window while __free_pages_prepare() (page
zeroing, KASAN poisoning, memcg uncharging, etc.) and free_one_page() are
still running where the pages are not yet PageBuddy in the buddy allocator,
but zone_rcu_free_waiting(zone) already reads 0 and returns false.
Although rcu_free_sync() flushes rwork.work and expedite works to wait for
folios taken off the per-CPU lists, both alloc_contig_frozen_range_noprof()
and offline_pages() skip calling rcu_free_sync() when
zone_rcu_free_waiting() returns false, causing
alloc_contig_frozen_range_noprof() to return -EBUSY and offline_pages() to
skip synchronizing on the pass that migrated folios.
[1] https://lore.kernel.org/r/20261019154913.1765974-8-kas@kernel.org
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104130118.2507513-1-kas@kernel.org?part=7
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 08/18] mm/compaction: free the migrated folios that wait for a grace period
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (6 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 07/18] mm: free the pages waiting for a grace period before isolation fails Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-05 13:03 ` sashiko-bot
2026-11-04 13:01 ` [RFC PATCH v2 09/18] mm/memory-failure: handle folios with PG_rcu_free Kiryl Shutsemau (Meta)
` (9 subsequent siblings)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
A block that compaction has migrated every folio out of becomes free
once those folios are back in the buddy lists. A migrated folio that
had PG_rcu_free comes back only after a grace period, so compaction
finds the block still in use, migrates more blocks than it needs and
fails more often. defrag_mode relies on compaction making whole free
blocks, and there it shows up as allocation stalls.
When compaction moves past a block it migrated from and the zone has
pages waiting, free them before it drains the per-cpu lists and checks
whether it is done. The drain costs an expedited grace period, so
async compaction, which must not block, skips it.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/compaction.c | 8 ++++++++
mm/page_alloc.c | 2 +-
mm/page_alloc.h | 1 +
3 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/mm/compaction.c b/mm/compaction.c
index a049415512c6..2acb87a3446d 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2760,6 +2760,14 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
block_start_pfn(cc->migrate_pfn, cc->order);
if (last_migrated_pfn < current_block_start) {
+ /*
+ * Folios migrated away from lockless readers
+ * are freed after a grace period. Async
+ * compaction does not wait for it.
+ */
+ if (cc->mode != MIGRATE_ASYNC &&
+ zone_rcu_free_waiting(cc->zone))
+ rcu_free_drain();
lru_add_drain_cpu_zone(cc->zone);
/* No more flushing until we migrate again */
last_migrated_pfn = 0;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index eea287ab2a01..afd4343ad572 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1457,7 +1457,7 @@ static void rcu_free_defer(struct folio *folio)
* callers want to allocate the pages next. A concurrent drain holds the
* lock until it has freed what it took.
*/
-static void rcu_free_drain(void)
+void rcu_free_drain(void)
{
static DEFINE_MUTEX(lock);
LIST_HEAD(list);
diff --git a/mm/page_alloc.h b/mm/page_alloc.h
index 764e3f43b3b9..4f5f06adbfdc 100644
--- a/mm/page_alloc.h
+++ b/mm/page_alloc.h
@@ -235,6 +235,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid,
alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
void free_frozen_pages(struct page *page, unsigned int order);
void free_unref_folios(struct folio_batch *fbatch);
+void rcu_free_drain(void);
void rcu_free_sync(void);
/* Whether @zone has pages waiting for a grace period */
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 08/18] mm/compaction: free the migrated folios that wait for a grace period
2026-11-04 13:01 ` [RFC PATCH v2 08/18] mm/compaction: free the migrated folios that wait for a grace period Kiryl Shutsemau (Meta)
@ 2026-11-05 13:03 ` sashiko-bot
0 siblings, 0 replies; 27+ messages in thread
From: sashiko-bot @ 2026-11-05 13:03 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
David Hildenbrand, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
kernel-team
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] mm/compaction: compact_zone() misses pages captured in rcu_free_drain()
--
commit 0e470d35ade0bceb30271d7e57f7ca44bff99bf7
Author: Kiryl Shutsemau (Meta) <kas@kernel.org>
mm/compaction: free the migrated folios that wait for a grace period
This commit drains deferred RCU-freed folios in compact_zone() when non-async
compaction moves past a block it migrated from and the zone has pages waiting
for a grace period.
> diff --git a/mm/compaction.c b/mm/compaction.c
> index a049415512c67..2acb87a3446df 100644
> --- a/mm/compaction.c
> +++ b/mm/compaction.c
> @@ -2760,6 +2760,14 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
> block_start_pfn(cc->migrate_pfn, cc->order);
>
> if (last_migrated_pfn < current_block_start) {
> + /*
> + * Folios migrated away from lockless readers
> + * are freed after a grace period. Async
> + * compaction does not wait for it.
> + */
> + if (cc->mode != MIGRATE_ASYNC &&
> + zone_rcu_free_waiting(cc->zone))
> + rcu_free_drain();
> lru_add_drain_cpu_zone(cc->zone);
> /* No more flushing until we migrate again */
> last_migrated_pfn = 0;
[Severity: Medium]
Can compact_zone() miss a page that gets captured into capc->page during
rcu_free_drain() or lru_add_drain_cpu_zone()?
During direct compaction, current->capture_control points to capc. When a
migrated block contains a folio with PG_rcu_free set, migrate_folio_done() ->
folio_put() -> __free_pages_prepare() defers freeing it via rcu_free_defer().
Because the block has not coalesced yet, capc->page is still NULL when
compact_zone() checks it before the check_drain label:
mm/compaction.c:compact_zone() {
...
/* Stop if a page has been captured */
if (capc && capc->page) {
ret = COMPACT_SUCCESS;
break;
}
check_drain:
...
}
When check_drain then runs rcu_free_drain() and lru_add_drain_cpu_zone(), the
deferred pages are freed in current's context through __free_one_page(). Once
the block coalesces to capc->order, compaction_capture() stores the page in
capc->page and __free_one_page() returns early without adding the page to
zone->free_area[order]:
mm/page_alloc.c:__free_one_page() {
...
if (compaction_capture(capc, page, order, migratetype)) {
account_freepages(zone, -(1 << order), migratetype);
return;
}
...
}
After check_drain finishes, the while loop condition calls
compact_finished() -> __compact_finished(), which only checks whether
cc->zone->free_area[order] is non-empty and does not check capc->page. Since
the captured page was not placed on the free list, compact_finished() returns
COMPACT_CONTINUE and compact_zone() proceeds to isolate and migrate another
block even though the allocation has already been captured.
Should compact_zone() recheck capc && capc->page after draining (or check it
in __compact_finished())?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104130118.2507513-1-kas@kernel.org?part=8
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 09/18] mm/memory-failure: handle folios with PG_rcu_free
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (7 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 08/18] mm/compaction: free the migrated folios that wait for a grace period Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-05 13:03 ` sashiko-bot
2026-11-04 13:01 ` [RFC PATCH v2 10/18] mm: have folio_mark_accessed() return whether the folio is hot Kiryl Shutsemau (Meta)
` (8 subsequent siblings)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
A page whose free waits for an RCU grace period has refcount 0 and is
neither on the LRU nor in the buddy allocator. memory_failure() and
soft_offline_page() cannot handle such a page: get_any_page() retries a
few times and gives up. Soft offline of the page fails, and a hard
error is ignored. If the page is part of a large folio, the folio is
later freed with the poisoned page in it, and that page can be handed
out again.
Free the waiting pages first when the page looks like one of them, and
again on each retry in get_any_page(): the page can be freed while it
retries. memory_failure() has to do it before it sets PG_hwpoison. The
free of a poisoned order-0 page drops the page instead of returning it
to the buddy allocator, so a later drain would still leave the error
ignored.
A folio with PG_rcu_free that is still in use may be copied by a reader
that holds no reference. When memory_failure() succeeds, its caller can
unmap the page from the direct map under such a reader. Wait for a
grace period before returning success. By then the page has left the
page cache, or is an order-0 folio with PG_hwpoison, which a reader that
starts after the grace period has to check.
Wait at the end, not right after setting PG_hwpoison. A large folio is
split later, and until the new folios are in the page cache, a reader
can still copy the poisoned page as part of the large one.
Both callers hold mf_mutex and can wait for the grace period.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/memory-failure.c | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 7dde7c3c9a0c..398463b2e82d 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -1419,6 +1419,19 @@ static int __get_hwpoison_page(struct page *page, unsigned long flags)
return 0;
}
+/* A page waiting for a grace period is not in the buddy yet: free it */
+static void mf_free_rcu_pages(struct page *p)
+{
+ if (!zone_rcu_free_waiting(page_zone(p)))
+ return;
+ if (page_count(p))
+ return;
+ if (is_free_buddy_page(p))
+ return;
+
+ rcu_free_sync();
+}
+
#define GET_PAGE_MAX_RETRY_NUM 3
static int get_any_page(struct page *p, unsigned long flags)
@@ -1443,6 +1456,7 @@ static int get_any_page(struct page *p, unsigned long flags)
}
try_again:
+ mf_free_rcu_pages(p);
if (!count_increased) {
ret = __get_hwpoison_page(p, flags);
if (!ret) {
@@ -2440,6 +2454,7 @@ int memory_failure(unsigned long pfn, int flags)
if (res != -ENOENT)
goto unlock_mutex;
+ mf_free_rcu_pages(p);
if (TestSetPageHWPoison(p)) {
res = -EHWPOISON;
if (flags & MF_ACTION_REQUIRED)
@@ -2447,6 +2462,13 @@ int memory_failure(unsigned long pfn, int flags)
if (flags & MF_COUNT_INCREASED)
put_page(p);
action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);
+ /*
+ * The memory_failure() that set PG_hwpoison may have failed
+ * without waiting for lockless readers, and the caller may
+ * unmap the page once this returns 0.
+ */
+ if (!res && folio_test_rcu_free(page_folio(p)))
+ synchronize_rcu();
goto unlock_mutex;
}
@@ -2612,6 +2634,13 @@ int memory_failure(unsigned long pfn, int flags)
identify_page_state:
res = identify_page_state(pfn, p, page_flags);
+ /*
+ * The caller may unmap the page once this returns 0. Lockless
+ * readers that started before the page got PG_hwpoison, or before a
+ * split left it in a folio of its own, may still be copying it.
+ */
+ if (!res && folio_test_rcu_free(folio))
+ synchronize_rcu();
mutex_unlock(&mf_mutex);
return res;
unlock_page:
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 09/18] mm/memory-failure: handle folios with PG_rcu_free
2026-11-04 13:01 ` [RFC PATCH v2 09/18] mm/memory-failure: handle folios with PG_rcu_free Kiryl Shutsemau (Meta)
@ 2026-11-05 13:03 ` sashiko-bot
0 siblings, 0 replies; 27+ messages in thread
From: sashiko-bot @ 2026-11-05 13:03 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
David Hildenbrand, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
kernel-team
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
- [High] mm/memory-failure: unsafe RCU wait in memory_failure() already-poisoned path
- [High] mm/memory-failure: mf_free_rcu_pages() skips sync while rcu_free_pages() runs
- [Medium] mm/memory-failure: order-0 PG_rcu_free page dropped after setting PG_hwpoison
--
commit 3d8069abfad786882a48d6ce85e757674239a9ca
Author: Kiryl Shutsemau (Meta) <kas@kernel.org>
mm/memory-failure: handle folios with PG_rcu_free
This commit updates memory_failure() and get_any_page() to drain pages
waiting for an RCU grace period via rcu_free_sync() and waits for a
grace period via synchronize_rcu() before returning success on a
PG_rcu_free folio.
> diff --git a/mm/memory-failure.c b/mm/memory-failure.c
> index 7dde7c3c9a0c5..398463b2e82d5 100644
> --- a/mm/memory-failure.c
> +++ b/mm/memory-failure.c
> @@ -1419,6 +1419,19 @@ static int __get_hwpoison_page(struct page *page, unsigned long flags)
> return 0;
> }
>
> +/* A page waiting for a grace period is not in the buddy yet: free it */
> +static void mf_free_rcu_pages(struct page *p)
> +{
> + if (!zone_rcu_free_waiting(page_zone(p)))
> + return;
> + if (page_count(p))
> + return;
> + if (is_free_buddy_page(p))
> + return;
> +
> + rcu_free_sync();
> +}
[Severity: High]
As reported by Sashiko on RFC [1], can mf_free_rcu_pages() return early
without calling rcu_free_sync() while rcu_free_pages() is still freeing
the target folio?
In mm/page_alloc.c:rcu_free_pages(), NR_FREE_RCU_PAGES is decremented
before the folios are actually freed to the buddy allocator:
mm/page_alloc.c:rcu_free_pages() {
list_for_each_entry_safe(folio, next, list, lru) {
nr += folio_nr_pages(folio);
mod_zone_page_state(folio_zone(folio), NR_FREE_RCU_PAGES,
-folio_nr_pages(folio));
if (direct) {
__free_pages_ok(&folio->page, folio_order(folio),
FPI_NONE);
cond_resched();
} else if (!folio_batch_add(&fbatch, folio)) {
free_unref_folios(&fbatch);
cond_resched_tasks_rcu_qs();
}
}
if (folio_batch_count(&fbatch))
free_unref_folios(&fbatch);
}
Similarly, during folio release, folio_put_testzero() drops _refcount to
0 before __free_pages_prepare() -> rcu_free_defer() increments
NR_FREE_RCU_PAGES.
When the last waiting folios in a zone are in either window,
page_count(p) is 0, is_free_buddy_page(p) is false, and
zone_rcu_free_waiting(page_zone(p)) evaluates to false.
If memory_failure() calls mf_free_rcu_pages(p) during this window, it
skips rcu_free_sync() and proceeds to TestSetPageHWPoison(p) before
__free_pages_prepare() runs. For an order-0 page, __free_pages_prepare()
then sees PageHWPoison(page) && !order and drops the page instead of
placing it in the buddy allocator, causing get_hwpoison_page() to fail
with -EIO and leaving the hardware error ignored as MF_IGNORED.
In addition, because memory_failure() calls mf_free_rcu_pages(p) before
get_hwpoison_page() calls zone_pcp_disable(), if rcu_free_workfn() or
rcu_free_expedite_workfn() already spliced p off the per-CPU list,
rcu_free_pages(..., false) -> free_unref_folios() will free p onto a
per-CPU pageset (PCP) list rather than the buddy allocator before
TestSetPageHWPoison(p) runs.
> +
> #define GET_PAGE_MAX_RETRY_NUM 3
>
> static int get_any_page(struct page *p, unsigned long flags)
> @@ -1443,6 +1456,7 @@ static int get_any_page(struct page *p, unsigned long flags)
> }
>
> try_again:
> + mf_free_rcu_pages(p);
> if (!count_increased) {
[Severity: Medium]
As reported by Sashiko on RFC [2], what happens in memory_failure() if an
in-use order-0 folio with PG_rcu_free is freed on another CPU after the
initial mf_free_rcu_pages(p) call?
In memory_failure(), the initial mf_free_rcu_pages(p) call returns
immediately when page_count(p) > 0:
mm/memory-failure.c:memory_failure() {
mf_free_rcu_pages(p);
if (TestSetPageHWPoison(p)) {
...
}
res = get_hwpoison_page(p, flags);
}
If another CPU drops the last reference on that order-0 folio just before
TestSetPageHWPoison(p), __free_pages_prepare() queues the folio via
rcu_free_defer(folio), and TestSetPageHWPoison(p) then sets PG_hwpoison on
p while it is on the RCU free list. When get_hwpoison_page() ->
get_any_page() then calls mf_free_rcu_pages(p) at try_again,
rcu_free_sync() drains p through __free_pages_ok() ->
__free_pages_prepare():
mm/page_alloc.c:__free_pages_prepare() {
if (unlikely(folio_test_rcu_free(folio)) &&
!(PageHWPoison(page) && !order)) {
...
rcu_free_defer(folio);
return false;
}
...
if (unlikely(PageHWPoison(page)) && !order) {
/* Do not let hwpoison pages hit pcplists/buddy */
...
return false;
}
}
Because PageHWPoison(p) is already set and order is 0,
__free_pages_prepare() drops the page instead of placing it in the buddy
allocator. And if the last reference is dropped after
TestSetPageHWPoison(p) while get_any_page() is retrying,
__free_pages_prepare() sees PageHWPoison(page) && !order, skips
rcu_free_defer(folio) without waiting for an RCU grace period, and also
drops the page.
In both cases, is_free_buddy_page(p) remains false, get_any_page() fails
with -EIO after retrying, and memory_failure() reports MF_IGNORED without
waiting for lockless RCU readers or accounting the page as recovered.
[ ... ]
> @@ -2440,6 +2454,7 @@ int memory_failure(unsigned long pfn, int flags)
> if (res != -ENOENT)
> goto unlock_mutex;
>
> + mf_free_rcu_pages(p);
> if (TestSetPageHWPoison(p)) {
> res = -EHWPOISON;
> if (flags & MF_ACTION_REQUIRED)
> @@ -2447,6 +2462,13 @@ int memory_failure(unsigned long pfn, int flags)
> if (flags & MF_COUNT_INCREASED)
> put_page(p);
> action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);
> + /*
> + * The memory_failure() that set PG_hwpoison may have failed
> + * without waiting for lockless readers, and the caller may
> + * unmap the page once this returns 0.
> + */
> + if (!res && folio_test_rcu_free(page_folio(p)))
> + synchronize_rcu();
> goto unlock_mutex;
> }
[Severity: High]
As reported by Sashiko on RFC [3], can this already-poisoned path in
memory_failure() return 0 and allow the caller (such as kill_me_maybe() ->
set_mce_nospec()) to unmap pfn from the kernel direct map while lockless
RCU page-cache readers can still access p?
There appear to be two issues when a previous memory_failure() call set
PG_hwpoison on p and then failed before isolating the page (for example,
in get_hwpoison_page() or try_to_split_thp_page()), and a second
memory_failure(pfn, MF_ACTION_REQUIRED) call enters the
TestSetPageHWPoison(p) branch with kill_accessing_process() returning 0:
1. In the TestSetPageHWPoison(p) branch, put_page(p) drops the
MF_COUNT_INCREASED reference before folio_test_rcu_free(page_folio(p)) is
checked, and no reference is held on p during kill_accessing_process() if
MF_COUNT_INCREASED was not set. If the folio's last reference is dropped
after mf_free_rcu_pages(p) (either by put_page(p) or concurrently on
another CPU), __free_pages_prepare() immediately clears PG_rcu_free before
deferring the free:
mm/page_alloc.c:__free_pages_prepare() {
if (unlikely(folio_test_rcu_free(folio)) &&
!(PageHWPoison(page) && !order)) {
VM_WARN_ON_ONCE(fpi_flags & FPI_NOLOCK);
__folio_clear_rcu_free(folio);
count_vm_events(PGFREE_RCU, 1 << order);
rcu_free_defer(folio);
return false;
}
}
When that happens, folio_test_rcu_free(page_folio(p)) evaluates to false
and skips synchronize_rcu() while the folio is still waiting for its RCU
grace period and lockless readers that looked up the folio before it was
freed are still copying from p.
2. If p is part of a large folio and the first memory_failure() failed in
get_hwpoison_page() (before folio_set_has_hwpoisoned(folio)) or in
try_to_split_thp_page():
mm/memory-failure.c:memory_failure() {
if (err || new_order) {
/* get folio again in case the original one is split */
folio = page_folio(p);
res = -EHWPOISON;
kill_procs_now(p, pfn, flags, folio);
put_page(p);
action_result(pfn, MF_MSG_UNSPLIT_THP, MF_FAILED);
goto unlock_mutex;
}
}
the unsplit large folio remains in the page cache with PG_hwpoison only
set on subpage p. Even after synchronize_rcu() completes, new lockless
page-cache readers can still copy the poisoned subpage as part of the
large folio while the caller unmaps p from the direct map.
[1] https://lore.kernel.org/r/20261019154913.1765974-9-kas@kernel.org
[2] https://lore.kernel.org/r/20261019154913.1765974-9-kas@kernel.org
[3] https://lore.kernel.org/r/20261019154913.1765974-9-kas@kernel.org
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104130118.2507513-1-kas@kernel.org?part=9
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 10/18] mm: have folio_mark_accessed() return whether the folio is hot
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (8 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 09/18] mm/memory-failure: handle folios with PG_rcu_free Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 11/18] mm/filemap: mark a folio accessed in one place in filemap_read() Kiryl Shutsemau (Meta)
` (7 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
Each call of folio_mark_accessed() moves a folio one step up what the
LRU records of its accesses. Once the folio is at the top, the call
leaves it as it was: the folio is as hot as the LRU can tell.
Return whether the call found the folio there. On the classic LRU
that is active, referenced and not idle. On the multi-gen LRU it is
PG_referenced, all the refs bits and PG_workingset, which
lru_gen_inc_refs() now reports.
An unevictable folio counts as hot once it is referenced, and on the
multi-gen LRU from the first call: it stays in memory in any case. A
dropbehind folio is not hot: it is dropped after the read.
No caller uses the value yet. Preparation for reading the folios that
it calls hot without a folio reference.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: shorter changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/swap.h | 2 +-
mm/folio.c | 46 ++++++++++++++++++++++++++++++--------------
2 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 5658a1634b85..67aa5271ae6c 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -295,7 +295,7 @@ extern unsigned long totalreserve_pages;
/* linux/mm/folio.c */
void folio_add_lru(struct folio *folio);
-void folio_mark_accessed(struct folio *folio);
+bool folio_mark_accessed(struct folio *folio);
void lru_add_drain_all(void);
enum lru_cache_drained {
diff --git a/mm/folio.c b/mm/folio.c
index 50a6dbe55998..6144b9cf301b 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -352,28 +352,34 @@ static void __lru_cache_activate_folio(struct folio *folio)
#ifdef CONFIG_LRU_GEN
-static void lru_gen_inc_refs(struct folio *folio)
+/*
+ * Return true if @folio is hot: unevictable, or its flags already held as
+ * many accesses as they can.
+ */
+static bool lru_gen_inc_refs(struct folio *folio)
{
unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
if (folio_test_unevictable(folio))
- return;
+ return true;
/* see the comment on LRU_REFS_FLAGS */
if (!folio_test_referenced(folio)) {
set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
- return;
+ return false;
}
do {
if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
- if (!folio_test_workingset(folio))
- folio_set_workingset(folio);
- return;
+ if (folio_test_workingset(folio))
+ return true;
+ folio_set_workingset(folio);
+ return false;
}
new_flags = old_flags + BIT(LRU_REFS_PGOFF);
} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+ return false;
}
static bool lru_gen_clear_refs(struct folio *folio)
@@ -396,8 +402,9 @@ static bool lru_gen_clear_refs(struct folio *folio)
#else /* !CONFIG_LRU_GEN */
-static void lru_gen_inc_refs(struct folio *folio)
+static bool lru_gen_inc_refs(struct folio *folio)
{
+ return false;
}
static bool lru_gen_clear_refs(struct folio *folio)
@@ -419,15 +426,20 @@ static bool lru_gen_clear_refs(struct folio *folio)
*
* When a newly allocated folio is not yet visible, so safe for non-atomic ops,
* __folio_set_referenced() may be substituted for folio_mark_accessed().
+ *
+ * Return: true if @folio is hot: it was already as accessed as the LRU
+ * records. On the classic LRU that is active or unevictable, referenced
+ * and not idle; on the multi-gen LRU, unevictable, or PG_referenced, all
+ * the refs bits and PG_workingset set. False for a dropbehind folio.
*/
-void folio_mark_accessed(struct folio *folio)
+bool folio_mark_accessed(struct folio *folio)
{
+ bool hot = false;
+
if (folio_test_dropbehind(folio))
- return;
- if (lru_gen_enabled()) {
- lru_gen_inc_refs(folio);
- return;
- }
+ return false;
+ if (lru_gen_enabled())
+ return lru_gen_inc_refs(folio);
if (!folio_test_referenced(folio)) {
folio_set_referenced(folio);
@@ -437,6 +449,7 @@ void folio_mark_accessed(struct folio *folio)
* this list is never rotated or maintained, so marking an
* unevictable page accessed has no effect.
*/
+ hot = true;
} else if (!folio_test_active(folio)) {
/*
* If the folio is on the LRU, queue it for activation via
@@ -450,9 +463,14 @@ void folio_mark_accessed(struct folio *folio)
__lru_cache_activate_folio(folio);
folio_clear_referenced(folio);
workingset_activation(folio);
+ } else {
+ hot = true;
}
- if (folio_test_idle(folio))
+ if (folio_test_idle(folio)) {
folio_clear_idle(folio);
+ hot = false;
+ }
+ return hot;
}
EXPORT_SYMBOL(folio_mark_accessed);
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 11/18] mm/filemap: mark a folio accessed in one place in filemap_read()
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (9 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 10/18] mm: have folio_mark_accessed() return whether the folio is hot Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 12/18] mm/filemap: set PG_rcu_free on the folios that reads keep coming back to Kiryl Shutsemau (Meta)
` (6 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
filemap_read() calls folio_mark_accessed() in two places. Before the
copy loop it marks the first folio of the batch, unless the previous
read through the file ended in that folio. In the loop it marks every
later folio.
The test made for the first folio also says "mark" for every later
one. The loop gets to a later folio only after the folio before it was
copied to its end, so last_pos is at the later folio's first byte and
pos_same_folio() is false.
Do both in one helper, filemap_read_accessed(), called for each folio
in the loop. For the first folio the call moves below the test of
end_offset against the folio's start, which is false for it: the folio
holds ki_pos, and end_offset is beyond ki_pos.
No change in behaviour intended. Preparation for deciding in the
helper which folios are worth reading without a reference.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: shorter changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/filemap.c | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
diff --git a/mm/filemap.c b/mm/filemap.c
index 00fd89cf6f55..3508eea44ad9 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2759,6 +2759,20 @@ static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio)
return (pos1 >> shift == pos2 >> shift);
}
+/*
+ * filemap_read() is about to copy from @folio at @pos. @last_pos is where
+ * the previous read of this open file ended, or how far this one has got.
+ *
+ * Reading on in the folio that the last read ended in is not another access
+ * to it: a folio read in small pieces is marked accessed once.
+ */
+static void filemap_read_accessed(struct folio *folio, loff_t pos,
+ loff_t last_pos)
+{
+ if (!pos_same_folio(pos, last_pos - 1, folio))
+ folio_mark_accessed(folio);
+}
+
static void filemap_end_dropbehind_read(struct folio *folio)
{
if (!folio_test_dropbehind(folio))
@@ -2844,14 +2858,6 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
*/
writably_mapped = mapping_writably_mapped(mapping);
- /*
- * When a read accesses the same folio several times, only
- * mark it as accessed the first time.
- */
- if (!pos_same_folio(iocb->ki_pos, last_pos - 1,
- fbatch.folios[0]))
- folio_mark_accessed(fbatch.folios[0]);
-
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
size_t fsize = folio_size(folio);
@@ -2862,8 +2868,7 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
if (end_offset < folio_pos(folio))
break;
- if (i > 0)
- folio_mark_accessed(folio);
+ filemap_read_accessed(folio, iocb->ki_pos, last_pos);
/*
* If users can be writing to this folio using arbitrary
* virtual addresses, take care of potential aliasing
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 12/18] mm/filemap: set PG_rcu_free on the folios that reads keep coming back to
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (10 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 11/18] mm/filemap: mark a folio accessed in one place in filemap_read() Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 13/18] mm/filemap: read from the page cache without folio references Kiryl Shutsemau (Meta)
` (5 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
A folio with PG_rcu_free can be read without a reference, but its
memory is held back for a grace period after it is freed. That trade
is worth making only for folios that readers look up again and again.
Flagging every page cache folio would also delay the free of streaming
reads, truncated files and reclaimed cold data, which no reader comes
back to.
Set the flag in filemap_read(), under the read's reference, when the
folio turns out to be hot:
- folio_mark_accessed() finds it as accessed as the LRU records, or
- the read goes back to bytes that the previous read through the same
open file has just read.
filemap_read() does not mark accessed the folio that the previous read
ended in, so re-reads through one open file do not make a folio hotter.
The second test is for that folio.
Leave alone a folio whose next read needs a reference anyway: not
uptodate, readahead or dropbehind. Leave alone unevictable folios too.
folio_mark_accessed() calls them hot after at most one access, so every
ramfs folio that is read would get the flag. An initramfs deleted at
switch_root would then have the free of all its folios deferred.
AS_RCU_READ records that an inode has had a folio flagged, so that a
lockless read can skip the inodes that never had one.
pgfree_rcu_mark in /proc/vmstat counts the pages that get the flag.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: count the marks, AS_RCU_READ, 64-bit only, shorter comment,
rewrote the changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/pagemap.h | 12 +++++++++++
include/linux/vm_event_item.h | 1 +
mm/filemap.c | 40 +++++++++++++++++++++++++++++++++--
mm/vmstat.c | 1 +
4 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 0adfa6605653..6c7f58803eee 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -210,6 +210,7 @@ enum mapping_flags {
AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM = 9,
AS_KERNEL_FILE = 10, /* mapping for a fake kernel file that shouldn't
account usage to user cgroups */
+ AS_RCU_READ = 11, /* a folio has had PG_rcu_free */
/* Bits 16-25 are used for FOLIO_ORDER */
AS_FOLIO_ORDER_BITS = 5,
AS_FOLIO_ORDER_MIN = 16,
@@ -269,6 +270,17 @@ static inline bool mapping_unevictable(const struct address_space *mapping)
return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);
}
+static inline void mapping_set_rcu_read(struct address_space *mapping)
+{
+ if (!test_bit(AS_RCU_READ, &mapping->flags))
+ set_bit(AS_RCU_READ, &mapping->flags);
+}
+
+static inline bool mapping_rcu_read(const struct address_space *mapping)
+{
+ return test_bit(AS_RCU_READ, &mapping->flags);
+}
+
static inline void mapping_set_exiting(struct address_space *mapping)
{
set_bit(AS_EXITING, &mapping->flags);
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 7626a3e3a07a..c1a63f070aa7 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -37,6 +37,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
FOR_ALL_ZONES(PGSCAN_SKIP)
PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
PGFREE_RCU, PGFREE_RCU_EXPEDITE, PGFREE_RCU_DRAIN,
+ PGFREE_RCU_MARK,
PGFAULT, PGMAJFAULT,
PGLAZYFREED,
PGREUSE,
diff --git a/mm/filemap.c b/mm/filemap.c
index 3508eea44ad9..b732ecaa78a9 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2759,18 +2759,54 @@ static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio)
return (pos1 >> shift == pos2 >> shift);
}
+/*
+ * Reads keep coming back to @folio: free it only after an RCU grace period
+ * from now on. Not if its next read needs a reference anyway. Not if it
+ * is unevictable: folio_mark_accessed() calls such a folio hot after at
+ * most one access, so every ramfs folio that is read would get the flag.
+ */
+static void filemap_mark_hot(struct folio *folio)
+{
+ struct address_space *mapping;
+
+ if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
+ return;
+ if (folio_test_rcu_free(folio) || !folio_test_uptodate(folio) ||
+ folio_test_readahead(folio) || folio_test_dropbehind(folio) ||
+ folio_test_unevictable(folio))
+ return;
+ if (folio_test_set_rcu_free(folio))
+ return;
+
+ count_vm_events(PGFREE_RCU_MARK, folio_nr_pages(folio));
+ /* NULL if the folio was truncated meanwhile */
+ mapping = READ_ONCE(folio->mapping);
+ if (mapping)
+ mapping_set_rcu_read(mapping);
+}
+
/*
* filemap_read() is about to copy from @folio at @pos. @last_pos is where
* the previous read of this open file ended, or how far this one has got.
*
* Reading on in the folio that the last read ended in is not another access
* to it: a folio read in small pieces is marked accessed once.
+ *
+ * The folio is hot when folio_mark_accessed() finds it as accessed as the
+ * LRU records, or when this open file goes back to data that it has just
+ * read.
*/
static void filemap_read_accessed(struct folio *folio, loff_t pos,
loff_t last_pos)
{
- if (!pos_same_folio(pos, last_pos - 1, folio))
- folio_mark_accessed(folio);
+ bool hot;
+
+ if (pos_same_folio(pos, last_pos - 1, folio))
+ hot = pos < last_pos;
+ else
+ hot = folio_mark_accessed(folio);
+ if (hot)
+ filemap_mark_hot(folio);
}
static void filemap_end_dropbehind_read(struct folio *folio)
diff --git a/mm/vmstat.c b/mm/vmstat.c
index d994de1fd739..c27caad017e3 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1347,6 +1347,7 @@ const char * const vmstat_text[] = {
[I(PGFREE_RCU)] = "pgfree_rcu",
[I(PGFREE_RCU_EXPEDITE)] = "pgfree_rcu_expedite",
[I(PGFREE_RCU_DRAIN)] = "pgfree_rcu_drain",
+ [I(PGFREE_RCU_MARK)] = "pgfree_rcu_mark",
[I(PGFAULT)] = "pgfault",
[I(PGMAJFAULT)] = "pgmajfault",
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 13/18] mm/filemap: read from the page cache without folio references
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (11 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 12/18] mm/filemap: set PG_rcu_free on the folios that reads keep coming back to Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 14/18] mm/filemap: read large folios " Kiryl Shutsemau (Meta)
` (4 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
linux-kernel, linux-fsdevel, fuse-devel, kernel-team,
Kiryl Shutsemau (Meta)
From: Linus Torvalds <torvalds@linux-foundation.org>
Small reads of one file from many CPUs all take and drop a reference on
the same folio, and its refcount bounces between the CPUs.
Before taking a reference, copy what can be copied under rcu_read_lock()
from folios with PG_rcu_free, with page faults disabled. A flagged
folio is not reused before rcu_read_unlock(), so a folio still in its
slot after the flag test is this file's page until then. A folio that
leaves the page cache during the copy reads as the file did when it
left. Read i_size between the flag and the second look at the slot: a
write that extends the file fills the slot before it moves i_size.
Leave to the slow path a folio that is not uptodate, needs readahead or
dropbehind, is not hot, or has a poisoned page. Leave large folios
there too: looking one up for every page is slower than one reference
for all of its pages. A split clears PG_head before the new folios
replace the old one in their slots, so copy an order-0 folio only at its
own index. Copy at most 512 KiB per rcu_read_lock() section, and skip
files that never had a folio flagged or that are writably mapped where
flush_dcache_folio() is needed.
Declare the folio batch in the loop: with stack auto-initialisation,
the compiler zeroes it where it is declared, and the copy without
references does not use it.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[kas: i_size before the second look at the slot, poisoned folio check,
large folios left to the slow path, 512 KiB sections, folio batch in
the loop, shorter comments and changelog]
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/filemap.c | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 175 insertions(+), 3 deletions(-)
diff --git a/mm/filemap.c b/mm/filemap.c
index b732ecaa78a9..e99705a66c1d 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2821,6 +2821,168 @@ static void filemap_end_dropbehind_read(struct folio *folio)
}
}
+/*
+ * Whether filemap_read_rcu_batch() can copy from @folio, or has to leave it
+ * to the slow path, which holds a reference for readahead, dropbehind and
+ * folio_mark_accessed(), and deals with poisoned pages. @last_index is the
+ * page the previous copy ended in.
+ */
+static bool filemap_read_rcu_can_copy(struct folio *folio, pgoff_t index,
+ pgoff_t last_index)
+{
+ if (!folio_test_uptodate(folio))
+ return false;
+ if (folio_test_readahead(folio) || folio_test_dropbehind(folio))
+ return false;
+ /* One reference covers all of a large folio's pages */
+ if (folio_test_large(folio))
+ return false;
+ /* A split clears PG_head before it puts the new folios in their slots */
+ if (folio->index != index)
+ return false;
+ /* memory_failure() waits only for readers that got past this */
+ if (folio_test_hwpoison(folio))
+ return false;
+
+ /*
+ * The slow path does not call folio_mark_accessed() on the folio that
+ * the previous read ended in, so copying from it skips nothing. The
+ * folio's size can change without a reference: compare pages instead.
+ */
+ if (index == last_index)
+ return true;
+
+ /* Otherwise only if folio_mark_accessed() would leave it as it is */
+ if (lru_gen_enabled()) {
+ unsigned long all = LRU_REFS_FLAGS | BIT(PG_workingset);
+
+ return folio_test_unevictable(folio) ||
+ (READ_ONCE(folio->flags.f) & all) == all;
+ }
+ return folio_test_referenced(folio) && !folio_test_idle(folio) &&
+ (folio_test_active(folio) || folio_test_unevictable(folio));
+}
+
+/*
+ * Return the folio at @pos if it is this file's page until rcu_read_unlock(),
+ * and read i_size into @isize. A folio with PG_rcu_free is freed only after
+ * an RCU grace period. Without a reference, the folio that xas_load()
+ * returned can be freed and its memory reused as another folio before the
+ * flag is tested, so look at the slot again after the flag.
+ */
+static struct folio *filemap_read_rcu_lookup(struct xa_state *xas,
+ struct inode *inode, loff_t pos, loff_t *isize)
+{
+ struct folio *folio;
+
+ xas_set(xas, pos >> PAGE_SHIFT);
+ folio = xas_load(xas);
+ if (xas_retry(xas, folio) || !folio || xa_is_value(folio))
+ return NULL;
+
+ /* Not folio_test_rcu_free(): it may not be a folio any more */
+ if (!(READ_ONCE(folio->flags.f) & __PG_RCU_FREE))
+ return NULL;
+ /* Read i_size and slot after the flag: it implies uptodate */
+ smp_rmb();
+ *isize = i_size_read(inode);
+ if (pos >= *isize)
+ return NULL;
+ /* i_size_read() is an acquire; a write fills the slot first */
+ if (xas_reload(xas) != folio)
+ return NULL;
+ /* Read the rest of the folio only once the slot holds it */
+ smp_rmb();
+ return folio;
+}
+
+/* Bytes one RCU read section copies at most: it holds up grace periods */
+#define FILEMAP_RCU_READ_MAX SZ_512K
+
+/*
+ * Copy from folios with PG_rcu_free without taking references, at most
+ * FILEMAP_RCU_READ_MAX bytes under one rcu_read_lock(). Return true if that
+ * much was copied and the read is not done. *last_index is the page the
+ * previous copy ended in, set as the slow path sets last_pos.
+ */
+static bool filemap_read_rcu_batch(struct kiocb *iocb, struct iov_iter *iter,
+ pgoff_t *last_index, size_t *copied)
+{
+ struct address_space *mapping = iocb->ki_filp->f_mapping;
+ struct inode *inode = mapping->host;
+ XA_STATE(xas, &mapping->i_pages, 0);
+ size_t budget = FILEMAP_RCU_READ_MAX;
+ bool more = false;
+
+ /* flush_dcache_folio() is not known to be safe without a reference */
+ if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE &&
+ mapping_writably_mapped(mapping))
+ return false;
+
+ /* A user fault cannot sleep in the RCU read section: it ends the copy */
+ rcu_read_lock();
+ pagefault_disable();
+ while (iov_iter_count(iter)) {
+ loff_t pos = iocb->ki_pos;
+ pgoff_t index = pos >> PAGE_SHIFT;
+ size_t offset = offset_in_page(pos);
+ struct folio *folio;
+ size_t bytes, n;
+ loff_t isize;
+
+ if (!budget) {
+ more = true;
+ break;
+ }
+ folio = filemap_read_rcu_lookup(&xas, inode, pos, &isize);
+ if (!folio)
+ break;
+ if (!filemap_read_rcu_can_copy(folio, index, *last_index))
+ break;
+
+ bytes = min_t(loff_t, isize - pos, iov_iter_count(iter));
+ bytes = min3(bytes, budget, PAGE_SIZE - offset);
+ /* Not _nofault(): its usercopy check reads the folio's order */
+ n = copy_page_to_iter(folio_page(folio, index - folio->index),
+ offset, bytes, iter);
+ *copied += n;
+ iocb->ki_pos += n;
+ budget -= n;
+ if (n)
+ *last_index = (pos + n - 1) >> PAGE_SHIFT;
+ if (n < bytes)
+ break;
+ }
+ pagefault_enable();
+ rcu_read_unlock();
+
+ return more;
+}
+
+/*
+ * Copy from the page cache without folio references, in batches, and
+ * return how much was copied. The slow path reads the rest.
+ */
+static size_t filemap_read_rcu(struct kiocb *iocb, struct iov_iter *iter,
+ loff_t last_pos)
+{
+ /* The page the previous read ended in; ULONG_MAX if none */
+ pgoff_t last_index = (last_pos - 1) >> PAGE_SHIFT;
+ size_t copied = 0;
+
+ /* Not on an inode that never had a folio flagged */
+ if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE) ||
+ !mapping_rcu_read(iocb->ki_filp->f_mapping))
+ return 0;
+
+ while (filemap_read_rcu_batch(iocb, iter, &last_index, &copied)) {
+ if (fatal_signal_pending(current))
+ break;
+ cond_resched();
+ }
+ return copied;
+}
+
/**
* filemap_read - Read data from the page cache.
* @iocb: The iocb to read.
@@ -2841,11 +3003,11 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
struct file_ra_state *ra = &filp->f_ra;
struct address_space *mapping = filp->f_mapping;
struct inode *inode = mapping->host;
- struct folio_batch fbatch;
int i, error = 0;
bool writably_mapped;
loff_t isize, end_offset;
loff_t last_pos = ra->prev_pos;
+ size_t fast_read;
if (unlikely(iocb->ki_pos < 0))
return -EINVAL;
@@ -2855,9 +3017,19 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
return 0;
iov_iter_truncate(iter, inode->i_sb->s_maxbytes - iocb->ki_pos);
- folio_batch_init(&fbatch);
+
+ fast_read = filemap_read_rcu(iocb, iter, last_pos);
+ if (fast_read) {
+ already_read += fast_read;
+ last_pos = iocb->ki_pos;
+ if (!iov_iter_count(iter))
+ goto out;
+ }
do {
+ struct folio_batch fbatch;
+
+ folio_batch_init(&fbatch);
cond_resched();
/*
@@ -2931,9 +3103,9 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
filemap_end_dropbehind_read(folio);
folio_put(folio);
}
- folio_batch_init(&fbatch);
} while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);
+out:
file_accessed(filp);
ra->prev_pos = last_pos;
return already_read ? already_read : error;
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 14/18] mm/filemap: read large folios without folio references
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (12 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 13/18] mm/filemap: read from the page cache without folio references Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-05 13:03 ` sashiko-bot
2026-11-04 13:01 ` [RFC PATCH v2 15/18] mm/filemap: don't store f_ra.prev_pos for FMODE_RANDOM files Kiryl Shutsemau (Meta)
` (3 subsequent siblings)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
A large folio is in all the slots of its range or in none. Once the
slot of the last page to copy holds it too, the pages in between are
this folio's: copy them after one lookup. The folio's size is read
without a reference and only picks the page to check; a split after the
check copies PG_rcu_free to the new folios. A split can also change the
order and PG_head under the reader: clamp the order, and read
PG_has_hwpoisoned without folio_test_has_hwpoisoned(), which asserts
PG_head.
Copy page by page with _copy_to_iter(): the hardened usercopy check of
copy_to_iter() reads the folio's order, which a split can change.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/filemap.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 64 insertions(+), 8 deletions(-)
diff --git a/mm/filemap.c b/mm/filemap.c
index e99705a66c1d..182ae7d5db56 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2834,15 +2834,16 @@ static bool filemap_read_rcu_can_copy(struct folio *folio, pgoff_t index,
return false;
if (folio_test_readahead(folio) || folio_test_dropbehind(folio))
return false;
- /* One reference covers all of a large folio's pages */
- if (folio_test_large(folio))
- return false;
/* A split clears PG_head before it puts the new folios in their slots */
- if (folio->index != index)
+ if (!folio_test_large(folio) && folio->index != index)
return false;
/* memory_failure() waits only for readers that got past this */
if (folio_test_hwpoison(folio))
return false;
+ /* Not folio_test_has_hwpoisoned(): it asserts PG_head, a split clears it */
+ if (folio_test_large(folio) &&
+ test_bit(PG_has_hwpoisoned, &folio->_flags_1))
+ return false;
/*
* The slow path does not call folio_mark_accessed() on the folio that
@@ -2896,6 +2897,62 @@ static struct folio *filemap_read_rcu_lookup(struct xa_state *xas,
return folio;
}
+/*
+ * How much of @bytes, starting @offset into page @index, can be copied from
+ * @folio. A folio is in all the slots of its range or in none: when the slot
+ * of the last page holds it too, so do the slots in between, and the pages
+ * are this folio's.
+ */
+static size_t filemap_read_rcu_size(struct xa_state *xas, struct folio *folio,
+ pgoff_t index, size_t offset, size_t bytes)
+{
+ pgoff_t end = index + DIV_ROUND_UP(offset + bytes, PAGE_SIZE);
+ unsigned int order;
+ pgoff_t next;
+
+ if (end - index > 1 && folio_test_large(folio)) {
+ /* A split can change it under us: it only picks the page to check */
+ order = min(folio_large_order(folio), MAX_PAGECACHE_ORDER);
+ next = folio->index + (1UL << order);
+ if (next > index && next < end)
+ end = next;
+ xas_set(xas, end - 1);
+ /* A split after this copies PG_rcu_free to the new folios */
+ if (xas_load(xas) == folio)
+ return min(bytes, (end - index) * PAGE_SIZE - offset);
+ }
+ return min(bytes, PAGE_SIZE - offset);
+}
+
+/*
+ * Copy @bytes from @folio, starting @offset into its page at @index. Page
+ * by page, as copy_page_to_iter() does, without its checks for each call.
+ * Not copy_to_iter(): its usercopy check reads the folio's order, which a
+ * split can change.
+ */
+static size_t filemap_read_rcu_copy(struct folio *folio, pgoff_t index,
+ size_t offset, size_t bytes, struct iov_iter *iter)
+{
+ struct page *page = folio_page(folio, index - folio->index);
+ size_t n = 0;
+
+ while (n < bytes) {
+ size_t len = min(bytes - n, PAGE_SIZE - offset);
+ size_t done;
+ void *kaddr;
+
+ kaddr = kmap_local_page(page);
+ done = _copy_to_iter(kaddr + offset, len, iter);
+ kunmap_local(kaddr);
+ n += done;
+ if (done < len)
+ break;
+ offset = 0;
+ page++;
+ }
+ return n;
+}
+
/* Bytes one RCU read section copies at most: it holds up grace periods */
#define FILEMAP_RCU_READ_MAX SZ_512K
@@ -2941,10 +2998,9 @@ static bool filemap_read_rcu_batch(struct kiocb *iocb, struct iov_iter *iter,
break;
bytes = min_t(loff_t, isize - pos, iov_iter_count(iter));
- bytes = min3(bytes, budget, PAGE_SIZE - offset);
- /* Not _nofault(): its usercopy check reads the folio's order */
- n = copy_page_to_iter(folio_page(folio, index - folio->index),
- offset, bytes, iter);
+ bytes = filemap_read_rcu_size(&xas, folio, index, offset,
+ min(bytes, budget));
+ n = filemap_read_rcu_copy(folio, index, offset, bytes, iter);
*copied += n;
iocb->ki_pos += n;
budget -= n;
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 14/18] mm/filemap: read large folios without folio references
2026-11-04 13:01 ` [RFC PATCH v2 14/18] mm/filemap: read large folios " Kiryl Shutsemau (Meta)
@ 2026-11-05 13:03 ` sashiko-bot
0 siblings, 0 replies; 27+ messages in thread
From: sashiko-bot @ 2026-11-05 13:03 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
David Hildenbrand, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
kernel-team
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] mm/filemap: lockless large folio read can bypass hwpoison check and copy poisoned subpage
--
commit 3f27d3e89354814e80ba93e8767ff8878253a593
Author: Kiryl Shutsemau (Meta) <kas@kernel.org>
mm/filemap: read large folios without folio references
This patch extends lockless RCU pagecache reads to support large folios
without taking folio references. It checks whether the last page of the
read range still points to the same folio in the XArray and copies data
page by page via _copy_to_iter().
> diff --git a/mm/filemap.c b/mm/filemap.c
> index e99705a66c1d4..182ae7d5db56a 100644
> --- a/mm/filemap.c
> +++ b/mm/filemap.c
> @@ -2834,15 +2834,16 @@ static bool filemap_read_rcu_can_copy(struct folio *folio, pgoff_t index,
> return false;
> if (folio_test_readahead(folio) || folio_test_dropbehind(folio))
> return false;
> - /* One reference covers all of a large folio's pages */
> - if (folio_test_large(folio))
> - return false;
> /* A split clears PG_head before it puts the new folios in their slots */
> - if (folio->index != index)
> + if (!folio_test_large(folio) && folio->index != index)
> return false;
> /* memory_failure() waits only for readers that got past this */
> if (folio_test_hwpoison(folio))
> return false;
> + /* Not folio_test_has_hwpoisoned(): it asserts PG_head, a split clears it */
> + if (folio_test_large(folio) &&
> + test_bit(PG_has_hwpoisoned, &folio->_flags_1))
> + return false;
>
> /*
> * The slow path does not call folio_mark_accessed() on the folio that
> @@ -2896,6 +2897,62 @@ static struct folio *filemap_read_rcu_lookup(struct xa_state *xas,
> return folio;
> }
>
> +/*
> + * How much of @bytes, starting @offset into page @index, can be copied from
> + * @folio. A folio is in all the slots of its range or in none: when the slot
> + * of the last page holds it too, so do the slots in between, and the pages
> + * are this folio's.
> + */
> +static size_t filemap_read_rcu_size(struct xa_state *xas, struct folio *folio,
> + pgoff_t index, size_t offset, size_t bytes)
> +{
> + pgoff_t end = index + DIV_ROUND_UP(offset + bytes, PAGE_SIZE);
> + unsigned int order;
> + pgoff_t next;
> +
> + if (end - index > 1 && folio_test_large(folio)) {
> + /* A split can change it under us: it only picks the page to check */
> + order = min(folio_large_order(folio), MAX_PAGECACHE_ORDER);
> + next = folio->index + (1UL << order);
> + if (next > index && next < end)
> + end = next;
> + xas_set(xas, end - 1);
> + /* A split after this copies PG_rcu_free to the new folios */
> + if (xas_load(xas) == folio)
> + return min(bytes, (end - index) * PAGE_SIZE - offset);
> + }
> + return min(bytes, PAGE_SIZE - offset);
> +}
> +
> +/*
> + * Copy @bytes from @folio, starting @offset into its page at @index. Page
> + * by page, as copy_page_to_iter() does, without its checks for each call.
> + * Not copy_to_iter(): its usercopy check reads the folio's order, which a
> + * split can change.
> + */
> +static size_t filemap_read_rcu_copy(struct folio *folio, pgoff_t index,
> + size_t offset, size_t bytes, struct iov_iter *iter)
> +{
> + struct page *page = folio_page(folio, index - folio->index);
> + size_t n = 0;
> +
> + while (n < bytes) {
> + size_t len = min(bytes - n, PAGE_SIZE - offset);
> + size_t done;
> + void *kaddr;
> +
> + kaddr = kmap_local_page(page);
> + done = _copy_to_iter(kaddr + offset, len, iter);
[Severity: High]
As reported by Sashiko on RFC [1], can a lockless reader in
filemap_read_rcu_batch() bypass the hwpoison checks in
filemap_read_rcu_can_copy() and copy from a hardware-poisoned tail subpage in
filemap_read_rcu_copy()?
When a hardware memory error hits a tail page p of a large folio,
memory_failure() sets PG_hwpoison on the tail struct page p rather than on
&folio->page, so folio_test_hwpoison(folio) in filemap_read_rcu_can_copy()
only checks subpage 0. Meanwhile, filemap_read_rcu_copy() copies each subpage
via _copy_to_iter() without checking PageHWPoison(page), relying on
filemap_read_rcu_can_copy() to reject the folio beforehand.
However, there appear to be three ways a poisoned tail subpage can get past
filemap_read_rcu_can_copy():
1. During a split, __split_folio_to_order() in mm/huge_memory.c clears
PG_has_hwpoisoned on the original folio at the start of the split, before the
new sub-folios are initialized, before ClearPageCompound() clears PG_head,
and before __folio_freeze_and_split_unmapped() updates the XArray slots in
mapping->i_pages:
mm/huge_memory.c:__split_folio_to_order() {
...
folio_clear_has_hwpoisoned(folio);
/* Check first new_nr_pages since the loop below skips them */
if (handle_hwpoison &&
page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages))
folio_set_has_hwpoisoned(folio);
...
}
While the XArray slots still point to folio, folio_test_large(folio) remains
true (including across splits to new_order > 0 when the poisoned subpage is
at index >= folio->index + (1UL << new_order)) while PG_has_hwpoisoned is 0.
A reader looking up the poisoned tail index or sizing a multi-page read
across the folio in filemap_read_rcu_size() will see the folio still in its
slots with PG_has_hwpoisoned cleared and proceed to copy the poisoned subpage.
2. filemap_read_rcu_can_copy() evaluates folio_test_large(folio) twice
without caching the result:
if (!folio_test_large(folio) && folio->index != index)
return false;
if (folio_test_hwpoison(folio))
return false;
if (folio_test_large(folio) &&
test_bit(PG_has_hwpoisoned, &folio->_flags_1))
return false;
If a concurrent split to order 0 clears PG_has_hwpoisoned and PG_head between
the two folio_test_large(folio) calls while reading at index != folio->index,
the first check sees folio_test_large(folio) as true (skipping the
folio->index != index check) and the second check sees it as false (skipping
the PG_has_hwpoisoned check). filemap_read_rcu_size() then returns a single
page and filemap_read_rcu_copy() copies from folio_page(folio, index -
folio->index).
3. In mm/memory-failure.c, memory_failure() sets PG_hwpoison on the tail page
p via TestSetPageHWPoison(p) before calling get_hwpoison_page() and before
setting PG_has_hwpoisoned on the folio. If get_hwpoison_page() fails (for
example, while the folio refcount is transiently frozen), PG_hwpoison remains
set on tail page p while PG_has_hwpoisoned is never set on the folio. If a
subsequent memory_failure() call hits TestSetPageHWPoison(p), it calls
synchronize_rcu() and returns 0 so the caller can unmap p from the kernel
direct map, after which a lockless reader copying from p in
filemap_read_rcu_copy() can trigger an unhandled kernel page fault.
Would checking PageHWPoison(page) on each subpage in filemap_read_rcu_copy()
(or before copying) avoid reading from a poisoned tail page across these
races?
[1] https://lore.kernel.org/r/20261019154913.1765974-16-kas@kernel.org
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104130118.2507513-1-kas@kernel.org?part=14
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 15/18] mm/filemap: don't store f_ra.prev_pos for FMODE_RANDOM files
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (13 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 14/18] mm/filemap: read large folios " Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 16/18] mm/shmem: count accesses to a folio the way filemap_read() does Kiryl Shutsemau (Meta)
` (2 subsequent siblings)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
filemap_read() writes the end of every read to file->f_ra.prev_pos, in
the cache line that fdget() and fdput() write when threads share the
file table.
Applications that read a file at random offsets from many threads, such
as RocksDB, set POSIX_FADV_RANDOM on it. Readahead then never reads
prev_pos: page_cache_sync_ra() forces the read first.
Skip the store for such files after a read that filemap_read_rcu() did
in full: without folio references, the file's line is what the threads
contend on. Keep it on the slow path, which takes folio references:
skipping it there made reads slower in my testing.
filemap_read() compares the start of a read with prev_pos, to tell a
read that goes on in the same folio from another access to it. On an
FMODE_RANDOM file it can now be stale; with many threads on one file it
was another thread's read already.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/filemap.c | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/mm/filemap.c b/mm/filemap.c
index 182ae7d5db56..5998de3d6697 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -3078,8 +3078,13 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
if (fast_read) {
already_read += fast_read;
last_pos = iocb->ki_pos;
- if (!iov_iter_count(iter))
- goto out;
+ if (!iov_iter_count(iter)) {
+ file_accessed(filp);
+ /* Readahead of a random file ignores it: skip the shared store */
+ if (!(filp->f_mode & FMODE_RANDOM))
+ ra->prev_pos = last_pos;
+ return already_read;
+ }
}
do {
@@ -3161,7 +3166,6 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
}
} while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);
-out:
file_accessed(filp);
ra->prev_pos = last_pos;
return already_read ? already_read : error;
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 16/18] mm/shmem: count accesses to a folio the way filemap_read() does
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (14 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 15/18] mm/filemap: don't store f_ra.prev_pos for FMODE_RANDOM files Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 17/18] mm/shmem: read without folio references Kiryl Shutsemau (Meta)
2026-11-04 13:01 ` [RFC PATCH v2 18/18] mm/filemap: add a debugfs switch for lockless page cache reads Kiryl Shutsemau (Meta)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
shmem_file_read_iter() marks a folio accessed only when a read starts at
the beginning of the folio. Small reads that start anywhere else never
count, so a folio that is read over and over at one offset ages as if it
were never read.
Count accesses with filemap_read_accessed(), as filemap_read() does: a
read marks the folio accessed unless it goes on in the folio that the
previous read through the file ended in. Reading the same bytes again
through one open file counts once, as for other files, where it used to
count each time it started at the beginning of the folio. The end of
the previous read is kept in f_ra.prev_pos, as for other files.
This also sets PG_rcu_free on the tmpfs folios that reads keep coming
back to. On a shmem folio the PG_readahead bit is PG_reclaim, and shmem
does no readahead, so do not let that bit keep the flag off. A write
marks a tmpfs folio accessed too, so a folio written in several pieces
can get the flag on its first read.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/filemap.c | 9 +++++----
mm/internal.h | 1 +
mm/shmem.c | 9 ++++-----
3 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/mm/filemap.c b/mm/filemap.c
index 5998de3d6697..65d414ad9d85 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2772,8 +2772,10 @@ static void filemap_mark_hot(struct folio *folio)
if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
return;
if (folio_test_rcu_free(folio) || !folio_test_uptodate(folio) ||
- folio_test_readahead(folio) || folio_test_dropbehind(folio) ||
- folio_test_unevictable(folio))
+ folio_test_dropbehind(folio) || folio_test_unevictable(folio))
+ return;
+ /* shmem does no readahead: there the bit is PG_reclaim */
+ if (folio_test_readahead(folio) && !folio_test_swapbacked(folio))
return;
if (folio_test_set_rcu_free(folio))
return;
@@ -2796,8 +2798,7 @@ static void filemap_mark_hot(struct folio *folio)
* LRU records, or when this open file goes back to data that it has just
* read.
*/
-static void filemap_read_accessed(struct folio *folio, loff_t pos,
- loff_t last_pos)
+void filemap_read_accessed(struct folio *folio, loff_t pos, loff_t last_pos)
{
bool hot;
diff --git a/mm/internal.h b/mm/internal.h
index 38b1165212c9..18391dc50418 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1248,6 +1248,7 @@ struct migration_target_control {
*/
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size);
+void filemap_read_accessed(struct folio *folio, loff_t pos, loff_t last_pos);
static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)
{
diff --git a/mm/shmem.c b/mm/shmem.c
index 848316eaa7f4..1c700fa9666d 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3309,6 +3309,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
struct file *file = iocb->ki_filp;
struct inode *inode = file_inode(file);
struct address_space *mapping = inode->i_mapping;
+ loff_t last_pos = file->f_ra.prev_pos;
pgoff_t index;
unsigned long offset;
int error = 0;
@@ -3378,11 +3379,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
flush_dcache_page(page);
}
- /*
- * Mark the folio accessed if we read the beginning.
- */
- if (!offset)
- folio_mark_accessed(folio);
+ filemap_read_accessed(folio, iocb->ki_pos, last_pos);
/*
* Ok, we have the page, and it's up-to-date, so
* now we can copy it to user space...
@@ -3410,6 +3407,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
retval += ret;
iocb->ki_pos += ret;
+ last_pos = iocb->ki_pos;
if (!iov_iter_count(to))
break;
@@ -3421,6 +3419,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
}
file_accessed(file);
+ file->f_ra.prev_pos = last_pos;
return retval ? retval : error;
}
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* [RFC PATCH v2 17/18] mm/shmem: read without folio references
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (15 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 16/18] mm/shmem: count accesses to a folio the way filemap_read() does Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
2026-11-05 13:03 ` sashiko-bot
2026-11-04 13:01 ` [RFC PATCH v2 18/18] mm/filemap: add a debugfs switch for lockless page cache reads Kiryl Shutsemau (Meta)
17 siblings, 1 reply; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
shmem_file_read_iter() takes a reference on every folio it reads, so
small reads of one tmpfs file from many CPUs bounce the folio's
refcount.
Copy with filemap_read_rcu() first, and continue with the existing loop
where it stops. Holes, swap entries and fallocated folios that are not
uptodate all end the copy. So does a hwpoisoned page, which shmem keeps
in the page cache and reports with -EIO.
This drops the folio lock as well as the reference. shmem_get_folio()
held it only for the lookup, and the copy ran after folio_unlock(), so a
read was never atomic against a write or a hole punch. Now a read also
does not wait for one that holds the lock, as filemap_read() does not.
shmem does no readahead, and on its folios the PG_readahead bit is
PG_reclaim, which drop_caches can leave set on dirty tmpfs folios. Do
not let it end the copy there.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/filemap.c | 7 ++++---
mm/internal.h | 2 ++
mm/shmem.c | 11 +++++++++--
3 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/mm/filemap.c b/mm/filemap.c
index 65d414ad9d85..e148490db4da 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2831,9 +2831,10 @@ static void filemap_end_dropbehind_read(struct folio *folio)
static bool filemap_read_rcu_can_copy(struct folio *folio, pgoff_t index,
pgoff_t last_index)
{
- if (!folio_test_uptodate(folio))
+ if (!folio_test_uptodate(folio) || folio_test_dropbehind(folio))
return false;
- if (folio_test_readahead(folio) || folio_test_dropbehind(folio))
+ /* shmem does no readahead: there the bit is PG_reclaim */
+ if (folio_test_readahead(folio) && !folio_test_swapbacked(folio))
return false;
/* A split clears PG_head before it puts the new folios in their slots */
if (!folio_test_large(folio) && folio->index != index)
@@ -3020,7 +3021,7 @@ static bool filemap_read_rcu_batch(struct kiocb *iocb, struct iov_iter *iter,
* Copy from the page cache without folio references, in batches, and
* return how much was copied. The slow path reads the rest.
*/
-static size_t filemap_read_rcu(struct kiocb *iocb, struct iov_iter *iter,
+size_t filemap_read_rcu(struct kiocb *iocb, struct iov_iter *iter,
loff_t last_pos)
{
/* The page the previous read ended in; ULONG_MAX if none */
diff --git a/mm/internal.h b/mm/internal.h
index 18391dc50418..ae9a4e511c82 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1249,6 +1249,8 @@ struct migration_target_control {
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size);
void filemap_read_accessed(struct folio *folio, loff_t pos, loff_t last_pos);
+size_t filemap_read_rcu(struct kiocb *iocb, struct iov_iter *iter,
+ loff_t last_pos);
static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)
{
diff --git a/mm/shmem.c b/mm/shmem.c
index 1c700fa9666d..8700b03a8a5e 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3313,7 +3313,14 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
pgoff_t index;
unsigned long offset;
int error = 0;
- ssize_t retval = 0;
+ ssize_t retval;
+
+ retval = filemap_read_rcu(iocb, to, last_pos);
+ if (retval) {
+ last_pos = iocb->ki_pos;
+ if (!iov_iter_count(to))
+ goto out;
+ }
for (;;) {
struct folio *folio = NULL;
@@ -3417,7 +3424,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
}
cond_resched();
}
-
+out:
file_accessed(file);
file->f_ra.prev_pos = last_pos;
return retval ? retval : error;
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [RFC PATCH v2 17/18] mm/shmem: read without folio references
2026-11-04 13:01 ` [RFC PATCH v2 17/18] mm/shmem: read without folio references Kiryl Shutsemau (Meta)
@ 2026-11-05 13:03 ` sashiko-bot
0 siblings, 0 replies; 27+ messages in thread
From: sashiko-bot @ 2026-11-05 13:03 UTC (permalink / raw)
To: Kiryl Shutsemau (Meta), Andrew Morton, Linus Torvalds,
David Hildenbrand, Matthew Wilcox, Vlastimil Babka,
Lorenzo Stoakes, Jan Kara, Hugh Dickins, Christian Brauner,
Alexander Viro, Harry Yoo
Cc: Miklos Szeredi, Miaohe Lin, Naoya Horiguchi, Oscar Salvador,
Liam R. Howlett, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Brendan Jackman, Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache,
Ryan Roberts, Dev Jain, Barry Song, Lance Yang, Usama Arif,
Kairui Song, Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie,
Wei Xu, Baoquan He, Paul E. McKenney, Pedro Falcato,
Luis Chamberlain, Ilya Gladyshev, Jaegeuk Kim, linux-mm,
kernel-team
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] mm: race in filemap_read_rcu_can_copy() copies hwpoisoned tail pages on split
--
commit 6b69ad4da5a3278d80ff53e3999062ede8b8a468
Author: Kiryl Shutsemau (Meta) <kas@kernel.org>
mm/shmem: read without folio references
This commit updates shmem_file_read_iter() to copy from the page cache
without taking folio references or locks via filemap_read_rcu() before
falling back to the slow path. It also updates filemap_read_rcu_can_copy()
to ignore PG_readahead on swapbacked folios where that bit is PG_reclaim.
> diff --git a/mm/shmem.c b/mm/shmem.c
> index 1c700fa9666d0..8700b03a8a5ee 100644
> --- a/mm/shmem.c
> +++ b/mm/shmem.c
> @@ -3313,7 +3313,14 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
> pgoff_t index;
> unsigned long offset;
> int error = 0;
> - ssize_t retval = 0;
> + ssize_t retval;
> +
> + retval = filemap_read_rcu(iocb, to, last_pos);
> + if (retval) {
> + last_pos = iocb->ki_pos;
> + if (!iov_iter_count(to))
> + goto out;
> + }
[Severity: High]
Can calling filemap_read_rcu() from shmem_file_read_iter() copy from a
hwpoisoned tail page instead of returning -EIO when a large shmem folio is
concurrently split?
Unlike regular filesystems, shmem_error_remove_folio() keeps hwpoisoned
folios in the page cache. When a large folio has a hwpoisoned tail page
(index > folio->index), PG_hwpoison is set on that tail subpage and
PG_has_hwpoisoned is set on the folio. If the folio is later split (for
example via truncate_inode_partial_folio(), shmem_unused_huge_shrink(), or
shmem_writeout(), or during memory_failure()), a concurrent reader in
shmem_file_read_iter() -> filemap_read_rcu() -> filemap_read_rcu_batch() can
bypass the hwpoison checks in three ways:
1. In mm/huge_memory.c:__split_folio_to_order(), PG_has_hwpoisoned is cleared
on the folio at the beginning of the split, before the loop over the subpages,
before ClearPageCompound() clears PG_head, and before
__folio_freeze_and_split_unmapped() updates the XArray slots:
mm/huge_memory.c:__split_folio_to_order() {
...
folio_clear_has_hwpoisoned(folio);
/* Check first new_nr_pages since the loop below skips them */
if (handle_hwpoison &&
page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages))
folio_set_has_hwpoisoned(folio);
...
if (new_order)
folio_set_order(folio, new_order);
else
ClearPageCompound(&folio->page);
}
During this window (or after a non-uniform split where new_order > 0 leaves
PG_head set on the first half while the poisoned subpage is in the second
half, before the XArray slots are updated), xas_load() still returns the
original folio with PG_head set and PG_has_hwpoisoned clear. In
filemap_read_rcu_can_copy(), folio_test_hwpoison(folio) only checks the head
page, so it returns true for the poisoned tail page, and
filemap_read_rcu_size() also sees xas_load(xas) == folio at end - 1, allowing
filemap_read_rcu_copy() to copy across the poisoned subpage.
2. In mm/filemap.c:filemap_read_rcu_can_copy(), folio_test_large(folio) is
tested twice without synchronization:
mm/filemap.c:filemap_read_rcu_can_copy() {
...
/* A split clears PG_head before it puts the new folios in their slots */
if (!folio_test_large(folio) && folio->index != index)
return false;
/* memory_failure() waits only for readers that got past this */
if (folio_test_hwpoison(folio))
return false;
/* Not folio_test_has_hwpoisoned(): it asserts PG_head, a split clears it */
if (folio_test_large(folio) &&
test_bit(PG_has_hwpoisoned, &folio->_flags_1))
return false;
...
}
If ClearPageCompound(&folio->page) runs between those two checks, the first
folio_test_large(folio) sees true (allowing folio->index != index) and the
second sees false (skipping the PG_has_hwpoisoned check).
3. Neither filemap_read_rcu_can_copy() nor filemap_read_rcu_copy() checks
PageHWPoison(page) on the individual subpages being copied:
mm/filemap.c:filemap_read_rcu_copy() {
struct page *page = folio_page(folio, index - folio->index);
size_t n = 0;
while (n < bytes) {
size_t len = min(bytes - n, PAGE_SIZE - offset);
size_t done;
void *kaddr;
kaddr = kmap_local_page(page);
done = _copy_to_iter(kaddr + offset, len, iter);
...
}
Could this copy corrupted data to userspace or trigger an unhandled kernel
machine check or page fault if the poisoned page was unmapped from the direct
map?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104130118.2507513-1-kas@kernel.org?part=17
^ permalink raw reply [flat|nested] 27+ messages in thread
* [RFC PATCH v2 18/18] mm/filemap: add a debugfs switch for lockless page cache reads
2026-11-04 13:00 [RFC PATCH v2 00/18] mm: read the page cache without folio references Kiryl Shutsemau (Meta)
` (16 preceding siblings ...)
2026-11-04 13:01 ` [RFC PATCH v2 17/18] mm/shmem: read without folio references Kiryl Shutsemau (Meta)
@ 2026-11-04 13:01 ` Kiryl Shutsemau (Meta)
17 siblings, 0 replies; 27+ messages in thread
From: Kiryl Shutsemau (Meta) @ 2026-11-04 13:01 UTC (permalink / raw)
To: Andrew Morton, Linus Torvalds, David Hildenbrand, Matthew Wilcox,
Vlastimil Babka, Lorenzo Stoakes, Jan Kara, Hugh Dickins,
Christian Brauner, Alexander Viro, Harry Yoo
Cc: Kiryl Shutsemau (Meta), Miklos Szeredi, Miaohe Lin,
Naoya Horiguchi, Oscar Salvador, Liam R. Howlett, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Brendan Jackman,
Johannes Weiner, Zi Yan, Baolin Wang, Nico Pache, Ryan Roberts,
Dev Jain, Barry Song, Lance Yang, Usama Arif, Kairui Song,
Qi Zheng, Shakeel Butt, Axel Rasmussen, Yuanchu Xie, Wei Xu,
Baoquan He, Paul E. McKenney, Pedro Falcato, Luis Chamberlain,
Ilya Gladyshev, Jaegeuk Kim, linux-mm, linux-kernel,
linux-fsdevel, fuse-devel, kernel-team
Reading hot folios without a reference defers the free of their memory
by an RCU grace period. A workload that loses on that trade, or a bug
in the lockless read, has no way to turn it off: CONFIG_PAGE_RCU_FREE is
on in every 64-bit kernel.
Add lockless_page_cache_read to debugfs. Writing 0 stops filemap_read()
and shmem_file_read_iter() from setting PG_rcu_free and from copying
without a reference. A static key keeps the test off the read path.
debugfs, not a sysctl: the switch is not ABI and can go once the reads
have proven themselves.
Folios that have the flag keep it, and their free stays deferred: a read
that tested the key before the switch may still be copying from them.
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
mm/Kconfig | 3 +++
mm/filemap.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++---
2 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/mm/Kconfig b/mm/Kconfig
index 6e5341b0bf64..768873ebc71c 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -733,6 +733,9 @@ config PAGE_RCU_FREE
32-bit. Not on PREEMPT_RT, where expedited grace periods are
normal ones, so freeing the waiting pages on demand would stall.
+ Writing 0 to lockless_page_cache_read in debugfs turns these
+ reads off.
+
config PHYS_ADDR_T_64BIT
def_bool 64BIT
diff --git a/mm/filemap.c b/mm/filemap.c
index e148490db4da..eff29d501f58 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -48,6 +48,7 @@
#include <linux/rcupdate_wait.h>
#include <linux/sched/mm.h>
#include <linux/sysctl.h>
+#include <linux/debugfs.h>
#include <linux/pgalloc.h>
#include <asm/tlbflush.h>
@@ -2759,6 +2760,50 @@ static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio)
return (pos1 >> shift == pos2 >> shift);
}
+#ifdef CONFIG_PAGE_RCU_FREE
+/* Writing 0 to lockless_page_cache_read in debugfs turns the reads off */
+static DEFINE_STATIC_KEY_TRUE(lockless_read_key);
+
+static bool lockless_read_enabled(void)
+{
+ return static_branch_likely(&lockless_read_key);
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int lockless_read_get(void *data, u64 *val)
+{
+ *val = static_key_enabled(&lockless_read_key);
+ return 0;
+}
+
+static int lockless_read_set(void *data, u64 val)
+{
+ if (val > 1)
+ return -EINVAL;
+ if (val)
+ static_branch_enable(&lockless_read_key);
+ else
+ static_branch_disable(&lockless_read_key);
+ return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(lockless_read_fops, lockless_read_get,
+ lockless_read_set, "%llu\n");
+
+static int __init lockless_read_debugfs_init(void)
+{
+ debugfs_create_file_unsafe("lockless_page_cache_read", 0644, NULL, NULL,
+ &lockless_read_fops);
+ return 0;
+}
+late_initcall(lockless_read_debugfs_init);
+#endif
+#else
+static bool lockless_read_enabled(void)
+{
+ return false;
+}
+#endif
+
/*
* Reads keep coming back to @folio: free it only after an RCU grace period
* from now on. Not if its next read needs a reference anyway. Not if it
@@ -2769,7 +2814,7 @@ static void filemap_mark_hot(struct folio *folio)
{
struct address_space *mapping;
- if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE))
+ if (!lockless_read_enabled())
return;
if (folio_test_rcu_free(folio) || !folio_test_uptodate(folio) ||
folio_test_dropbehind(folio) || folio_test_unevictable(folio))
@@ -3028,8 +3073,8 @@ size_t filemap_read_rcu(struct kiocb *iocb, struct iov_iter *iter,
pgoff_t last_index = (last_pos - 1) >> PAGE_SHIFT;
size_t copied = 0;
- /* Not on an inode that never had a folio flagged */
- if (!IS_ENABLED(CONFIG_PAGE_RCU_FREE) ||
+ /* Not when switched off, or on an inode that never had a folio flagged */
+ if (!lockless_read_enabled() ||
!mapping_rcu_read(iocb->ki_filp->f_mapping))
return 0;
--
2.54.0
^ permalink raw reply related [flat|nested] 27+ messages in thread