[RFC PATCH v12 based on v6.14-rc4 24/25] mm/vmscan: apply luf mechanism to unmapping during folio reclaim

Byungchul Park
2025-02-26T12:03:58Z

A new mechanism, LUF(Lazy Unmap Flush), defers tlb flush until folios
that have been unmapped and freed, eventually get allocated again. It's
safe for folios that had been mapped read only and were unmapped, since
the contents of the folios don't change while staying in pcp or buddy
so we can still read the data through the stale tlb entries.

Applied the mechanism to unmapping during folio reclaim.

Signed-off-by: Byungchul Park <byungchul@sk.com>
---
include/linux/rmap.h | 5 +++--
mm/rmap.c | 11 +++++++----
mm/vmscan.c | 37 ++++++++++++++++++++++++++++++++-----
3 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index cedba4812ccc7..854b41441d466 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -661,7 +661,7 @@ int folio_referenced(struct folio *, int is_locked,
struct mem_cgroup *memcg, unsigned long *vm_flags);

bool try_to_migrate(struct folio *folio, enum ttu_flags flags);
-void try_to_unmap(struct folio *, enum ttu_flags flags);
+bool try_to_unmap(struct folio *, enum ttu_flags flags);

int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
unsigned long end, struct page **pages,
@@ -794,8 +794,9 @@ static inline int folio_referenced(struct folio *folio, int is_locked,
return 0;
}

-static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
+static inline bool try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
+ return false;
}

static inline int folio_mkclean(struct folio *folio)
diff --git a/mm/rmap.c b/mm/rmap.c
index 284fc48aef2de..df350b4dfddd0 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2386,10 +2386,11 @@ static int folio_not_mapped(struct folio *folio)
* Tries to remove all the page table entries which are mapping this
* folio. It is the caller's responsibility to check if the folio is
* still mapped if needed (use TTU_SYNC to prevent accounting races).
+ * Return true if all the mappings are read-only, otherwise false.
*
* Context: Caller must hold the folio lock.
*/
-void try_to_unmap(struct folio *folio, enum ttu_flags flags)
+bool try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
struct rmap_walk_control rwc = {
.rmap_one = try_to_unmap_one,
@@ -2408,10 +2409,12 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags)
else
rmap_walk(folio, &rwc);

- if (can_luf_test())
+ if (can_luf_test()) {
fold_batch(tlb_ubc_luf, tlb_ubc_ro, true);
- else
- fold_batch(tlb_ubc, tlb_ubc_ro, true);
+ return true;
+ }
+ fold_batch(tlb_ubc, tlb_ubc_ro, true);
+ return false;
}

/*
diff --git a/mm/vmscan.c b/mm/vmscan.c
index a31a7cf87315f..065b40f36bbdd 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1092,14 +1092,17 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
struct reclaim_stat *stat, bool ignore_references)
{
struct folio_batch free_folios;
+ struct folio_batch free_folios_luf;
LIST_HEAD(ret_folios);
LIST_HEAD(demote_folios);
unsigned int nr_reclaimed = 0, nr_demoted = 0;
unsigned int pgactivate = 0;
bool do_demote_pass;
struct swap_iocb *plug = NULL;
+ unsigned short luf_key;

folio_batch_init(&free_folios);
+ folio_batch_init(&free_folios_luf);
memset(stat, 0, sizeof(*stat));
cond_resched();
do_demote_pass = can_demote(pgdat->node_id, sc);
@@ -1111,6 +1114,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
enum folio_references references = FOLIOREF_RECLAIM;
bool dirty, writeback;
unsigned int nr_pages;
+ bool can_luf = false;

cond_resched();

@@ -1344,7 +1348,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
if (folio_test_large(folio))
flags |= TTU_SYNC;

- try_to_unmap(folio, flags);
+ can_luf = try_to_unmap(folio, flags);
if (folio_mapped(folio)) {
stat->nr_unmap_fail += nr_pages;
if (!was_swapbacked &&
@@ -1488,6 +1492,8 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
* leave it off the LRU).
*/
nr_reclaimed += nr_pages;
+ if (can_luf)
+ luf_flush(fold_unmap_luf());
continue;
}
}
@@ -1520,6 +1526,19 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
nr_reclaimed += nr_pages;

folio_unqueue_deferred_split(folio);
+
+ if (can_luf) {
+ if (folio_batch_add(&free_folios_luf, folio) == 0) {
+ mem_cgroup_uncharge_folios(&free_folios);
+ mem_cgroup_uncharge_folios(&free_folios_luf);
+ luf_key = fold_unmap_luf();
+ try_to_unmap_flush();
+ free_unref_folios(&free_folios, 0);
+ free_unref_folios(&free_folios_luf, luf_key);
+ }
+ continue;
+ }
+
if (folio_batch_add(&free_folios, folio) == 0) {
mem_cgroup_uncharge_folios(&free_folios);
try_to_unmap_flush();
@@ -1554,9 +1573,21 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
list_add(&folio->lru, &ret_folios);
VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
folio_test_unevictable(folio), folio);
+ if (can_luf)
+ luf_flush(fold_unmap_luf());
}
/* 'folio_list' is always empty here */

+ /*
+ * Finalize this turn before demote_folio_list().
+ */
+ mem_cgroup_uncharge_folios(&free_folios);
+ mem_cgroup_uncharge_folios(&free_folios_luf);
+ luf_key = fold_unmap_luf();
+ try_to_unmap_flush();
+ free_unref_folios(&free_folios, 0);
+ free_unref_folios(&free_folios_luf, luf_key);
+
/* Migrate folios selected for demotion */
nr_demoted = demote_folio_list(&demote_folios, pgdat);
nr_reclaimed += nr_demoted;
@@ -1590,10 +1621,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,

pgactivate = stat->nr_activate[0] + stat->nr_activate[1];

- mem_cgroup_uncharge_folios(&free_folios);
- try_to_unmap_flush();
- free_unref_folios(&free_folios, 0);
-
list_splice(&ret_folios, folio_list);
count_vm_events(PGACTIVATE, pgactivate);

--
2.17.1