[PATCH 0/2] mm: zswap: free cold writeback folios promptly

[PATCH 0/2] mm: zswap: free cold writeback folios promptly

[Alexandre Ghiti]

When zswap writes an entry back, it allocates an order-0 swap cache folio,
decompresses into it, and issues the write. The folio is cold by
construction, yet today it is left on the LRU for page reclaim to find and
free later. That wastes a reclaim scan and keeps cold memory resident
longer than necessary.

Free the folio as soon as writeback completes instead:

  Patch 1 - synchronous-IO devices (zram, brd, pmem, ...) complete writeback
            in the calling context, so the folio is freed directly there.

  Patch 2 - asynchronous block devices and filesystem-backed swap complete
            writeback in IRQ/softirq context, where the folio cannot be
            freed (that needs the folio and swap cluster locks and may
            sleep). Allocate the folio off the LRU and defer the free to a
            workqueue, which folio_end_writeback() hands it off to.

Results
-------
Paired baseline vs series on async swap (NVMe, Patch 2 deferred path). Each
workload runs confined to a memory cgroup (memory.max) small enough to force
zswap shrinker writeback.

Kernel build (defconfig, make -j4; memory.max = 600M):

  metric            baseline       series      delta
  pgrotated           441028         2521     -99.4%
  pgsteal_direct     3524343      2869004     -18.6%
  pgscan_direct      8393791      7765008      -7.5%
  zswpwb              705129       699019      -0.9%
  build time (s)        1155         1114      -3.6%

Of the 699019 folios written back, 698990 (99.996%) were freed promptly on
writeback completion; only 28 fell back to reclaim.

MySQL/OLTP (sysbench, 10 tables x 1M rows, 512M buffer pool, 8 threads, 300s;
memory.max = 256M):

  metric            baseline       series      delta
  transactions/s      153.87       163.30      +6.1%
  p95 latency (ms)    157.42       145.82      -7.4%
  avg latency (ms)     52.09        49.05      -5.9%
  pgrotated           743738        22460     -97.0%
  pgsteal_direct     6886490      5445278     -20.9%
  pgscan_direct     13462510     10820730     -19.6%

Alexandre Ghiti (2):
  mm: zswap: free synchronous-IO writeback folios directly
  mm: zswap: deferred dropbehind free of writeback folios

 include/linux/zswap.h |  2 ++
 mm/filemap.c          | 20 +++++++++++++
 mm/swap.h             |  3 +-
 mm/swap_state.c       | 17 +++++++----
 mm/zswap.c            | 66 +++++++++++++++++++++++++++++++++++++++++--
 5 files changed, 98 insertions(+), 10 deletions(-)


base-commit: 626acb37cd445144f321f1b64cac9a93760fa716
-- 
2.53.0-Meta

[PATCH 1/2] mm: zswap: free synchronous-IO writeback folios directly

[Alexandre Ghiti]

When zswap writes an entry back, it allocates a swap cache folio,
decompresses the entry into it and writes it out. That folio is cold by
construction, but it is currently left on the LRU for page reclaim to find
and free later. This wastes a reclaim scan and keeps cold memory resident
longer than necessary.

For synchronous-IO swap devices writeback completes in the calling context,
so the folio can be freed right after the write rather than left behind; do
that. Because it is freed directly rather than through reclaim, it is
allocated off the LRU: dropping the last reference on a folio still on the
LRU would trip the free-time page-flag checks. A folio that a concurrent
swapin has meanwhile claimed is left in place and reclaimed as usual.

Asynchronous and filesystem-backed swap complete writeback in interrupt
context, where the folio cannot be freed; they are handled in a later
change.

Signed-off-by: Alexandre Ghiti <alex@ghiti.fr>
---
 mm/swap.h       |  3 ++-
 mm/swap_state.c | 17 +++++++++++------
 mm/zswap.c      | 36 ++++++++++++++++++++++++++++++++++--
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/mm/swap.h b/mm/swap.h
index 77d2d14eda42..c617e5a0257f 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -306,7 +306,8 @@ void *swap_cache_get_shadow(swp_entry_t entry);
 void swap_cache_del_folio(struct folio *folio);
 struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask,
 				     unsigned long orders, struct vm_fault *vmf,
-				     struct mempolicy *mpol, pgoff_t ilx);
+				     struct mempolicy *mpol, pgoff_t ilx,
+				     bool skip_lru);
 /* Below helpers require the caller to lock and pass in the swap cluster. */
 void __swap_cache_add_folio(struct swap_cluster_info *ci,
 			    struct folio *folio, swp_entry_t entry);
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 9c3a5cf99778..048efc7ca612 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -403,7 +403,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
 static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
 					swp_entry_t targ_entry, gfp_t gfp,
 					unsigned int order, struct vm_fault *vmf,
-					struct mempolicy *mpol, pgoff_t ilx)
+					struct mempolicy *mpol, pgoff_t ilx,
+					bool skip_lru)
 {
 	int err;
 	swp_entry_t entry;
@@ -484,7 +485,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
 	lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages);
 
 	/* Caller will initiate read into locked new_folio */
-	folio_add_lru(folio);
+	if (!skip_lru)
+		folio_add_lru(folio);
 	return folio;
 }
 
@@ -507,7 +509,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
  */
 struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
 				     unsigned long orders, struct vm_fault *vmf,
-				     struct mempolicy *mpol, pgoff_t ilx)
+				     struct mempolicy *mpol, pgoff_t ilx,
+				     bool skip_lru)
 {
 	int order, err;
 	struct folio *ret;
@@ -522,7 +525,7 @@ struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
 
 	do {
 		ret = __swap_cache_alloc(ci, targ_entry, gfp, order,
-					 vmf, mpol, ilx);
+					 vmf, mpol, ilx, skip_lru);
 		if (!IS_ERR(ret))
 			break;
 		err = PTR_ERR(ret);
@@ -643,7 +646,8 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
 		folio = swap_cache_get_folio(entry);
 		if (folio)
 			return folio;
-		folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx);
+		folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx,
+					       false);
 	} while (PTR_ERR(folio) == -EEXIST);
 
 	if (IS_ERR_OR_NULL(folio))
@@ -683,7 +687,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
 		folio = swap_cache_get_folio(entry);
 		if (folio)
 			return folio;
-		folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx);
+		folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx,
+					       false);
 	} while (PTR_ERR(folio) == -EEXIST);
 
 	if (IS_ERR(folio))
diff --git a/mm/zswap.c b/mm/zswap.c
index 761cd699e0a3..3c494c5a671c 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -972,6 +972,30 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio)
 /*********************************
 * writeback code
 **********************************/
+static void zswap_writeback_free_folio(struct folio *folio)
+{
+	folio_lock(folio);
+
+	/* The folio was allocated off the LRU and nothing re-adds it here. */
+	VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
+
+	/*
+	 * Gate remove_mapping() on folio_test_swapcache(): a racing swapin may
+	 * have freed the swap slot (folio_free_swap()) and dropped the folio from
+	 * the cache, and remove_mapping() must not run on a non-swapcache folio
+	 * (it would trip __remove_mapping()'s mapping == folio_mapping() check).
+	 */
+	if (folio_test_swapcache(folio) &&
+	    remove_mapping(swap_address_space(folio->swap), folio))
+		goto out;
+
+	/* Raced: the folio is now owned by the swapin; put it back on the LRU. */
+	folio_add_lru(folio);
+out:
+	folio_unlock(folio);
+	folio_put(folio);
+}
+
 /*
  * Attempts to free an entry by adding a folio to the swap cache,
  * decompressing the entry data into the folio, and issuing a
@@ -992,16 +1016,18 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	struct folio *folio;
 	struct mempolicy *mpol;
 	struct swap_info_struct *si;
+	bool sync;
 	int ret = 0;
 
 	/* try to allocate swap cache folio */
 	si = get_swap_device(swpentry);
 	if (!si)
 		return -EEXIST;
+	sync = data_race(si->flags & SWP_SYNCHRONOUS_IO);
 
 	mpol = get_task_policy(current);
 	folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol,
-				       NO_INTERLEAVE_INDEX);
+				       NO_INTERLEAVE_INDEX, sync);
 	put_swap_device(si);
 
 	/*
@@ -1046,11 +1072,17 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	folio_mark_uptodate(folio);
 
 	/* move it to the tail of the inactive list after end_writeback */
-	folio_set_reclaim(folio);
+	if (!sync)
+		folio_set_reclaim(folio);
 
 	/* start writeback */
 	__swap_writepage(folio, NULL);
 
+	if (sync) {
+		zswap_writeback_free_folio(folio);
+		return 0;
+	}
+
 out:
 	if (ret) {
 		swap_cache_del_folio(folio);
-- 
2.53.0-Meta

Re: [PATCH 1/2] mm: zswap: free synchronous-IO writeback folios directly

[Kairui Song]

On Sat, Jul 18, 2026 at 11:36:39AM +0800, Alexandre Ghiti wrote:
> When zswap writes an entry back, it allocates a swap cache folio,
> decompresses the entry into it and writes it out. That folio is cold by
> construction, but it is currently left on the LRU for page reclaim to find
> and free later. This wastes a reclaim scan and keeps cold memory resident
> longer than necessary.
> 
> For synchronous-IO swap devices writeback completes in the calling context,
> so the folio can be freed right after the write rather than left behind; do
> that. Because it is freed directly rather than through reclaim, it is
> allocated off the LRU: dropping the last reference on a folio still on the
> LRU would trip the free-time page-flag checks. A folio that a concurrent
> swapin has meanwhile claimed is left in place and reclaimed as usual.
> 
> Asynchronous and filesystem-backed swap complete writeback in interrupt
> context, where the folio cannot be freed; they are handled in a later
> change.
> 
> Signed-off-by: Alexandre Ghiti <alex@ghiti.fr>
> ---
>  mm/swap.h       |  3 ++-
>  mm/swap_state.c | 17 +++++++++++------
>  mm/zswap.c      | 36 ++++++++++++++++++++++++++++++++++--
>  3 files changed, 47 insertions(+), 9 deletions(-)
> 
> diff --git a/mm/swap.h b/mm/swap.h
> index 77d2d14eda42..c617e5a0257f 100644
> --- a/mm/swap.h
> +++ b/mm/swap.h
> @@ -306,7 +306,8 @@ void *swap_cache_get_shadow(swp_entry_t entry);
>  void swap_cache_del_folio(struct folio *folio);
>  struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask,
>  				     unsigned long orders, struct vm_fault *vmf,
> -				     struct mempolicy *mpol, pgoff_t ilx);
> +				     struct mempolicy *mpol, pgoff_t ilx,
> +				     bool skip_lru);
>  /* Below helpers require the caller to lock and pass in the swap cluster. */
>  void __swap_cache_add_folio(struct swap_cluster_info *ci,
>  			    struct folio *folio, swp_entry_t entry);
> diff --git a/mm/swap_state.c b/mm/swap_state.c
> index 9c3a5cf99778..048efc7ca612 100644
> --- a/mm/swap_state.c
> +++ b/mm/swap_state.c
> @@ -403,7 +403,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
>  static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
>  					swp_entry_t targ_entry, gfp_t gfp,
>  					unsigned int order, struct vm_fault *vmf,
> -					struct mempolicy *mpol, pgoff_t ilx)
> +					struct mempolicy *mpol, pgoff_t ilx,
> +					bool skip_lru)
>  {
>  	int err;
>  	swp_entry_t entry;
> @@ -484,7 +485,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
>  	lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages);
>  
>  	/* Caller will initiate read into locked new_folio */
> -	folio_add_lru(folio);
> +	if (!skip_lru)
> +		folio_add_lru(folio);
>  	return folio;
>  }
>  
> @@ -507,7 +509,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
>   */
>  struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
>  				     unsigned long orders, struct vm_fault *vmf,
> -				     struct mempolicy *mpol, pgoff_t ilx)
> +				     struct mempolicy *mpol, pgoff_t ilx,
> +				     bool skip_lru)

Compared to the bool skip_lru here, will this work better? (not tested)

diff --git a/mm/swap_state.c b/mm/swap_state.c
index 8afd0b2d7c27..7ff5254b6ff8 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -487,9 +487,6 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
 
        node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages);
        lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages);
-
-       /* Caller will initiate read into locked new_folio */
-       folio_add_lru(folio);
        return folio;
 }
 
@@ -651,6 +648,8 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
                if (folio)
                        return folio;
                folio = swap_cache_alloc_folio(entry, gfp, orders, NULL, mpol, ilx);
+               if (!IS_ERR(folio))
+                       folio_add_lru(folio);
        } while (PTR_ERR(folio) == -EEXIST);
 
        if (IS_ERR_OR_NULL(folio))
@@ -692,6 +691,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp,
                if (folio)
                        return folio;
                folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx);
+               if (!IS_ERR(folio))
+                       folio_add_lru(folio);
        } while (PTR_ERR(folio) == -EEXIST);

It's identical code wise, just fewer arguments and changes.

Prehaps also rename swap_cache_alloc_folio to __swap_cache_alloc_folio
and update kdoc that caller need to ensure the folio won't be leaked off-LRU
and unreclaimable now.

[PATCH 2/2] mm: zswap: deferred dropbehind free of writeback folios

[Alexandre Ghiti]

The previous change frees zswap writeback folios promptly for
synchronous-IO swap, where writeback completes in the calling context.
Asynchronous block devices and filesystem-backed swap complete
writeback in interrupt context, where the folio cannot be freed, so
those cold folios are still left on the LRU for reclaim to clean up
later.

Free them promptly as well by deferring the work to a workqueue that
runs once writeback has completed.

Suggested-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Alexandre Ghiti <alex@ghiti.fr>
---
 include/linux/zswap.h |  2 ++
 mm/filemap.c          | 20 ++++++++++++++++++++
 mm/zswap.c            | 34 +++++++++++++++++++++++++++++++---
 3 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/include/linux/zswap.h b/include/linux/zswap.h
index 30c193a1207e..23dcde274964 100644
--- a/include/linux/zswap.h
+++ b/include/linux/zswap.h
@@ -35,6 +35,7 @@ void zswap_lruvec_state_init(struct lruvec *lruvec);
 void zswap_folio_swapin(struct folio *folio);
 bool zswap_is_enabled(void);
 bool zswap_never_enabled(void);
+void zswap_writeback_dropbehind_folio(struct folio *folio);
 #else
 
 struct zswap_lruvec_state {};
@@ -58,6 +59,7 @@ static inline void zswap_swapoff(int type) {}
 static inline void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) {}
 static inline void zswap_lruvec_state_init(struct lruvec *lruvec) {}
 static inline void zswap_folio_swapin(struct folio *folio) {}
+static inline void zswap_writeback_dropbehind_folio(struct folio *folio) {}
 
 static inline bool zswap_is_enabled(void)
 {
diff --git a/mm/filemap.c b/mm/filemap.c
index dc3a0e960b9f..548e465b5b9a 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -21,6 +21,7 @@
 #include <linux/gfp.h>
 #include <linux/mm.h>
 #include <linux/swap.h>
+#include <linux/zswap.h>
 #include <linux/leafops.h>
 #include <linux/syscalls.h>
 #include <linux/mman.h>
@@ -1680,6 +1681,8 @@ EXPORT_SYMBOL_GPL(folio_end_writeback_no_dropbehind);
  */
 void folio_end_writeback(struct folio *folio)
 {
+	bool swap_dropbehind;
+
 	VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio);
 
 	/*
@@ -1689,7 +1692,24 @@ void folio_end_writeback(struct folio *folio)
 	 * reused before the folio_wake_bit().
 	 */
 	folio_get(folio);
+
+	/*
+	 * zswap writeback folios are off-LRU, so we must prevent a racing
+	 * swapin from removing the folio from the swap cache and keeping it
+	 * off-LRU: the writeback flag allows that. Afterwards a swapin may win
+	 * the race, but the folio is already queued and the worker puts it back
+	 * on the LRU in that case.
+	 */
+	swap_dropbehind = folio_test_swapcache(folio) &&
+			  folio_test_dropbehind(folio);
+
 	folio_end_writeback_no_dropbehind(folio);
+
+	if (swap_dropbehind) {
+		zswap_writeback_dropbehind_folio(folio);
+		return;
+	}
+
 	folio_end_dropbehind(folio);
 	folio_put(folio);
 }
diff --git a/mm/zswap.c b/mm/zswap.c
index 3c494c5a671c..d185da85bb68 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -34,6 +34,7 @@
 #include <linux/writeback.h>
 #include <linux/pagemap.h>
 #include <linux/workqueue.h>
+#include <linux/llist.h>
 #include <linux/list_lru.h>
 #include <linux/zsmalloc.h>
 
@@ -990,12 +991,39 @@ static void zswap_writeback_free_folio(struct folio *folio)
 		goto out;
 
 	/* Raced: the folio is now owned by the swapin; put it back on the LRU. */
+	folio_clear_dropbehind(folio);
 	folio_add_lru(folio);
 out:
 	folio_unlock(folio);
 	folio_put(folio);
 }
 
+static DEFINE_PER_CPU(struct llist_head, zswap_dropbehind_llist);
+
+static void zswap_dropbehind_workfn(struct work_struct *work)
+{
+	struct llist_node *pos, *next;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		pos = llist_del_all(per_cpu_ptr(&zswap_dropbehind_llist, cpu));
+		llist_for_each_safe(pos, next, pos) {
+			struct folio *folio = container_of((struct list_head *)pos,
+							   struct folio, lru);
+			zswap_writeback_free_folio(folio);
+		}
+	}
+}
+
+static DECLARE_WORK(zswap_dropbehind_work, zswap_dropbehind_workfn);
+
+void zswap_writeback_dropbehind_folio(struct folio *folio)
+{
+	llist_add((struct llist_node *)&folio->lru,
+		  this_cpu_ptr(&zswap_dropbehind_llist));
+	schedule_work(&zswap_dropbehind_work);
+}
+
 /*
  * Attempts to free an entry by adding a folio to the swap cache,
  * decompressing the entry data into the folio, and issuing a
@@ -1027,7 +1055,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 
 	mpol = get_task_policy(current);
 	folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol,
-				       NO_INTERLEAVE_INDEX, sync);
+				       NO_INTERLEAVE_INDEX, true);
 	put_swap_device(si);
 
 	/*
@@ -1071,9 +1099,9 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	/* folio is up to date */
 	folio_mark_uptodate(folio);
 
-	/* move it to the tail of the inactive list after end_writeback */
+	/* Free the folio once writeback completes; see folio_end_writeback(). */
 	if (!sync)
-		folio_set_reclaim(folio);
+		folio_set_dropbehind(folio);
 
 	/* start writeback */
 	__swap_writepage(folio, NULL);
-- 
2.53.0-Meta