* [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
@ 2026-07-27 14:04 ` Baoquan He
2026-07-27 15:05 ` Nhat Pham
2026-07-27 16:09 ` Chris Li
2026-07-27 14:04 ` [RFC PATCH 04/11] mm, swap: add xswap grow trigger on cluster allocation Baoquan He
` (8 subsequent siblings)
9 siblings, 2 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:04 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
Implement dynamic cluster_info array growth for xswap devices using a
VM_SPARSE vmalloc area:
1. xswap_map_clusters(): Allocate physical pages and map them into
the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().
2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via
vm_area_unmap_pages() (used by the error/teardown paths, shrink
comes later).
3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE)
for the cluster_info array, lazily mapping only the initial chunk.
4. free_swap_cluster_info(): Refactor to take swap_info_struct*.
For xswap, unmap all clusters and free_vm_area().
5. swapoff: Remove snapshot locals; move p->max/p->cluster_info
clearing after free_swap_cluster_info().
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 216 insertions(+), 12 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 143088dae07d..6d9c95ed09bd 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -49,6 +49,24 @@
#include "internal.h"
#include "swap.h"
+#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation. It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+ max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
+static int xswap_map_clusters(struct swap_info_struct *si,
+ unsigned long start_idx, unsigned long nr);
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+ unsigned long start_idx, unsigned long nr);
+#endif
+
static void swap_range_alloc(struct swap_info_struct *si,
unsigned int nr_entries);
static bool folio_swapcache_freeable(struct folio *folio);
@@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
BUG_ON(si->flags & SWP_WRITEOK);
+#ifdef CONFIG_XSWAP
+ /*
+ * xswap clusters beyond nr_clusters_mapped have been unmapped
+ * by the shrinker and their vmalloc pages are no longer
+ * accessible. Only iterate over currently mapped clusters.
+ */
+ if (si->flags & SWP_XSWAP)
+ end = min(end, READ_ONCE(si->nr_clusters_mapped) *
+ SWAPFILE_CLUSTER);
+#endif
+
for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
ci = swap_cluster_lock(si, offset);
swap_cluster_unlock(ci);
}
}
-static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
- unsigned long maxpages)
+static void free_swap_cluster_info(struct swap_info_struct *si)
{
+ struct swap_cluster_info *cluster_info = si->cluster_info;
+ unsigned long maxpages = si->max;
struct swap_cluster_info *ci;
- int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
+ int i, nr_clusters;
if (!cluster_info)
return;
+
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ /* Unmap all mapped clusters and free the VM_SPARSE area */
+ if (si->nr_clusters_mapped > 0)
+ xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+ free_vm_area(si->cluster_vm);
+ si->cluster_vm = NULL;
+ si->nr_clusters = 0;
+ si->nr_clusters_mapped = 0;
+ return;
+ }
+#endif
+
+ nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
for (i = 0; i < nr_clusters; i++) {
ci = cluster_info + i;
/* Cluster with bad marks count will have a remaining table */
@@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
{
struct swap_info_struct *p = NULL;
- struct swap_cluster_info *cluster_info;
struct file *swap_file, *victim;
struct address_space *mapping;
struct inode *inode;
- unsigned int maxpages;
int err, found = 0;
if (!capable(CAP_SYS_ADMIN))
@@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
swap_file = p->swap_file;
p->swap_file = NULL;
- maxpages = p->max;
- cluster_info = p->cluster_info;
- p->max = 0;
- p->cluster_info = NULL;
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
arch_swap_invalidate_area(p->type);
@@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
mutex_unlock(&swapon_mutex);
kfree(p->global_cluster);
p->global_cluster = NULL;
- free_swap_cluster_info(cluster_info, maxpages);
+ free_swap_cluster_info(p);
+ p->max = 0;
+ p->cluster_info = NULL;
inode = mapping->host;
@@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
return maxpages;
}
+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+ unsigned long start_idx, unsigned long nr)
+{
+ unsigned long start_addr = (unsigned long)si->cluster_info +
+ (size_t)start_idx * sizeof(struct swap_cluster_info);
+ unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+ /*
+ * vm_area_map_pages() requires that start and end be page-aligned.
+ * If start_addr falls within a page that was already mapped by a
+ * previous batch (grow path), round it up to skip the already-mapped
+ * partial page. Always round end_addr up so the vmap page table walk
+ * terminates correctly (the walk loop exits when addr == end, and addr
+ * advances by PAGE_SIZE each iteration).
+ */
+ unsigned long vm_start = PAGE_ALIGN(start_addr);
+ unsigned long vm_end = PAGE_ALIGN(end_addr);
+ unsigned long npages;
+ struct page **pages;
+ unsigned long i;
+
+ if (vm_start >= vm_end) {
+ /* All requested clusters fall within already-mapped pages. */
+ for (i = start_idx; i < start_idx + nr; i++)
+ spin_lock_init(&si->cluster_info[i].lock);
+ WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+ return 0;
+ }
+
+ npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+ pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+ if (!pages)
+ return -ENOMEM;
+
+ for (i = 0; i < npages; i++) {
+ /*
+ * __GFP_ZERO is critical: cluster_info structs contain pointer
+ * fields (extend_table, zero_bitmap, memcg_table, table) that
+ * must start as NULL. Without zeroing, stale data from a
+ * previous user of the page would look like valid pointers.
+ */
+ pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!pages[i])
+ goto fail;
+ }
+
+ if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
+ i = npages; /* free all pages on failure */
+ goto fail;
+ }
+
+ kfree(pages);
+
+ /* Initialize spinlocks for newly mapped clusters */
+ for (i = start_idx; i < start_idx + nr; i++)
+ spin_lock_init(&si->cluster_info[i].lock);
+
+ /*
+ * Pairs with READ_ONCE() in shrink/grow paths.
+ */
+ WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+ return 0;
+
+fail:
+ while (i > 0) {
+ i--;
+ if (pages[i])
+ __free_page(pages[i]);
+ }
+ kfree(pages);
+ return -ENOMEM;
+}
+
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+ unsigned long start_idx, unsigned long nr)
+{
+ unsigned long start_addr = (unsigned long)si->cluster_info +
+ (size_t)start_idx * sizeof(struct swap_cluster_info);
+ unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+ /*
+ * Round to page boundaries: start up (skip partial page that may
+ * contain clusters still in use before start_idx), end up so the
+ * entire range is covered. vm_area_unmap_pages() operates on
+ * whole pages.
+ */
+ unsigned long vm_start = PAGE_ALIGN(start_addr);
+ unsigned long vm_end = PAGE_ALIGN(end_addr);
+
+ if (vm_start >= vm_end)
+ goto out;
+
+ vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
+ /*
+ * vm_area_unmap_pages() only clears PTEs; it does not free the
+ * physical pages. Walk the page table to find and free them.
+ */
+ /* TODO: free backing pages via page table walk or tracking bitmap */
+ /*
+ * Pairs with READ_ONCE() in shrink/grow paths.
+ */
+ out:
+ WRITE_ONCE(si->nr_clusters_mapped, start_idx);
+}
+#endif /* CONFIG_XSWAP */
+
static int setup_swap_clusters_info(struct swap_info_struct *si,
union swap_header *swap_header,
unsigned long maxpages)
@@ -3573,6 +3720,63 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
int err = -ENOMEM;
unsigned long i;
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
+ struct vm_struct *vm;
+
+ vm = get_vm_area(size, VM_SPARSE);
+ if (!vm)
+ goto err;
+
+ cluster_info = vm->addr;
+ si->cluster_vm = vm;
+ si->nr_clusters = nr_clusters;
+ si->cluster_info = cluster_info;
+
+ /* Map the initial chunk (at least cluster 0) */
+ if (xswap_map_clusters(si, 0, min_t(unsigned long,
+ XSWAP_GROW_CLUSTERS, nr_clusters)))
+ goto err_free_vm;
+
+ /* xswap: only cluster 0 slot 0 is bad */
+ err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
+ if (err)
+ goto err_unmap;
+
+ INIT_LIST_HEAD(&si->free_clusters);
+ INIT_LIST_HEAD(&si->full_clusters);
+ INIT_LIST_HEAD(&si->discard_clusters);
+ for (i = 0; i < SWAP_NR_ORDERS; i++) {
+ INIT_LIST_HEAD(&si->nonfull_clusters[i]);
+ INIT_LIST_HEAD(&si->frag_clusters[i]);
+ }
+
+ /* Mark mapped clusters: cluster 0 has 1 bad slot, rest free */
+ for (i = 0; i < si->nr_clusters_mapped; i++) {
+ struct swap_cluster_info *ci = &cluster_info[i];
+
+ if (i == 0) {
+ ci->flags = CLUSTER_FLAG_NONFULL;
+ list_add_tail(&ci->list, &si->nonfull_clusters[0]);
+ } else {
+ ci->flags = CLUSTER_FLAG_FREE;
+ list_add_tail(&ci->list, &si->free_clusters);
+ }
+ }
+
+ return 0;
+
+err_unmap:
+ xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+err_free_vm:
+ free_vm_area(si->cluster_vm);
+ si->cluster_vm = NULL;
+ si->cluster_info = NULL;
+ return err;
+ }
+#endif /* CONFIG_XSWAP */
+
cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
if (!cluster_info)
goto err;
@@ -3640,7 +3844,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
si->cluster_info = cluster_info;
return 0;
err:
- free_swap_cluster_info(cluster_info, maxpages);
+ free_swap_cluster_info(si);
return err;
}
@@ -3852,7 +4056,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
si->global_cluster = NULL;
inode = NULL;
destroy_swap_extents(si, swap_file);
- free_swap_cluster_info(si->cluster_info, si->max);
+ free_swap_cluster_info(si);
si->cluster_info = NULL;
/*
* Clear the SWP_USED flag after all resources are freed so
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc
2026-07-27 14:04 ` [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc Baoquan He
@ 2026-07-27 15:05 ` Nhat Pham
2026-07-27 16:09 ` Chris Li
1 sibling, 0 replies; 17+ messages in thread
From: Nhat Pham @ 2026-07-27 15:05 UTC (permalink / raw)
To: Baoquan He
Cc: linux-mm, akpm, chrisl, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel
On Mon, Jul 27, 2026 at 7:05 AM Baoquan He <baoquan.he@linux.dev> wrote:
>
> Implement dynamic cluster_info array growth for xswap devices using a
> VM_SPARSE vmalloc area:
>
> 1. xswap_map_clusters(): Allocate physical pages and map them into
> the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().
>
> 2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via
> vm_area_unmap_pages() (used by the error/teardown paths, shrink
> comes later).
>
> 3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE)
> for the cluster_info array, lazily mapping only the initial chunk.
>
> 4. free_swap_cluster_info(): Refactor to take swap_info_struct*.
> For xswap, unmap all clusters and free_vm_area().
>
> 5. swapoff: Remove snapshot locals; move p->max/p->cluster_info
> clearing after free_swap_cluster_info().
>
> Signed-off-by: Baoquan He <baoquan.he@linux.dev>
I'll give my 2 cents on the direction we're pursuing later. I've done
a lot of thinking over the past weeks, and I have some new concerns
now regarding how we plan to land this joint venture :)
That said, this is a new piece of extension, done per my request
(kernel-driven dynamic swap address space growth - thanks for taking
my concerns seriously), and I've also evaluated this vmalloc-based
data structure in the past week, so I'll put in my inquiries/concerns.
> ---
> mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
> 1 file changed, 216 insertions(+), 12 deletions(-)
>
> diff --git a/mm/swapfile.c b/mm/swapfile.c
> index 143088dae07d..6d9c95ed09bd 100644
> --- a/mm/swapfile.c
> +++ b/mm/swapfile.c
> @@ -49,6 +49,24 @@
> #include "internal.h"
> #include "swap.h"
>
> +#ifdef CONFIG_XSWAP
> +/*
> + * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
> + *
> + * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
> + * operation. It is set to the number of cluster_info structs that
> + * fit in a single page (at least 16), so that the vmalloc page table
> + * overhead is proportional to the number of clusters mapped.
> + */
> +#define XSWAP_GROW_CLUSTERS \
> + max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
> +
> +static int xswap_map_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr);
> +static void xswap_unmap_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr);
> +#endif
> +
> static void swap_range_alloc(struct swap_info_struct *si,
> unsigned int nr_entries);
> static bool folio_swapcache_freeable(struct folio *folio);
> @@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
>
> BUG_ON(si->flags & SWP_WRITEOK);
>
> +#ifdef CONFIG_XSWAP
> + /*
> + * xswap clusters beyond nr_clusters_mapped have been unmapped
> + * by the shrinker and their vmalloc pages are no longer
> + * accessible. Only iterate over currently mapped clusters.
> + */
> + if (si->flags & SWP_XSWAP)
> + end = min(end, READ_ONCE(si->nr_clusters_mapped) *
> + SWAPFILE_CLUSTER);
> +#endif
> +
> for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
> ci = swap_cluster_lock(si, offset);
> swap_cluster_unlock(ci);
> }
> }
>
> -static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
> - unsigned long maxpages)
> +static void free_swap_cluster_info(struct swap_info_struct *si)
> {
> + struct swap_cluster_info *cluster_info = si->cluster_info;
> + unsigned long maxpages = si->max;
> struct swap_cluster_info *ci;
> - int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
> + int i, nr_clusters;
>
> if (!cluster_info)
> return;
> +
> +#ifdef CONFIG_XSWAP
> + if (si->flags & SWP_XSWAP) {
> + /* Unmap all mapped clusters and free the VM_SPARSE area */
> + if (si->nr_clusters_mapped > 0)
> + xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
> + free_vm_area(si->cluster_vm);
> + si->cluster_vm = NULL;
> + si->nr_clusters = 0;
> + si->nr_clusters_mapped = 0;
> + return;
> + }
> +#endif
> +
> + nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
> for (i = 0; i < nr_clusters; i++) {
> ci = cluster_info + i;
> /* Cluster with bad marks count will have a remaining table */
> @@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
> SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
> {
> struct swap_info_struct *p = NULL;
> - struct swap_cluster_info *cluster_info;
> struct file *swap_file, *victim;
> struct address_space *mapping;
> struct inode *inode;
> - unsigned int maxpages;
> int err, found = 0;
>
> if (!capable(CAP_SYS_ADMIN))
> @@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
>
> swap_file = p->swap_file;
> p->swap_file = NULL;
> - maxpages = p->max;
> - cluster_info = p->cluster_info;
> - p->max = 0;
> - p->cluster_info = NULL;
> spin_unlock(&p->lock);
> spin_unlock(&swap_lock);
> arch_swap_invalidate_area(p->type);
> @@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
> mutex_unlock(&swapon_mutex);
> kfree(p->global_cluster);
> p->global_cluster = NULL;
> - free_swap_cluster_info(cluster_info, maxpages);
> + free_swap_cluster_info(p);
> + p->max = 0;
> + p->cluster_info = NULL;
>
> inode = mapping->host;
>
> @@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
> return maxpages;
> }
>
> +#ifdef CONFIG_XSWAP
> +static int xswap_map_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr)
> +{
> + unsigned long start_addr = (unsigned long)si->cluster_info +
> + (size_t)start_idx * sizeof(struct swap_cluster_info);
> + unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
> + /*
> + * vm_area_map_pages() requires that start and end be page-aligned.
> + * If start_addr falls within a page that was already mapped by a
> + * previous batch (grow path), round it up to skip the already-mapped
> + * partial page. Always round end_addr up so the vmap page table walk
> + * terminates correctly (the walk loop exits when addr == end, and addr
> + * advances by PAGE_SIZE each iteration).
> + */
> + unsigned long vm_start = PAGE_ALIGN(start_addr);
> + unsigned long vm_end = PAGE_ALIGN(end_addr);
> + unsigned long npages;
> + struct page **pages;
> + unsigned long i;
> +
> + if (vm_start >= vm_end) {
> + /* All requested clusters fall within already-mapped pages. */
> + for (i = start_idx; i < start_idx + nr; i++)
> + spin_lock_init(&si->cluster_info[i].lock);
> + WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
> + return 0;
> + }
> +
> + npages = (vm_end - vm_start) >> PAGE_SHIFT;
> +
> + pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
> + if (!pages)
> + return -ENOMEM;
> +
> + for (i = 0; i < npages; i++) {
> + /*
> + * __GFP_ZERO is critical: cluster_info structs contain pointer
> + * fields (extend_table, zero_bitmap, memcg_table, table) that
> + * must start as NULL. Without zeroing, stale data from a
> + * previous user of the page would look like valid pointers.
> + */
> + pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
> + if (!pages[i])
> + goto fail;
> + }
> +
> + if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
> + i = npages; /* free all pages on failure */
> + goto fail;
I was evaluating whether your vmalloc array can be extended to support
kernel-driven dynamic growth at least (either as a slot-in replacement
for xarray, or as a follow-up optimization if it's too complicated),
and I stumble this mapping action.
Seems like it does not take any GFP flag argument, and under the hood
it calls GFP_KERNEL. Would this be safe in the swap allocation path
(where you slotted it in in patch 4)? Kairui used a more precise set
of flags in this path, for e.g for swap table allocation:
ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
GFP_KERNEL);
I'm guessing this has to do with the fact that we often entered
swapping out paths with PF_MEMALLOC... Is there any risk of deadlock
etc.?
Kairui, any thoughts?
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc
2026-07-27 14:04 ` [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc Baoquan He
2026-07-27 15:05 ` Nhat Pham
@ 2026-07-27 16:09 ` Chris Li
1 sibling, 0 replies; 17+ messages in thread
From: Chris Li @ 2026-07-27 16:09 UTC (permalink / raw)
To: Baoquan He
Cc: linux-mm, akpm, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel
On Mon, Jul 27, 2026 at 7:05 AM Baoquan He <baoquan.he@linux.dev> wrote:
>
> Implement dynamic cluster_info array growth for xswap devices using a
> VM_SPARSE vmalloc area:
>
> 1. xswap_map_clusters(): Allocate physical pages and map them into
> the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().
>
> 2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via
> vm_area_unmap_pages() (used by the error/teardown paths, shrink
> comes later).
>
> 3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE)
> for the cluster_info array, lazily mapping only the initial chunk.
>
> 4. free_swap_cluster_info(): Refactor to take swap_info_struct*.
> For xswap, unmap all clusters and free_vm_area().
>
> 5. swapoff: Remove snapshot locals; move p->max/p->cluster_info
> clearing after free_swap_cluster_info().
I feel that we should provide more generic support for dynamically
growing an array of element size A to size N. Cluster array is just a
user of such an API. There might be more users. This patch hardcodes
the map/umpa of clusters. What about other users of such a dynamic
array?
Another consideration is that the map/unmap logic, while more generic,
adds quite a bit of complexity to an approach intended only for
growth. Any reason we want map/unmap rathern than simple grow from the
current si->max?
I read a bit more about the patch. It seems you need the unmap for the
error cleanup path. In that case I suspect just make cluster a
kmem_cache allocation, cluster array just a pointer. The cluster
pointer array should just grow, not shrink. That might be overall
simpler.
I'm worried someone might abuse the cluster unmap causing unwanted side effects.
I see the other reason might be you want the cluster code to be
exactly the same when XSWAP is off. For that I think it is acceptiale
to turn cluster into kmem_cache allocated and cluster array as
pointers array as seperate patch as the base even when XSWAP is not
configed.
Chris
> Signed-off-by: Baoquan He <baoquan.he@linux.dev>
> ---
> mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
> 1 file changed, 216 insertions(+), 12 deletions(-)
>
> diff --git a/mm/swapfile.c b/mm/swapfile.c
> index 143088dae07d..6d9c95ed09bd 100644
> --- a/mm/swapfile.c
> +++ b/mm/swapfile.c
> @@ -49,6 +49,24 @@
> #include "internal.h"
> #include "swap.h"
>
> +#ifdef CONFIG_XSWAP
> +/*
> + * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
> + *
> + * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
> + * operation. It is set to the number of cluster_info structs that
> + * fit in a single page (at least 16), so that the vmalloc page table
> + * overhead is proportional to the number of clusters mapped.
> + */
> +#define XSWAP_GROW_CLUSTERS \
> + max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
> +
> +static int xswap_map_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr);
> +static void xswap_unmap_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr);
> +#endif
> +
> static void swap_range_alloc(struct swap_info_struct *si,
> unsigned int nr_entries);
> static bool folio_swapcache_freeable(struct folio *folio);
> @@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
>
> BUG_ON(si->flags & SWP_WRITEOK);
>
> +#ifdef CONFIG_XSWAP
> + /*
> + * xswap clusters beyond nr_clusters_mapped have been unmapped
> + * by the shrinker and their vmalloc pages are no longer
> + * accessible. Only iterate over currently mapped clusters.
> + */
> + if (si->flags & SWP_XSWAP)
> + end = min(end, READ_ONCE(si->nr_clusters_mapped) *
> + SWAPFILE_CLUSTER);
> +#endif
> +
> for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
> ci = swap_cluster_lock(si, offset);
> swap_cluster_unlock(ci);
> }
> }
>
> -static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
> - unsigned long maxpages)
> +static void free_swap_cluster_info(struct swap_info_struct *si)
> {
> + struct swap_cluster_info *cluster_info = si->cluster_info;
> + unsigned long maxpages = si->max;
> struct swap_cluster_info *ci;
> - int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
> + int i, nr_clusters;
>
> if (!cluster_info)
> return;
> +
> +#ifdef CONFIG_XSWAP
> + if (si->flags & SWP_XSWAP) {
> + /* Unmap all mapped clusters and free the VM_SPARSE area */
> + if (si->nr_clusters_mapped > 0)
> + xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
> + free_vm_area(si->cluster_vm);
> + si->cluster_vm = NULL;
> + si->nr_clusters = 0;
> + si->nr_clusters_mapped = 0;
> + return;
> + }
> +#endif
> +
> + nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
> for (i = 0; i < nr_clusters; i++) {
> ci = cluster_info + i;
> /* Cluster with bad marks count will have a remaining table */
> @@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
> SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
> {
> struct swap_info_struct *p = NULL;
> - struct swap_cluster_info *cluster_info;
> struct file *swap_file, *victim;
> struct address_space *mapping;
> struct inode *inode;
> - unsigned int maxpages;
> int err, found = 0;
>
> if (!capable(CAP_SYS_ADMIN))
> @@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
>
> swap_file = p->swap_file;
> p->swap_file = NULL;
> - maxpages = p->max;
> - cluster_info = p->cluster_info;
> - p->max = 0;
> - p->cluster_info = NULL;
> spin_unlock(&p->lock);
> spin_unlock(&swap_lock);
> arch_swap_invalidate_area(p->type);
> @@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
> mutex_unlock(&swapon_mutex);
> kfree(p->global_cluster);
> p->global_cluster = NULL;
> - free_swap_cluster_info(cluster_info, maxpages);
> + free_swap_cluster_info(p);
> + p->max = 0;
> + p->cluster_info = NULL;
>
> inode = mapping->host;
>
> @@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
> return maxpages;
> }
>
> +#ifdef CONFIG_XSWAP
> +static int xswap_map_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr)
> +{
> + unsigned long start_addr = (unsigned long)si->cluster_info +
> + (size_t)start_idx * sizeof(struct swap_cluster_info);
> + unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
> + /*
> + * vm_area_map_pages() requires that start and end be page-aligned.
> + * If start_addr falls within a page that was already mapped by a
> + * previous batch (grow path), round it up to skip the already-mapped
> + * partial page. Always round end_addr up so the vmap page table walk
> + * terminates correctly (the walk loop exits when addr == end, and addr
> + * advances by PAGE_SIZE each iteration).
> + */
> + unsigned long vm_start = PAGE_ALIGN(start_addr);
> + unsigned long vm_end = PAGE_ALIGN(end_addr);
> + unsigned long npages;
> + struct page **pages;
> + unsigned long i;
> +
> + if (vm_start >= vm_end) {
> + /* All requested clusters fall within already-mapped pages. */
> + for (i = start_idx; i < start_idx + nr; i++)
> + spin_lock_init(&si->cluster_info[i].lock);
> + WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
> + return 0;
> + }
> +
> + npages = (vm_end - vm_start) >> PAGE_SHIFT;
> +
> + pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
> + if (!pages)
> + return -ENOMEM;
> +
> + for (i = 0; i < npages; i++) {
> + /*
> + * __GFP_ZERO is critical: cluster_info structs contain pointer
> + * fields (extend_table, zero_bitmap, memcg_table, table) that
> + * must start as NULL. Without zeroing, stale data from a
> + * previous user of the page would look like valid pointers.
> + */
> + pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
> + if (!pages[i])
> + goto fail;
> + }
> +
> + if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
> + i = npages; /* free all pages on failure */
> + goto fail;
> + }
> +
> + kfree(pages);
> +
> + /* Initialize spinlocks for newly mapped clusters */
> + for (i = start_idx; i < start_idx + nr; i++)
> + spin_lock_init(&si->cluster_info[i].lock);
> +
> + /*
> + * Pairs with READ_ONCE() in shrink/grow paths.
> + */
> + WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
> + return 0;
> +
> +fail:
> + while (i > 0) {
> + i--;
> + if (pages[i])
> + __free_page(pages[i]);
> + }
> + kfree(pages);
> + return -ENOMEM;
> +}
> +
> +static void xswap_unmap_clusters(struct swap_info_struct *si,
> + unsigned long start_idx, unsigned long nr)
> +{
> + unsigned long start_addr = (unsigned long)si->cluster_info +
> + (size_t)start_idx * sizeof(struct swap_cluster_info);
> + unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
> + /*
> + * Round to page boundaries: start up (skip partial page that may
> + * contain clusters still in use before start_idx), end up so the
> + * entire range is covered. vm_area_unmap_pages() operates on
> + * whole pages.
> + */
> + unsigned long vm_start = PAGE_ALIGN(start_addr);
> + unsigned long vm_end = PAGE_ALIGN(end_addr);
> +
> + if (vm_start >= vm_end)
> + goto out;
> +
> + vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
> + /*
> + * vm_area_unmap_pages() only clears PTEs; it does not free the
> + * physical pages. Walk the page table to find and free them.
> + */
> + /* TODO: free backing pages via page table walk or tracking bitmap */
> + /*
> + * Pairs with READ_ONCE() in shrink/grow paths.
> + */
> + out:
> + WRITE_ONCE(si->nr_clusters_mapped, start_idx);
> +}
> +#endif /* CONFIG_XSWAP */
> +
> static int setup_swap_clusters_info(struct swap_info_struct *si,
> union swap_header *swap_header,
> unsigned long maxpages)
> @@ -3573,6 +3720,63 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
> int err = -ENOMEM;
> unsigned long i;
>
> +#ifdef CONFIG_XSWAP
> + if (si->flags & SWP_XSWAP) {
> + unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
> + struct vm_struct *vm;
> +
> + vm = get_vm_area(size, VM_SPARSE);
> + if (!vm)
> + goto err;
> +
> + cluster_info = vm->addr;
> + si->cluster_vm = vm;
> + si->nr_clusters = nr_clusters;
> + si->cluster_info = cluster_info;
> +
> + /* Map the initial chunk (at least cluster 0) */
> + if (xswap_map_clusters(si, 0, min_t(unsigned long,
> + XSWAP_GROW_CLUSTERS, nr_clusters)))
> + goto err_free_vm;
> +
> + /* xswap: only cluster 0 slot 0 is bad */
> + err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
> + if (err)
> + goto err_unmap;
> +
> + INIT_LIST_HEAD(&si->free_clusters);
> + INIT_LIST_HEAD(&si->full_clusters);
> + INIT_LIST_HEAD(&si->discard_clusters);
> + for (i = 0; i < SWAP_NR_ORDERS; i++) {
> + INIT_LIST_HEAD(&si->nonfull_clusters[i]);
> + INIT_LIST_HEAD(&si->frag_clusters[i]);
> + }
> +
> + /* Mark mapped clusters: cluster 0 has 1 bad slot, rest free */
> + for (i = 0; i < si->nr_clusters_mapped; i++) {
> + struct swap_cluster_info *ci = &cluster_info[i];
> +
> + if (i == 0) {
> + ci->flags = CLUSTER_FLAG_NONFULL;
> + list_add_tail(&ci->list, &si->nonfull_clusters[0]);
> + } else {
> + ci->flags = CLUSTER_FLAG_FREE;
> + list_add_tail(&ci->list, &si->free_clusters);
> + }
> + }
> +
> + return 0;
> +
> +err_unmap:
> + xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
> +err_free_vm:
> + free_vm_area(si->cluster_vm);
> + si->cluster_vm = NULL;
> + si->cluster_info = NULL;
> + return err;
> + }
> +#endif /* CONFIG_XSWAP */
> +
> cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
> if (!cluster_info)
> goto err;
> @@ -3640,7 +3844,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
> si->cluster_info = cluster_info;
> return 0;
> err:
> - free_swap_cluster_info(cluster_info, maxpages);
> + free_swap_cluster_info(si);
> return err;
> }
>
> @@ -3852,7 +4056,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
> si->global_cluster = NULL;
> inode = NULL;
> destroy_swap_extents(si, swap_file);
> - free_swap_cluster_info(si->cluster_info, si->max);
> + free_swap_cluster_info(si);
> si->cluster_info = NULL;
> /*
> * Clear the SWP_USED flag after all resources are freed so
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 17+ messages in thread
* [RFC PATCH 04/11] mm, swap: add xswap grow trigger on cluster allocation
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
2026-07-27 14:04 ` [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc Baoquan He
@ 2026-07-27 14:04 ` Baoquan He
2026-07-27 14:04 ` [RFC PATCH 05/11] mm, swap: add xswap_try_shrink and shrink trigger on cluster free Baoquan He
` (7 subsequent siblings)
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:04 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
When cluster_alloc_swap_entry() fails to find a free cluster and
the xswap device still has room to grow, expand the mapped range
by XSWAP_GROW_CLUSTERS clusters.
Since xswap is always SWP_SOLIDSTATE, no locks need to be dropped
before calling xswap_map_clusters() — global_cluster_lock is never
held on this path.
The grow sequence:
1. Check nr_clusters_mapped < nr_clusters and free list empty
2. Call xswap_map_clusters() to allocate and map more physical pages
3. Add newly mapped clusters to si->free_clusters under si->lock
4. Retry allocation from the fresh free clusters
This makes the xswap cluster space grow transparently as swap usage
increases, without any userspace intervention.
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
mm/swapfile.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 103 insertions(+), 3 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 6d9c95ed09bd..8a048b2897ce 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -65,6 +65,7 @@ static int xswap_map_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr);
static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr);
+static int xswap_check_mapped(pte_t *pte, unsigned long addr, void *data);
#endif
static void swap_range_alloc(struct swap_info_struct *si,
@@ -1204,6 +1205,48 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
if (found)
goto done;
}
+
+#ifdef CONFIG_XSWAP
+ /*
+ * For xswap: if no free cluster was found and more clusters
+ * can be mapped, grow the cluster_info array and retry.
+ */
+ if (!found && (si->flags & SWP_XSWAP) &&
+ READ_ONCE(si->nr_clusters_mapped) < READ_ONCE(si->nr_clusters) &&
+ list_empty(&si->free_clusters)) {
+ unsigned long nr_new = min(READ_ONCE(si->nr_clusters) -
+ READ_ONCE(si->nr_clusters_mapped),
+ XSWAP_GROW_CLUSTERS);
+ unsigned long start = READ_ONCE(si->nr_clusters_mapped);
+ unsigned long i;
+
+ if (!xswap_map_clusters(si, start, nr_new)) {
+ unsigned long added = 0;
+
+ spin_lock(&si->lock);
+ for (i = start; i < start + nr_new; i++) {
+ struct swap_cluster_info *ci = &si->cluster_info[i];
+ spin_lock(&ci->lock);
+ /*
+ * A concurrent grower may have already added
+ * these clusters to the free list. Only add
+ * clusters that are still off-list (NONE).
+ */
+ if (ci->flags == CLUSTER_FLAG_NONE) {
+ ci->flags = CLUSTER_FLAG_FREE;
+ list_add_tail(&ci->list, &si->free_clusters);
+ added++;
+ }
+ spin_unlock(&ci->lock);
+ }
+ spin_unlock(&si->lock);
+
+ /* Retry allocation from the free list */
+ found = alloc_swap_scan_list(si, &si->free_clusters,
+ folio, false);
+ }
+ }
+#endif
done:
if (!(si->flags & SWP_SOLIDSTATE))
spin_unlock(&si->global_cluster_lock);
@@ -3652,9 +3695,37 @@ static int xswap_map_clusters(struct swap_info_struct *si,
goto fail;
}
- if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
- i = npages; /* free all pages on failure */
- goto fail;
+ /*
+ * Check if the target pages are already mapped by a concurrent
+ * grower. We must do this after page allocation because
+ * alloc_page(GFP_KERNEL) can sleep, opening a race window.
+ * If someone already mapped these pages, free ours and continue.
+ */
+ if (apply_to_existing_page_range(&init_mm, vm_start,
+ vm_end - vm_start,
+ xswap_check_mapped, NULL)) {
+ i = npages;
+ goto fail_nounmap;
+ }
+
+ {
+ int err = vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages);
+
+ if (err) {
+ /*
+ * -EBUSY means the PTEs are already present:
+ * another thread raced with us and mapped the
+ * same pages between our check above and this
+ * call. Treat as success — free our unused
+ * pages and continue.
+ */
+ if (err == -EBUSY) {
+ i = npages;
+ goto fail_nounmap;
+ }
+ i = npages; /* free all pages on failure */
+ goto fail;
+ }
}
kfree(pages);
@@ -3669,6 +3740,26 @@ static int xswap_map_clusters(struct swap_info_struct *si,
WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
return 0;
+fail_nounmap:
+ /*
+ * Pages already mapped by a concurrent grower. Free our unused
+ * pages, then fall through to initialize spinlocks. The vmalloc
+ * PTEs now point to the concurrent grower's pages.
+ */
+ while (i > 0) {
+ i--;
+ if (pages[i])
+ __free_page(pages[i]);
+ }
+ kfree(pages);
+
+ /* Initialize spinlocks for newly mapped clusters */
+ for (i = start_idx; i < start_idx + nr; i++)
+ spin_lock_init(&si->cluster_info[i].lock);
+
+ WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+ return 0;
+
fail:
while (i > 0) {
i--;
@@ -3709,6 +3800,15 @@ static void xswap_unmap_clusters(struct swap_info_struct *si,
out:
WRITE_ONCE(si->nr_clusters_mapped, start_idx);
}
+
+/*
+ * Callback for apply_to_existing_page_range(): return 1 to stop at the
+ * first present PTE, signalling that the range is already mapped.
+ */
+static int xswap_check_mapped(pte_t *pte, unsigned long addr, void *data)
+{
+ return 1;
+}
#endif /* CONFIG_XSWAP */
static int setup_swap_clusters_info(struct swap_info_struct *si,
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 05/11] mm, swap: add xswap_try_shrink and shrink trigger on cluster free
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
2026-07-27 14:04 ` [RFC PATCH 03/11] mm, swap: add xswap cluster grow via VM_SPARSE vmalloc Baoquan He
2026-07-27 14:04 ` [RFC PATCH 04/11] mm, swap: add xswap grow trigger on cluster allocation Baoquan He
@ 2026-07-27 14:04 ` Baoquan He
2026-07-27 14:04 ` [RFC PATCH 06/11] mm, swap: free backing pages in xswap_unmap_clusters Baoquan He
` (6 subsequent siblings)
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:04 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
Add xswap_try_shrink() — the shrink logic that scans backwards from
the tail to find contiguous free clusters, then unmaps full pages
when >= XSWAP_GROW_CLUSTERS free clusters accumulate.
Wire the trigger in __free_cluster(): after a cluster is released to
the free list, call xswap_try_shrink() to attempt tail shrinking.
Also update the XSWAP_GROW_CLUSTERS comment to reflect both grow
and shrink semantics.
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
mm/swapfile.c | 48 ++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 44 insertions(+), 4 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 8a048b2897ce..e0f5fe64de2a 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -51,11 +51,13 @@
#ifdef CONFIG_XSWAP
/*
- * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ * xswap: dynamically grow and shrink the cluster_info array via a
+ * VM_SPARSE area.
*
- * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
- * operation. It is set to the number of cluster_info structs that
- * fit in a single page (at least 16), so that the vmalloc page table
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map/unmap in one
+ * grow/shrink operation. It is set to the number of cluster_info
+ * structs that fit in a single page (at least 16), so that the vmalloc
+ * page table
* overhead is proportional to the number of clusters mapped.
*/
#define XSWAP_GROW_CLUSTERS \
@@ -66,6 +68,7 @@ static int xswap_map_clusters(struct swap_info_struct *si,
static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr);
static int xswap_check_mapped(pte_t *pte, unsigned long addr, void *data);
+static void xswap_try_shrink(struct swap_info_struct *si);
#endif
static void swap_range_alloc(struct swap_info_struct *si,
@@ -629,6 +632,9 @@ static void __free_cluster(struct swap_info_struct *si, struct swap_cluster_info
swap_cluster_free_table(ci);
move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
ci->order = 0;
+#ifdef CONFIG_XSWAP
+ xswap_try_shrink(si);
+#endif
}
/*
@@ -3809,6 +3815,40 @@ static int xswap_check_mapped(pte_t *pte, unsigned long addr, void *data)
{
return 1;
}
+
+/*
+ * Try to shrink the cluster_info tail: unmap contiguous free clusters
+ * at the end of the mapped range.
+ */
+static void xswap_try_shrink(struct swap_info_struct *si)
+{
+ struct swap_cluster_info *ci;
+ unsigned long last, idx;
+
+ if (!(si->flags & SWP_XSWAP))
+ return;
+ if (READ_ONCE(si->nr_clusters_mapped) <= 1) /* keep cluster 0 */
+ return;
+
+ /* Find the last non-free cluster from the tail */
+ last = READ_ONCE(si->nr_clusters_mapped);
+ while (last > 1) {
+ idx = last - 1;
+ ci = &si->cluster_info[idx];
+ if (ci->count || ci->flags != CLUSTER_FLAG_FREE)
+ break;
+ last = idx;
+ }
+
+ if (last == si->nr_clusters_mapped)
+ return; /* nothing to shrink */
+
+ /* Only unmap if we can free at least one full page of clusters */
+ if (si->nr_clusters_mapped - last < XSWAP_GROW_CLUSTERS)
+ return;
+
+ xswap_unmap_clusters(si, last, si->nr_clusters_mapped - last);
+}
#endif /* CONFIG_XSWAP */
static int setup_swap_clusters_info(struct swap_info_struct *si,
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 06/11] mm, swap: free backing pages in xswap_unmap_clusters
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (2 preceding siblings ...)
2026-07-27 14:04 ` [RFC PATCH 05/11] mm, swap: add xswap_try_shrink and shrink trigger on cluster free Baoquan He
@ 2026-07-27 14:04 ` Baoquan He
2026-07-27 14:04 ` [RFC PATCH 07/11] mm, swap: add nr_free_tail for O(1) xswap shrink detection Baoquan He
` (5 subsequent siblings)
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:04 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
vm_area_unmap_pages() only clears PTEs and frees intermediate page
table pages — it does not free the backing physical pages allocated
by xswap_map_clusters().
Fix this by walking the page table with apply_to_existing_page_range()
before the unmap to collect all struct pages in the range. After
vunmap_range() clears the PTEs, free the collected pages via
__free_page().
Use a simple xswap_page_data collector callback: for each present PTE,
collect pte_page() into a dynamically allocated array. The array is
freed after the pages are released.
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
mm/swapfile.c | 51 +++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 45 insertions(+), 6 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index e0f5fe64de2a..7613ce231d70 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -3776,6 +3776,23 @@ static int xswap_map_clusters(struct swap_info_struct *si,
return -ENOMEM;
}
+struct xswap_page_data {
+ struct page **pages;
+ int nr;
+ int max;
+};
+
+static int xswap_collect_page(pte_t *pte, unsigned long addr, void *data)
+{
+ struct xswap_page_data *xpd = data;
+
+ if (!pte_present(*pte))
+ return 0;
+ if (xpd->nr < xpd->max)
+ xpd->pages[xpd->nr++] = pte_page(*pte);
+ return 0;
+}
+
static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr)
{
@@ -3785,21 +3802,43 @@ static void xswap_unmap_clusters(struct swap_info_struct *si,
/*
* Round to page boundaries: start up (skip partial page that may
* contain clusters still in use before start_idx), end up so the
- * entire range is covered. vm_area_unmap_pages() operates on
- * whole pages.
+ * entire range is covered. vm_area_unmap_pages() and
+ * apply_to_existing_page_range() operate on whole pages.
*/
unsigned long vm_start = PAGE_ALIGN(start_addr);
unsigned long vm_end = PAGE_ALIGN(end_addr);
+ unsigned long size;
+ unsigned long npages;
+ struct xswap_page_data xpd;
+ int i;
if (vm_start >= vm_end)
goto out;
- vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
+ size = vm_end - vm_start;
+ npages = size >> PAGE_SHIFT;
+
/*
- * vm_area_unmap_pages() only clears PTEs; it does not free the
- * physical pages. Walk the page table to find and free them.
+ * Walk the page table to collect physical pages before unmapping.
+ * vm_area_unmap_pages() only clears PTEs and frees intermediate
+ * page table pages — it does not free backing pages.
*/
- /* TODO: free backing pages via page table walk or tracking bitmap */
+ xpd.pages = kmalloc_array(npages, sizeof(*xpd.pages), GFP_KERNEL);
+ if (xpd.pages) {
+ xpd.nr = 0;
+ xpd.max = npages;
+ apply_to_existing_page_range(&init_mm, vm_start, size,
+ xswap_collect_page, &xpd);
+ }
+
+ vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
+
+ /* Free the collected backing pages */
+ if (xpd.pages) {
+ for (i = 0; i < xpd.nr; i++)
+ __free_page(xpd.pages[i]);
+ kfree(xpd.pages);
+ }
/*
* Pairs with READ_ONCE() in shrink/grow paths.
*/
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 07/11] mm, swap: add nr_free_tail for O(1) xswap shrink detection
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (3 preceding siblings ...)
2026-07-27 14:04 ` [RFC PATCH 06/11] mm, swap: free backing pages in xswap_unmap_clusters Baoquan He
@ 2026-07-27 14:04 ` Baoquan He
2026-07-27 14:04 ` [RFC PATCH 08/11] mm, swap: add adjustable runtime ceiling (nr_clusters) for xswap Baoquan He
` (4 subsequent siblings)
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:04 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
Track contiguous free clusters at the tail of the mapped range in
si->nr_free_tail, maintained across alloc/free/grow paths. This
eliminates the backwards scan on every shrink check.
Three paths maintain the counter:
1. xswap_update_free_tail(): called on cluster free. If the freed
cluster is adjacent to the existing tail boundary, increment and
extend backwards to include already-free clusters now connected.
2. xswap_trim_free_tail(): called on cluster allocation. If the
allocated cluster lies within the tail free region, truncate the
count to end just before it.
3. Grow path: nr_free_tail += nr_new — all newly mapped clusters
are immediately free.
xswap_try_shrink() simplifies to a threshold check:
if nr_free_tail >= XSWAP_GROW_CLUSTERS → unmap
Setup initializes nr_free_tail = nr_clusters_mapped - 1 (all but
cluster 0 are free at the tail).
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
include/linux/swap.h | 1 +
mm/swapfile.c | 121 +++++++++++++++++++++++++++++++++++++------
2 files changed, 105 insertions(+), 17 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 5f6b14d30758..c54c921378ab 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -252,6 +252,7 @@ struct swap_info_struct {
struct vm_struct *cluster_vm; /* VM_SPARSE area for xswap dynamic cluster_info */
unsigned long nr_clusters; /* total cluster count for xswap */
unsigned long nr_clusters_mapped; /* currently mapped cluster count */
+ unsigned long nr_free_tail; /* contiguous free clusters at tail */
#endif
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 7613ce231d70..1eb44f818b40 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -68,6 +68,9 @@ static int xswap_map_clusters(struct swap_info_struct *si,
static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr);
static int xswap_check_mapped(pte_t *pte, unsigned long addr, void *data);
+static void xswap_trim_free_tail(struct swap_info_struct *si, unsigned long idx);
+static void xswap_update_free_tail(struct swap_info_struct *si,
+ unsigned long freed_idx);
static void xswap_try_shrink(struct swap_info_struct *si);
#endif
@@ -633,6 +636,7 @@ static void __free_cluster(struct swap_info_struct *si, struct swap_cluster_info
move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
ci->order = 0;
#ifdef CONFIG_XSWAP
+ xswap_update_free_tail(si, ci - si->cluster_info);
xswap_try_shrink(si);
#endif
}
@@ -981,6 +985,9 @@ static bool __swap_cluster_alloc_entries(struct swap_info_struct *si,
if (cluster_is_empty(ci))
ci->order = order;
ci->count += nr_pages;
+#ifdef CONFIG_XSWAP
+ xswap_trim_free_tail(si, cluster_index(si, ci));
+#endif
swap_range_alloc(si, nr_pages);
return true;
@@ -1246,6 +1253,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
spin_unlock(&ci->lock);
}
spin_unlock(&si->lock);
+ WRITE_ONCE(si->nr_free_tail,
+ READ_ONCE(si->nr_free_tail) + nr_new);
/* Retry allocation from the free list */
found = alloc_swap_scan_list(si, &si->free_clusters,
@@ -3856,37 +3865,115 @@ static int xswap_check_mapped(pte_t *pte, unsigned long addr, void *data)
}
/*
- * Try to shrink the cluster_info tail: unmap contiguous free clusters
- * at the end of the mapped range.
+ * Maintain si->nr_free_tail, the number of contiguous free clusters at
+ * the tail of the mapped range. Called when a cluster at @freed_idx is
+ * freed. Provides O(1) shrink detection: if nr_free_tail is non-zero,
+ * the tail can be unmapped without scanning cluster_info[].
+ *
+ * Only increments when @freed_idx is the cluster immediately before the
+ * existing tail region. Then scans backwards for already-free clusters
+ * now connected to the tail, bounded by XSWAP_GROW_CLUSTERS at a time.
*/
-static void xswap_try_shrink(struct swap_info_struct *si)
+static void xswap_update_free_tail(struct swap_info_struct *si,
+ unsigned long freed_idx)
{
+ unsigned long nr_mapped, nr_tail, tid, i;
struct swap_cluster_info *ci;
- unsigned long last, idx;
if (!(si->flags & SWP_XSWAP))
return;
- if (READ_ONCE(si->nr_clusters_mapped) <= 1) /* keep cluster 0 */
+
+ nr_mapped = READ_ONCE(si->nr_clusters_mapped);
+ nr_tail = READ_ONCE(si->nr_free_tail);
+
+ /* Protect against concurrent shrink that races past us */
+ if (nr_tail >= nr_mapped)
return;
- /* Find the last non-free cluster from the tail */
- last = READ_ONCE(si->nr_clusters_mapped);
- while (last > 1) {
- idx = last - 1;
- ci = &si->cluster_info[idx];
- if (ci->count || ci->flags != CLUSTER_FLAG_FREE)
+ tid = nr_mapped - nr_tail - 1;
+
+ /* Only the cluster immediately before the tail region counts */
+ if (freed_idx != tid)
+ return;
+
+ nr_tail++;
+ WRITE_ONCE(si->nr_free_tail, nr_tail);
+
+ /* Extend: include already-free clusters now connected to the tail */
+ for (i = 1; i < XSWAP_GROW_CLUSTERS; i++) {
+ nr_mapped = READ_ONCE(si->nr_clusters_mapped);
+ nr_tail = READ_ONCE(si->nr_free_tail);
+ if (nr_tail >= nr_mapped - 1)
+ break; /* reached cluster 0 */
+ tid = nr_mapped - nr_tail - 1;
+ ci = &si->cluster_info[tid];
+
+ if (READ_ONCE(ci->count) ||
+ READ_ONCE(ci->flags) != CLUSTER_FLAG_FREE)
break;
- last = idx;
+ nr_tail++;
+ WRITE_ONCE(si->nr_free_tail, nr_tail);
}
+}
+
+/*
+ * Trim si->nr_free_tail when a cluster in the tail region is allocated.
+ * @idx: index of the cluster being allocated.
+ */
+static void xswap_trim_free_tail(struct swap_info_struct *si, unsigned long idx)
+{
+ unsigned long nr_mapped, nr_tail, tail_start;
- if (last == si->nr_clusters_mapped)
- return; /* nothing to shrink */
+ if (!(si->flags & SWP_XSWAP))
+ return;
+
+ /*
+ * nr_clusters_mapped and nr_free_tail are read locklessly;
+ * concurrent updates may cause nr_free_tail to be trimmed
+ * slightly less than ideally, which is harmless.
+ */
+ nr_mapped = READ_ONCE(si->nr_clusters_mapped);
+ nr_tail = READ_ONCE(si->nr_free_tail);
+ tail_start = nr_mapped - nr_tail;
+ if (idx >= tail_start)
+ WRITE_ONCE(si->nr_free_tail, nr_mapped - idx - 1);
+}
+
+/*
+ * Try to shrink the cluster_info tail. Uses si->nr_free_tail which
+ * is maintained incrementally during alloc/free — no scanning needed.
+ */
+static void xswap_try_shrink(struct swap_info_struct *si)
+{
+ unsigned long start_idx, nr_unmap, i;
+ struct swap_cluster_info *ci;
+
+ if (!(si->flags & SWP_XSWAP))
+ return;
+ if (si->nr_free_tail < XSWAP_GROW_CLUSTERS)
+ return;
+
+ nr_unmap = round_down(si->nr_free_tail, XSWAP_GROW_CLUSTERS);
+ start_idx = si->nr_clusters_mapped - nr_unmap;
+
+ /* Verify the tail clusters are still free before unmapping */
+ spin_lock(&si->lock);
+ for (i = start_idx; i < si->nr_clusters_mapped; i++) {
+ ci = &si->cluster_info[i];
+ if (ci->flags != CLUSTER_FLAG_FREE) {
+ nr_unmap = i - start_idx;
+ break;
+ }
+ list_del(&ci->list);
+ ci->flags = CLUSTER_FLAG_NONE;
+ }
+ spin_unlock(&si->lock);
- /* Only unmap if we can free at least one full page of clusters */
- if (si->nr_clusters_mapped - last < XSWAP_GROW_CLUSTERS)
+ if (nr_unmap < XSWAP_GROW_CLUSTERS)
return;
- xswap_unmap_clusters(si, last, si->nr_clusters_mapped - last);
+ xswap_unmap_clusters(si, start_idx, nr_unmap);
+ si->nr_free_tail -= nr_unmap;
}
#endif /* CONFIG_XSWAP */
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 08/11] mm, swap: add adjustable runtime ceiling (nr_clusters) for xswap
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (4 preceding siblings ...)
2026-07-27 14:04 ` [RFC PATCH 07/11] mm, swap: add nr_free_tail for O(1) xswap shrink detection Baoquan He
@ 2026-07-27 14:04 ` Baoquan He
2026-07-27 14:05 ` [RFC PATCH 09/11] mm, swap: add debugfs knob for xswap per-device cluster limit Baoquan He
` (3 subsequent siblings)
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:04 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
Split the xswap cluster limit into two fields:
- nr_clusters_max: immutable hard limit set at swapon from swap header
- nr_clusters: current growth ceiling, adjustable at runtime (≤ nr_clusters_max)
The grow path already uses nr_clusters as the ceiling. Shrink now also
respects it: when nr_clusters drops below nr_clusters_mapped, shrinking
fires on free until the mapped count reaches the ceiling. When
nr_clusters == nr_clusters_max (default), shrink is effectively
disabled — all growth and no shrink.
At swapon, nr_clusters starts at nr_clusters_max (full size).
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
include/linux/swap.h | 3 ++-
mm/swapfile.c | 32 +++++++++++++++++++++++---------
2 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index c54c921378ab..e4512aeead36 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -250,7 +250,8 @@ struct swap_info_struct {
struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
#ifdef CONFIG_XSWAP
struct vm_struct *cluster_vm; /* VM_SPARSE area for xswap dynamic cluster_info */
- unsigned long nr_clusters; /* total cluster count for xswap */
+ unsigned long nr_clusters_max; /* upper limit from swap header */
+ unsigned long nr_clusters; /* current growth ceiling (≤ nr_clusters_max) */
unsigned long nr_clusters_mapped; /* currently mapped cluster count */
unsigned long nr_free_tail; /* contiguous free clusters at tail */
#endif
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 1eb44f818b40..f57823d6449f 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -72,6 +72,7 @@ static void xswap_trim_free_tail(struct swap_info_struct *si, unsigned long idx)
static void xswap_update_free_tail(struct swap_info_struct *si,
unsigned long freed_idx);
static void xswap_try_shrink(struct swap_info_struct *si);
+
#endif
static void swap_range_alloc(struct swap_info_struct *si,
@@ -3151,6 +3152,7 @@ static void free_swap_cluster_info(struct swap_info_struct *si)
xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
free_vm_area(si->cluster_vm);
si->cluster_vm = NULL;
+ si->nr_clusters_max = 0;
si->nr_clusters = 0;
si->nr_clusters_mapped = 0;
return;
@@ -3945,20 +3947,31 @@ static void xswap_trim_free_tail(struct swap_info_struct *si, unsigned long idx)
*/
static void xswap_try_shrink(struct swap_info_struct *si)
{
- unsigned long start_idx, nr_unmap, i;
+ unsigned long nr_mapped, nr_ceiling, nr_tail, nr_unmap;
+ unsigned long start_idx, i;
struct swap_cluster_info *ci;
if (!(si->flags & SWP_XSWAP))
return;
- if (si->nr_free_tail < XSWAP_GROW_CLUSTERS)
+
+ nr_mapped = READ_ONCE(si->nr_clusters_mapped);
+ nr_ceiling = READ_ONCE(si->nr_clusters);
+ nr_tail = READ_ONCE(si->nr_free_tail);
+
+ if (nr_mapped <= nr_ceiling)
+ return;
+ if (nr_tail < XSWAP_GROW_CLUSTERS)
return;
- nr_unmap = round_down(si->nr_free_tail, XSWAP_GROW_CLUSTERS);
- start_idx = si->nr_clusters_mapped - nr_unmap;
+ nr_unmap = min(round_down(nr_tail, XSWAP_GROW_CLUSTERS),
+ nr_mapped - nr_ceiling);
+ if (nr_unmap < XSWAP_GROW_CLUSTERS)
+ return;
+ start_idx = nr_mapped - nr_unmap;
/* Verify the tail clusters are still free before unmapping */
spin_lock(&si->lock);
- for (i = start_idx; i < si->nr_clusters_mapped; i++) {
+ for (i = start_idx; i < nr_mapped; i++) {
ci = &si->cluster_info[i];
if (ci->flags != CLUSTER_FLAG_FREE) {
nr_unmap = i - start_idx;
@@ -3973,7 +3986,7 @@ static void xswap_try_shrink(struct swap_info_struct *si)
return;
xswap_unmap_clusters(si, start_idx, nr_unmap);
- si->nr_free_tail -= nr_unmap;
+ WRITE_ONCE(si->nr_free_tail, nr_tail - nr_unmap);
}
#endif /* CONFIG_XSWAP */
@@ -3997,6 +4010,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
cluster_info = vm->addr;
si->cluster_vm = vm;
+ si->nr_clusters_max = nr_clusters;
si->nr_clusters = nr_clusters;
si->cluster_info = cluster_info;
@@ -4031,6 +4045,9 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
}
}
+ /* All mapped clusters except cluster 0 are free at the tail */
+ si->nr_free_tail = si->nr_clusters_mapped - 1;
+
return 0;
err_unmap:
@@ -4434,7 +4451,6 @@ void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
static int __init swapfile_init(void)
{
swapfile_maximum_size = arch_max_swapfile_size();
-
/*
* Once a cluster is freed, it's swap table content is read
* only, and all swap cache readers (swap_cache_*) verifies
@@ -4444,12 +4460,10 @@ static int __init swapfile_init(void)
swap_table_cachep = kmem_cache_create("swap_table",
sizeof(struct swap_table),
0, SLAB_PANIC | SLAB_TYPESAFE_BY_RCU, NULL);
-
#ifdef CONFIG_MIGRATION
if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
swap_migration_ad_supported = true;
#endif /* CONFIG_MIGRATION */
-
return 0;
}
subsys_initcall(swapfile_init);
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 09/11] mm, swap: add debugfs knob for xswap per-device cluster limit
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (5 preceding siblings ...)
2026-07-27 14:04 ` [RFC PATCH 08/11] mm, swap: add adjustable runtime ceiling (nr_clusters) for xswap Baoquan He
@ 2026-07-27 14:05 ` Baoquan He
2026-07-27 14:05 ` [RFC PATCH 10/11] mm, swap: defer xswap shrink to workqueue to avoid lock recursion Baoquan He
` (2 subsequent siblings)
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:05 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
Add a per-device debugfs file for runtime adjustment of xswap cluster
limit:
/sys/kernel/debug/xswap/type<N>_cluster_limit
Reading shows the current ceiling (in clusters); writing sets it
(clamped to [0, nr_clusters_max]). Setting below nr_clusters_mapped
triggers an immediate shrink check via xswap_try_shrink().
The debugfs entry is created at swapon and removed at swapoff.
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
include/linux/swap.h | 1 +
mm/swapfile.c | 97 +++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index e4512aeead36..ebcd5eaa7924 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -254,6 +254,7 @@ struct swap_info_struct {
unsigned long nr_clusters; /* current growth ceiling (≤ nr_clusters_max) */
unsigned long nr_clusters_mapped; /* currently mapped cluster count */
unsigned long nr_free_tail; /* contiguous free clusters at tail */
+ struct dentry *debugfs_entry; /* debugfs: type<N>_max_clusters */
#endif
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
diff --git a/mm/swapfile.c b/mm/swapfile.c
index f57823d6449f..af62981506d3 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -48,6 +48,9 @@
#include "swap_table.h"
#include "internal.h"
#include "swap.h"
+#include <linux/debugfs.h>
+
+static DEFINE_SPINLOCK(swap_lock);
#ifdef CONFIG_XSWAP
/*
@@ -63,6 +66,8 @@
#define XSWAP_GROW_CLUSTERS \
max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+static struct dentry *xswap_debugfs_root;
+
static int xswap_map_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr);
static void xswap_unmap_clusters(struct swap_info_struct *si,
@@ -73,6 +78,90 @@ static void xswap_update_free_tail(struct swap_info_struct *si,
unsigned long freed_idx);
static void xswap_try_shrink(struct swap_info_struct *si);
+/*
+ * debugfs read/write for per-device max cluster count.
+ * Shows/sets si->nr_clusters (current growth ceiling), clamped to
+ * [0, si->nr_clusters_max].
+ */
+static ssize_t xswap_max_clusters_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct swap_info_struct *si = file->private_data;
+ char tmp[32];
+ int len;
+
+ len = snprintf(tmp, sizeof(tmp), "%lu\n", READ_ONCE(si->nr_clusters));
+ return simple_read_from_buffer(buf, count, ppos, tmp, len);
+}
+
+static ssize_t xswap_max_clusters_write(struct file *file,
+ const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct swap_info_struct *si = file->private_data;
+ unsigned long val, new_pages;
+ int err;
+
+ err = kstrtoul_from_user(buf, count, 0, &val);
+ if (err)
+ return err;
+
+ if (val > si->nr_clusters_max)
+ val = si->nr_clusters_max;
+
+ spin_lock(&si->lock);
+ si->nr_clusters = val;
+ spin_unlock(&si->lock);
+
+ /* Keep the visible swap size in sync with the new ceiling. */
+ new_pages = min_t(unsigned long, val * SWAPFILE_CLUSTER, si->max);
+ if (new_pages)
+ new_pages--;
+ if (new_pages != si->pages) {
+ long delta = (long)new_pages - (long)si->pages;
+
+ spin_lock(&swap_lock);
+ si->pages = new_pages;
+ atomic_long_add(delta, &nr_swap_pages);
+ total_swap_pages += delta;
+ spin_unlock(&swap_lock);
+ }
+
+ /*
+ * Lowering the ceiling may make tail clusters eligible for
+ * shrinking. Trigger an immediate check.
+ */
+ xswap_try_shrink(si);
+
+ return count;
+}
+
+static const struct file_operations xswap_debugfs_fops = {
+ .read = xswap_max_clusters_read,
+ .write = xswap_max_clusters_write,
+ .open = simple_open,
+ .llseek = default_llseek,
+};
+
+static void xswap_debugfs_add(struct swap_info_struct *si)
+{
+ char name[32];
+
+ if (!xswap_debugfs_root)
+ return;
+
+ snprintf(name, sizeof(name), "type%d_cluster_limit", si->type);
+ si->debugfs_entry = debugfs_create_file(name, 0644, xswap_debugfs_root,
+ si, &xswap_debugfs_fops);
+}
+
+static void xswap_debugfs_del(struct swap_info_struct *si)
+{
+ debugfs_remove(si->debugfs_entry);
+ si->debugfs_entry = NULL;
+}
+
+
#endif
static void swap_range_alloc(struct swap_info_struct *si,
@@ -89,7 +178,6 @@ static void move_cluster(struct swap_info_struct *si,
*
* Also protects swap_active_head total_swap_pages, and the SWP_WRITEOK flag.
*/
-static DEFINE_SPINLOCK(swap_lock);
static unsigned int nr_swapfiles;
atomic_long_t nr_swap_pages;
atomic_t nr_real_swapfiles;
@@ -3147,6 +3235,7 @@ static void free_swap_cluster_info(struct swap_info_struct *si)
#ifdef CONFIG_XSWAP
if (si->flags & SWP_XSWAP) {
+ xswap_debugfs_del(si);
/* Unmap all mapped clusters and free the VM_SPARSE area */
if (si->nr_clusters_mapped > 0)
xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
@@ -4048,6 +4137,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
/* All mapped clusters except cluster 0 are free at the tail */
si->nr_free_tail = si->nr_clusters_mapped - 1;
+ xswap_debugfs_add(si);
return 0;
err_unmap:
@@ -4464,6 +4554,11 @@ static int __init swapfile_init(void)
if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
swap_migration_ad_supported = true;
#endif /* CONFIG_MIGRATION */
+
+#ifdef CONFIG_XSWAP
+ xswap_debugfs_root = debugfs_create_dir("xswap", NULL);
+#endif
+
return 0;
}
subsys_initcall(swapfile_init);
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 10/11] mm, swap: defer xswap shrink to workqueue to avoid lock recursion
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (6 preceding siblings ...)
2026-07-27 14:05 ` [RFC PATCH 09/11] mm, swap: add debugfs knob for xswap per-device cluster limit Baoquan He
@ 2026-07-27 14:05 ` Baoquan He
2026-07-27 14:05 ` [RFC PATCH 11/11] mm, swap: serialize xswap map/unmap with a mutex Baoquan He
2026-07-27 15:26 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Chris Li
9 siblings, 0 replies; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:05 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
xswap_try_shrink() was called directly from __free_cluster() while
holding ci->lock. The shrink path calls xswap_unmap_clusters()
which unmaps vmalloc pages backing cluster_info, and on return
swap_cache_del_folio() tries swap_cluster_unlock(ci) on the now-
unmapped address — crashing on a not-present page.
Replace direct calls with schedule_work() so shrink runs in an
independent workqueue context where no cluster locks are held.
Use cancel_work_sync() during swapoff to ensure no pending shrink
work races with the VM area teardown.
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
include/linux/swap.h | 1 +
mm/swapfile.c | 18 ++++++++++++------
2 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index ebcd5eaa7924..ebae01fbd100 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -255,6 +255,7 @@ struct swap_info_struct {
unsigned long nr_clusters_mapped; /* currently mapped cluster count */
unsigned long nr_free_tail; /* contiguous free clusters at tail */
struct dentry *debugfs_entry; /* debugfs: type<N>_max_clusters */
+ struct work_struct xswap_shrink_work; /* deferred shrink trigger */
#endif
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
diff --git a/mm/swapfile.c b/mm/swapfile.c
index af62981506d3..b41ff2c594fc 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -127,11 +127,8 @@ static ssize_t xswap_max_clusters_write(struct file *file,
spin_unlock(&swap_lock);
}
- /*
- * Lowering the ceiling may make tail clusters eligible for
- * shrinking. Trigger an immediate check.
- */
- xswap_try_shrink(si);
+ /* Shrink trigger: lowering the ceiling may free tail clusters. */
+ schedule_work(&si->xswap_shrink_work);
return count;
}
@@ -726,7 +723,7 @@ static void __free_cluster(struct swap_info_struct *si, struct swap_cluster_info
ci->order = 0;
#ifdef CONFIG_XSWAP
xswap_update_free_tail(si, ci - si->cluster_info);
- xswap_try_shrink(si);
+ schedule_work(&si->xswap_shrink_work);
#endif
}
@@ -3236,6 +3233,7 @@ static void free_swap_cluster_info(struct swap_info_struct *si)
#ifdef CONFIG_XSWAP
if (si->flags & SWP_XSWAP) {
xswap_debugfs_del(si);
+ cancel_work_sync(&si->xswap_shrink_work);
/* Unmap all mapped clusters and free the VM_SPARSE area */
if (si->nr_clusters_mapped > 0)
xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
@@ -4030,6 +4028,13 @@ static void xswap_trim_free_tail(struct swap_info_struct *si, unsigned long idx)
WRITE_ONCE(si->nr_free_tail, nr_mapped - idx - 1);
}
+static void xswap_shrink_work_fn(struct work_struct *work)
+{
+ struct swap_info_struct *si = container_of(work,
+ struct swap_info_struct, xswap_shrink_work);
+ xswap_try_shrink(si);
+}
+
/*
* Try to shrink the cluster_info tail. Uses si->nr_free_tail which
* is maintained incrementally during alloc/free — no scanning needed.
@@ -4137,6 +4142,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
/* All mapped clusters except cluster 0 are free at the tail */
si->nr_free_tail = si->nr_clusters_mapped - 1;
+ INIT_WORK(&si->xswap_shrink_work, xswap_shrink_work_fn);
xswap_debugfs_add(si);
return 0;
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [RFC PATCH 11/11] mm, swap: serialize xswap map/unmap with a mutex
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (7 preceding siblings ...)
2026-07-27 14:05 ` [RFC PATCH 10/11] mm, swap: defer xswap shrink to workqueue to avoid lock recursion Baoquan He
@ 2026-07-27 14:05 ` Baoquan He
2026-07-27 15:13 ` Nhat Pham
2026-07-27 15:26 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Chris Li
9 siblings, 1 reply; 17+ messages in thread
From: Baoquan He @ 2026-07-27 14:05 UTC (permalink / raw)
To: linux-mm
Cc: akpm, chrisl, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel, Baoquan He
Concurrent xswap grow operations race on vm_area_map_pages(),
triggering WARN_ON(!pte_none) in the vmap page table walk when
one thread sees a PTE already set by another. The -EBUSY safety
net catches it but the warning is noisy and unavoidable without
serialization.
Add a per-device mutex (xswap_lock) held across xswap_map_clusters
and xswap_unmap_clusters so that only one thread can modify the
VM_SPARSE page table at a time. The shrink path is already
deferred to a workqueue so it does not contend with itself.
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
---
include/linux/swap.h | 1 +
mm/swapfile.c | 17 +++++++++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index ebae01fbd100..b0fe0475bba7 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -256,6 +256,7 @@ struct swap_info_struct {
unsigned long nr_free_tail; /* contiguous free clusters at tail */
struct dentry *debugfs_entry; /* debugfs: type<N>_max_clusters */
struct work_struct xswap_shrink_work; /* deferred shrink trigger */
+ struct mutex xswap_lock; /* serialize map/unmap operations */
#endif
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
diff --git a/mm/swapfile.c b/mm/swapfile.c
index b41ff2c594fc..89c6a6b03a9d 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -3773,11 +3773,14 @@ static int xswap_map_clusters(struct swap_info_struct *si,
struct page **pages;
unsigned long i;
+ mutex_lock(&si->xswap_lock);
+
if (vm_start >= vm_end) {
/* All requested clusters fall within already-mapped pages. */
for (i = start_idx; i < start_idx + nr; i++)
spin_lock_init(&si->cluster_info[i].lock);
WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+ mutex_unlock(&si->xswap_lock);
return 0;
}
@@ -3842,6 +3845,7 @@ static int xswap_map_clusters(struct swap_info_struct *si,
* Pairs with READ_ONCE() in shrink/grow paths.
*/
WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+ mutex_unlock(&si->xswap_lock);
return 0;
fail_nounmap:
@@ -3862,6 +3866,7 @@ static int xswap_map_clusters(struct swap_info_struct *si,
spin_lock_init(&si->cluster_info[i].lock);
WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+ mutex_unlock(&si->xswap_lock);
return 0;
fail:
@@ -3871,6 +3876,7 @@ static int xswap_map_clusters(struct swap_info_struct *si,
__free_page(pages[i]);
}
kfree(pages);
+ mutex_unlock(&si->xswap_lock);
return -ENOMEM;
}
@@ -3910,8 +3916,12 @@ static void xswap_unmap_clusters(struct swap_info_struct *si,
struct xswap_page_data xpd;
int i;
- if (vm_start >= vm_end)
+ mutex_lock(&si->xswap_lock);
+
+ if (vm_start >= vm_end) {
+ mutex_unlock(&si->xswap_lock);
goto out;
+ }
size = vm_end - vm_start;
npages = size >> PAGE_SHIFT;
@@ -3937,10 +3947,12 @@ static void xswap_unmap_clusters(struct swap_info_struct *si,
__free_page(xpd.pages[i]);
kfree(xpd.pages);
}
+
+ mutex_unlock(&si->xswap_lock);
+out:
/*
* Pairs with READ_ONCE() in shrink/grow paths.
*/
- out:
WRITE_ONCE(si->nr_clusters_mapped, start_idx);
}
@@ -4142,6 +4154,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
/* All mapped clusters except cluster 0 are free at the tail */
si->nr_free_tail = si->nr_clusters_mapped - 1;
+ mutex_init(&si->xswap_lock);
INIT_WORK(&si->xswap_shrink_work, xswap_shrink_work_fn);
xswap_debugfs_add(si);
return 0;
--
2.54.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [RFC PATCH 11/11] mm, swap: serialize xswap map/unmap with a mutex
2026-07-27 14:05 ` [RFC PATCH 11/11] mm, swap: serialize xswap map/unmap with a mutex Baoquan He
@ 2026-07-27 15:13 ` Nhat Pham
0 siblings, 0 replies; 17+ messages in thread
From: Nhat Pham @ 2026-07-27 15:13 UTC (permalink / raw)
To: Baoquan He
Cc: linux-mm, akpm, chrisl, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel
On Mon, Jul 27, 2026 at 7:06 AM Baoquan He <baoquan.he@linux.dev> wrote:
>
> Concurrent xswap grow operations race on vm_area_map_pages(),
> triggering WARN_ON(!pte_none) in the vmap page table walk when
> one thread sees a PTE already set by another. The -EBUSY safety
> net catches it but the warning is noisy and unavoidable without
> serialization.
>
> Add a per-device mutex (xswap_lock) held across xswap_map_clusters
> and xswap_unmap_clusters so that only one thread can modify the
> VM_SPARSE page table at a time. The shrink path is already
> deferred to a workqueue so it does not contend with itself.
>
> Signed-off-by: Baoquan He <baoquan.he@linux.dev>
Seems like this should be squashed into one of the older patches, no?
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct
2026-07-27 14:04 ` [RFC PATCH 02/11] mm, swap: add CONFIG_XSWAP and xswap fields to swap_info_struct Baoquan He
` (8 preceding siblings ...)
2026-07-27 14:05 ` [RFC PATCH 11/11] mm, swap: serialize xswap map/unmap with a mutex Baoquan He
@ 2026-07-27 15:26 ` Chris Li
9 siblings, 0 replies; 17+ messages in thread
From: Chris Li @ 2026-07-27 15:26 UTC (permalink / raw)
To: Baoquan He
Cc: linux-mm, akpm, nphamcs, kasong, baohua, youngjun.park, hannes,
yosry, david, shikemeng, chengming.zhou, linux-kernel
On Mon, Jul 27, 2026 at 7:05 AM Baoquan He <baoquan.he@linux.dev> wrote:
>
> Add CONFIG_XSWAP Kconfig option (depends on SWAP && 64BIT) for
> extendable (virtual) swap device support.
>
> Add three fields to struct swap_info_struct under CONFIG_XSWAP:
> - cluster_vm: the VM_SPARSE vm_struct backing the cluster_info array
> - nr_clusters: total number of clusters in the xswap address space
> - nr_clusters_mapped: number of clusters currently mapped (lazy grow)
>
> These fields enable lazy vmalloc-based dynamic cluster management
> where the cluster_info array is backed by a sparse vmalloc area that
> grows on demand and shrinks when clusters are freed.
>
> Signed-off-by: Baoquan He <baoquan.he@linux.dev>
> ---
> include/linux/swap.h | 5 +++++
> mm/Kconfig | 9 +++++++++
> 2 files changed, 14 insertions(+)
>
> diff --git a/include/linux/swap.h b/include/linux/swap.h
> index 80c83dd8804f..5f6b14d30758 100644
> --- a/include/linux/swap.h
> +++ b/include/linux/swap.h
> @@ -248,6 +248,11 @@ struct swap_info_struct {
> signed char type; /* strange name for an index */
> unsigned int max; /* size of this swap device */
> struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
> +#ifdef CONFIG_XSWAP
> + struct vm_struct *cluster_vm; /* VM_SPARSE area for xswap dynamic cluster_info */
> + unsigned long nr_clusters; /* total cluster count for xswap */
> + unsigned long nr_clusters_mapped; /* currently mapped cluster count */
> +#endif
We might need more than one dynamic kvmalloc grow array. Maybe
abstract this into a struct so it can be reused for arrays other than
clusters as well. The abstraction can also include the element size so
it can have a generic function to calculate how many pages it needs
for N elements.
Chris
^ permalink raw reply [flat|nested] 17+ messages in thread