From: Baoquan He <baoquan.he@linux.dev>
To: linux-mm@kvack.org
Cc: akpm@linux-foundation.org, david@kernel.org, chrisl@kernel.org,
kasong@tencent.com, nphamcs@gmail.com, baohua@kernel.org,
youngjun.park@lge.com, corbet@lwn.net, hannes@cmpxchg.org,
chengming.zhou@linux.dev, kunwu.chan@gmail.com,
klarasmodin@gmail.com, linux-kernel@vger.kernel.org,
Baoquan He <hebaoquan@kylinos.cn>
Subject: [PATCH v5 04/19] mm, swap: use a VM_SPARSE cluster array for xswap devices
Date: Mon, 9 Nov 2026 13:54:40 +0800 [thread overview]
Message-ID: <20261109055455.536925-5-baoquan.he@linux.dev> (raw)
In-Reply-To: <20261109055455.536925-1-baoquan.he@linux.dev>
From: Baoquan He <hebaoquan@kylinos.cn>
An xswap device does not have to allocate cluster_info for its whole
address space at once. Back it with a VM_SPARSE area and map it in
chunks. At setup, map only the first chunk. The grow path maps more
chunks when they are needed.
Setup, free, and the two cluster_info scans must change together. If
only setup changes, free would free the wrong object, and the scans
would read cluster_info entries that are not mapped yet.
The map helper rejects a stale start_idx and cleans up a partial mapping
if it fails. The grow path added later depends on this check.
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
mm/swapfile.c | 152 ++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 141 insertions(+), 11 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index afd0e8718544..00c7b398bafe 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -50,6 +50,17 @@
#include "swap.h"
#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation. It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+ max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
/* Batch size for the fallback unmap; keeps the stack array small. */
#define XSWAP_UNMAP_BATCH_PAGES 16
#define XSWAP_UNMAP_BATCH_CLUSTERS \
@@ -2825,10 +2836,27 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
unsigned int prev)
{
struct swap_cluster_info *ci;
- unsigned long i, end;
+ unsigned long i, cluster_end, end;
unsigned int ci_off;
unsigned long swp_tb;
+ end = si->max;
+#ifdef CONFIG_XSWAP
+ /*
+ * An xswap device maps its cluster_info in chunks as it grows, so
+ * the tail past nr_clusters_mapped has nothing behind it.
+ */
+ if (si->flags & SWP_XSWAP) {
+ unsigned long mapped_end;
+
+ /* Pairs with the smp_store_release() in xswap_map_clusters(). */
+ mapped_end = smp_load_acquire(&si->nr_clusters_mapped) *
+ SWAPFILE_CLUSTER;
+ if (mapped_end < end)
+ end = mapped_end;
+ }
+#endif
+
/*
* No need for swap_lock here: we're just looking
* for whether an entry is in use, not modifying it; false
@@ -2836,11 +2864,11 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
* allocations from this area (while holding swap_lock).
*/
i = prev + 1;
- while (i < si->max) {
+ while (i < end) {
ci = __swap_offset_to_cluster(si, i);
- end = min_t(unsigned long,
- ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER,
- si->max);
+ cluster_end = min_t(unsigned long,
+ ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER,
+ end);
/*
* An empty cluster has no slot in use, so skip it whole.
@@ -2850,13 +2878,13 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
* enough, unlike in every other cluster_is_empty() caller.
*/
if (!READ_ONCE(ci->count)) {
- i = end;
+ i = cluster_end;
cond_resched();
continue;
}
ci_off = i % SWAPFILE_CLUSTER;
- for (; i < end; ci_off++, i++) {
+ for (; i < cluster_end; ci_off++, i++) {
swp_tb = swap_table_get(ci, ci_off);
if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb))
return i;
@@ -2866,7 +2894,6 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
return 0;
}
-
static int try_to_unuse(unsigned int type)
{
struct mm_struct *prev_mm;
@@ -3164,6 +3191,14 @@ static void wait_for_allocation(struct swap_info_struct *si)
BUG_ON(si->flags & SWP_WRITEOK);
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ /* Pairs with the smp_store_release() in xswap_map_clusters(). */
+ end = min(end, smp_load_acquire(&si->nr_clusters_mapped) *
+ SWAPFILE_CLUSTER);
+ }
+#endif
+
for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
ci = swap_cluster_lock(si, offset);
swap_cluster_unlock(ci);
@@ -3175,11 +3210,43 @@ static void free_swap_cluster_info(struct swap_info_struct *si)
struct swap_cluster_info *cluster_info = si->cluster_info;
unsigned long maxpages = si->max;
struct swap_cluster_info *ci;
- int i, nr_clusters;
+ unsigned long i, nr_clusters;
if (!cluster_info)
return;
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ unsigned long nr_mapped;
+
+ /*
+ * Cluster 0 keeps the bad header slot, so it never empties
+ * and __free_cluster() never frees its table.
+ */
+ /* Pairs with the smp_store_release() in xswap_map_clusters(). */
+ nr_mapped = smp_load_acquire(&si->nr_clusters_mapped);
+ for (i = 0; i < nr_mapped; i++) {
+ ci = &cluster_info[i];
+ spin_lock(&ci->lock);
+ if (cluster_table_is_alloced(ci)) {
+ swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, true);
+ swap_cluster_free_table(ci);
+ }
+ spin_unlock(&ci->lock);
+ }
+ /*
+ * free_vm_area() drops the mapping without freeing the
+ * pages, and the unmap helper handles an empty range.
+ */
+ xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+ free_vm_area(si->cluster_vm);
+ si->cluster_vm = NULL;
+ si->cluster_info = NULL;
+ si->nr_clusters_mapped = 0;
+ return;
+ }
+#endif
+
nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
for (i = 0; i < nr_clusters; i++) {
ci = cluster_info + i;
@@ -3678,7 +3745,7 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
}
#ifdef CONFIG_XSWAP
-static int __maybe_unused xswap_map_clusters(struct swap_info_struct *si,
+static int xswap_map_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr)
{
unsigned long start_addr = (unsigned long)si->cluster_info +
@@ -3824,7 +3891,7 @@ static void xswap_unmap_range(struct swap_info_struct *si,
}
/* Caller must hold si->xswap_lock. Cannot fail. */
-static void __maybe_unused xswap_unmap_clusters(struct swap_info_struct *si,
+static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr)
{
unsigned long start_addr = (unsigned long)si->cluster_info +
@@ -3902,6 +3969,69 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
int err = -ENOMEM;
unsigned long i;
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
+ struct vm_struct *vm;
+
+ vm = get_vm_area(size, VM_SPARSE);
+ if (!vm)
+ goto err;
+
+ cluster_info = vm->addr;
+ si->cluster_vm = vm;
+ si->nr_clusters_max = nr_clusters;
+ si->cluster_info = cluster_info;
+
+ /* Must be initialized before xswap_map_clusters() locks it. */
+ mutex_init(&si->xswap_lock);
+
+ if (xswap_map_clusters(si, 0, min_t(unsigned long,
+ XSWAP_GROW_CLUSTERS, nr_clusters)))
+ goto err_free_vm;
+
+ /* xswap: only cluster 0 slot 0 is bad */
+ err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
+ if (err)
+ goto err_unmap;
+
+ INIT_LIST_HEAD(&si->free_clusters);
+ INIT_LIST_HEAD(&si->full_clusters);
+ INIT_LIST_HEAD(&si->discard_clusters);
+ for (i = 0; i < SWAP_NR_ORDERS; i++) {
+ INIT_LIST_HEAD(&si->nonfull_clusters[i]);
+ INIT_LIST_HEAD(&si->frag_clusters[i]);
+ }
+
+ /*
+ * Cluster 0 holds the header slot, which is marked bad, so it
+ * is not entirely free. si->max is cluster aligned, so no
+ * cluster holds a slot past the end of the device.
+ */
+ for (i = 0; i < si->nr_clusters_mapped; i++) {
+ struct swap_cluster_info *ci = &cluster_info[i];
+
+ if (ci->count) {
+ ci->flags = CLUSTER_FLAG_NONFULL;
+ list_add_tail(&ci->list, &si->nonfull_clusters[0]);
+ } else {
+ ci->flags = CLUSTER_FLAG_FREE;
+ list_add_tail(&ci->list, &si->free_clusters);
+ }
+ }
+
+ return 0;
+
+err_unmap:
+ xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+err_free_vm:
+ free_vm_area(si->cluster_vm);
+ si->cluster_vm = NULL;
+ si->cluster_info = NULL;
+ return err;
+ }
+#endif /* CONFIG_XSWAP */
+
cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
if (!cluster_info)
goto err;
--
2.54.0
next prev parent reply other threads:[~2026-11-09 5:56 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-09 5:54 [PATCH v5 00/19] mm, swap: extendable swap devices (xswap phase I) Baoquan He
2026-11-09 5:54 ` [PATCH v5 01/19] mm: xswap support for zswap Baoquan He
2026-11-10 5:57 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 02/19] mm, swap: refactor free_swap_cluster_info to take swap_info_struct Baoquan He
2026-11-09 5:54 ` [PATCH v5 03/19] mm, swap: add VM_SPARSE cluster-array helpers for xswap Baoquan He
2026-11-09 5:54 ` Baoquan He [this message]
2026-11-10 5:58 ` [PATCH v5 04/19] mm, swap: use a VM_SPARSE cluster array for xswap devices sashiko-bot
2026-11-09 5:54 ` [PATCH v5 05/19] mm, swap: widen swap_info_struct max/pages to unsigned long Baoquan He
2026-11-09 5:54 ` [PATCH v5 06/19] mm, swap: add a sysfs interface to create xswap devices Baoquan He
2026-11-09 5:54 ` [PATCH v5 07/19] mm, swap: refactor swapoff and add xswap_destroy Baoquan He
2026-11-09 5:54 ` [PATCH v5 08/19] mm, swap: require zswap for xswap devices Baoquan He
2026-11-09 5:54 ` [PATCH v5 09/19] mm, swap: use a stable id " Baoquan He
2026-11-10 5:58 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 10/19] mm, swap: let the command line set the xswap device size Baoquan He
2026-11-09 5:54 ` [PATCH v5 11/19] mm, swap: add CONFIG_XSWAP_DEFAULT_MAX Baoquan He
2026-11-09 5:54 ` [PATCH v5 12/19] mm, swap: let swapoff destroy xswap devices by name Baoquan He
2026-11-10 5:58 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 13/19] mm, swap: add xswap grow trigger on cluster allocation Baoquan He
2026-11-10 5:58 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 14/19] mm, swap: add debugfs counters for xswap Baoquan He
2026-11-09 5:54 ` [PATCH v5 15/19] mm, swap: make xswap cluster_info accesses RCU-safe Baoquan He
2026-11-10 5:58 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 16/19] mm, swap: prepare xswap_unmap_clusters for range shrinking Baoquan He
2026-11-09 5:54 ` [PATCH v5 17/19] mm, swap: add xswap shrink trigger and defer it to a workqueue Baoquan He
2026-11-10 5:58 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 18/19] mm, swap: do not give an xswap slot to a cgroup that cannot zswap Baoquan He
2026-11-10 5:58 ` sashiko-bot
2026-11-09 5:54 ` [PATCH v5 19/19] mm, swap: skip the overcommit probe when removing xswap Baoquan He
2026-11-09 6:15 ` [PATCH v5 00/19] mm, swap: extendable swap devices (xswap phase I) Baoquan He
2026-11-12 7:25 ` [syzbot ci] " syzbot ci
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261109055455.536925-5-baoquan.he@linux.dev \
--to=baoquan.he@linux.dev \
--cc=akpm@linux-foundation.org \
--cc=baohua@kernel.org \
--cc=chengming.zhou@linux.dev \
--cc=chrisl@kernel.org \
--cc=corbet@lwn.net \
--cc=david@kernel.org \
--cc=hannes@cmpxchg.org \
--cc=hebaoquan@kylinos.cn \
--cc=kasong@tencent.com \
--cc=klarasmodin@gmail.com \
--cc=kunwu.chan@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=nphamcs@gmail.com \
--cc=youngjun.park@lge.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.