From: Nhat Pham <nphamcs@gmail.com>
To: akpm@linux-foundation.org
Cc: chrisl@kernel.org, kasong@tencent.com, hannes@cmpxchg.org,
mhocko@kernel.org, roman.gushchin@linux.dev,
shakeel.butt@linux.dev, yosry@kernel.org, david@kernel.org,
muchun.song@linux.dev, shikemeng@huaweicloud.com,
baoquan.he@linux.dev, baohua@kernel.org, youngjun.park@lge.com,
chengming.zhou@linux.dev, ljs@kernel.org, liam@infradead.org,
vbabka@kernel.org, rppt@kernel.org, surenb@google.com,
qi.zheng@linux.dev, axelrasmussen@google.com, yuanchu@google.com,
weixugc@google.com, riel@surriel.com, gourry@gourry.net,
haowenchao22@gmail.com, corbet@lwn.net, hughd@google.com,
baolin.wang@linux.alibaba.com, tj@kernel.org, mkoutny@suse.com,
skhan@linuxfoundation.org, kunwu.chan@linux.dev,
kernel-team@meta.com, nphamcs@gmail.com, linux-mm@kvack.org,
linux-kernel@vger.kernel.org, linux-doc@vger.kernel.org,
cgroups@vger.kernel.org
Subject: [PATCH v4 10/11] mm, swap: defer memcg_table allocation for physical swap clusters
Date: Tue, 25 Aug 2026 08:32:36 -0700 [thread overview]
Message-ID: <20260825153238.2695446-11-nphamcs@gmail.com> (raw)
In-Reply-To: <20260825153238.2695446-1-nphamcs@gmail.com>
Stop allocating a memcg table for every physical swap cluster that only
ever holds vswap backings. The table costs SWAPFILE_CLUSTER *
sizeof(unsigned short) per cluster, 1 KB per 2 MB of swap on a 64-bit
kernel with 4 KB pages. On a vswap-heavy workload, where zswap writeback
is the only consumer of physical swap, that is the common case.
Such clusters never have their memcg_table read or written: vswap-layer
charging records on the vswap cluster's table, not the physical one.
Allocate eagerly only where the table is known to be needed: every
vswap cluster, and, when vswap is off, every physical cluster, since
all of its slots then map directly into the PTEs. A physical cluster
otherwise allocates on its first direct-use slot, and skips entirely
if it only holds vswap backings. That deferred allocation can fail,
though rarely.
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
mm/swapfile.c | 85 +++++++++++++++++++++++++++++++++++++++------------
1 file changed, 66 insertions(+), 19 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 3b10173cddc3..8c80544a7d25 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -471,7 +471,8 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci)
swap_cluster_free_table_folio_rcu_cb);
}
-static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
+static int swap_cluster_alloc_table(struct swap_info_struct *si,
+ struct swap_cluster_info *ci, gfp_t gfp)
{
struct swap_table *table = NULL;
struct folio *folio;
@@ -494,7 +495,14 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
rcu_assign_pointer(ci->table, table);
#ifdef CONFIG_MEMCG
- if (!mem_cgroup_disabled()) {
+ /*
+ * A physical cluster under vswap may hold only vswap backings, which
+ * record their memcg on the vswap cluster's table, not this one. Such
+ * clusters defer memcg_table allocation until they hand out a slot
+ * that maps directly into the PTEs.
+ */
+ if ((!vswap_is_enabled() || swap_is_vswap(si)) &&
+ !mem_cgroup_disabled()) {
VM_WARN_ON_ONCE(ci->memcg_table);
ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp);
if (!ci->memcg_table) {
@@ -565,8 +573,8 @@ swap_cluster_populate(struct swap_info_struct *si,
lockdep_assert_held(&si->global_cluster_lock);
lockdep_assert_held(&ci->lock);
- if (!swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
- __GFP_NOWARN))
+ if (!swap_cluster_alloc_table(si, ci, __GFP_HIGH | __GFP_NOMEMALLOC |
+ __GFP_NOWARN))
return ci;
/*
@@ -579,8 +587,8 @@ swap_cluster_populate(struct swap_info_struct *si,
spin_unlock(&si->global_cluster_lock);
local_unlock(&percpu_swap_cluster.lock);
- ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
- GFP_KERNEL);
+ ret = swap_cluster_alloc_table(si, ci, __GFP_HIGH | __GFP_NOMEMALLOC |
+ GFP_KERNEL);
/*
* Back to atomic context. We might have migrated to a new CPU with a
@@ -857,7 +865,7 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
ci = cluster_info + idx;
/* Need to allocate swap table first for initial bad slot marking. */
- if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL))
+ if (!ci->count && swap_cluster_alloc_table(si, ci, GFP_KERNEL))
return -ENOMEM;
spin_lock(&ci->lock);
/* Check for duplicated bad swap slots. */
@@ -1079,7 +1087,9 @@ static bool __swap_cluster_alloc_entries(struct swap_info_struct *si,
/* Try use a new cluster for current CPU and allocate from it. */
static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
struct swap_cluster_info *ci,
- struct folio *folio, unsigned long offset)
+ struct folio *folio,
+ unsigned long offset,
+ bool *nomem)
{
unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER);
@@ -1108,6 +1118,24 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
if (!ret)
continue;
}
+#ifdef CONFIG_MEMCG
+ /*
+ * Lazy-allocate memcg_table on the first direct-use slot of a
+ * physical cluster.
+ */
+ if (vswap_is_enabled() && folio &&
+ !folio_test_swapcache(folio) && !mem_cgroup_disabled() &&
+ !ci->memcg_table) {
+ ci->memcg_table = kzalloc_obj(*ci->memcg_table,
+ GFP_ATOMIC | __GFP_NOMEMALLOC |
+ __GFP_NOWARN);
+ if (!ci->memcg_table) {
+ if (nomem)
+ *nomem = true;
+ goto out;
+ }
+ }
+#endif
if (!__swap_cluster_alloc_entries(si, ci, folio, offset % SWAPFILE_CLUSTER))
break;
found = offset;
@@ -1117,7 +1145,15 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
break;
}
out:
- relocate_cluster(si, ci);
+ /*
+ * On a discard-capable device, relocating a cluster whose memcg_table
+ * allocation failed queues a discard for slots that were never used,
+ * which folio_alloc_phys_swap() reads as progress and retries on.
+ */
+ if (nomem && *nomem && !ci->count)
+ __free_cluster(si, ci);
+ else
+ relocate_cluster(si, ci);
swap_cluster_unlock(ci);
if (swap_is_vswap(si)) {
this_cpu_write(percpu_vswap_cluster.offset[order], next);
@@ -1138,7 +1174,13 @@ static unsigned int alloc_swap_scan_list(struct swap_info_struct *si,
bool scan_all)
{
unsigned int found = SWAP_ENTRY_INVALID;
+ bool nomem = false;
+ /*
+ * In rare cases alloc_swap_scan_cluster() can fail due to
+ * memcg_table allocation failure. Short-circuit to avoid looping
+ * over the list indefinitely.
+ */
do {
struct swap_cluster_info *ci = isolate_lock_cluster(si, list);
unsigned long offset;
@@ -1146,10 +1188,10 @@ static unsigned int alloc_swap_scan_list(struct swap_info_struct *si,
if (!ci)
break;
offset = cluster_offset(si, ci);
- found = alloc_swap_scan_cluster(si, ci, folio, offset);
+ found = alloc_swap_scan_cluster(si, ci, folio, offset, &nomem);
if (found)
break;
- } while (scan_all);
+ } while (scan_all && !nomem);
return found;
}
@@ -1170,7 +1212,7 @@ static unsigned int vswap_alloc_cluster(struct swap_info_struct *si,
spin_lock_init(&ci_dyn->ci.lock);
INIT_LIST_HEAD(&ci_dyn->ci.list);
- if (swap_cluster_alloc_table(&ci_dyn->ci, GFP_ATOMIC)) {
+ if (swap_cluster_alloc_table(si, &ci_dyn->ci, GFP_ATOMIC)) {
kfree(ci_dyn);
return SWAP_ENTRY_INVALID;
}
@@ -1196,7 +1238,7 @@ static unsigned int vswap_alloc_cluster(struct swap_info_struct *si,
}
offset = cluster_offset(si, ci);
- return alloc_swap_scan_cluster(si, ci, folio, offset);
+ return alloc_swap_scan_cluster(si, ci, folio, offset, NULL);
}
static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force)
@@ -1303,7 +1345,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
if (cluster_is_usable(ci, order)) {
if (cluster_is_empty(ci))
offset = cluster_offset(si, ci);
- found = alloc_swap_scan_cluster(si, ci, folio, offset);
+ found = alloc_swap_scan_cluster(si, ci, folio, offset,
+ NULL);
} else {
swap_cluster_unlock(ci);
}
@@ -1347,7 +1390,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
if (order < PMD_ORDER) {
/*
* Scan only one fragment cluster is good enough. Order 0
- * allocation will surely success, and large allocation
+ * allocation will surely success unless the memcg table
+ * allocation fails, which is rare, and large allocation
* failure is not critical. Scanning one cluster still
* keeps the list rotated and reclaimed (for clean swap cache).
*/
@@ -1363,7 +1407,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
for (int o = 1; o < SWAP_NR_ORDERS; o++) {
/*
* Clusters here have at least one usable slots and can't fail order 0
- * allocation, but reclaim may drop si->lock and race with another user.
+ * allocation, but reclaim may drop si->lock and race with another user,
+ * and the memcg table allocation may fail.
*/
found = alloc_swap_scan_list(si, &si->frag_clusters[o], folio, true);
if (found)
@@ -1586,7 +1631,7 @@ static swp_entry_t swap_alloc_fast(struct folio *folio)
if (ci && cluster_is_usable(ci, order)) {
if (cluster_is_empty(ci))
offset = cluster_offset(si, ci);
- found = alloc_swap_scan_cluster(si, ci, folio, offset);
+ found = alloc_swap_scan_cluster(si, ci, folio, offset, NULL);
} else if (ci) {
swap_cluster_unlock(ci);
}
@@ -1969,7 +2014,8 @@ static bool vswap_alloc(struct folio *folio)
if (ci && cluster_is_usable(ci, order)) {
if (cluster_is_empty(ci))
offset = cluster_offset(vswap_si, ci);
- alloc_swap_scan_cluster(vswap_si, ci, folio, offset);
+ alloc_swap_scan_cluster(vswap_si, ci, folio, offset,
+ NULL);
} else if (ci) {
swap_cluster_unlock(ci);
}
@@ -2836,7 +2882,8 @@ swp_entry_t swap_alloc_hibernation_slot(int type)
if (pcp_si == si && pcp_offset) {
ci = swap_cluster_lock(si, pcp_offset);
if (cluster_is_usable(ci, 0))
- offset = alloc_swap_scan_cluster(si, ci, NULL, pcp_offset);
+ offset = alloc_swap_scan_cluster(si, ci, NULL,
+ pcp_offset, NULL);
else
swap_cluster_unlock(ci);
}
--
2.53.0-Meta
next prev parent reply other threads:[~2026-08-25 15:33 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-08-25 15:32 [PATCH v4 00/11] Virtual Swap Space (Swap Table Edition) Nhat Pham
2026-08-25 15:32 ` [PATCH v4 01/11] mm, swap: add virtual swap device infrastructure Nhat Pham
2026-08-25 15:32 ` [PATCH v4 02/11] mm, swap: support zswap and zero-filled swap pages as vswap backends Nhat Pham
2026-08-25 15:32 ` [PATCH v4 03/11] mm, swap: prepare the swap IO path for vswap Nhat Pham
2026-08-25 15:32 ` [PATCH v4 04/11] mm, swap: support physical swap as a vswap backend Nhat Pham
2026-08-25 15:32 ` [PATCH v4 05/11] mm, swap: enable THP swapin for vswap entries Nhat Pham
2026-08-25 15:32 ` [PATCH v4 06/11] mm, swap: write back vswap zswap entries to physical swap Nhat Pham
2026-08-25 15:32 ` [PATCH v4 07/11] mm, swap: reclaim physical slots backing cache-only vswap entries Nhat Pham
2026-08-25 15:32 ` [PATCH v4 08/11] mm, swap: only charge physical swap entries Nhat Pham
2026-08-25 15:32 ` [PATCH v4 09/11] mm, swap: add debugfs counters for vswap Nhat Pham
2026-08-25 15:32 ` Nhat Pham [this message]
2026-08-25 15:32 ` [PATCH v4 11/11] mm, swap: widen swap_info_struct max/pages to unsigned long Nhat Pham
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260825153238.2695446-11-nphamcs@gmail.com \
--to=nphamcs@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=axelrasmussen@google.com \
--cc=baohua@kernel.org \
--cc=baolin.wang@linux.alibaba.com \
--cc=baoquan.he@linux.dev \
--cc=cgroups@vger.kernel.org \
--cc=chengming.zhou@linux.dev \
--cc=chrisl@kernel.org \
--cc=corbet@lwn.net \
--cc=david@kernel.org \
--cc=gourry@gourry.net \
--cc=hannes@cmpxchg.org \
--cc=haowenchao22@gmail.com \
--cc=hughd@google.com \
--cc=kasong@tencent.com \
--cc=kernel-team@meta.com \
--cc=kunwu.chan@linux.dev \
--cc=liam@infradead.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=mhocko@kernel.org \
--cc=mkoutny@suse.com \
--cc=muchun.song@linux.dev \
--cc=qi.zheng@linux.dev \
--cc=riel@surriel.com \
--cc=roman.gushchin@linux.dev \
--cc=rppt@kernel.org \
--cc=shakeel.butt@linux.dev \
--cc=shikemeng@huaweicloud.com \
--cc=skhan@linuxfoundation.org \
--cc=surenb@google.com \
--cc=tj@kernel.org \
--cc=vbabka@kernel.org \
--cc=weixugc@google.com \
--cc=yosry@kernel.org \
--cc=youngjun.park@lge.com \
--cc=yuanchu@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox