* [PATCH v4 3/9] mm: update vma_modify_flags() to handle residual flags, document
From: Ahmed Elaidy @ 2026-05-15 12:42 UTC (permalink / raw)
To: stable
Cc: linux-mm, akpm, ljs, avagin, Lorenzo Stoakes, Pedro Falcato,
Vlastimil Babka, Baolin Wang, Barry Song,
David Hildenbrand (Red Hat), Dev Jain, Jann Horn, Jonathan Corbet,
Lance Yang, Liam Howlett, Masami Hiramatsu (Google),
Mathieu Desnoyers, Michal Hocko, Mike Rapoport, Nico Pache,
Ryan Roberts, Steven Rostedt, Suren Baghdasaryan, Zi Yan,
Ahmed Elaidy
In-Reply-To: <20260515124218.151966-2-elaidya225@gmail.com>
From: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
The vma_modify_*() family of functions each either perform splits, a merge
or no changes at all in preparation for the requested modification to
occur.
When doing so for a VMA flags change, we currently don't account for any
flags which may remain (for instance, VM_SOFTDIRTY) despite the requested
change in the case that a merge succeeded.
This is made more important by subsequent patches which will introduce the
concept of sticky VMA flags which rely on this behaviour.
This patch fixes this by passing the VMA flags parameter as a pointer and
updating it accordingly on merge and updating callers to accommodate for
this.
Additionally, while we are here, we add kdocs for each of the
vma_modify_*() functions, as the fact that the requested modification is
not performed is confusing so it is useful to make this abundantly clear.
We also update the VMA userland tests to account for this change.
Link: https://lkml.kernel.org/r/23b5b549b0eaefb2922625626e58c2a352f3e93c.1763460113.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrei Vagin <avagin@gmail.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: David Hildenbrand (Red Hat) <david@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
(cherry picked from commit 9119d6c2095bb20292cb9812dd70d37f17e3bd37)
Signed-off-by: Ahmed Elaidy <elaidya225@gmail.com>
Cc: stable@vger.kernel.org # 6.18.x
---
mm/madvise.c | 2 +-
mm/mlock.c | 2 +-
mm/mprotect.c | 2 +-
mm/mseal.c | 7 +-
mm/vma.c | 56 ++++++++--------
mm/vma.h | 140 +++++++++++++++++++++++++++++-----------
tools/testing/vma/vma.c | 3 +-
7 files changed, 143 insertions(+), 69 deletions(-)
diff --git a/mm/madvise.c b/mm/madvise.c
index fb1c86e630b6..0b3280752bfb 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -167,7 +167,7 @@ static int madvise_update_vma(vm_flags_t new_flags,
range->start, range->end, anon_name);
else
vma = vma_modify_flags(&vmi, madv_behavior->prev, vma,
- range->start, range->end, new_flags);
+ range->start, range->end, &new_flags);
if (IS_ERR(vma))
return PTR_ERR(vma);
diff --git a/mm/mlock.c b/mm/mlock.c
index bb0776f5ef7c..2f699c3497a5 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -478,7 +478,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
/* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
goto out;
- vma = vma_modify_flags(vmi, *prev, vma, start, end, newflags);
+ vma = vma_modify_flags(vmi, *prev, vma, start, end, &newflags);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto out;
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 988c366137d5..fa818cd58201 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -813,7 +813,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
newflags &= ~VM_ACCOUNT;
}
- vma = vma_modify_flags(vmi, *pprev, vma, start, end, newflags);
+ vma = vma_modify_flags(vmi, *pprev, vma, start, end, &newflags);
if (IS_ERR(vma)) {
error = PTR_ERR(vma);
goto fail;
diff --git a/mm/mseal.c b/mm/mseal.c
index c561f0ea93e8..3d2f06046e90 100644
--- a/mm/mseal.c
+++ b/mm/mseal.c
@@ -69,9 +69,10 @@ static int mseal_apply(struct mm_struct *mm,
const unsigned long curr_end = MIN(vma->vm_end, end);
if (!(vma->vm_flags & VM_SEALED)) {
- vma = vma_modify_flags(&vmi, prev, vma,
- curr_start, curr_end,
- vma->vm_flags | VM_SEALED);
+ vm_flags_t vm_flags = vma->vm_flags | VM_SEALED;
+
+ vma = vma_modify_flags(&vmi, prev, vma, curr_start,
+ curr_end, &vm_flags);
if (IS_ERR(vma))
return PTR_ERR(vma);
vm_flags_set(vma, VM_SEALED);
diff --git a/mm/vma.c b/mm/vma.c
index 5815ae9e5770..06609f4116b4 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -1676,25 +1676,35 @@ static struct vm_area_struct *vma_modify(struct vma_merge_struct *vmg)
return vma;
}
-struct vm_area_struct *vma_modify_flags(
- struct vma_iterator *vmi, struct vm_area_struct *prev,
- struct vm_area_struct *vma, unsigned long start, unsigned long end,
- vm_flags_t vm_flags)
+struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ vm_flags_t *vm_flags_ptr)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
+ const vm_flags_t vm_flags = *vm_flags_ptr;
+ struct vm_area_struct *ret;
vmg.vm_flags = vm_flags;
- return vma_modify(&vmg);
+ ret = vma_modify(&vmg);
+ if (IS_ERR(ret))
+ return ret;
+
+ /*
+ * For a merge to succeed, the flags must match those requested. For
+ * flags which do not obey typical merge rules (i.e. do not need to
+ * match), we must let the caller know about them.
+ */
+ if (vmg.state == VMA_MERGE_SUCCESS)
+ *vm_flags_ptr = ret->vm_flags;
+ return ret;
}
-struct vm_area_struct
-*vma_modify_name(struct vma_iterator *vmi,
- struct vm_area_struct *prev,
- struct vm_area_struct *vma,
- unsigned long start,
- unsigned long end,
- struct anon_vma_name *new_name)
+struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ struct anon_vma_name *new_name)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
@@ -1703,12 +1713,10 @@ struct vm_area_struct
return vma_modify(&vmg);
}
-struct vm_area_struct
-*vma_modify_policy(struct vma_iterator *vmi,
- struct vm_area_struct *prev,
- struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- struct mempolicy *new_pol)
+struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ struct mempolicy *new_pol)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
@@ -1717,14 +1725,10 @@ struct vm_area_struct
return vma_modify(&vmg);
}
-struct vm_area_struct
-*vma_modify_flags_uffd(struct vma_iterator *vmi,
- struct vm_area_struct *prev,
- struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- vm_flags_t vm_flags,
- struct vm_userfaultfd_ctx new_ctx,
- bool give_up_on_oom)
+struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end, vm_flags_t vm_flags,
+ struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
diff --git a/mm/vma.h b/mm/vma.h
index d73e1b324bfd..1f2d11bb08b4 100644
--- a/mm/vma.h
+++ b/mm/vma.h
@@ -266,47 +266,115 @@ void remove_vma(struct vm_area_struct *vma);
void unmap_region(struct ma_state *mas, struct vm_area_struct *vma,
struct vm_area_struct *prev, struct vm_area_struct *next);
-/* We are about to modify the VMA's flags. */
-__must_check struct vm_area_struct
-*vma_modify_flags(struct vma_iterator *vmi,
+/**
+ * vma_modify_flags() - Peform any necessary split/merge in preparation for
+ * setting VMA flags to *@vm_flags in the range @start to @end contained within
+ * @vma.
+ * @vmi: Valid VMA iterator positioned at @vma.
+ * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
+ * @vma: The VMA containing the range @start to @end to be updated.
+ * @start: The start of the range to update. May be offset within @vma.
+ * @end: The exclusive end of the range to update, may be offset within @vma.
+ * @vm_flags_ptr: A pointer to the VMA flags that the @start to @end range is
+ * about to be set to. On merge, this will be updated to include any additional
+ * flags which remain in place.
+ *
+ * IMPORTANT: The actual modification being requested here is NOT applied,
+ * rather the VMA is perhaps split, perhaps merged to accommodate the change,
+ * and the caller is expected to perform the actual modification.
+ *
+ * In order to account for VMA flags which may persist (e.g. soft-dirty), the
+ * @vm_flags_ptr parameter points to the requested flags which are then updated
+ * so the caller, should they overwrite any existing flags, correctly retains
+ * these.
+ *
+ * Returns: A VMA which contains the range @start to @end ready to have its
+ * flags altered to *@vm_flags.
+ */
+__must_check struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ vm_flags_t *vm_flags_ptr);
+
+/**
+ * vma_modify_name() - Peform any necessary split/merge in preparation for
+ * setting anonymous VMA name to @new_name in the range @start to @end contained
+ * within @vma.
+ * @vmi: Valid VMA iterator positioned at @vma.
+ * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
+ * @vma: The VMA containing the range @start to @end to be updated.
+ * @start: The start of the range to update. May be offset within @vma.
+ * @end: The exclusive end of the range to update, may be offset within @vma.
+ * @new_name: The anonymous VMA name that the @start to @end range is about to
+ * be set to.
+ *
+ * IMPORTANT: The actual modification being requested here is NOT applied,
+ * rather the VMA is perhaps split, perhaps merged to accommodate the change,
+ * and the caller is expected to perform the actual modification.
+ *
+ * Returns: A VMA which contains the range @start to @end ready to have its
+ * anonymous VMA name changed to @new_name.
+ */
+__must_check struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi,
struct vm_area_struct *prev, struct vm_area_struct *vma,
unsigned long start, unsigned long end,
- vm_flags_t vm_flags);
-
-/* We are about to modify the VMA's anon_name. */
-__must_check struct vm_area_struct
-*vma_modify_name(struct vma_iterator *vmi,
- struct vm_area_struct *prev,
- struct vm_area_struct *vma,
- unsigned long start,
- unsigned long end,
- struct anon_vma_name *new_name);
-
-/* We are about to modify the VMA's memory policy. */
-__must_check struct vm_area_struct
-*vma_modify_policy(struct vma_iterator *vmi,
- struct vm_area_struct *prev,
- struct vm_area_struct *vma,
+ struct anon_vma_name *new_name);
+
+/**
+ * vma_modify_policy() - Peform any necessary split/merge in preparation for
+ * setting NUMA policy to @new_pol in the range @start to @end contained
+ * within @vma.
+ * @vmi: Valid VMA iterator positioned at @vma.
+ * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
+ * @vma: The VMA containing the range @start to @end to be updated.
+ * @start: The start of the range to update. May be offset within @vma.
+ * @end: The exclusive end of the range to update, may be offset within @vma.
+ * @new_pol: The NUMA policy that the @start to @end range is about to be set
+ * to.
+ *
+ * IMPORTANT: The actual modification being requested here is NOT applied,
+ * rather the VMA is perhaps split, perhaps merged to accommodate the change,
+ * and the caller is expected to perform the actual modification.
+ *
+ * Returns: A VMA which contains the range @start to @end ready to have its
+ * NUMA policy changed to @new_pol.
+ */
+__must_check struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct mempolicy *new_pol);
-/* We are about to modify the VMA's flags and/or uffd context. */
-__must_check struct vm_area_struct
-*vma_modify_flags_uffd(struct vma_iterator *vmi,
- struct vm_area_struct *prev,
- struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- vm_flags_t vm_flags,
- struct vm_userfaultfd_ctx new_ctx,
- bool give_up_on_oom);
-
-__must_check struct vm_area_struct
-*vma_merge_new_range(struct vma_merge_struct *vmg);
-
-__must_check struct vm_area_struct
-*vma_merge_extend(struct vma_iterator *vmi,
- struct vm_area_struct *vma,
- unsigned long delta);
+/**
+ * vma_modify_flags_uffd() - Peform any necessary split/merge in preparation for
+ * setting VMA flags to @vm_flags and UFFD context to @new_ctx in the range
+ * @start to @end contained within @vma.
+ * @vmi: Valid VMA iterator positioned at @vma.
+ * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
+ * @vma: The VMA containing the range @start to @end to be updated.
+ * @start: The start of the range to update. May be offset within @vma.
+ * @end: The exclusive end of the range to update, may be offset within @vma.
+ * @vm_flags: The VMA flags that the @start to @end range is about to be set to.
+ * @new_ctx: The userfaultfd context that the @start to @end range is about to
+ * be set to.
+ * @give_up_on_oom: If an out of memory condition occurs on merge, simply give
+ * up on it and treat the merge as best-effort.
+ *
+ * IMPORTANT: The actual modification being requested here is NOT applied,
+ * rather the VMA is perhaps split, perhaps merged to accommodate the change,
+ * and the caller is expected to perform the actual modification.
+ *
+ * Returns: A VMA which contains the range @start to @end ready to have its VMA
+ * flags changed to @vm_flags and its userfaultfd context changed to @new_ctx.
+ */
+__must_check struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi,
+ struct vm_area_struct *prev, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end, vm_flags_t vm_flags,
+ struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom);
+
+__must_check struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg);
+
+__must_check struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
+ struct vm_area_struct *vma, unsigned long delta);
void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
diff --git a/tools/testing/vma/vma.c b/tools/testing/vma/vma.c
index 656e1c75b711..fd37ce3b2628 100644
--- a/tools/testing/vma/vma.c
+++ b/tools/testing/vma/vma.c
@@ -339,6 +339,7 @@ static bool test_simple_modify(void)
struct mm_struct mm = {};
struct vm_area_struct *init_vma = alloc_vma(&mm, 0, 0x3000, 0, vm_flags);
VMA_ITERATOR(vmi, &mm, 0x1000);
+ vm_flags_t flags = VM_READ | VM_MAYREAD;
ASSERT_FALSE(attach_vma(&mm, init_vma));
@@ -347,7 +348,7 @@ static bool test_simple_modify(void)
* performs the merge/split only.
*/
vma = vma_modify_flags(&vmi, init_vma, init_vma,
- 0x1000, 0x2000, VM_READ | VM_MAYREAD);
+ 0x1000, 0x2000, &flags);
ASSERT_NE(vma, NULL);
/* We modify the provided VMA, and on split allocate new VMAs. */
ASSERT_EQ(vma, init_vma);
--
2.54.0
^ permalink raw reply related
* [PATCH v4 2/9] mm: add atomic VMA flags and set VM_MAYBE_GUARD as such
From: Ahmed Elaidy @ 2026-05-15 12:42 UTC (permalink / raw)
To: stable
Cc: linux-mm, akpm, ljs, avagin, Lorenzo Stoakes, Pedro Falcato,
Vlastimil Babka, David Hildenbrand (Red Hat), Lance Yang,
Baolin Wang, Barry Song, Dev Jain, Jann Horn, Jonathan Corbet,
Liam Howlett, Masami Hiramatsu (Google), Mathieu Desnoyers,
Michal Hocko, Mike Rapoport, Nico Pache, Ryan Roberts,
Steven Rostedt, Suren Baghdasaryan, Zi Yan, Ahmed Elaidy
In-Reply-To: <20260515124218.151966-2-elaidya225@gmail.com>
From: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
This patch adds the ability to atomically set VMA flags with only the mmap
read/VMA read lock held.
As this could be hugely problematic for VMA flags in general given that
all other accesses are non-atomic and serialised by the mmap/VMA locks, we
implement this with a strict allow-list - that is, only designated flags
are allowed to do this.
We make VM_MAYBE_GUARD one of these flags.
Link: https://lkml.kernel.org/r/97e57abed09f2663077ed7a36fb8206e243171a9.1763460113.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: David Hildenbrand (Red Hat) <david@kernel.org>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Cc: Andrei Vagin <avagin@gmail.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
(cherry picked from commit 568822502383acd57d7cc1c72ee43932c45a9524)
Signed-off-by: Ahmed Elaidy <elaidya225@gmail.com>
Cc: stable@vger.kernel.org # 6.18.x
---
include/linux/mm.h | 44 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index f1787efaedc5..a96c99066351 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -501,6 +501,9 @@ extern unsigned int kobjsize(const void *objp);
/* This mask represents all the VMA flag bits used by mlock */
#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
+/* These flags can be updated atomically via VMA/mmap read lock. */
+#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD
+
/* Arch-specific flags to clear when updating VM flags on protection change */
#ifndef VM_ARCH_CLEAR
# define VM_ARCH_CLEAR VM_NONE
@@ -843,6 +846,47 @@ static inline void vm_flags_mod(struct vm_area_struct *vma,
__vm_flags_mod(vma, set, clear);
}
+static inline bool __vma_flag_atomic_valid(struct vm_area_struct *vma,
+ int bit)
+{
+ const vm_flags_t mask = BIT(bit);
+
+ /* Only specific flags are permitted */
+ if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED)))
+ return false;
+
+ return true;
+}
+
+/*
+ * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific
+ * valid flags are allowed to do this.
+ */
+static inline void vma_flag_set_atomic(struct vm_area_struct *vma, int bit)
+{
+ /* mmap read lock/VMA read lock must be held. */
+ if (!rwsem_is_locked(&vma->vm_mm->mmap_lock))
+ vma_assert_locked(vma);
+
+ if (__vma_flag_atomic_valid(vma, bit))
+ set_bit(bit, &ACCESS_PRIVATE(vma, __vm_flags));
+}
+
+/*
+ * Test for VMA flag atomically. Requires no locks. Only specific valid flags
+ * are allowed to do this.
+ *
+ * This is necessarily racey, so callers must ensure that serialisation is
+ * achieved through some other means, or that races are permissible.
+ */
+static inline bool vma_flag_test_atomic(struct vm_area_struct *vma, int bit)
+{
+ if (__vma_flag_atomic_valid(vma, bit))
+ return test_bit(bit, &vma->vm_flags);
+
+ return false;
+}
+
static inline void vma_set_anonymous(struct vm_area_struct *vma)
{
vma->vm_ops = NULL;
--
2.54.0
^ permalink raw reply related
* [PATCH v4 1/9] mm: introduce VM_MAYBE_GUARD and make visible in /proc/$pid/smaps
From: Ahmed Elaidy @ 2026-05-15 12:42 UTC (permalink / raw)
To: stable
Cc: linux-mm, akpm, ljs, avagin, Lorenzo Stoakes, Pedro Falcato,
Vlastimil Babka, David Hildenbrand (Red Hat), Lance Yang,
Baolin Wang, Barry Song, Dev Jain, Jann Horn, Jonathan Corbet,
Liam Howlett, Masami Hiramatsu (Google), Mathieu Desnoyers,
Michal Hocko, Mike Rapoport, Nico Pache, Ryan Roberts,
Steven Rostedt, Suren Baghdasaryan, Zi Yan, Ahmed Elaidy
In-Reply-To: <20260515124218.151966-2-elaidya225@gmail.com>
From: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Patch series "introduce VM_MAYBE_GUARD and make it sticky", v4.
Currently, guard regions are not visible to users except through
/proc/$pid/pagemap, with no explicit visibility at the VMA level.
This makes the feature less useful, as it isn't entirely apparent which
VMAs may have these entries present, especially when performing actions
which walk through memory regions such as those performed by CRIU.
This series addresses this issue by introducing the VM_MAYBE_GUARD flag
which fulfils this role, updating the smaps logic to display an entry for
these.
The semantics of this flag are that a guard region MAY be present if set
(we cannot be sure, as we can't efficiently track whether an
MADV_GUARD_REMOVE finally removes all the guard regions in a VMA) - but if
not set the VMA definitely does NOT have any guard regions present.
It's problematic to establish this flag without further action, because
that means that VMAs with guard regions in them become non-mergeable with
adjacent VMAs for no especially good reason.
To work around this, this series also introduces the concept of 'sticky'
VMA flags - that is flags which:
a. if set in one VMA and not in another still permit those VMAs to be
merged (if otherwise compatible).
b. When they are merged, the resultant VMA must have the flag set.
The VMA logic is updated to propagate these flags correctly.
Additionally, VM_MAYBE_GUARD being an explicit VMA flag allows us to solve
an issue with file-backed guard regions - previously these established an
anon_vma object for file-backed mappings solely to have vma_needs_copy()
correctly propagate guard region mappings to child processes.
We introduce a new flag alias VM_COPY_ON_FORK (which currently only
specifies VM_MAYBE_GUARD) and update vma_needs_copy() to check explicitly
for this flag and to copy page tables if it is present, which resolves
this issue.
Additionally, we add the ability for allow-listed VMA flags to be
atomically writable with only mmap/VMA read locks held.
The only flag we allow so far is VM_MAYBE_GUARD, which we carefully ensure
does not cause any races by being allowed to do so.
This allows us to maintain guard region installation as a read-locked
operation and not endure the overhead of obtaining a write lock here.
Finally we introduce extensive VMA userland tests to assert that the
sticky VMA logic behaves correctly as well as guard region self tests to
assert that smaps visibility is correctly implemented.
This patch (of 9):
Currently, if a user needs to determine if guard regions are present in a
range, they have to scan all VMAs (or have knowledge of which ones might
have guard regions).
Since commit 8e2f2aeb8b48 ("fs/proc/task_mmu: add guard region bit to
pagemap") and the related commit a516403787e0 ("fs/proc: extend the
PAGEMAP_SCAN ioctl to report guard regions"), users can use either
/proc/$pid/pagemap or the PAGEMAP_SCAN functionality to perform this
operation at a virtual address level.
This is not ideal, and it gives no visibility at a /proc/$pid/smaps level
that guard regions exist in ranges.
This patch remedies the situation by establishing a new VMA flag,
VM_MAYBE_GUARD, to indicate that a VMA may contain guard regions (it is
uncertain because we cannot reasonably determine whether a
MADV_GUARD_REMOVE call has removed all of the guard regions in a VMA, and
additionally VMAs may change across merge/split).
We utilise 0x800 for this flag which makes it available to 32-bit
architectures also, a flag that was previously used by VM_DENYWRITE, which
was removed in commit 8d0920bde5eb ("mm: remove VM_DENYWRITE") and hasn't
bee reused yet.
We also update the smaps logic and documentation to identify these VMAs.
Another major use of this functionality is that we can use it to identify
that we ought to copy page tables on fork.
We do not actually implement usage of this flag in mm/madvise.c yet as we
need to allow some VMA flags to be applied atomically under mmap/VMA read
lock in order to avoid the need to acquire a write lock for this purpose.
Link: https://lkml.kernel.org/r/cover.1763460113.git.ljs@kernel.org
Link: https://lkml.kernel.org/r/cf8ef821eba29b6c5b5e138fffe95d6dcabdedb9.1763460113.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: David Hildenbrand (Red Hat) <david@kernel.org>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Cc: Andrei Vagin <avagin@gmail.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
(cherry picked from commit 5dba5cc2e0ffa76f2f6c8922a04469dc9602c396)
Signed-off-by: Ahmed Elaidy <elaidya225@gmail.com>
Cc: stable@vger.kernel.org # 6.18.x
---
Documentation/filesystems/proc.rst | 5 +++--
fs/proc/task_mmu.c | 1 +
include/linux/mm.h | 3 +++
include/trace/events/mmflags.h | 1 +
mm/memory.c | 4 ++++
tools/testing/vma/vma_internal.h | 1 +
6 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst
index 0b86a8022fa1..8256e857e2d7 100644
--- a/Documentation/filesystems/proc.rst
+++ b/Documentation/filesystems/proc.rst
@@ -553,7 +553,7 @@ otherwise.
kernel flags associated with the particular virtual memory area in two letter
encoded manner. The codes are the following:
- == =======================================
+ == =============================================================
rd readable
wr writeable
ex executable
@@ -591,7 +591,8 @@ encoded manner. The codes are the following:
sl sealed
lf lock on fault pages
dp always lazily freeable mapping
- == =======================================
+ gu maybe contains guard regions (if not set, definitely doesn't)
+ == =============================================================
Note that there is no guarantee that every flag and associated mnemonic will
be present in all further kernel releases. Things get changed, the flags may
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index b490245ff9be..4c5adfd4fc1f 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1159,6 +1159,7 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
[ilog2(VM_MAYSHARE)] = "ms",
[ilog2(VM_GROWSDOWN)] = "gd",
[ilog2(VM_PFNMAP)] = "pf",
+ [ilog2(VM_MAYBE_GUARD)] = "gu",
[ilog2(VM_LOCKED)] = "lo",
[ilog2(VM_IO)] = "io",
[ilog2(VM_SEQ_READ)] = "sr",
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 1e74eb7267ac..f1787efaedc5 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -269,6 +269,8 @@ extern struct rw_semaphore nommu_region_sem;
extern unsigned int kobjsize(const void *objp);
#endif
+#define VM_MAYBE_GUARD_BIT 11
+
/*
* vm_flags in vm_area_struct, see mm_types.h.
* When changing, update also include/trace/events/mmflags.h
@@ -294,6 +296,7 @@ extern unsigned int kobjsize(const void *objp);
#define VM_UFFD_MISSING 0
#endif /* CONFIG_MMU */
#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
+#define VM_MAYBE_GUARD BIT(VM_MAYBE_GUARD_BIT) /* The VMA maybe contains guard regions. */
#define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
#define VM_LOCKED 0x00002000
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index aa441f593e9a..a6e5a44c9b42 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -213,6 +213,7 @@ IF_HAVE_PG_ARCH_3(arch_3)
{VM_UFFD_MISSING, "uffd_missing" }, \
IF_HAVE_UFFD_MINOR(VM_UFFD_MINOR, "uffd_minor" ) \
{VM_PFNMAP, "pfnmap" }, \
+ {VM_MAYBE_GUARD, "maybe_guard" }, \
{VM_UFFD_WP, "uffd_wp" }, \
{VM_LOCKED, "locked" }, \
{VM_IO, "io" }, \
diff --git a/mm/memory.c b/mm/memory.c
index 94bf107a47ca..dde20cd5fa5b 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1478,6 +1478,10 @@ vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
if (src_vma->anon_vma)
return true;
+ /* Guard regions have modified page tables that require copying. */
+ if (src_vma->vm_flags & VM_MAYBE_GUARD)
+ return true;
+
/*
* Don't copy ptes where a page fault will fill them correctly. Fork
* becomes much lighter when there are big shared or private readonly
diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h
index dc976a285ad2..c87bcc9013f5 100644
--- a/tools/testing/vma/vma_internal.h
+++ b/tools/testing/vma/vma_internal.h
@@ -56,6 +56,7 @@ extern unsigned long dac_mmap_min_addr;
#define VM_MAYEXEC 0x00000040
#define VM_GROWSDOWN 0x00000100
#define VM_PFNMAP 0x00000400
+#define VM_MAYBE_GUARD 0x00000800
#define VM_LOCKED 0x00002000
#define VM_IO 0x00004000
#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
--
2.54.0
^ permalink raw reply related
* [PATCH 6.18.y v4 0/9] mm: backport sticky VMA flags and soft-dirty fix
From: Ahmed Elaidy @ 2026-05-15 12:42 UTC (permalink / raw)
To: stable; +Cc: linux-mm, akpm, ljs, avagin, Ahmed Elaidy
This series backports the sticky VMA flags infrastructure and the
VM_SOFTDIRTY-on-merge fix to linux-6.18.y.
Motivation: CRIU incremental dump/restore can hit a missing-parent-pagemap
failure when VM_SOFTDIRTY is lost during VMA merge operations.
Patch 8 is the target fix:
mm: propagate VM_SOFTDIRTY on merge
The preceding patches provide required dependencies on 6.18.y and are included
to preserve upstream behavior, as requested by maintainers for stable backports.
Changes since v3:
- Reverted to sending the full 9-patch series as requested by Greg KH and Lorenzo.
- Updated Lorenzo's email to ljs@kernel.org across all patches.
- Added Cc: stable@vger.kernel.org # 6.18.x to all patches.
- Added Fixes tag for soft-dirty merging in Patch 8.
Lorenzo Stoakes (9):
mm: introduce VM_MAYBE_GUARD and make visible in /proc/$pid/smaps
mm: add atomic VMA flags and set VM_MAYBE_GUARD as such
mm: update vma_modify_flags() to handle residual flags, document
mm: implement sticky VMA flags
mm: introduce copy-on-fork VMAs and make VM_MAYBE_GUARD one
mm: set the VM_MAYBE_GUARD flag on guard region install
tools/testing/vma: add VMA sticky userland tests
mm: propagate VM_SOFTDIRTY on merge
testing/selftests/mm: add soft-dirty merge self-test
Documentation/filesystems/proc.rst | 5 +-
fs/proc/task_mmu.c | 1 +
include/linux/mm.h | 100 +++++++++++++++++
include/trace/events/mmflags.h | 1 +
mm/khugepaged.c | 71 +++++++-----
mm/madvise.c | 24 +++--
mm/memory.c | 14 +--
mm/mlock.c | 2 +-
mm/mprotect.c | 2 +-
mm/mseal.c | 7 +-
mm/vma.c | 81 +++++++-------
mm/vma.h | 138 +++++++++++++++++-------
tools/testing/selftests/mm/soft-dirty.c | 127 +++++++++++++++++++++-
tools/testing/vma/vma.c | 92 ++++++++++++++--
tools/testing/vma/vma_internal.h | 49 +++++++++
15 files changed, 579 insertions(+), 135 deletions(-)
--
2.54.0
^ permalink raw reply
* Re: [PATCH v4 4/5] ksm: Optimize rmap_walk_ksm by passing a suitable address range
From: Lorenzo Stoakes @ 2026-05-15 12:28 UTC (permalink / raw)
To: xu.xin16; +Cc: david, akpm, hughd, linux-mm, linux-kernel, michel
In-Reply-To: <20260515151344118ESEm1fT7x-Gd6kodR8VVq@zte.com.cn>
On Fri, May 15, 2026 at 03:13:44PM +0800, xu.xin16@zte.com.cn wrote:
> > > diff --git a/mm/ksm.c b/mm/ksm.c
> > > index 0299a53ba7c9..a13184d00759 100644
> > > --- a/mm/ksm.c
> > > +++ b/mm/ksm.c
> > > @@ -3200,6 +3200,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
> > > hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
> > > /* Ignore the stable/unstable/sqnr flags */
> > > const unsigned long addr = rmap_item->address & PAGE_MASK;
> > > + const unsigned long vm_pgoff = rmap_item->vm_pgoff;
> > > struct anon_vma *anon_vma = rmap_item->anon_vma;
> > > struct anon_vma_chain *vmac;
> > > struct vm_area_struct *vma;
> > > @@ -3213,8 +3214,12 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
> > > anon_vma_lock_read(anon_vma);
> > > }
> > >
> > > + /*
> > > + * Currently KSM folios are order-0 normal pages, so pgoff_end
> > > + * should be the same as pgoff_start.
> > > + */
> > > anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
> > > - 0, ULONG_MAX) {
> > > + vm_pgoff, vm_pgoff) {
> >
> > But vm_pgoff would just correspond to the start of the VMA, not where the page
> > is actually mapped?
> >
> > I'd assume you really want the linear page index of the original page?
>
> Right. I've reconsidered and realized that using vm_pgoff is indeed unstable.
Your email client is inserting (kinda) HTML :) & apos ; -> ' please tell it to
behave :P
>
> My initial idea was: as long as we can find the VMA that maps this page,
> it's sufficient for anon_vma_interval_tree_foreach() to check whether
> "vm_pgoff <= pgoff of the original page <= (vm_pgoff + vma_pages(v) - 1)".
>
> However, the flaw here is that the VMA may be split(e.g., due to madvise or mprotect),
> causing vma_pages(v) to change, thereby making this condition no longer satisfied.
>
> Indeed, it's better to use the linear page index of the original page.
Yup :)
Partially mapped large folios would cause weirdness also but KSM uses order-0
right? So probably not a thing.
>
> I'll send v5 to correct this.
>
> >
> > --
> > Cheers,
> >
> > David
> >
Cheers, Lorenzo
^ permalink raw reply
* [PATCH] kho: test: include more variety of allocations
From: Michal Clapinski @ 2026-05-15 12:13 UTC (permalink / raw)
To: Pasha Tatashin, Mike Rapoport, Pratyush Yadav, Alexander Graf,
kexec, linux-mm
Cc: Michal Clapinski
Test early (before kho_init) and late allocations.
Also test allocations on primary and secondary boot, since kho_scratch
behaves differently.
Signed-off-by: Michal Clapinski <mclapinski@google.com>
---
lib/test_kho.c | 167 +++++++++++++++++++-------
tools/testing/selftests/kho/init.c | 36 +++++-
tools/testing/selftests/kho/vmtest.sh | 36 +++++-
3 files changed, 188 insertions(+), 51 deletions(-)
diff --git a/lib/test_kho.c b/lib/test_kho.c
index aa6a0956bb8b..90a49f7cc9ab 100644
--- a/lib/test_kho.c
+++ b/lib/test_kho.c
@@ -23,13 +23,17 @@
#include <net/checksum.h>
-#define KHO_TEST_MAGIC 0x4b484f21 /* KHO! */
-#define KHO_TEST_FDT "kho_test"
#define KHO_TEST_COMPAT "kho-test-v1"
static long max_mem = (PAGE_SIZE << MAX_PAGE_ORDER) * 2;
module_param(max_mem, long, 0644);
+static bool second_boot;
+module_param(second_boot, bool, 0644);
+
+static bool third_boot;
+module_param(third_boot, bool, 0644);
+
struct kho_test_state {
unsigned int nr_folios;
struct folio **folios;
@@ -40,7 +44,25 @@ struct kho_test_state {
__wsum csum;
};
-static struct kho_test_state kho_test_state;
+struct kho_superstate {
+ struct kho_test_state kho_test_state;
+ const char *kho_test_fdt;
+ int kho_test_magic;
+};
+
+static struct kho_superstate kho_superstate[] = {
+ {{}, "kho_test0", 0x4b484f30}, /* KHO0 */
+ {{}, "kho_test1", 0x4b484f31},
+ {{}, "kho_test2", 0x4b484f32},
+ {{}, "kho_test3", 0x4b484f33},
+};
+
+enum superstate_index {
+ FIRST_BOOT_EARLY_ALLOC,
+ FIRST_BOOT_LATE_ALLOC,
+ SECOND_BOOT_EARLY_ALLOC,
+ SECOND_BOOT_LATE_ALLOC,
+};
static void kho_test_unpreserve_data(struct kho_test_state *state)
{
@@ -94,10 +116,11 @@ static int kho_test_preserve_data(struct kho_test_state *state)
return err;
}
-static int kho_test_prepare_fdt(struct kho_test_state *state, ssize_t fdt_size)
+static int kho_test_prepare_fdt(struct kho_superstate *superstate, ssize_t fdt_size)
{
+ struct kho_test_state *state = &superstate->kho_test_state;
const char compatible[] = KHO_TEST_COMPAT;
- unsigned int magic = KHO_TEST_MAGIC;
+ unsigned int magic = superstate->kho_test_magic;
void *fdt = folio_address(state->fdt);
int err;
@@ -121,10 +144,11 @@ static int kho_test_prepare_fdt(struct kho_test_state *state, ssize_t fdt_size)
return err;
}
-static int kho_test_preserve(struct kho_test_state *state)
+static int kho_test_preserve(struct kho_superstate *superstate)
{
ssize_t fdt_size;
int err;
+ struct kho_test_state *state = &superstate->kho_test_state;
fdt_size = state->nr_folios * sizeof(phys_addr_t) + PAGE_SIZE;
state->fdt = folio_alloc(GFP_KERNEL, get_order(fdt_size));
@@ -139,11 +163,11 @@ static int kho_test_preserve(struct kho_test_state *state)
if (err)
goto err_unpreserve_fdt;
- err = kho_test_prepare_fdt(state, fdt_size);
+ err = kho_test_prepare_fdt(superstate, fdt_size);
if (err)
goto err_unpreserve_data;
- err = kho_add_subtree(KHO_TEST_FDT, folio_address(state->fdt),
+ err = kho_add_subtree(superstate->kho_test_fdt, folio_address(state->fdt),
fdt_totalsize(folio_address(state->fdt)));
if (err)
goto err_unpreserve_data;
@@ -202,14 +226,12 @@ static int kho_test_generate_data(struct kho_test_state *state)
return -ENOMEM;
}
-static int kho_test_save(void)
+static int kho_test_alloc(struct kho_test_state *state)
{
- struct kho_test_state *state = &kho_test_state;
struct folio **folios;
unsigned long max_nr;
int err;
- max_mem = PAGE_ALIGN(max_mem);
max_nr = max_mem >> PAGE_SHIFT;
folios = kvmalloc_objs(*state->folios, max_nr);
@@ -221,10 +243,6 @@ static int kho_test_save(void)
if (err)
goto err_free_folios;
- err = kho_test_preserve(state);
- if (err)
- goto err_free_folios;
-
return 0;
err_free_folios:
@@ -232,6 +250,18 @@ static int kho_test_save(void)
return err;
}
+static int kho_test_alloc_and_preserve(int nr)
+{
+ struct kho_test_state *state = &kho_superstate[nr].kho_test_state;
+ int err;
+
+ err = kho_test_alloc(state);
+ if (err)
+ return err;
+
+ return kho_test_preserve(&kho_superstate[nr]);
+}
+
static int kho_test_restore_data(const void *fdt, int node)
{
const struct kho_vmalloc *folios_info_phys;
@@ -284,12 +314,31 @@ static int kho_test_restore_data(const void *fdt, int node)
return 0;
}
-static int kho_test_restore(phys_addr_t fdt_phys)
+static int kho_test_early_alloc(void)
+{
+ if (third_boot)
+ return 0;
+ else if (second_boot)
+ return kho_test_alloc(&kho_superstate[SECOND_BOOT_EARLY_ALLOC].kho_test_state);
+ else
+ return kho_test_alloc(&kho_superstate[FIRST_BOOT_EARLY_ALLOC].kho_test_state);
+}
+core_initcall(kho_test_early_alloc);
+
+static int kho_test_restore(int nr)
{
- void *fdt = phys_to_virt(fdt_phys);
- const unsigned int *magic;
int node, len, err;
+ phys_addr_t fdt_phys;
+ void *fdt;
+ const unsigned int *magic;
+ err = kho_retrieve_subtree(kho_superstate[nr].kho_test_fdt, &fdt_phys, NULL);
+ if (err) {
+ pr_err("failed to retrieve %s FDT: %d\n", kho_superstate[nr].kho_test_fdt, err);
+ return err;
+ }
+
+ fdt = phys_to_virt(fdt_phys);
node = fdt_path_offset(fdt, "/");
if (node < 0)
return -EINVAL;
@@ -301,62 +350,90 @@ static int kho_test_restore(phys_addr_t fdt_phys)
if (!magic || len != sizeof(*magic))
return -EINVAL;
- if (*magic != KHO_TEST_MAGIC)
+ if (*magic != kho_superstate[nr].kho_test_magic)
return -EINVAL;
err = kho_test_restore_data(fdt, node);
if (err)
- return err;
+ pr_err("KHO restore failed\n");
+ else
+ pr_info("KHO restore succeeded\n");
+
+ return err;
+}
+
+extern struct kho_scratch *kho_scratch;
+extern unsigned int kho_scratch_cnt;
+static int check_cma(void)
+{
+ for (int i = 0; i < kho_scratch_cnt; i++) {
+ unsigned long base_pfn = PHYS_PFN(kho_scratch[i].addr);
+ unsigned long count = kho_scratch[i].size >> PAGE_SHIFT;
+ unsigned long pfn;
+
+ for (pfn = base_pfn; pfn < base_pfn + count;
+ pfn += pageblock_nr_pages)
+ if (get_pageblock_migratetype(pfn_to_page(pfn)) != MIGRATE_CMA) {
+ pr_err("KHO wrong migratetype\n");
+ return 1;
+ }
+ }
return 0;
}
static int __init kho_test_init(void)
{
- phys_addr_t fdt_phys;
int err;
if (!kho_is_enabled())
return 0;
- err = kho_retrieve_subtree(KHO_TEST_FDT, &fdt_phys, NULL);
- if (!err) {
- err = kho_test_restore(fdt_phys);
- if (err)
- pr_err("KHO restore failed\n");
- else
- pr_info("KHO restore succeeded\n");
-
- return err;
- }
+ if (check_cma())
+ return -EINVAL;
- if (err != -ENOENT) {
- pr_warn("failed to retrieve %s FDT: %d\n", KHO_TEST_FDT, err);
- return err;
+ if (third_boot) {
+ err = kho_test_restore(SECOND_BOOT_EARLY_ALLOC);
+ err |= kho_test_restore(SECOND_BOOT_LATE_ALLOC);
+ } else if (second_boot) {
+ err = kho_test_restore(FIRST_BOOT_EARLY_ALLOC);
+ err |= kho_test_restore(FIRST_BOOT_LATE_ALLOC);
+
+ err |= kho_test_preserve(&kho_superstate[SECOND_BOOT_EARLY_ALLOC]);
+ err |= kho_test_alloc_and_preserve(SECOND_BOOT_LATE_ALLOC);
+ } else {
+ err = kho_test_preserve(&kho_superstate[FIRST_BOOT_EARLY_ALLOC]);
+ err |= kho_test_alloc_and_preserve(FIRST_BOOT_LATE_ALLOC);
}
-
- return kho_test_save();
+ return err;
}
module_init(kho_test_init);
-static void kho_test_cleanup(void)
+static void kho_test_cleanup(struct kho_test_state *state)
{
+ kho_remove_subtree(folio_address(state->fdt));
+
/* unpreserve and free the data stored in folios */
- kho_test_unpreserve_data(&kho_test_state);
- for (int i = 0; i < kho_test_state.nr_folios; i++)
- folio_put(kho_test_state.folios[i]);
+ kho_test_unpreserve_data(state);
+ for (int i = 0; i < state->nr_folios; i++)
+ folio_put(state->folios[i]);
- kvfree(kho_test_state.folios);
+ kvfree(state->folios);
/* Unpreserve and release the FDT folio */
- kho_unpreserve_folio(kho_test_state.fdt);
- folio_put(kho_test_state.fdt);
+ kho_unpreserve_folio(state->fdt);
+ folio_put(state->fdt);
}
static void __exit kho_test_exit(void)
{
- kho_remove_subtree(folio_address(kho_test_state.fdt));
- kho_test_cleanup();
+ if (second_boot) {
+ kho_test_cleanup(&kho_superstate[SECOND_BOOT_EARLY_ALLOC].kho_test_state);
+ kho_test_cleanup(&kho_superstate[SECOND_BOOT_LATE_ALLOC].kho_test_state);
+ } else {
+ kho_test_cleanup(&kho_superstate[FIRST_BOOT_EARLY_ALLOC].kho_test_state);
+ kho_test_cleanup(&kho_superstate[FIRST_BOOT_LATE_ALLOC].kho_test_state);
+ }
}
module_exit(kho_test_exit);
diff --git a/tools/testing/selftests/kho/init.c b/tools/testing/selftests/kho/init.c
index 88a41b6eba95..18cf180c3322 100644
--- a/tools/testing/selftests/kho/init.c
+++ b/tools/testing/selftests/kho/init.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
+#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
@@ -12,6 +13,17 @@
#define COMMAND_LINE_SIZE 2048
#define KERNEL_IMAGE "/kernel"
+#define INITRD_IMAGE "/initrd"
+
+void console_log(char *log)
+{
+ static int fd;
+
+ if (!fd)
+ fd = open("/dev/console", O_WRONLY);
+
+ write(fd, log, strlen(log) + 1);
+}
static int mount_filesystems(void)
{
@@ -29,11 +41,14 @@ static long kexec_file_load(int kernel_fd, int initrd_fd,
cmdline, flags);
}
+#define CMDLINE_SECOND " test_kho.second_boot=1"
+#define CMDLINE_THIRD " test_kho.third_boot=1"
+
static int kexec_load(void)
{
char cmdline[COMMAND_LINE_SIZE];
ssize_t len;
- int fd, err;
+ int fd, err, initfd;
fd = open("/proc/cmdline", O_RDONLY);
if (fd < 0)
@@ -50,6 +65,25 @@ static int kexec_load(void)
if (fd < 0)
return -1;
+ if (!strstr(cmdline, CMDLINE_SECOND)) {
+ console_log("init: first boot detected\n");
+
+ strcat(cmdline, CMDLINE_SECOND);
+ len = strlen(cmdline) + 1;
+ initfd = open(INITRD_IMAGE, O_RDONLY);
+ if (initfd < 0)
+ return -1;
+
+ err = kexec_file_load(fd, initfd, len, cmdline, 0);
+ close(fd);
+ close(initfd);
+ return err ? : 0;
+ }
+
+ console_log("init: second boot detected\n");
+ strcat(cmdline, CMDLINE_THIRD);
+ len = strlen(cmdline) + 1;
+
err = kexec_file_load(fd, -1, len, cmdline, KEXEC_FILE_NO_INITRAMFS);
close(fd);
diff --git a/tools/testing/selftests/kho/vmtest.sh b/tools/testing/selftests/kho/vmtest.sh
index 0014bd76e88d..e7bd1b84bbe8 100755
--- a/tools/testing/selftests/kho/vmtest.sh
+++ b/tools/testing/selftests/kho/vmtest.sh
@@ -43,6 +43,14 @@ function fail() {
local msg=${1:-""}
ktap_test_fail "$msg"
+
+ local serial_log="$tmp_dir/qemu.serial"
+ if [[ -f "$serial_log" ]]; then
+ echo "=== QEMU Serial Output ===" >&2
+ cat "$serial_log" >&2
+ echo "==========================" >&2
+ fi
+
exit "$KSFT_FAIL"
}
@@ -84,11 +92,11 @@ EOF
function mkinitrd() {
local kernel=$1
+ local initrd_temp="$tmp_dir/initrd_temp.cpio"
- "$CROSS_COMPILE"gcc -s -static -Os -nostdinc -nostdlib \
+ "$CROSS_COMPILE"gcc -s -static -Os \
-fno-asynchronous-unwind-tables -fno-ident \
-I "$headers_dir/include" \
- -I "$kernel_dir/tools/include/nolibc" \
-o "$tmp_dir/init" "$test_dir/init.c"
cat > "$tmp_dir/cpio_list" <<EOF
@@ -98,6 +106,18 @@ dir /debugfs 0755 0 0
nod /dev/console 0600 0 0 c 5 1
file /init $tmp_dir/init 0755 0 0
file /kernel $kernel 0644 0 0
+EOF
+
+ "$build_dir/usr/gen_init_cpio" "$tmp_dir/cpio_list" > "$initrd_temp"
+
+ cat > "$tmp_dir/cpio_list" <<EOF
+dir /dev 0755 0 0
+dir /proc 0755 0 0
+dir /debugfs 0755 0 0
+nod /dev/console 0600 0 0 c 5 1
+file /init $tmp_dir/init 0755 0 0
+file /kernel $kernel 0644 0 0
+file /initrd $initrd_temp 0755 0 0
EOF
"$build_dir/usr/gen_init_cpio" "$tmp_dir/cpio_list" > "$initrd"
@@ -109,16 +129,22 @@ function run_qemu() {
local kernel=$3
local serial="$tmp_dir/qemu.serial"
- cmdline="$cmdline kho=on panic=-1"
+ # 2GiB of preserved mem to exhaust memory initialized early
+ cmdline="$cmdline kho=on panic=-1 test_kho.max_mem=2147483648 \
+ ignore_loglevel earlyprintk=serial,ttyS0,115200 printk.time=1"
- $qemu_cmd -m 1G -smp 2 -no-reboot -nographic -nodefaults \
+ $qemu_cmd -m 8G -smp 2 -no-reboot -nographic -nodefaults \
-accel kvm -accel hvf -accel tcg \
-serial file:"$serial" \
-append "$cmdline" \
-kernel "$kernel" \
-initrd "$initrd"
- grep "KHO restore succeeded" "$serial" &> /dev/null || fail "KHO failed"
+ count="$(grep --text "KHO restore succeeded" $serial | wc -l)"
+ echo Successful restores: "$count"
+ if [[ $count -ne 4 ]]; then
+ fail "KHO failed"
+ fi
}
function target_to_arch() {
--
2.54.0.563.g4f69b47b94-goog
^ permalink raw reply related
* [PATCH 6/6] mm/swap: remove SWP_FS_OPS
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
In-Reply-To: <20260515120019.4015143-1-hch@lst.de>
Provide a swap_fs_activate helper that directly sets up swap_fs_ops,
and a flag in struct swap_ops to indicate of NOFS swapping is allowed.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
Documentation/filesystems/locking.rst | 5 +++--
Documentation/filesystems/vfs.rst | 4 ++--
fs/nfs/file.c | 4 +---
fs/smb/client/file.c | 4 +---
include/linux/swap.h | 6 +++++-
mm/page_io.c | 9 ++++++++-
mm/swap.h | 5 ++++-
mm/swapfile.c | 2 --
mm/vmscan.c | 14 ++++++--------
9 files changed, 30 insertions(+), 23 deletions(-)
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 8421ea21bd35..70481bdc031d 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -355,13 +355,14 @@ should perform any validation and preparation necessary to ensure that
writes can be performed with minimal memory allocation. It should call
add_swap_extent(), or the helper iomap_swapfile_activate(), and return
the number of extents added. If IO should be submitted through
-->swap_rw(), it should set SWP_FS_OPS, otherwise IO will be submitted
+->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted
directly to the block device ``sis->bdev``.
->swap_deactivate() will be called in the sys_swapoff()
path after ->swap_activate() returned success.
-->swap_rw will be called for swap IO if SWP_FS_OPS was set by ->swap_activate().
+->swap_rw will be called for swap IO if swap_fs_activate was called by
+->swap_activate().
file_lock_operations
====================
diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst
index 7c753148af88..e7677423a20f 100644
--- a/Documentation/filesystems/vfs.rst
+++ b/Documentation/filesystems/vfs.rst
@@ -977,7 +977,7 @@ cache in your filesystem. The following members are defined:
can be performed with minimal memory allocation. It should call
add_swap_extent(), or the helper iomap_swapfile_activate(), and
return the number of extents added. If IO should be submitted
- through ->swap_rw(), it should set SWP_FS_OPS, otherwise IO will
+ through ->swap_rw(), it should call swap_fs_activate, otherwise IO will
be submitted directly to the block device ``sis->bdev``.
``swap_deactivate``
@@ -985,7 +985,7 @@ cache in your filesystem. The following members are defined:
successful.
``swap_rw``
- Called to read or write swap pages when SWP_FS_OPS is set.
+ Called to read or write swap pages when swap_fs_activate was called.
The File Object
===============
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index 25048a3c2364..8172c9972b46 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -589,7 +589,7 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file,
ret = rpc_clnt_swap_activate(clnt);
if (ret)
return ret;
- ret = add_swap_extent(sis, 0, sis->max, 0);
+ ret = swap_fs_activate(sis);
if (ret < 0) {
rpc_clnt_swap_deactivate(clnt);
return ret;
@@ -599,8 +599,6 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file,
if (cl->rpc_ops->enable_swap)
cl->rpc_ops->enable_swap(inode);
-
- sis->flags |= SWP_FS_OPS;
return ret;
}
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index 664a2c223089..74c2748484ff 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -3327,9 +3327,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis,
* but we could add call to grab a byte range lock to prevent others
* from reading or writing the file
*/
-
- sis->flags |= SWP_FS_OPS;
- return add_swap_extent(sis, 0, sis->max, 0);
+ return swap_fs_activate(sis);
}
static void cifs_swap_deactivate(struct file *file)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 0da33b803348..15790544ca3e 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -208,7 +208,6 @@ enum {
SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */
SWP_BLKDEV = (1 << 6), /* its a block device */
SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */
- SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */
SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */
SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
@@ -404,6 +403,7 @@ extern void __meminit kswapd_stop(int nid);
#ifdef CONFIG_SWAP
+int swap_fs_activate(struct swap_info_struct *sis);
int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
unsigned long nr_pages, sector_t start_block);
int generic_swapfile_activate(struct swap_info_struct *, struct file *,
@@ -528,6 +528,10 @@ static inline bool folio_free_swap(struct folio *folio)
return false;
}
+static inline int swap_fs_activate(struct swap_info_struct *sis)
+{
+ return -EINVAL;
+}
static inline int add_swap_extent(struct swap_info_struct *sis,
unsigned long start_page,
unsigned long nr_pages, sector_t start_block)
diff --git a/mm/page_io.c b/mm/page_io.c
index 4678a8af9f96..46eed28ee261 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -625,12 +625,19 @@ static bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
swap_dev_pos(prev_folio->swap) + prev_folio_size;
}
-const struct swap_ops swap_fs_ops = {
+static const struct swap_ops swap_fs_ops = {
+ .flags = SWAP_OPS_F_NOFS,
.submit_write = swap_fs_submit_write,
.submit_read = swap_fs_submit_read,
.can_merge = swap_fs_can_merge,
};
+int swap_fs_activate(struct swap_info_struct *sis)
+{
+ sis->ops = &swap_fs_ops;
+ return add_swap_extent(sis, 0, sis->max, 0);
+}
+
void swap_write_submit(struct swap_io_ctx *ctx)
{
if (!ctx->sio)
diff --git a/mm/swap.h b/mm/swap.h
index aaf774fd03b4..b70dd4178baa 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -58,7 +58,11 @@ struct swap_io_ctx {
struct swap_info_struct *sis;
};
+#define SWAP_OPS_F_NOFS (1U << 0)
+
struct swap_ops {
+ unsigned int flags;
+
bool (*can_merge)(struct folio *folio, struct folio *prev_folio,
size_t prev_folio_size);
void (*submit_write)(struct swap_io_ctx *ctx);
@@ -503,7 +507,6 @@ static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
#endif /* CONFIG_SWAP */
extern const struct swap_ops swap_bdev_ops;
-extern const struct swap_ops swap_fs_ops;
int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
struct list_head *folio_list);
diff --git a/mm/swapfile.c b/mm/swapfile.c
index fce69a91e7b4..7b44caf6a0e8 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -2797,8 +2797,6 @@ static int setup_swap_extents(struct swap_info_struct *sis,
ret = mapping->a_ops->swap_activate(sis, swap_file, span);
if (ret < 0)
return ret;
- if (sis->flags & SWP_FS_OPS)
- sis->ops = &swap_fs_ops;
sis->flags |= SWP_ACTIVATED;
return ret;
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 56cd59e27447..d0bc145098e0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1035,16 +1035,14 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
{
if (gfp_mask & __GFP_FS)
return true;
- if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO))
- return false;
/*
- * We can "enter_fs" for swap-cache with only __GFP_IO
- * providing this isn't SWP_FS_OPS.
- * ->flags can be updated non-atomically,
- * but that will never affect SWP_FS_OPS, so the data_race
- * is safe.
+ * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by
+ * a swapfile that requires GFP_NOFS I/O.
*/
- return !data_race(folio_swap_flags(folio) & SWP_FS_OPS);
+ if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) &&
+ !(__swap_entry_to_info(folio->swap)->ops->flags & SWAP_OPS_F_NOFS))
+ return true;
+ return false;
}
/*
--
2.53.0
^ permalink raw reply related
* [PATCH 5/6] mm/swap: use swap_ops to register swap device's methods
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
In-Reply-To: <20260515120019.4015143-1-hch@lst.de>
From: Baoquan He <baoquan.he@linux.dev>
This simplifies codes and makes logic clearer. And also makes later any
new swap device type being added easier to handle.
Currently there are two types of swap devices: fs and bdev.
Suggested-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Baoquan He <baoquan.he@linux.dev>
[hch: updated for the new submit and can_merge abstraction]
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/swap.h | 1 +
mm/page_io.c | 63 ++++++++++++++++++++++++++++----------------
mm/swap.h | 10 +++++++
mm/swapfile.c | 4 +++
4 files changed, 55 insertions(+), 23 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 7a09df6977a5..0da33b803348 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -282,6 +282,7 @@ struct swap_info_struct {
struct work_struct reclaim_work; /* reclaim worker */
struct list_head discard_clusters; /* discard clusters list */
struct plist_node avail_list; /* entry in swap_avail_head */
+ const struct swap_ops *ops;
};
static inline swp_entry_t page_swap_entry(struct page *page)
diff --git a/mm/page_io.c b/mm/page_io.c
index bbd8cf47d20d..4678a8af9f96 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -309,19 +309,7 @@ static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio)
if (ctx->sis != sis)
return false;
-
- if (sis->flags & SWP_FS_OPS) {
- if (swap_dev_pos(folio->swap) !=
- swap_dev_pos(prev_folio->swap) + prev_folio_size)
- return false;
- } else {
- if (swap_folio_sector(folio) !=
- swap_folio_sector(prev_folio) +
- (prev_folio_size >> SECTOR_SHIFT))
- return false;
- }
-
- return true;
+ return sis->ops->can_merge(folio, prev_folio, prev_folio_size);
}
static void swap_add_page(struct swap_io_ctx *ctx, struct folio *folio, int rw)
@@ -585,6 +573,20 @@ static void swap_bdev_submit_read(struct swap_io_ctx *ctx)
}
}
+static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size)
+{
+ return swap_folio_sector(folio) ==
+ swap_folio_sector(prev_folio) +
+ (prev_folio_size >> SECTOR_SHIFT);
+}
+
+const struct swap_ops swap_bdev_ops = {
+ .submit_write = swap_bdev_submit_write,
+ .submit_read = swap_bdev_submit_read,
+ .can_merge = swap_bdev_can_merge,
+};
+
static void swap_fs_submit(struct swap_io_ctx *ctx, int rw)
{
struct swap_iocb *sio = ctx->sio;
@@ -606,15 +608,34 @@ static void swap_fs_submit(struct swap_io_ctx *ctx, int rw)
sio->iocb.ki_complete(&sio->iocb, ret);
}
+static void swap_fs_submit_write(struct swap_io_ctx *ctx)
+{
+ swap_fs_submit(ctx, WRITE);
+}
+
+static void swap_fs_submit_read(struct swap_io_ctx *ctx)
+{
+ swap_fs_submit(ctx, READ);
+}
+
+static bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size)
+{
+ return swap_dev_pos(folio->swap) ==
+ swap_dev_pos(prev_folio->swap) + prev_folio_size;
+}
+
+const struct swap_ops swap_fs_ops = {
+ .submit_write = swap_fs_submit_write,
+ .submit_read = swap_fs_submit_read,
+ .can_merge = swap_fs_can_merge,
+};
+
void swap_write_submit(struct swap_io_ctx *ctx)
{
if (!ctx->sio)
return;
-
- if (ctx->sis->flags & SWP_FS_OPS)
- swap_fs_submit(ctx, WRITE);
- else
- swap_bdev_submit_write(ctx);
+ ctx->sis->ops->submit_write(ctx);
ctx->sio = NULL;
ctx->sis = NULL;
}
@@ -623,11 +644,7 @@ void swap_read_submit(struct swap_io_ctx *ctx)
{
if (!ctx->sio)
return;
-
- if (ctx->sis->flags & SWP_FS_OPS)
- swap_fs_submit(ctx, READ);
- else
- swap_bdev_submit_read(ctx);
+ ctx->sis->ops->submit_read(ctx);
ctx->sio = NULL;
ctx->sis = NULL;
}
diff --git a/mm/swap.h b/mm/swap.h
index b359735be3c5..aaf774fd03b4 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -58,6 +58,13 @@ struct swap_io_ctx {
struct swap_info_struct *sis;
};
+struct swap_ops {
+ bool (*can_merge)(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size);
+ void (*submit_write)(struct swap_io_ctx *ctx);
+ void (*submit_read)(struct swap_io_ctx *ctx);
+};
+
#ifdef CONFIG_SWAP
#include <linux/swapops.h> /* for swp_offset */
#include <linux/blk_types.h> /* for bio_end_io_t */
@@ -495,6 +502,9 @@ static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
}
#endif /* CONFIG_SWAP */
+extern const struct swap_ops swap_bdev_ops;
+extern const struct swap_ops swap_fs_ops;
+
int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
struct list_head *folio_list);
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 27dbce0d1e1e..fce69a91e7b4 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -2785,6 +2785,8 @@ static int setup_swap_extents(struct swap_info_struct *sis,
if (ret)
return ret;
+ sis->ops = &swap_bdev_ops;
+
if (S_ISBLK(inode->i_mode)) {
ret = add_swap_extent(sis, 0, sis->max, 0);
*span = sis->pages;
@@ -2795,6 +2797,8 @@ static int setup_swap_extents(struct swap_info_struct *sis,
ret = mapping->a_ops->swap_activate(sis, swap_file, span);
if (ret < 0)
return ret;
+ if (sis->flags & SWP_FS_OPS)
+ sis->ops = &swap_fs_ops;
sis->flags |= SWP_ACTIVATED;
return ret;
}
--
2.53.0
^ permalink raw reply related
* Re: [PATCH 6.18.y v1 0/9] mm: backport sticky VMA flags and soft-dirty fix
From: Lorenzo Stoakes @ 2026-05-15 12:00 UTC (permalink / raw)
To: Ahmed Elaidy; +Cc: stable, linux-mm, akpm, avagin
In-Reply-To: <agcG2uKHhlt85FaV@lucifer>
On Fri, May 15, 2026 at 12:44:55PM +0100, Lorenzo Stoakes wrote:
> Hi,
>
> Just a heads up that I generally don't read kernel mail sent to my work address,
> as I changed my email setup significantly and use ljs@kernel.org for everything
> upstream!
>
> Understandable given the original patches obviously used it but just FYI :)
>
> Cheers, Lorenzo
>
> On Sat, Apr 25, 2026 at 12:12:34AM +0300, Ahmed Elaidy wrote:
> > This series backports the sticky VMA flags infrastructure and the
> > VM_SOFTDIRTY-on-merge fix to linux-6.18.y.
> >
> > Motivation: CRIU incremental dump/restore can hit a missing-parent-pagemap
> > failure when VM_SOFTDIRTY is lost during VMA merge operations.
> >
> > Patch 8 is the target fix:
> > mm: propagate VM_SOFTDIRTY on merge
> >
> > The preceding patches provide required dependencies on 6.18.y and are included
> > to preserve upstream behavior.
> >
> > Backport notes:
> > - Non-trivial context conflicts were resolved in:
> > - mm/mseal.c
> > - mm/vma.c
> > - Conflict resolution keeps upstream semantics; no intentional behavior
> > changes beyond context adaptation for 6.18.y.
Thanks for doing this, had a quick look through the series and all LGTM!
Cheers, Lorenzo
> >
> > Cc: stable@vger.kernel.org
> >
> >
> >
> > Lorenzo Stoakes (9):
> > mm: introduce VM_MAYBE_GUARD and make visible in /proc/$pid/smaps
> > mm: add atomic VMA flags and set VM_MAYBE_GUARD as such
> > mm: update vma_modify_flags() to handle residual flags, document
> > mm: implement sticky VMA flags
> > mm: introduce copy-on-fork VMAs and make VM_MAYBE_GUARD one
> > mm: set the VM_MAYBE_GUARD flag on guard region install
> > tools/testing/vma: add VMA sticky userland tests
> > mm: propagate VM_SOFTDIRTY on merge
> > testing/selftests/mm: add soft-dirty merge self-test
> >
> > Documentation/filesystems/proc.rst | 5 +-
> > fs/proc/task_mmu.c | 1 +
> > include/linux/mm.h | 100 +++++++++++++++++
> > include/trace/events/mmflags.h | 1 +
> > mm/khugepaged.c | 71 +++++++-----
> > mm/madvise.c | 24 +++--
> > mm/memory.c | 14 +--
> > mm/mlock.c | 2 +-
> > mm/mprotect.c | 2 +-
> > mm/mseal.c | 7 +-
> > mm/vma.c | 81 +++++++-------
> > mm/vma.h | 138 +++++++++++++++++-------
> > tools/testing/selftests/mm/soft-dirty.c | 127 +++++++++++++++++++++-
> > tools/testing/vma/vma.c | 92 ++++++++++++++--
> > tools/testing/vma/vma_internal.h | 49 +++++++++
> > 15 files changed, 579 insertions(+), 135 deletions(-)
> >
> > --
> > 2.53.0
^ permalink raw reply
* [PATCH 3/6] mm/swap: intoduce struct swap_io_ctx
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
In-Reply-To: <20260515120019.4015143-1-hch@lst.de>
Generalize the context currently provided by double pointers to struct
swap_iocb to an on-stack context. This cleans up the code and prepares
for adding more fields and supporting batching multiple folios into a
single bio for block-based swap as well.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
mm/madvise.c | 16 +++++++--------
mm/page_io.c | 54 +++++++++++++++++++++++++++----------------------
mm/shmem.c | 13 ++++++++----
mm/swap.h | 36 ++++++++++++++-------------------
mm/swap_state.c | 40 +++++++++++++++++++-----------------
mm/vmscan.c | 15 +++++++-------
6 files changed, 91 insertions(+), 83 deletions(-)
diff --git a/mm/madvise.c b/mm/madvise.c
index 69708e953cf5..9ca82af8799a 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
unsigned long end, struct mm_walk *walk)
{
struct vm_area_struct *vma = walk->private;
- struct swap_iocb *splug = NULL;
+ struct swap_io_ctx ctx = {};
pte_t *ptep = NULL;
spinlock_t *ptl;
unsigned long addr;
@@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
pte_unmap_unlock(ptep, ptl);
ptep = NULL;
- folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
- vma, addr, &splug);
+ folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE,
+ vma, addr);
if (folio)
folio_put(folio);
}
if (ptep)
pte_unmap_unlock(ptep, ptl);
- swap_read_unplug(splug);
+ swap_read_submit(&ctx);
cond_resched();
return 0;
@@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,
XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
pgoff_t end_index = linear_page_index(vma, end) - 1;
struct folio *folio;
- struct swap_iocb *splug = NULL;
+ struct swap_io_ctx ctx = {};
rcu_read_lock();
xas_for_each(&xas, folio, end_index) {
@@ -257,15 +257,15 @@ static void shmem_swapin_range(struct vm_area_struct *vma,
xas_pause(&xas);
rcu_read_unlock();
- folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping),
- vma, addr, &splug);
+ folio = read_swap_cache_async(&ctx, entry,
+ mapping_gfp_mask(mapping), vma, addr);
if (folio)
folio_put(folio);
rcu_read_lock();
}
rcu_read_unlock();
- swap_read_unplug(splug);
+ swap_read_submit(&ctx);
}
#endif /* CONFIG_SWAP */
diff --git a/mm/page_io.c b/mm/page_io.c
index 70cea9e24d2f..a78efc9909c8 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -237,7 +237,7 @@ static void swap_zeromap_folio_clear(struct folio *folio)
* We may have stale swap cache pages in memory: notice
* them here and get rid of the unnecessary final write.
*/
-int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
+int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
{
int ret = 0;
@@ -285,7 +285,7 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
}
rcu_read_unlock();
- __swap_writepage(folio, swap_plug);
+ __swap_writepage(ctx, folio);
return 0;
out_unlock:
folio_unlock(folio);
@@ -375,9 +375,9 @@ static void sio_write_complete(struct kiocb *iocb, long ret)
mempool_free(sio, sio_pool);
}
-static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
+static void swap_writepage_fs(struct swap_io_ctx *ctx, struct folio *folio)
{
- struct swap_iocb *sio = swap_plug ? *swap_plug : NULL;
+ struct swap_iocb *sio = ctx->sio;
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
struct file *swap_file = sis->swap_file;
loff_t pos = swap_dev_pos(folio->swap);
@@ -388,7 +388,7 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
if (sio) {
if (sio->iocb.ki_filp != swap_file ||
sio->iocb.ki_pos + sio->len != pos) {
- swap_write_unplug(sio);
+ swap_write_submit(ctx);
sio = NULL;
}
}
@@ -403,12 +403,11 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);
sio->len += folio_size(folio);
sio->pages += 1;
- if (sio->pages == ARRAY_SIZE(sio->bvec) || !swap_plug) {
- swap_write_unplug(sio);
+ if (sio->pages == ARRAY_SIZE(sio->bvec)) {
+ swap_write_submit(ctx);
sio = NULL;
}
- if (swap_plug)
- *swap_plug = sio;
+ ctx->sio = sio;
}
static void swap_writepage_bdev_sync(struct folio *folio,
@@ -448,7 +447,7 @@ static void swap_writepage_bdev_async(struct folio *folio,
submit_bio(bio);
}
-void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
+void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
{
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
@@ -459,7 +458,7 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
* is safe.
*/
if (data_race(sis->flags & SWP_FS_OPS))
- swap_writepage_fs(folio, swap_plug);
+ swap_writepage_fs(ctx, folio);
/*
* ->flags can be updated non-atomically,
* but that will never affect SWP_SYNCHRONOUS_IO, so the data_race
@@ -471,16 +470,21 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
swap_writepage_bdev_async(folio, sis);
}
-void swap_write_unplug(struct swap_iocb *sio)
+void swap_write_submit(struct swap_io_ctx *ctx)
{
struct iov_iter from;
+ struct swap_iocb *sio = ctx->sio;
struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
int ret;
+ if (!ctx)
+ return;
+
iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);
ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
if (ret != -EIOCBQUEUED)
sio_write_complete(&sio->iocb, ret);
+ ctx->sio = NULL;
}
static void sio_read_complete(struct kiocb *iocb, long ret)
@@ -539,18 +543,16 @@ static bool swap_read_folio_zeromap(struct folio *folio)
return true;
}
-static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
+static void swap_read_folio_fs(struct swap_io_ctx *ctx, struct folio *folio)
{
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
- struct swap_iocb *sio = NULL;
+ struct swap_iocb *sio = ctx->sio;
loff_t pos = swap_dev_pos(folio->swap);
- if (plug)
- sio = *plug;
if (sio) {
if (sio->iocb.ki_filp != sis->swap_file ||
sio->iocb.ki_pos + sio->len != pos) {
- swap_read_unplug(sio);
+ swap_read_submit(ctx);
sio = NULL;
}
}
@@ -565,12 +567,11 @@ static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);
sio->len += folio_size(folio);
sio->pages += 1;
- if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {
- swap_read_unplug(sio);
+ if (sio->pages == ARRAY_SIZE(sio->bvec)) {
+ swap_read_submit(ctx);
sio = NULL;
}
- if (plug)
- *plug = sio;
+ ctx->sio = sio;
}
static void swap_read_folio_bdev_sync(struct folio *folio,
@@ -610,7 +611,7 @@ static void swap_read_folio_bdev_async(struct folio *folio,
submit_bio(bio);
}
-void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
+void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
{
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
@@ -645,7 +646,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
zswap_folio_swapin(folio);
if (data_race(sis->flags & SWP_FS_OPS)) {
- swap_read_folio_fs(folio, plug);
+ swap_read_folio_fs(ctx, folio);
} else if (synchronous) {
swap_read_folio_bdev_sync(folio, sis);
} else {
@@ -660,14 +661,19 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
delayacct_swapin_end();
}
-void __swap_read_unplug(struct swap_iocb *sio)
+void swap_read_submit(struct swap_io_ctx *ctx)
{
struct iov_iter from;
+ struct swap_iocb *sio = ctx->sio;
struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
int ret;
+ if (!sio)
+ return;
+
iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);
ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
if (ret != -EIOCBQUEUED)
sio_read_complete(&sio->iocb, ret);
+ ctx->sio = NULL;
}
diff --git a/mm/shmem.c b/mm/shmem.c
index b8becbd4beaf..a9c1694d2755 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1584,13 +1584,13 @@ int shmem_unuse(unsigned int type)
/**
* shmem_writeout - Write the folio to swap
+ * @plug: swap I/O context
* @folio: The folio to write
- * @plug: swap plug
* @folio_list: list to put back folios on split
*
* Move the folio from the page cache to the swap cache.
*/
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
+int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
struct list_head *folio_list)
{
struct address_space *mapping = folio->mapping;
@@ -1702,7 +1702,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));
BUG_ON(folio_mapped(folio));
- error = swap_writeout(folio, plug);
+ error = swap_writeout(ctx, folio);
if (error != AOP_WRITEPAGE_ACTIVATE) {
/* folio has been unlocked */
return error;
@@ -1741,7 +1741,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
int shmem_write_folio(struct folio *folio)
{
- return shmem_writeout(folio, NULL, NULL);
+ struct swap_io_ctx ctx = {};
+ int err;
+
+ err = shmem_writeout(&ctx, folio, NULL);
+ swap_write_submit(&ctx);
+ return err;
}
EXPORT_SYMBOL_GPL(shmem_write_folio);
diff --git a/mm/swap.h b/mm/swap.h
index b6db72fb9879..3ec35b6d629f 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -4,7 +4,6 @@
#include <linux/atomic.h> /* for atomic_long_t */
struct mempolicy;
-struct swap_iocb;
extern int page_cluster;
@@ -54,6 +53,10 @@ enum swap_cluster_flags {
CLUSTER_FLAG_MAX,
};
+struct swap_io_ctx {
+ struct swap_iocb *sio;
+};
+
#ifdef CONFIG_SWAP
#include <linux/swapops.h> /* for swp_offset */
#include <linux/blk_types.h> /* for bio_end_io_t */
@@ -216,17 +219,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si,
/* linux/mm/page_io.c */
int sio_pool_init(void);
-struct swap_iocb;
-void swap_read_folio(struct folio *folio, struct swap_iocb **plug);
-void __swap_read_unplug(struct swap_iocb *plug);
-static inline void swap_read_unplug(struct swap_iocb *plug)
-{
- if (unlikely(plug))
- __swap_read_unplug(plug);
-}
-void swap_write_unplug(struct swap_iocb *sio);
-int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug);
-void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug);
+void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio);
+void swap_read_submit(struct swap_io_ctx *ctx);
+void swap_write_submit(struct swap_io_ctx *ctx);
+int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio);
+void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio);
/* linux/mm/swap_state.c */
extern struct address_space swap_space __read_mostly;
@@ -293,9 +290,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
void show_swap_cache_info(void);
void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
-struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
- struct vm_area_struct *vma, unsigned long addr,
- struct swap_iocb **plug);
+struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
+ gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr);
struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
struct mempolicy *mpol, pgoff_t ilx);
struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
@@ -353,7 +349,6 @@ static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
}
#else /* CONFIG_SWAP */
-struct swap_iocb;
static inline struct swap_cluster_info *swap_cluster_lock(
struct swap_info_struct *si, pgoff_t offset, bool irq)
{
@@ -399,11 +394,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page)
{
}
-static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
+static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
{
}
-static inline void swap_write_unplug(struct swap_iocb *sio)
+static inline void swap_write_submit(struct swap_io_ctx *ctx)
{
}
@@ -443,8 +438,7 @@ static inline void swap_update_readahead(struct folio *folio,
{
}
-static inline int swap_writeout(struct folio *folio,
- struct swap_iocb **swap_plug)
+static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
{
return 0;
}
@@ -500,7 +494,7 @@ static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
}
#endif /* CONFIG_SWAP */
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
+int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
struct list_head *folio_list);
#endif /* _MM_SWAP_H */
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 1415a5c54a43..abc26414368d 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -573,14 +573,17 @@ struct folio *swap_cache_alloc_folio(swp_entry_t entry, gfp_t gfp_mask,
*/
struct folio *swapin_folio(swp_entry_t entry, struct folio *folio)
{
+ struct swap_io_ctx ctx = {};
struct folio *swapcache;
pgoff_t offset = swp_offset(entry);
unsigned long nr_pages = folio_nr_pages(folio);
entry = swp_entry(swp_type(entry), round_down(offset, nr_pages));
swapcache = __swap_cache_prepare_and_add(entry, folio, 0, true);
- if (swapcache == folio)
- swap_read_folio(folio, NULL);
+ if (swapcache == folio) {
+ swap_read_folio(&ctx, folio);
+ swap_read_submit(&ctx);
+ }
return swapcache;
}
@@ -590,9 +593,8 @@ struct folio *swapin_folio(swp_entry_t entry, struct folio *folio)
* A failure return means that either the page allocation failed or that
* the swap entry is no longer in use.
*/
-struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
- struct vm_area_struct *vma, unsigned long addr,
- struct swap_iocb **plug)
+struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
+ gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr)
{
struct swap_info_struct *si;
bool page_allocated;
@@ -610,7 +612,7 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
mpol_cond_put(mpol);
if (page_allocated)
- swap_read_folio(folio, plug);
+ swap_read_folio(ctx, folio);
put_swap_device(si);
return folio;
@@ -704,8 +706,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
unsigned long start_offset, end_offset;
unsigned long mask;
struct swap_info_struct *si = __swap_entry_to_info(entry);
+ struct swap_io_ctx ctx = {};
struct blk_plug plug;
- struct swap_iocb *splug = NULL;
bool page_allocated;
mask = swapin_nr_pages(offset) - 1;
@@ -729,7 +731,7 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
if (!folio)
continue;
if (page_allocated) {
- swap_read_folio(folio, &splug);
+ swap_read_folio(&ctx, folio);
if (offset != entry_offset) {
folio_set_readahead(folio);
count_vm_event(SWAP_RA);
@@ -738,14 +740,15 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
folio_put(folio);
}
blk_finish_plug(&plug);
- swap_read_unplug(splug);
+ swap_read_submit(&ctx);
lru_add_drain(); /* Push any new pages onto the LRU now */
skip:
- /* The page was likely read above, so no need for plugging here */
folio = swap_cache_alloc_folio(entry, gfp_mask, mpol, ilx,
&page_allocated);
- if (unlikely(page_allocated))
- swap_read_folio(folio, NULL);
+ if (unlikely(page_allocated)) {
+ swap_read_folio(&ctx, folio);
+ swap_read_submit(&ctx);
+ }
return folio;
}
@@ -806,8 +809,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf)
{
+ struct swap_io_ctx ctx = {};
struct blk_plug plug;
- struct swap_iocb *splug = NULL;
struct folio *folio;
pte_t *pte = NULL, pentry;
int win;
@@ -854,7 +857,7 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
if (!folio)
continue;
if (page_allocated) {
- swap_read_folio(folio, &splug);
+ swap_read_folio(&ctx, folio);
if (addr != vmf->address) {
folio_set_readahead(folio);
count_vm_event(SWAP_RA);
@@ -865,14 +868,15 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
if (pte)
pte_unmap(pte);
blk_finish_plug(&plug);
- swap_read_unplug(splug);
+ swap_read_submit(&ctx);
lru_add_drain();
skip:
- /* The folio was likely read above, so no need for plugging here */
folio = swap_cache_alloc_folio(targ_entry, gfp_mask, mpol, targ_ilx,
&page_allocated);
- if (unlikely(page_allocated))
- swap_read_folio(folio, NULL);
+ if (unlikely(page_allocated)) {
+ swap_read_folio(&ctx, folio);
+ swap_read_submit(&ctx);
+ }
return folio;
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index dc0d4312ac6c..56cd59e27447 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -617,8 +617,8 @@ typedef enum {
/*
* pageout is called by shrink_folio_list() for each dirty folio.
*/
-static pageout_t pageout(struct folio *folio, struct address_space *mapping,
- struct swap_iocb **plug, struct list_head *folio_list)
+static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping,
+ struct folio *folio, struct list_head *folio_list)
{
int res;
@@ -654,9 +654,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,
* the split out folios get added back to folio_list.
*/
if (shmem_mapping(mapping))
- res = shmem_writeout(folio, plug, folio_list);
+ res = shmem_writeout(ctx, folio, folio_list);
else
- res = swap_writeout(folio, plug);
+ res = swap_writeout(ctx, folio);
if (res < 0)
handle_write_error(mapping, folio, res);
@@ -1061,7 +1061,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
unsigned int nr_reclaimed = 0, nr_demoted = 0;
unsigned int pgactivate = 0;
bool do_demote_pass;
- struct swap_iocb *plug = NULL;
+ struct swap_io_ctx ctx = {};
folio_batch_init(&free_folios);
memset(stat, 0, sizeof(*stat));
@@ -1392,7 +1392,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
* starts and then write it out here.
*/
try_to_unmap_flush_dirty();
- switch (pageout(folio, mapping, &plug, folio_list)) {
+ switch (pageout(&ctx, mapping, folio, folio_list)) {
case PAGE_KEEP:
goto keep_locked;
case PAGE_ACTIVATE:
@@ -1582,8 +1582,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
list_splice(&ret_folios, folio_list);
count_vm_events(PGACTIVATE, pgactivate);
- if (plug)
- swap_write_unplug(plug);
+ swap_write_submit(&ctx);
return nr_reclaimed;
}
--
2.53.0
^ permalink raw reply related
* [PATCH 4/6] mm/swap: also use struct swap_iocb for block I/O
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
In-Reply-To: <20260515120019.4015143-1-hch@lst.de>
Block I/O benefits from batching just as much as remote file systems.
Extent struct swap_iocb to support building a bio on the fly as well,
and rewrite the block based swap code for it. This especially benefits
submit_bio based drivers that do not have the block plugging available,
but also saves allocating extra bios for blk-mq drivers.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
mm/page_io.c | 506 +++++++++++++++++++++++---------------------------
mm/swap.h | 1 +
mm/swapfile.c | 9 +-
3 files changed, 235 insertions(+), 281 deletions(-)
diff --git a/mm/page_io.c b/mm/page_io.c
index a78efc9909c8..bbd8cf47d20d 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -27,54 +27,6 @@
#include <linux/zswap.h>
#include "swap.h"
-static void __end_swap_bio_write(struct bio *bio)
-{
- struct folio *folio = bio_first_folio_all(bio);
-
- if (bio->bi_status) {
- /*
- * We failed to write the page out to swap-space.
- * Re-dirty the page in order to avoid it being reclaimed.
- * Also print a dire warning that things will go BAD (tm)
- * very quickly.
- *
- * Also clear PG_reclaim to avoid folio_rotate_reclaimable()
- */
- folio_mark_dirty(folio);
- pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
- MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
- (unsigned long long)bio->bi_iter.bi_sector);
- folio_clear_reclaim(folio);
- }
- folio_end_writeback(folio);
-}
-
-static void end_swap_bio_write(struct bio *bio)
-{
- __end_swap_bio_write(bio);
- bio_put(bio);
-}
-
-static void __end_swap_bio_read(struct bio *bio)
-{
- struct folio *folio = bio_first_folio_all(bio);
-
- if (bio->bi_status) {
- pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
- MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
- (unsigned long long)bio->bi_iter.bi_sector);
- } else {
- folio_mark_uptodate(folio);
- }
- folio_unlock(folio);
-}
-
-static void end_swap_bio_read(struct bio *bio)
-{
- __end_swap_bio_read(bio);
- bio_put(bio);
-}
-
int generic_swapfile_activate(struct swap_info_struct *sis,
struct file *swap_file,
sector_t *span)
@@ -325,9 +277,12 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
struct swap_iocb {
- struct kiocb iocb;
+ union {
+ struct kiocb iocb;
+ struct bio bio;
+ };
struct bio_vec bvec[SWAP_CLUSTER_MAX];
- int pages;
+ int nr_vecs;
int len;
};
static mempool_t *sio_pool;
@@ -345,172 +300,68 @@ int sio_pool_init(void)
return 0;
}
-static void sio_write_complete(struct kiocb *iocb, long ret)
+static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio)
{
- struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
- struct page *page = sio->bvec[0].bv_page;
- int p;
+ struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
+ struct bio_vec *last_bv = &ctx->sio->bvec[ctx->sio->nr_vecs - 1];
+ struct folio *prev_folio = page_folio(last_bv->bv_page);
+ size_t prev_folio_size = folio_size(prev_folio);
- if (ret != sio->len) {
- /*
- * In the case of swap-over-nfs, this can be a
- * temporary failure if the system has limited
- * memory for allocating transmit buffers.
- * Mark the page dirty and avoid
- * folio_rotate_reclaimable but rate-limit the
- * messages.
- */
- pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
- ret, swap_dev_pos(page_swap_entry(page)));
- for (p = 0; p < sio->pages; p++) {
- page = sio->bvec[p].bv_page;
- set_page_dirty(page);
- ClearPageReclaim(page);
- }
- }
+ if (ctx->sis != sis)
+ return false;
- for (p = 0; p < sio->pages; p++)
- end_page_writeback(sio->bvec[p].bv_page);
+ if (sis->flags & SWP_FS_OPS) {
+ if (swap_dev_pos(folio->swap) !=
+ swap_dev_pos(prev_folio->swap) + prev_folio_size)
+ return false;
+ } else {
+ if (swap_folio_sector(folio) !=
+ swap_folio_sector(prev_folio) +
+ (prev_folio_size >> SECTOR_SHIFT))
+ return false;
+ }
- mempool_free(sio, sio_pool);
+ return true;
}
-static void swap_writepage_fs(struct swap_io_ctx *ctx, struct folio *folio)
+static void swap_add_page(struct swap_io_ctx *ctx, struct folio *folio, int rw)
{
- struct swap_iocb *sio = ctx->sio;
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
- struct file *swap_file = sis->swap_file;
- loff_t pos = swap_dev_pos(folio->swap);
+ struct swap_iocb *sio = ctx->sio;
- count_swpout_vm_event(folio);
- folio_start_writeback(folio);
- folio_unlock(folio);
- if (sio) {
- if (sio->iocb.ki_filp != swap_file ||
- sio->iocb.ki_pos + sio->len != pos) {
+ if (sio && !swap_can_merge(ctx, folio)) {
+ if (rw == WRITE)
swap_write_submit(ctx);
- sio = NULL;
- }
+ else
+ swap_read_submit(ctx);
+ sio = ctx->sio;
}
+
if (!sio) {
- sio = mempool_alloc(sio_pool, GFP_NOIO);
- init_sync_kiocb(&sio->iocb, swap_file);
- sio->iocb.ki_complete = sio_write_complete;
- sio->iocb.ki_pos = pos;
- sio->pages = 0;
+ ctx->sis = sis;
+ ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO);
+ sio->nr_vecs = 0;
sio->len = 0;
}
- bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);
+ bvec_set_folio(&sio->bvec[sio->nr_vecs], folio, folio_size(folio), 0);
sio->len += folio_size(folio);
- sio->pages += 1;
- if (sio->pages == ARRAY_SIZE(sio->bvec)) {
- swap_write_submit(ctx);
- sio = NULL;
+ sio->nr_vecs += 1;
+ if (sio->nr_vecs == ARRAY_SIZE(sio->bvec)) {
+ if (rw == WRITE)
+ swap_write_submit(ctx);
+ else
+ swap_read_submit(ctx);
}
- ctx->sio = sio;
}
-static void swap_writepage_bdev_sync(struct folio *folio,
- struct swap_info_struct *sis)
-{
- struct bio_vec bv;
- struct bio bio;
-
- bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_WRITE | REQ_SWAP);
- bio.bi_iter.bi_sector = swap_folio_sector(folio);
- bio_add_folio_nofail(&bio, folio, folio_size(folio), 0);
-
- bio_associate_blkg_from_page(&bio, folio);
- count_swpout_vm_event(folio);
-
- folio_start_writeback(folio);
- folio_unlock(folio);
-
- submit_bio_wait(&bio);
- __end_swap_bio_write(&bio);
-}
-
-static void swap_writepage_bdev_async(struct folio *folio,
- struct swap_info_struct *sis)
+void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
{
- struct bio *bio;
-
- bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE | REQ_SWAP, GFP_NOIO);
- bio->bi_iter.bi_sector = swap_folio_sector(folio);
- bio->bi_end_io = end_swap_bio_write;
- bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
+ VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
- bio_associate_blkg_from_page(bio, folio);
count_swpout_vm_event(folio);
folio_start_writeback(folio);
folio_unlock(folio);
- submit_bio(bio);
-}
-
-void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
-{
- struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
-
- VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
- /*
- * ->flags can be updated non-atomically,
- * but that will never affect SWP_FS_OPS, so the data_race
- * is safe.
- */
- if (data_race(sis->flags & SWP_FS_OPS))
- swap_writepage_fs(ctx, folio);
- /*
- * ->flags can be updated non-atomically,
- * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race
- * is safe.
- */
- else if (data_race(sis->flags & SWP_SYNCHRONOUS_IO))
- swap_writepage_bdev_sync(folio, sis);
- else
- swap_writepage_bdev_async(folio, sis);
-}
-
-void swap_write_submit(struct swap_io_ctx *ctx)
-{
- struct iov_iter from;
- struct swap_iocb *sio = ctx->sio;
- struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
- int ret;
-
- if (!ctx)
- return;
-
- iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);
- ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
- if (ret != -EIOCBQUEUED)
- sio_write_complete(&sio->iocb, ret);
- ctx->sio = NULL;
-}
-
-static void sio_read_complete(struct kiocb *iocb, long ret)
-{
- struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
- int p;
-
- if (ret == sio->len) {
- for (p = 0; p < sio->pages; p++) {
- struct folio *folio = page_folio(sio->bvec[p].bv_page);
-
- count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
- count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
- folio_mark_uptodate(folio);
- folio_unlock(folio);
- }
- count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
- } else {
- for (p = 0; p < sio->pages; p++) {
- struct folio *folio = page_folio(sio->bvec[p].bv_page);
-
- folio_unlock(folio);
- }
- pr_alert_ratelimited("Read-error on swap-device\n");
- }
- mempool_free(sio, sio_pool);
+ swap_add_page(ctx, folio, WRITE);
}
static bool swap_read_folio_zeromap(struct folio *folio)
@@ -543,74 +394,6 @@ static bool swap_read_folio_zeromap(struct folio *folio)
return true;
}
-static void swap_read_folio_fs(struct swap_io_ctx *ctx, struct folio *folio)
-{
- struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
- struct swap_iocb *sio = ctx->sio;
- loff_t pos = swap_dev_pos(folio->swap);
-
- if (sio) {
- if (sio->iocb.ki_filp != sis->swap_file ||
- sio->iocb.ki_pos + sio->len != pos) {
- swap_read_submit(ctx);
- sio = NULL;
- }
- }
- if (!sio) {
- sio = mempool_alloc(sio_pool, GFP_KERNEL);
- init_sync_kiocb(&sio->iocb, sis->swap_file);
- sio->iocb.ki_pos = pos;
- sio->iocb.ki_complete = sio_read_complete;
- sio->pages = 0;
- sio->len = 0;
- }
- bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);
- sio->len += folio_size(folio);
- sio->pages += 1;
- if (sio->pages == ARRAY_SIZE(sio->bvec)) {
- swap_read_submit(ctx);
- sio = NULL;
- }
- ctx->sio = sio;
-}
-
-static void swap_read_folio_bdev_sync(struct folio *folio,
- struct swap_info_struct *sis)
-{
- struct bio_vec bv;
- struct bio bio;
-
- bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ);
- bio.bi_iter.bi_sector = swap_folio_sector(folio);
- bio_add_folio_nofail(&bio, folio, folio_size(folio), 0);
- /*
- * Keep this task valid during swap readpage because the oom killer may
- * attempt to access it in the page fault retry time check.
- */
- get_task_struct(current);
- count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
- count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
- count_vm_events(PSWPIN, folio_nr_pages(folio));
- submit_bio_wait(&bio);
- __end_swap_bio_read(&bio);
- put_task_struct(current);
-}
-
-static void swap_read_folio_bdev_async(struct folio *folio,
- struct swap_info_struct *sis)
-{
- struct bio *bio;
-
- bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL);
- bio->bi_iter.bi_sector = swap_folio_sector(folio);
- bio->bi_end_io = end_swap_bio_read;
- bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
- count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
- count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
- count_vm_events(PSWPIN, folio_nr_pages(folio));
- submit_bio(bio);
-}
-
void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
{
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
@@ -644,14 +427,7 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
/* We have to read from slower devices. Increase zswap protection. */
zswap_folio_swapin(folio);
-
- if (data_race(sis->flags & SWP_FS_OPS)) {
- swap_read_folio_fs(ctx, folio);
- } else if (synchronous) {
- swap_read_folio_bdev_sync(folio, sis);
- } else {
- swap_read_folio_bdev_async(folio, sis);
- }
+ swap_add_page(ctx, folio, READ);
finish:
if (workingset) {
@@ -661,19 +437,197 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
delayacct_swapin_end();
}
-void swap_read_submit(struct swap_io_ctx *ctx)
+static void sio_write_end(struct swap_iocb *sio, bool failed)
+{
+ int p;
+
+ for (p = 0; p < sio->nr_vecs; p++) {
+ struct page *page = sio->bvec[p].bv_page;
+
+ if (failed) {
+ set_page_dirty(page);
+ ClearPageReclaim(page);
+ }
+ end_page_writeback(page);
+ }
+ mempool_free(sio, sio_pool);
+}
+
+static void sio_write_complete(struct kiocb *iocb, long ret)
+{
+ struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
+ bool failed = ret != sio->len;
+
+ if (failed) {
+ struct page *page = sio->bvec[0].bv_page;
+
+ /*
+ * In the case of swap-over-nfs, this can be a temporary failure
+ * if the system has limited memory for allocating transmit
+ * buffers. Mark the page dirty and avoid
+ * folio_rotate_reclaimable but rate-limit the messages.
+ */
+ pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
+ ret, swap_dev_pos(page_swap_entry(page)));
+ }
+
+ sio_write_end(sio, failed);
+}
+
+static void end_swap_bio_write(struct bio *bio)
+{
+ struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
+ bool failed = !!bio->bi_status;
+
+ if (failed)
+ pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
+ MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
+ (unsigned long long)bio->bi_iter.bi_sector);
+ sio_write_end(sio, failed);
+}
+
+static void sio_read_end(struct swap_iocb *sio)
+{
+ int p;
+
+ for (p = 0; p < sio->nr_vecs; p++) {
+ struct folio *folio = page_folio(sio->bvec[p].bv_page);
+
+ count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
+ count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
+ folio_mark_uptodate(folio);
+ folio_unlock(folio);
+ }
+ count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
+ mempool_free(sio, sio_pool);
+}
+
+static void sio_read_fail(struct swap_iocb *sio)
+{
+ int p;
+
+ for (p = 0; p < sio->nr_vecs; p++)
+ folio_unlock(page_folio(sio->bvec[p].bv_page));
+ mempool_free(sio, sio_pool);
+}
+
+static void sio_read_complete(struct kiocb *iocb, long ret)
+{
+ struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
+
+ if (ret != sio->len) {
+ pr_alert_ratelimited("Read-error on swap-device\n");
+ sio_read_fail(sio);
+ return;
+ }
+
+ sio_read_end(sio);
+}
+
+static void end_swap_bio_read(struct bio *bio)
+{
+ struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
+
+ if (bio->bi_status) {
+ pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
+ MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
+ (unsigned long long)bio->bi_iter.bi_sector);
+ sio_read_fail(sio);
+ return;
+ }
+
+ sio_read_end(sio);
+}
+
+static void swap_bdev_submit_write(struct swap_io_ctx *ctx)
+{
+ struct swap_iocb *sio = ctx->sio;
+ struct bio *bio = &sio->bio;
+
+ bio_init(bio, ctx->sis->bdev, sio->bvec, ARRAY_SIZE(sio->bvec),
+ REQ_OP_WRITE | REQ_SWAP);
+ bio->bi_iter.bi_size = sio->len;
+ bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
+ bio_associate_blkg_from_page(bio, bio_first_folio_all(bio));
+
+ if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
+ submit_bio_wait(bio);
+ end_swap_bio_write(bio);
+ } else {
+ bio->bi_end_io = end_swap_bio_write;
+ submit_bio(bio);
+ }
+}
+
+static void swap_bdev_submit_read(struct swap_io_ctx *ctx)
+{
+ struct swap_iocb *sio = ctx->sio;
+ struct bio *bio = &sio->bio;
+
+ bio_init(bio, ctx->sis->bdev, sio->bvec, ARRAY_SIZE(sio->bvec),
+ REQ_OP_READ);
+ bio->bi_iter.bi_size = sio->len;
+ bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
+
+ if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
+ /*
+ * Keep this task valid during swap readpage because the oom
+ * killer may attempt to access it in the page fault retry
+ * time check.
+ */
+ get_task_struct(current);
+ submit_bio_wait(bio);
+ end_swap_bio_read(bio);
+ put_task_struct(current);
+ } else {
+ bio->bi_end_io = end_swap_bio_read;
+ submit_bio(bio);
+ }
+}
+
+static void swap_fs_submit(struct swap_io_ctx *ctx, int rw)
{
- struct iov_iter from;
struct swap_iocb *sio = ctx->sio;
struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
+ struct iov_iter iter;
int ret;
- if (!sio)
- return;
+ init_sync_kiocb(&sio->iocb, ctx->sis->swap_file);
+ sio->iocb.ki_pos = swap_dev_pos(page_folio(sio->bvec[0].bv_page)->swap);
+ if (rw == WRITE)
+ sio->iocb.ki_complete = sio_write_complete;
+ else
+ sio->iocb.ki_complete = sio_read_complete;
- iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);
- ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
+ iov_iter_bvec(&iter, rw == WRITE ? ITER_SOURCE : ITER_DEST,
+ sio->bvec, sio->nr_vecs, sio->len);
+ ret = mapping->a_ops->swap_rw(&sio->iocb, &iter);
if (ret != -EIOCBQUEUED)
- sio_read_complete(&sio->iocb, ret);
+ sio->iocb.ki_complete(&sio->iocb, ret);
+}
+
+void swap_write_submit(struct swap_io_ctx *ctx)
+{
+ if (!ctx->sio)
+ return;
+
+ if (ctx->sis->flags & SWP_FS_OPS)
+ swap_fs_submit(ctx, WRITE);
+ else
+ swap_bdev_submit_write(ctx);
+ ctx->sio = NULL;
+ ctx->sis = NULL;
+}
+
+void swap_read_submit(struct swap_io_ctx *ctx)
+{
+ if (!ctx->sio)
+ return;
+
+ if (ctx->sis->flags & SWP_FS_OPS)
+ swap_fs_submit(ctx, READ);
+ else
+ swap_bdev_submit_read(ctx);
ctx->sio = NULL;
+ ctx->sis = NULL;
}
diff --git a/mm/swap.h b/mm/swap.h
index 3ec35b6d629f..b359735be3c5 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -55,6 +55,7 @@ enum swap_cluster_flags {
struct swap_io_ctx {
struct swap_iocb *sio;
+ struct swap_info_struct *sis;
};
#ifdef CONFIG_SWAP
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 9174f1eeffb0..27dbce0d1e1e 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -2781,6 +2781,10 @@ static int setup_swap_extents(struct swap_info_struct *sis,
struct inode *inode = mapping->host;
int ret;
+ ret = sio_pool_init();
+ if (ret)
+ return ret;
+
if (S_ISBLK(inode->i_mode)) {
ret = add_swap_extent(sis, 0, sis->max, 0);
*span = sis->pages;
@@ -2792,11 +2796,6 @@ static int setup_swap_extents(struct swap_info_struct *sis,
if (ret < 0)
return ret;
sis->flags |= SWP_ACTIVATED;
- if ((sis->flags & SWP_FS_OPS) &&
- sio_pool_init() != 0) {
- destroy_swap_extents(sis, swap_file);
- return -ENOMEM;
- }
return ret;
}
--
2.53.0
^ permalink raw reply related
* [PATCH 2/6] mm: merge writeout into pageout
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
In-Reply-To: <20260515120019.4015143-1-hch@lst.de>
writeout is only called from pageout, and a straight flow at the end, so
merge the two functions.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
mm/vmscan.c | 63 ++++++++++++++++++++++++-----------------------------
1 file changed, 29 insertions(+), 34 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bd1b1aa12581..dc0d4312ac6c 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -614,45 +614,14 @@ typedef enum {
PAGE_CLEAN,
} pageout_t;
-static pageout_t writeout(struct folio *folio, struct address_space *mapping,
- struct swap_iocb **plug, struct list_head *folio_list)
-{
- int res;
-
- folio_set_reclaim(folio);
-
- /*
- * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled
- * or we failed to allocate contiguous swap entries, in which case
- * the split out folios get added back to folio_list.
- */
- if (shmem_mapping(mapping))
- res = shmem_writeout(folio, plug, folio_list);
- else
- res = swap_writeout(folio, plug);
-
- if (res < 0)
- handle_write_error(mapping, folio, res);
- if (res == AOP_WRITEPAGE_ACTIVATE) {
- folio_clear_reclaim(folio);
- return PAGE_ACTIVATE;
- }
-
- /* synchronous write? */
- if (!folio_test_writeback(folio))
- folio_clear_reclaim(folio);
-
- trace_mm_vmscan_write_folio(folio);
- node_stat_add_folio(folio, NR_VMSCAN_WRITE);
- return PAGE_SUCCESS;
-}
-
/*
* pageout is called by shrink_folio_list() for each dirty folio.
*/
static pageout_t pageout(struct folio *folio, struct address_space *mapping,
struct swap_iocb **plug, struct list_head *folio_list)
{
+ int res;
+
/*
* We no longer attempt to writeback filesystem folios here, other
* than tmpfs/shmem. That's taken care of in page-writeback.
@@ -676,7 +645,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,
return PAGE_ACTIVATE;
if (!folio_clear_dirty_for_io(folio))
return PAGE_CLEAN;
- return writeout(folio, mapping, plug, folio_list);
+
+ folio_set_reclaim(folio);
+
+ /*
+ * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled
+ * or we failed to allocate contiguous swap entries, in which case
+ * the split out folios get added back to folio_list.
+ */
+ if (shmem_mapping(mapping))
+ res = shmem_writeout(folio, plug, folio_list);
+ else
+ res = swap_writeout(folio, plug);
+
+ if (res < 0)
+ handle_write_error(mapping, folio, res);
+ if (res == AOP_WRITEPAGE_ACTIVATE) {
+ folio_clear_reclaim(folio);
+ return PAGE_ACTIVATE;
+ }
+
+ /* synchronous write? */
+ if (!folio_test_writeback(folio))
+ folio_clear_reclaim(folio);
+
+ trace_mm_vmscan_write_folio(folio);
+ node_stat_add_folio(folio, NR_VMSCAN_WRITE);
+ return PAGE_SUCCESS;
}
/*
--
2.53.0
^ permalink raw reply related
* [PATCH 1/6] shmem: provide a shmem_write_folio wrapper
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
In-Reply-To: <20260515120019.4015143-1-hch@lst.de>
Provide a wrapper for the shmem abuses in drm to preparare for swap I/O
refactoring by keepin swap_iocb handling entirely contained in mm/.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +-
drivers/gpu/drm/ttm/ttm_backup.c | 2 +-
include/linux/shmem_fs.h | 5 +----
mm/shmem.c | 7 ++++++-
mm/swap.h | 4 ++++
5 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 06543ae60706..ef9440166295 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping)
if (folio_mapped(folio))
folio_redirty_for_writepage(&wbc, folio);
else
- error = shmem_writeout(folio, NULL, NULL);
+ error = shmem_write_folio(folio);
}
}
diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c
index 81df4cb5606b..c5b813a563e7 100644
--- a/drivers/gpu/drm/ttm/ttm_backup.c
+++ b/drivers/gpu/drm/ttm/ttm_backup.c
@@ -117,7 +117,7 @@ ttm_backup_backup_page(struct file *backup, struct page *page,
if (writeback && !folio_mapped(to_folio) &&
folio_clear_dirty_for_io(to_folio)) {
folio_set_reclaim(to_folio);
- ret = shmem_writeout(to_folio, NULL, NULL);
+ ret = shmem_write_folio(to_folio);
if (!folio_test_writeback(to_folio))
folio_clear_reclaim(to_folio);
/*
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 93a0ba872ebe..ab404effa879 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -12,8 +12,6 @@
#include <linux/userfaultfd_k.h>
#include <linux/bits.h>
-struct swap_iocb;
-
/* inode in-kernel data */
#ifdef CONFIG_TMPFS_QUOTA
@@ -122,8 +120,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)
void shmem_unlock_mapping(struct address_space *mapping);
struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask);
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
- struct list_head *folio_list);
+int shmem_write_folio(struct folio *folio);
void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
int shmem_unuse(unsigned int type);
diff --git a/mm/shmem.c b/mm/shmem.c
index 3b5dc21b323c..b8becbd4beaf 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1738,7 +1738,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
folio_mark_dirty(folio);
return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */
}
-EXPORT_SYMBOL_GPL(shmem_writeout);
+
+int shmem_write_folio(struct folio *folio)
+{
+ return shmem_writeout(folio, NULL, NULL);
+}
+EXPORT_SYMBOL_GPL(shmem_write_folio);
#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
diff --git a/mm/swap.h b/mm/swap.h
index a77016f2423b..b6db72fb9879 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -499,4 +499,8 @@ static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
return 0;
}
#endif /* CONFIG_SWAP */
+
+int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
+ struct list_head *folio_list);
+
#endif /* _MM_SWAP_H */
--
2.53.0
^ permalink raw reply related
* RFC: better block swap batching and a different take on swap_ops
From: Christoph Hellwig @ 2026-05-15 12:00 UTC (permalink / raw)
Cc: baoquan.he, akpm, chrisl, usama.arif, kasong, nphamcs, shikemeng,
youngjun.park, linux-mm
Hi all,
this series makes use of the swap_iocb for block as well so that it
doesn't do inefficient single-bio I/O, and then rebases the swap_ops
from Baoquan on top of the now very different method structure.
This is very hot off the press and has only survived very basic testing.
Diffstat:
Documentation/filesystems/locking.rst | 5
Documentation/filesystems/vfs.rst | 4
drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2
drivers/gpu/drm/ttm/ttm_backup.c | 2
fs/nfs/file.c | 4
fs/smb/client/file.c | 4
include/linux/shmem_fs.h | 5
include/linux/swap.h | 7
mm/madvise.c | 16
mm/page_io.c | 544 ++++++++++++++----------------
mm/shmem.c | 18
mm/swap.h | 52 +-
mm/swap_state.c | 40 +-
mm/swapfile.c | 11
mm/vmscan.c | 88 ++--
15 files changed, 402 insertions(+), 400 deletions(-)
^ permalink raw reply
* Re: [PATCH 6.18.y v3] mm: fix VM_SOFTDIRTY propagation on VMA merge
From: Lorenzo Stoakes @ 2026-05-15 11:52 UTC (permalink / raw)
To: Greg KH; +Cc: Ahmed Elaidy, Andrei Vagin, stable, akpm, linux-mm
In-Reply-To: <2026051531-failing-nectar-83bf@gregkh>
On Fri, May 15, 2026 at 11:22:02AM +0200, Greg KH wrote:
> On Mon, May 04, 2026 at 10:54:47PM +0300, Ahmed Elaidy wrote:
> > During VMA merging, such as through mprotect(), VM_SOFTDIRTY flags could be
> > lost. This breaks tools relying on soft-dirty tracking, such as CRIU
> > incremental dump/restore.
> >
> > Upstream resolved this using a broader VM_STICKY infrastructure (commit
> > bf14d4a05387 "mm: propagate VM_SOFTDIRTY on merge"). To minimize churn and
> > risk in the stable 6.18.y tree, this patch skips backporting the entire
> > VM_STICKY series (9 patches). Instead, it introduces a minimal standalone fix.
>
> 9 patches is nothing. Please just backport the whole thing, especially
> as we will be maintaining this kernel for a long time. We want what is
> upstream for future issues/fixes, right?
FWIW, agreed!
>
> thanks,
>
> greg k-h
Cheers, Lorenzo
^ permalink raw reply
* Re: [PATCH 6.18.y v1 0/9] mm: backport sticky VMA flags and soft-dirty fix
From: Lorenzo Stoakes @ 2026-05-15 11:44 UTC (permalink / raw)
To: Ahmed Elaidy; +Cc: stable, linux-mm, akpm, avagin
In-Reply-To: <20260424211315.1072123-1-elaidya225@gmail.com>
Hi,
Just a heads up that I generally don't read kernel mail sent to my work address,
as I changed my email setup significantly and use ljs@kernel.org for everything
upstream!
Understandable given the original patches obviously used it but just FYI :)
Cheers, Lorenzo
On Sat, Apr 25, 2026 at 12:12:34AM +0300, Ahmed Elaidy wrote:
> This series backports the sticky VMA flags infrastructure and the
> VM_SOFTDIRTY-on-merge fix to linux-6.18.y.
>
> Motivation: CRIU incremental dump/restore can hit a missing-parent-pagemap
> failure when VM_SOFTDIRTY is lost during VMA merge operations.
>
> Patch 8 is the target fix:
> mm: propagate VM_SOFTDIRTY on merge
>
> The preceding patches provide required dependencies on 6.18.y and are included
> to preserve upstream behavior.
>
> Backport notes:
> - Non-trivial context conflicts were resolved in:
> - mm/mseal.c
> - mm/vma.c
> - Conflict resolution keeps upstream semantics; no intentional behavior
> changes beyond context adaptation for 6.18.y.
>
> Cc: stable@vger.kernel.org
>
>
>
> Lorenzo Stoakes (9):
> mm: introduce VM_MAYBE_GUARD and make visible in /proc/$pid/smaps
> mm: add atomic VMA flags and set VM_MAYBE_GUARD as such
> mm: update vma_modify_flags() to handle residual flags, document
> mm: implement sticky VMA flags
> mm: introduce copy-on-fork VMAs and make VM_MAYBE_GUARD one
> mm: set the VM_MAYBE_GUARD flag on guard region install
> tools/testing/vma: add VMA sticky userland tests
> mm: propagate VM_SOFTDIRTY on merge
> testing/selftests/mm: add soft-dirty merge self-test
>
> Documentation/filesystems/proc.rst | 5 +-
> fs/proc/task_mmu.c | 1 +
> include/linux/mm.h | 100 +++++++++++++++++
> include/trace/events/mmflags.h | 1 +
> mm/khugepaged.c | 71 +++++++-----
> mm/madvise.c | 24 +++--
> mm/memory.c | 14 +--
> mm/mlock.c | 2 +-
> mm/mprotect.c | 2 +-
> mm/mseal.c | 7 +-
> mm/vma.c | 81 +++++++-------
> mm/vma.h | 138 +++++++++++++++++-------
> tools/testing/selftests/mm/soft-dirty.c | 127 +++++++++++++++++++++-
> tools/testing/vma/vma.c | 92 ++++++++++++++--
> tools/testing/vma/vma_internal.h | 49 +++++++++
> 15 files changed, 579 insertions(+), 135 deletions(-)
>
> --
> 2.53.0
^ permalink raw reply
* Re: [PATCH v2 2/2] mm: huge_memory: refactor thpsize_shmem_enabled_show() with helper arrays
From: Lorenzo Stoakes @ 2026-05-15 11:36 UTC (permalink / raw)
To: ranxiaokai627
Cc: akpm, baolin.wang, hughd, leitao, linux-kernel, linux-mm,
ran.xiaokai
In-Reply-To: <20260515060441.53094-1-ranxiaokai627@163.com>
On Fri, May 15, 2026 at 06:04:41AM +0000, ranxiaokai627@163.com wrote:
> >(As I said in the 1/2)
> >
> >Please don't send 2/2 in response to 1/2, and use a cover letter if you send
> >more than 1 patch!
>
> Thanks for the guidance.
> I will do that in the next verison.
>
> >On Wed, May 13, 2026 at 09:45:08AM +0000, ranxiaokai627@163.com wrote:
> >> From: Ran Xiaokai <ran.xiaokai@zte.com.cn>
> >>
> >> Replace the hardcoded if/else chain of test_bit() calls and string
> >> literals in thpsize_shmem_enabled_show() with a loop over
> >> huge_shmem_orders_by_mode[] and huge_shmem_enabled_mode_strings[] arrays.
> >>
> >> This makes thpsize_shmem_enabled_show() consistent with
> >> thpsize_shmem_enabled_store() and eliminates duplicated mode name strings.
> >>
> >> Signed-off-by: Ran Xiaokai <ran.xiaokai@zte.com.cn>
> >
> >The logic looks good, I wish we could de-duplicate. But for now maybe better to
> >get this refactored first.
> >
> >So:
> >
> >Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
> >
> >> ---
> >> mm/shmem.c | 36 +++++++++++++++++++++++-------------
> >> 1 file changed, 23 insertions(+), 13 deletions(-)
> >>
> >> diff --git a/mm/shmem.c b/mm/shmem.c
> >> index 60cb10854f11..086762e6de71 100644
> >> --- a/mm/shmem.c
> >> +++ b/mm/shmem.c
> >> @@ -5553,20 +5553,30 @@ static ssize_t thpsize_shmem_enabled_show(struct kobject *kobj,
> >> struct kobj_attribute *attr, char *buf)
> >> {
> >> int order = to_thpsize(kobj)->order;
> >> - const char *output;
> >> -
> >> - if (test_bit(order, &huge_shmem_orders_always))
> >> - output = "[always] inherit within_size advise never";
> >> - else if (test_bit(order, &huge_shmem_orders_inherit))
> >> - output = "always [inherit] within_size advise never";
> >> - else if (test_bit(order, &huge_shmem_orders_within_size))
> >> - output = "always inherit [within_size] advise never";
> >> - else if (test_bit(order, &huge_shmem_orders_madvise))
> >> - output = "always inherit within_size [advise] never";
> >> - else
> >> - output = "always inherit within_size advise [never]";
> >> + int active = HUGE_SHMEM_ENABLED_NEVER;
> >> + int len = 0;
> >> + int i;
> >> +
> >> + for (i = 0; i < ARRAY_SIZE(huge_shmem_orders_by_mode); i++) {
> >> + if (test_bit(order, huge_shmem_orders_by_mode[i])) {
> >> + active = i;
> >> + break;
> >> + }
> >> + }
> >> +
> >> + for (i = 0; i < ARRAY_SIZE(huge_shmem_enabled_mode_strings); i++) {
> >> + if (i == active)
> >> + len += sysfs_emit_at(buf, len, "[%s] ",
> >> + huge_shmem_enabled_mode_strings[i]);
> >> + else
> >> + len += sysfs_emit_at(buf, len, "%s ",
> >> + huge_shmem_enabled_mode_strings[i]);
> >> + }
> >> +
> >> + /* Replace trailing space with newline */
> >> + buf[len - 1] = '\n';
> >>
> >> - return sysfs_emit(buf, "%s\n", output);
> >> + return len;
> >> }
> >
> >This is pretty mcuh a one-for-one copy/pasta of defrag_show(), I don't love that
> >we have the exact same code duplicated across two files like that.
> >
> >You could write something like:
> >
> >static ssize_t thp_sysfs_enabled_show(struct kobject *kobj,
> > struct kobj_attribute *attr, char *buf,
> > const char *names, int names_len,
> > const char *orders_by_mode, int orders_by_mode_len,
> > int default_mode)
> >{
> > ...
> >}
> >
> >To abstract it, but that's kind of a horrible signature isn't it? :)
> >
> >Could use a helper struct, but that feels a bit overkill for this hmm...
> >
> >Really I wonder if we shouldn't have this in huge_memory.c anyway, it's a bit of
> >a weird thing to put it in mm/shmem.c, it's more huge pages than shmem imo.
> >
> >Anyway. The logic itself looks fine so LGTM!
>
> Yes, after this patch is applied, the read/write handlers for the
> shmem_enabled and enabled interfaces will have a lot of duplicated code.
> I will continue to investigate whether we can abstract a more generic
> function to handle both interfaces.
> Introducing a helper struct as a parameter is a good inspiration.
Well it'd probably be overkill :)
For the time being, let's not do that, and just get this change in (with other
changes suggested applied of course), so send a respin without that please.
I think it's more important to address the hideous duplication we have _right
now_ rather than optimising deduplicating this code :) we can always do that
later.
Cheers, Lorenzo
>
> >>
> >> static ssize_t thpsize_shmem_enabled_store(struct kobject *kobj,
> >> --
> >> 2.25.1
> >>
> >>
> >
> >Cheers, Lorenzo
>
^ permalink raw reply
* Re: [PATCH v2 1/2] mm: huge_memory: refactor thpsize_shmem_enabled_store() with sysfs_match_string()
From: Lorenzo Stoakes @ 2026-05-15 11:34 UTC (permalink / raw)
To: ranxiaokai627
Cc: akpm, baolin.wang, hughd, leitao, linux-kernel, linux-mm,
ran.xiaokai
In-Reply-To: <20260515072336.53287-1-ranxiaokai627@163.com>
On Fri, May 15, 2026 at 07:23:36AM +0000, ranxiaokai627@163.com wrote:
> >On Wed, May 13, 2026 at 09:45:07AM +0000, ranxiaokai627@163.com wrote:
> >> From: Ran Xiaokai <ran.xiaokai@zte.com.cn>
> >>
> >> Inspired by commit 82d9ff648c6c ("mm: huge_memory: refactor
> >> anon_enabled_store() with set_anon_enabled_mode()"), refactor
> >> thpsize_shmem_enabled_store() using sysfs_match_string().
> >> This eliminates the duplicated spin_lock/unlock(), set/clear_bit(),
> >> calls across all branches, reducing code duplication.
> >
> >> + /* Do not override huge allocation policy with non-PMD sized mTHP */
> >> + if (mode == HUGE_SHMEM_ENABLED_INHERIT &&
> >> + shmem_huge == SHMEM_HUGE_FORCE && !is_pmd_order(order))
> >> + return -EINVAL;
> >>
> >> - if (ret > 0) {
> >> - int err = start_stop_khugepaged();
> >> + spin_lock(&huge_shmem_orders_lock);
> >> + for (m = 0; m < ARRAY_SIZE(huge_shmem_orders_by_mode); m++) {
> >> + if (m == mode)
> >> + changed |= !__test_and_set_bit(order, huge_shmem_orders_by_mode[m]);
> >> + else
> >> + changed |= __test_and_clear_bit(order, huge_shmem_orders_by_mode[m]);
> >> + }
> >> + spin_unlock(&huge_shmem_orders_lock);
> >
> >You're copy/pasta'ing anon_enabled_store() but not the nicer refactoring that's
> >there.
> >
> >Please split this out like set_anon_enabled_mode() does. And put:
> >
> > static unsigned long *enabled_orders[] = {
> > &huge_shmem_orders_always,
> > &huge_shmem_orders_inherit,
> > &huge_shmem_orders_within_size,
> > &huge_shmem_orders_madvise,
> > };
> >
> >At the start of it like that does also.
> >
> >Please don't reproduce the single letter var name though :)
> >
>
> Arrays huge_shmem_orders_by_mode are shared by both store() and
> show() in patch2, so we should keeping at file scope to avoid duplication,
> what do you think ?
Ah yeah ok, in that case keep that separated out, but the other points in the
review stand!
Thanks :)
>
> >> +};
> >
> >>
> >> + if (changed) {
> >> + err = start_stop_khugepaged();
> >> if (err)
> >> ret = err;
> >
> >
> >> + } else {
> >> + /*
> >> + * Recalculate watermarks even when the mode hasn't changed
> >> + * to preserve the legacy behavior, as this is always called
> >> + * inside start_stop_khugepaged().
> >> + */
> >> + set_recommended_min_free_kbytes();
> >> }
> >> +
> >> return ret;
> >
> >return count;
> >
> >> }
> >>
> >> --
> >> 2.25.1
> >>
> >>
> >
> >Cheers, Lorenzo
>
^ permalink raw reply
* Re: improve the swap_activate interface
From: Christoph Hellwig @ 2026-05-15 11:33 UTC (permalink / raw)
To: Steve French
Cc: Christoph Hellwig, Andrew Morton, Chris Li, Kairui Song,
Christian Brauner, Darrick J . Wong, Jens Axboe, David Sterba,
Theodore Ts'o, Jaegeuk Kim, Chao Yu, Trond Myklebust,
Anna Schumaker, Namjae Jeon, Hyunchul Lee, Steve French,
Paulo Alcantara, Carlos Maiolino, Damien Le Moal, Naohiro Aota,
linux-xfs, linux-fsdevel, linux-doc, linux-mm, linux-block,
linux-btrfs, linux-ext4, linux-f2fs-devel, linux-nfs, linux-cifs
In-Reply-To: <CAH2r5msnYVb3hhXHwqDVHGGC1h4E6mLCRS4ktCrQoD9zdUW81g@mail.gmail.com>
On Wed, May 13, 2026 at 03:34:03PM -0500, Steve French wrote:
> I just tried this on 7.1-rc3 with the swap patches (full kernel build,
> on Ubuntu 25,10) and boot failed with out of memory which I had never
> seen before. Any idea how to workaround this with the swap patch
> series, or is there a fix for this in the swap series already?
Is that a failure with the patches or also with the baseline?
^ permalink raw reply
* [PATCH v4 12/12] mm, swap: merge zeromap into swap table
From: Kairui Song via B4 Relay @ 2026-05-15 9:54 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, linux-kernel, cgroups, Kairui Song,
Lorenzo Stoakes, Yosry Ahmed, Qi Zheng
In-Reply-To: <20260515-swap-table-p4-v4-0-f1b49e845a8d@tencent.com>
From: Kairui Song <kasong@tencent.com>
By allocating one additional bit in the swap table entry's flags field
alongside the count, we can store the zeromap inline
For 64 bit systems, zeromap will store in the swap table, avoiding zeromap
allocation. It reduces the allocated memory. That is the happy path.
For certain 32-bit archs, there might not be enough bits in the swap
table to contain both PFN and flags. Therefore, conditionally let each
cluster have a zeromap field at build time, and use that instead.
If the swapfile cluster is not fully used, it will still save memory for
zeromap. The empty cluster does not allocate a zeromap. In the worst case,
all cluster are fully populated. We will use memory similar to the
previous zeromap implementation.
A few macros were moved to different headers for build time struct
definition.
Acked-by: Chris Li <chrisl@kernel.org>
Reviewed-by: Youngjun Park <youngjun.park@lge.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/swap.h | 1 -
mm/memory.c | 11 +----
mm/page_io.c | 61 +++++++++++++++++++++++----
mm/swap.h | 51 +++++++++--------------
mm/swap_state.c | 14 ++++---
mm/swap_table.h | 115 +++++++++++++++++++++++++++++++++++++--------------
mm/swapfile.c | 54 +++++++++++-------------
7 files changed, 191 insertions(+), 116 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 203bbe23ba1f..6d72778e6cc3 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -253,7 +253,6 @@ struct swap_info_struct {
struct plist_node list; /* entry in swap_active_head */
signed char type; /* strange name for an index */
unsigned int max; /* size of this swap device */
- unsigned long *zeromap; /* kvmalloc'ed bitmap to track zero pages */
struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
diff --git a/mm/memory.c b/mm/memory.c
index 56f9e38ee891..860b2aabec39 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4611,13 +4611,11 @@ static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
- * Check if the PTEs within a range are contiguous swap entries
- * and have consistent swapcache, zeromap.
+ * Check if the PTEs within a range are contiguous swap entries.
*/
static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
{
unsigned long addr;
- softleaf_t entry;
int idx;
pte_t pte;
@@ -4627,18 +4625,13 @@ static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
return false;
- entry = softleaf_from_pte(pte);
- if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
- return false;
-
/*
* swap_read_folio() can't handle the case a large folio is hybridly
* from different backends. And they are likely corner cases. Similar
* things might be added once zswap support large folios.
*/
- if (unlikely(swap_zeromap_batch(entry, nr_pages, NULL) != nr_pages))
+ if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
return false;
-
return true;
}
diff --git a/mm/page_io.c b/mm/page_io.c
index 7ed76592e20d..f2d8fe7fd057 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -26,6 +26,7 @@
#include <linux/delayacct.h>
#include <linux/zswap.h>
#include "swap.h"
+#include "swap_table.h"
static void __end_swap_bio_write(struct bio *bio)
{
@@ -204,15 +205,20 @@ static bool is_folio_zero_filled(struct folio *folio)
static void swap_zeromap_folio_set(struct folio *folio)
{
struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
- struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
int nr_pages = folio_nr_pages(folio);
+ struct swap_cluster_info *ci;
swp_entry_t entry;
unsigned int i;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
+ ci = swap_cluster_get_and_lock(folio);
for (i = 0; i < folio_nr_pages(folio); i++) {
entry = page_swap_entry(folio_page(folio, i));
- set_bit(swp_offset(entry), sis->zeromap);
+ __swap_table_set_zero(ci, swp_cluster_offset(entry));
}
+ swap_cluster_unlock(ci);
count_vm_events(SWPOUT_ZERO, nr_pages);
if (objcg) {
@@ -223,14 +229,19 @@ static void swap_zeromap_folio_set(struct folio *folio)
static void swap_zeromap_folio_clear(struct folio *folio)
{
- struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
+ struct swap_cluster_info *ci;
swp_entry_t entry;
unsigned int i;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
+ ci = swap_cluster_get_and_lock(folio);
for (i = 0; i < folio_nr_pages(folio); i++) {
entry = page_swap_entry(folio_page(folio, i));
- clear_bit(swp_offset(entry), sis->zeromap);
+ __swap_table_clear_zero(ci, swp_cluster_offset(entry));
}
+ swap_cluster_unlock(ci);
}
/*
@@ -255,10 +266,9 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
}
/*
- * Use a bitmap (zeromap) to avoid doing IO for zero-filled pages.
- * The bits in zeromap are protected by the locked swapcache folio
- * and atomic updates are used to protect against read-modify-write
- * corruption due to other zero swap entries seeing concurrent updates.
+ * Use the swap table zero mark to avoid doing IO for zero-filled
+ * pages. The zero mark is protected by the cluster lock, which is
+ * acquired internally by swap_zeromap_folio_set/clear.
*/
if (is_folio_zero_filled(folio)) {
swap_zeromap_folio_set(folio);
@@ -509,19 +519,52 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
mempool_free(sio, sio_pool);
}
+/*
+ * Return the count of contiguous swap entries that share the same
+ * zeromap status as the starting entry. If is_zerop is not NULL,
+ * it will return the zeromap status of the starting entry.
+ *
+ * Context: Caller must ensure the cluster containing the entries
+ * that will be checked won't be freed.
+ */
+static int swap_zeromap_batch(swp_entry_t entry, int max_nr,
+ bool *is_zerop)
+{
+ int i;
+ bool is_zero;
+ unsigned int ci_start = swp_cluster_offset(entry);
+ struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
+
+ VM_WARN_ON_ONCE(ci_start + max_nr > SWAPFILE_CLUSTER);
+
+ rcu_read_lock();
+ is_zero = __swap_table_test_zero(ci, ci_start);
+ for (i = 1; i < max_nr; i++)
+ if (is_zero != __swap_table_test_zero(ci, ci_start + i))
+ break;
+ rcu_read_unlock();
+ if (is_zerop)
+ *is_zerop = is_zero;
+
+ return i;
+}
+
static bool swap_read_folio_zeromap(struct folio *folio)
{
int nr_pages = folio_nr_pages(folio);
struct obj_cgroup *objcg;
bool is_zeromap;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
/*
* Swapping in a large folio that is partially in the zeromap is not
* currently handled. Return true without marking the folio uptodate so
* that an IO error is emitted (e.g. do_swap_page() will sigbus).
+ * Folio lock stabilizes the cluster and map, so the check is safe.
*/
if (WARN_ON_ONCE(swap_zeromap_batch(folio->swap, nr_pages,
- &is_zeromap) != nr_pages))
+ &is_zeromap) != nr_pages))
return true;
if (!is_zeromap)
diff --git a/mm/swap.h b/mm/swap.h
index 5b2f095fff6e..81c06aae7ccd 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -3,12 +3,29 @@
#define _MM_SWAP_H
#include <linux/atomic.h> /* for atomic_long_t */
+#include <linux/mm.h> /* for PAGE_SHIFT */
struct mempolicy;
struct swap_iocb;
struct swap_memcg_table;
extern int page_cluster;
+#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
+#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
+#elif defined(MAX_PHYSMEM_BITS)
+#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
+#else
+#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
+#endif
+
+/* Swap table marker, 0x1 means shadow, 0x2 means PFN (SWP_TB_PFN_MARK) */
+#define SWAP_CACHE_PFN_MARK_BITS 2
+/* At least 2 bits are needed to distinguish SWP_TB_COUNT_MAX, 1 and 0 */
+#define SWAP_COUNT_MIN_BITS 2
+/* If there are enough bits besides PFN and marker, store zero flag inline */
+#define SWAP_TABLE_HAS_ZEROFLAG ((BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - \
+ SWAP_CACHE_PFN_BITS) > SWAP_COUNT_MIN_BITS)
+
#ifdef CONFIG_THP_SWAP
#define SWAPFILE_CLUSTER HPAGE_PMD_NR
#define swap_entry_order(order) (order)
@@ -41,6 +58,9 @@ struct swap_cluster_info {
unsigned int *extend_table; /* For large swap count, protected by ci->lock */
#ifdef CONFIG_MEMCG
struct swap_memcg_table *memcg_table; /* Swap table entries' cgroup record */
+#endif
+#if !SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long *zero_bitmap;
#endif
struct list_head list;
};
@@ -314,31 +334,6 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
return __swap_entry_to_info(folio->swap)->flags;
}
-/*
- * Return the count of contiguous swap entries that share the same
- * zeromap status as the starting entry. If is_zeromap is not NULL,
- * it will return the zeromap status of the starting entry.
- */
-static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
- bool *is_zeromap)
-{
- struct swap_info_struct *sis = __swap_entry_to_info(entry);
- unsigned long start = swp_offset(entry);
- unsigned long end = start + max_nr;
- bool first_bit;
-
- first_bit = test_bit(start, sis->zeromap);
- if (is_zeromap)
- *is_zeromap = first_bit;
-
- if (max_nr <= 1)
- return max_nr;
- if (first_bit)
- return find_next_zero_bit(sis->zeromap, end, start) - start;
- else
- return find_next_bit(sis->zeromap, end, start) - start;
-}
-
#else /* CONFIG_SWAP */
struct swap_iocb;
static inline struct swap_cluster_info *swap_cluster_lock(
@@ -476,11 +471,5 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
{
return 0;
}
-
-static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
- bool *has_zeromap)
-{
- return 0;
-}
#endif /* CONFIG_SWAP */
#endif /* _MM_SWAP_H */
diff --git a/mm/swap_state.c b/mm/swap_state.c
index c899d1d69b52..7701fa4b981c 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -160,6 +160,7 @@ static int __swap_cache_add_check(struct swap_cluster_info *ci,
{
unsigned int ci_off, ci_end;
unsigned long old_tb;
+ bool is_zero;
lockdep_assert_held(&ci->lock);
@@ -184,12 +185,14 @@ static int __swap_cache_add_check(struct swap_cluster_info *ci,
if (nr == 1)
return 0;
+ is_zero = __swap_table_test_zero(ci, ci_off);
ci_off = round_down(ci_off, nr);
ci_end = ci_off + nr;
do {
old_tb = __swap_table_get(ci, ci_off);
if (unlikely(swp_tb_is_folio(old_tb) ||
!__swp_tb_get_count(old_tb) ||
+ is_zero != __swap_table_test_zero(ci, ci_off) ||
(memcg_id && *memcg_id != __swap_cgroup_get(ci, ci_off))))
return -EBUSY;
} while (++ci_off < ci_end);
@@ -213,7 +216,7 @@ static void __swap_cache_do_add_folio(struct swap_cluster_info *ci,
do {
old_tb = __swap_table_get(ci, ci_off);
VM_WARN_ON_ONCE(swp_tb_is_folio(old_tb));
- __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_count(old_tb)));
+ __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb)));
} while (++ci_off < ci_end);
folio_ref_add(folio, nr_pages);
@@ -249,7 +252,6 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci,
struct folio *folio,
swp_entry_t entry, void *shadow)
{
- int count;
unsigned long old_tb;
struct swap_info_struct *si;
unsigned int ci_start, ci_off, ci_end;
@@ -269,13 +271,13 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci,
old_tb = __swap_table_get(ci, ci_off);
WARN_ON_ONCE(!swp_tb_is_folio(old_tb) ||
swp_tb_to_folio(old_tb) != folio);
- count = __swp_tb_get_count(old_tb);
- if (count)
+ if (__swp_tb_get_count(old_tb))
folio_swapped = true;
else
need_free = true;
/* If shadow is NULL, we set an empty shadow. */
- __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow, count));
+ __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow,
+ __swp_tb_get_flags(old_tb)));
} while (++ci_off < ci_end);
folio->swap.val = 0;
@@ -369,7 +371,7 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
do {
old_tb = __swap_table_get(ci, ci_off);
WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || swp_tb_to_folio(old_tb) != old);
- __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_count(old_tb)));
+ __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb)));
} while (++ci_off < ci_end);
/*
diff --git a/mm/swap_table.h b/mm/swap_table.h
index b4e1100f8296..e6613e62f8d0 100644
--- a/mm/swap_table.h
+++ b/mm/swap_table.h
@@ -26,12 +26,14 @@ struct swap_memcg_table {
* Swap table entry type and bits layouts:
*
* NULL: |---------------- 0 ---------------| - Free slot
- * Shadow: | SWAP_COUNT |---- SHADOW_VAL ---|1| - Swapped out slot
- * PFN: | SWAP_COUNT |------ PFN -------|10| - Cached slot
+ * Shadow: |SWAP_COUNT|Z|---- SHADOW_VAL ---|1| - Swapped out slot
+ * PFN: |SWAP_COUNT|Z|------ PFN -------|10| - Cached slot
* Pointer: |----------- Pointer ----------|100| - (Unused)
* Bad: |------------- 1 -------------|1000| - Bad slot
*
- * SWAP_COUNT is `SWP_TB_COUNT_BITS` long, each entry is an atomic long.
+ * COUNT is `SWP_TB_COUNT_BITS` long, Z is the `SWP_TB_ZERO_FLAG` bit,
+ * and together they form the `SWP_TB_FLAGS_BITS` wide flags field.
+ * Each entry is an atomic long.
*
* Usages:
*
@@ -54,14 +56,6 @@ struct swap_memcg_table {
* - Bad: Swap slot is reserved, protects swap header or holes on swap devices.
*/
-#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
-#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
-#elif defined(MAX_PHYSMEM_BITS)
-#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
-#else
-#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
-#endif
-
/* NULL Entry, all 0 */
#define SWP_TB_NULL 0UL
@@ -69,22 +63,26 @@ struct swap_memcg_table {
#define SWP_TB_SHADOW_MARK 0b1UL
/* Cached: PFN */
-#define SWP_TB_PFN_BITS (SWAP_CACHE_PFN_BITS + SWP_TB_PFN_MARK_BITS)
+#define SWP_TB_PFN_BITS (SWAP_CACHE_PFN_BITS + SWAP_CACHE_PFN_MARK_BITS)
#define SWP_TB_PFN_MARK 0b10UL
-#define SWP_TB_PFN_MARK_BITS 2
-#define SWP_TB_PFN_MARK_MASK (BIT(SWP_TB_PFN_MARK_BITS) - 1)
+#define SWP_TB_PFN_MARK_MASK (BIT(SWAP_CACHE_PFN_MARK_BITS) - 1)
-/* SWAP_COUNT part for PFN or shadow, the width can be shrunk or extended */
-#define SWP_TB_COUNT_BITS min(4, BITS_PER_LONG - SWP_TB_PFN_BITS)
+/* Flags: For PFN or shadow, contains SWAP_COUNT, width changes */
+#define SWP_TB_FLAGS_BITS min(5, BITS_PER_LONG - SWP_TB_PFN_BITS)
+#define SWP_TB_COUNT_BITS (SWP_TB_FLAGS_BITS - SWAP_TABLE_HAS_ZEROFLAG)
+#define SWP_TB_FLAGS_MASK (~((~0UL) >> SWP_TB_FLAGS_BITS))
#define SWP_TB_COUNT_MASK (~((~0UL) >> SWP_TB_COUNT_BITS))
+#define SWP_TB_FLAGS_SHIFT (BITS_PER_LONG - SWP_TB_FLAGS_BITS)
#define SWP_TB_COUNT_SHIFT (BITS_PER_LONG - SWP_TB_COUNT_BITS)
#define SWP_TB_COUNT_MAX ((1 << SWP_TB_COUNT_BITS) - 1)
+/* The first flag is zero bit (SWAP_TABLE_HAS_ZEROFLAG) */
+#define SWP_TB_ZERO_FLAG BIT(BITS_PER_LONG - SWP_TB_FLAGS_BITS)
/* Bad slot: ends with 0b1000 and rests of bits are all 1 */
#define SWP_TB_BAD ((~0UL) << 3)
/* Macro for shadow offset calculation */
-#define SWAP_COUNT_SHIFT SWP_TB_COUNT_BITS
+#define SWAP_COUNT_SHIFT SWP_TB_FLAGS_BITS
/*
* Helpers for casting one type of info into a swap table entry.
@@ -102,40 +100,47 @@ static inline unsigned long __count_to_swp_tb(unsigned char count)
* used (count > 0 && count < SWP_TB_COUNT_MAX), and
* overflow (count == SWP_TB_COUNT_MAX).
*/
- BUILD_BUG_ON(SWP_TB_COUNT_MAX < 2 || SWP_TB_COUNT_BITS < 2);
+ BUILD_BUG_ON(SWP_TB_COUNT_BITS < SWAP_COUNT_MIN_BITS);
VM_WARN_ON(count > SWP_TB_COUNT_MAX);
return ((unsigned long)count) << SWP_TB_COUNT_SHIFT;
}
-static inline unsigned long pfn_to_swp_tb(unsigned long pfn, unsigned int count)
+static inline unsigned long __flags_to_swp_tb(unsigned char flags)
+{
+ BUILD_BUG_ON(SWP_TB_FLAGS_BITS > BITS_PER_BYTE);
+ VM_WARN_ON(flags >> SWP_TB_FLAGS_BITS);
+ return ((unsigned long)flags) << SWP_TB_FLAGS_SHIFT;
+}
+
+static inline unsigned long pfn_to_swp_tb(unsigned long pfn, unsigned char flags)
{
unsigned long swp_tb;
BUILD_BUG_ON(sizeof(unsigned long) != sizeof(void *));
BUILD_BUG_ON(SWAP_CACHE_PFN_BITS >
- (BITS_PER_LONG - SWP_TB_PFN_MARK_BITS - SWP_TB_COUNT_BITS));
+ (BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - SWP_TB_FLAGS_BITS));
- swp_tb = (pfn << SWP_TB_PFN_MARK_BITS) | SWP_TB_PFN_MARK;
- VM_WARN_ON_ONCE(swp_tb & SWP_TB_COUNT_MASK);
+ swp_tb = (pfn << SWAP_CACHE_PFN_MARK_BITS) | SWP_TB_PFN_MARK;
+ VM_WARN_ON_ONCE(swp_tb & SWP_TB_FLAGS_MASK);
- return swp_tb | __count_to_swp_tb(count);
+ return swp_tb | __flags_to_swp_tb(flags);
}
-static inline unsigned long folio_to_swp_tb(struct folio *folio, unsigned int count)
+static inline unsigned long folio_to_swp_tb(struct folio *folio, unsigned char flags)
{
- return pfn_to_swp_tb(folio_pfn(folio), count);
+ return pfn_to_swp_tb(folio_pfn(folio), flags);
}
-static inline unsigned long shadow_to_swp_tb(void *shadow, unsigned int count)
+static inline unsigned long shadow_to_swp_tb(void *shadow, unsigned char flags)
{
BUILD_BUG_ON((BITS_PER_XA_VALUE + 1) !=
BITS_PER_BYTE * sizeof(unsigned long));
BUILD_BUG_ON((unsigned long)xa_mk_value(0) != SWP_TB_SHADOW_MARK);
VM_WARN_ON_ONCE(shadow && !xa_is_value(shadow));
- VM_WARN_ON_ONCE(shadow && ((unsigned long)shadow & SWP_TB_COUNT_MASK));
+ VM_WARN_ON_ONCE(shadow && ((unsigned long)shadow & SWP_TB_FLAGS_MASK));
- return (unsigned long)shadow | __count_to_swp_tb(count) | SWP_TB_SHADOW_MARK;
+ return (unsigned long)shadow | SWP_TB_SHADOW_MARK | __flags_to_swp_tb(flags);
}
/*
@@ -173,14 +178,14 @@ static inline bool swp_tb_is_countable(unsigned long swp_tb)
static inline struct folio *swp_tb_to_folio(unsigned long swp_tb)
{
VM_WARN_ON(!swp_tb_is_folio(swp_tb));
- return pfn_folio((swp_tb & ~SWP_TB_COUNT_MASK) >> SWP_TB_PFN_MARK_BITS);
+ return pfn_folio((swp_tb & ~SWP_TB_FLAGS_MASK) >> SWAP_CACHE_PFN_MARK_BITS);
}
static inline void *swp_tb_to_shadow(unsigned long swp_tb)
{
VM_WARN_ON(!swp_tb_is_shadow(swp_tb));
/* No shift needed, xa_value is stored as it is in the lower bits. */
- return (void *)(swp_tb & ~SWP_TB_COUNT_MASK);
+ return (void *)(swp_tb & ~SWP_TB_FLAGS_MASK);
}
static inline unsigned char __swp_tb_get_count(unsigned long swp_tb)
@@ -189,6 +194,12 @@ static inline unsigned char __swp_tb_get_count(unsigned long swp_tb)
return ((swp_tb & SWP_TB_COUNT_MASK) >> SWP_TB_COUNT_SHIFT);
}
+static inline unsigned char __swp_tb_get_flags(unsigned long swp_tb)
+{
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ return ((swp_tb & SWP_TB_FLAGS_MASK) >> SWP_TB_FLAGS_SHIFT);
+}
+
static inline int swp_tb_get_count(unsigned long swp_tb)
{
if (swp_tb_is_countable(swp_tb))
@@ -253,6 +264,50 @@ static inline unsigned long swap_table_get(struct swap_cluster_info *ci,
return swp_tb;
}
+static inline void __swap_table_set_zero(struct swap_cluster_info *ci,
+ unsigned int ci_off)
+{
+#if SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long swp_tb = __swap_table_get(ci, ci_off);
+
+ BUILD_BUG_ON(SWP_TB_ZERO_FLAG & ~SWP_TB_FLAGS_MASK);
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ swp_tb |= SWP_TB_ZERO_FLAG;
+ __swap_table_set(ci, ci_off, swp_tb);
+#else
+ lockdep_assert_held(&ci->lock);
+ __set_bit(ci_off, ci->zero_bitmap);
+#endif
+}
+
+static inline bool __swap_table_test_zero(struct swap_cluster_info *ci,
+ unsigned int ci_off)
+{
+#if SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long swp_tb = __swap_table_get(ci, ci_off);
+
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ return !!(swp_tb & SWP_TB_ZERO_FLAG);
+#else
+ return test_bit(ci_off, ci->zero_bitmap);
+#endif
+}
+
+static inline void __swap_table_clear_zero(struct swap_cluster_info *ci,
+ unsigned int ci_off)
+{
+#if SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long swp_tb = __swap_table_get(ci, ci_off);
+
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ swp_tb &= ~SWP_TB_ZERO_FLAG;
+ __swap_table_set(ci, ci_off, swp_tb);
+#else
+ lockdep_assert_held(&ci->lock);
+ __clear_bit(ci_off, ci->zero_bitmap);
+#endif
+}
+
#ifdef CONFIG_MEMCG
static inline void __swap_cgroup_set(struct swap_cluster_info *ci,
unsigned int ci_off, unsigned long nr, unsigned short id)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 095e9c953e49..a9a1e477fec9 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -427,6 +427,11 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci)
ci->memcg_table = NULL;
#endif
+#if !SWAP_TABLE_HAS_ZEROFLAG
+ kfree(ci->zero_bitmap);
+ ci->zero_bitmap = NULL;
+#endif
+
table = (struct swap_table *)rcu_access_pointer(ci->table);
if (!table)
return;
@@ -469,13 +474,21 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
VM_WARN_ON_ONCE(ci->memcg_table);
ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp);
if (!ci->memcg_table)
- ret = -ENOMEM;
+ goto err_free;
}
#endif
- if (ret)
- swap_cluster_free_table(ci);
- return ret;
+#if !SWAP_TABLE_HAS_ZEROFLAG
+ VM_WARN_ON_ONCE(ci->zero_bitmap);
+ ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp);
+ if (!ci->zero_bitmap)
+ goto err_free;
+#endif
+ return 0;
+
+err_free:
+ swap_cluster_free_table(ci);
+ return -ENOMEM;
}
/*
@@ -928,8 +941,8 @@ static bool __swap_cluster_alloc_entries(struct swap_info_struct *si,
order = 0;
nr_pages = 1;
swap_cluster_assert_empty(ci, ci_off, 1, false);
- /* Sets a fake shadow as placeholder */
- __swap_table_set(ci, ci_off, shadow_to_swp_tb(NULL, 1));
+ /* Fake shadow placeholder with no flag, hibernation does not use the zeromap */
+ __swap_table_set(ci, ci_off, __swp_tb_mk_count(shadow_to_swp_tb(NULL, 0), 1));
} else {
/* Allocation without folio is only possible with hibernation */
WARN_ON_ONCE(1);
@@ -1302,14 +1315,8 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
void (*swap_slot_free_notify)(struct block_device *, unsigned long);
unsigned int i;
- /*
- * Use atomic clear_bit operations only on zeromap instead of non-atomic
- * bitmap_clear to prevent adjacent bits corruption due to simultaneous writes.
- */
- for (i = 0; i < nr_entries; i++) {
- clear_bit(offset + i, si->zeromap);
+ for (i = 0; i < nr_entries; i++)
zswap_invalidate(swp_entry(si->type, offset + i));
- }
if (si->flags & SWP_BLKDEV)
swap_slot_free_notify =
@@ -1894,7 +1901,11 @@ void __swap_cluster_free_entries(struct swap_info_struct *si,
* ref, or after swap cache is dropped
*/
VM_WARN_ON(!swp_tb_is_shadow(old_tb) || __swp_tb_get_count(old_tb) > 1);
+
+ /* Resetting the slot to NULL also clears the inline flags. */
__swap_table_set(ci, ci_off, null_to_swp_tb());
+ if (!SWAP_TABLE_HAS_ZEROFLAG)
+ __swap_table_clear_zero(ci, ci_off);
/*
* Uncharge swap slots by memcg in batches. Consecutive
@@ -3088,7 +3099,6 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
{
struct swap_info_struct *p = NULL;
- unsigned long *zeromap;
struct swap_cluster_info *cluster_info;
struct file *swap_file, *victim;
struct address_space *mapping;
@@ -3184,8 +3194,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
swap_file = p->swap_file;
p->swap_file = NULL;
- zeromap = p->zeromap;
- p->zeromap = NULL;
maxpages = p->max;
cluster_info = p->cluster_info;
p->max = 0;
@@ -3197,7 +3205,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
mutex_unlock(&swapon_mutex);
kfree(p->global_cluster);
p->global_cluster = NULL;
- kvfree(zeromap);
free_swap_cluster_info(cluster_info, maxpages);
inode = mapping->host;
@@ -3729,17 +3736,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
if (error)
goto bad_swap_unlock_inode;
- /*
- * Use kvmalloc_array instead of bitmap_zalloc as the allocation order might
- * be above MAX_PAGE_ORDER incase of a large swap file.
- */
- si->zeromap = kvmalloc_array(BITS_TO_LONGS(maxpages), sizeof(long),
- GFP_KERNEL | __GFP_ZERO);
- if (!si->zeromap) {
- error = -ENOMEM;
- goto bad_swap_unlock_inode;
- }
-
if (si->bdev && bdev_stable_writes(si->bdev))
si->flags |= SWP_STABLE_WRITES;
@@ -3841,8 +3837,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
destroy_swap_extents(si, swap_file);
free_swap_cluster_info(si->cluster_info, si->max);
si->cluster_info = NULL;
- kvfree(si->zeromap);
- si->zeromap = NULL;
/*
* Clear the SWP_USED flag after all resources are freed so
* alloc_swap_info can reuse this si safely.
--
2.54.0
^ permalink raw reply related
* [PATCH v4 08/12] mm, swap: delay and unify memcg lookup and charging for swapin
From: Kairui Song via B4 Relay @ 2026-05-15 9:54 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, linux-kernel, cgroups, Kairui Song,
Lorenzo Stoakes, Yosry Ahmed, Qi Zheng
In-Reply-To: <20260515-swap-table-p4-v4-0-f1b49e845a8d@tencent.com>
From: Kairui Song <kasong@tencent.com>
Instead of checking the cgroup private ID during page table walk in
swap_pte_batch(), move the memcg lookup into __swap_cache_add_check()
under the cluster lock.
The first pre-alloc check is speculative and skips the memcg check since
the post-alloc stable check ensures all slots covered by the folio
belong to the same memcg. It is very rare for contiguous and aligned
entries across a contiguous region of a page table of the same process
or shmem mapping to belong to different memcgs.
This also prepares for recording the memcg info in the cluster's table.
Also make the order check and fallback more compact.
There should be no user-observable behavior change.
Acked-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/memcontrol.h | 6 +++---
mm/internal.h | 10 +---------
mm/memcontrol.c | 10 ++++------
mm/swap_state.c | 28 +++++++++++++++++++---------
4 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 7d08128de1fd..a013f37f24aa 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -646,8 +646,8 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,
int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp);
-int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
- gfp_t gfp, swp_entry_t entry);
+int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id,
+ struct mm_struct *mm, gfp_t gfp);
void __mem_cgroup_uncharge(struct folio *folio);
@@ -1137,7 +1137,7 @@ static inline int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp)
}
static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,
- struct mm_struct *mm, gfp_t gfp, swp_entry_t entry)
+ unsigned short id, struct mm_struct *mm, gfp_t gfp)
{
return 0;
}
diff --git a/mm/internal.h b/mm/internal.h
index 5a2ddcf68e0b..9d2fec696bd6 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -451,24 +451,16 @@ static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte)
{
pte_t expected_pte = pte_next_swp_offset(pte);
const pte_t *end_ptep = start_ptep + max_nr;
- const softleaf_t entry = softleaf_from_pte(pte);
pte_t *ptep = start_ptep + 1;
- unsigned short cgroup_id;
VM_WARN_ON(max_nr < 1);
- VM_WARN_ON(!softleaf_is_swap(entry));
+ VM_WARN_ON(!softleaf_is_swap(softleaf_from_pte(pte)));
- cgroup_id = lookup_swap_cgroup_id(entry);
while (ptep < end_ptep) {
- softleaf_t entry;
-
pte = ptep_get(ptep);
if (!pte_same(pte, expected_pte))
break;
- entry = softleaf_from_pte(pte);
- if (lookup_swap_cgroup_id(entry) != cgroup_id)
- break;
expected_pte = pte_next_swp_offset(expected_pte);
ptep++;
}
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a28a68eed7ba..4f940cf22ffe 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -5070,27 +5070,25 @@ int mem_cgroup_charge_hugetlb(struct folio *folio, gfp_t gfp)
/**
* mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin.
- * @folio: folio to charge.
+ * @folio: the folio to charge
+ * @id: memory cgroup id
* @mm: mm context of the victim
* @gfp: reclaim mode
- * @entry: swap entry for which the folio is allocated
*
* This function charges a folio allocated for swapin. Please call this before
* adding the folio to the swapcache.
*
* Returns 0 on success. Otherwise, an error code is returned.
*/
-int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
- gfp_t gfp, swp_entry_t entry)
+int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id,
+ struct mm_struct *mm, gfp_t gfp)
{
struct mem_cgroup *memcg;
- unsigned short id;
int ret;
if (mem_cgroup_disabled())
return 0;
- id = lookup_swap_cgroup_id(entry);
rcu_read_lock();
memcg = mem_cgroup_from_private_id(id);
if (!memcg || !css_tryget_online(&memcg->css))
diff --git a/mm/swap_state.c b/mm/swap_state.c
index cdb7859eb502..75339640160a 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -142,17 +142,21 @@ void *swap_cache_get_shadow(swp_entry_t entry)
* @ci: The locked swap cluster
* @targ_entry: The target swap entry to check, will be rounded down by @nr
* @nr: Number of slots to check, must be a power of 2
- * @shadowp: Returns the shadow value if one exists in the range.
+ * @shadowp: Returns the shadow value if one exists in the range
+ * @memcg_id: Returns the memory cgroup id, NULL to ignore cgroup check
*
* Check if all slots covered by given range have a swap count >= 1.
- * Retrieves the shadow if there is one.
+ * Retrieves the shadow if there is one. If @memcg_id is not NULL, also
+ * checks if all slots belong to the same cgroup and return the cgroup
+ * private id.
*
* Context: Caller must lock the cluster.
* Return: 0 if success, error code if failed.
*/
static int __swap_cache_add_check(struct swap_cluster_info *ci,
swp_entry_t targ_entry,
- unsigned long nr, void **shadowp)
+ unsigned long nr, void **shadowp,
+ unsigned short *memcg_id)
{
unsigned int ci_off, ci_end;
unsigned long old_tb;
@@ -172,19 +176,24 @@ static int __swap_cache_add_check(struct swap_cluster_info *ci,
return -EEXIST;
if (!__swp_tb_get_count(old_tb))
return -ENOENT;
- if (swp_tb_is_shadow(old_tb) && shadowp)
+ if (shadowp && swp_tb_is_shadow(old_tb))
*shadowp = swp_tb_to_shadow(old_tb);
+ if (memcg_id)
+ *memcg_id = lookup_swap_cgroup_id(targ_entry);
if (nr == 1)
return 0;
+ targ_entry.val = round_down(targ_entry.val, nr);
ci_off = round_down(ci_off, nr);
ci_end = ci_off + nr;
do {
old_tb = __swap_table_get(ci, ci_off);
if (unlikely(swp_tb_is_folio(old_tb) ||
- !__swp_tb_get_count(old_tb)))
+ !__swp_tb_get_count(old_tb) ||
+ (memcg_id && *memcg_id != lookup_swap_cgroup_id(targ_entry))))
return -EBUSY;
+ targ_entry.val++;
} while (++ci_off < ci_end);
return 0;
@@ -400,6 +409,7 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
swp_entry_t entry;
struct folio *folio;
void *shadow = NULL;
+ unsigned short memcg_id;
unsigned long address, nr_pages = 1UL << order;
struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
@@ -408,7 +418,7 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
/* Check if the slot and range are available, skip allocation if not */
spin_lock(&ci->lock);
- err = __swap_cache_add_check(ci, targ_entry, nr_pages, NULL);
+ err = __swap_cache_add_check(ci, targ_entry, nr_pages, NULL, NULL);
spin_unlock(&ci->lock);
if (unlikely(err))
return ERR_PTR(err);
@@ -431,7 +441,7 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
/* Double check the range is still not in conflict */
spin_lock(&ci->lock);
- err = __swap_cache_add_check(ci, targ_entry, nr_pages, &shadow);
+ err = __swap_cache_add_check(ci, targ_entry, nr_pages, &shadow, &memcg_id);
if (unlikely(err)) {
spin_unlock(&ci->lock);
folio_put(folio);
@@ -443,8 +453,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
__swap_cache_do_add_folio(ci, folio, entry);
spin_unlock(&ci->lock);
- if (mem_cgroup_swapin_charge_folio(folio, vmf ? vmf->vma->vm_mm : NULL,
- gfp, entry)) {
+ if (mem_cgroup_swapin_charge_folio(folio, memcg_id,
+ vmf ? vmf->vma->vm_mm : NULL, gfp)) {
spin_lock(&ci->lock);
__swap_cache_do_del_folio(ci, folio, entry, shadow);
spin_unlock(&ci->lock);
--
2.54.0
^ permalink raw reply related
* [PATCH v4 11/12] mm/memcg: remove no longer used swap cgroup array
From: Kairui Song via B4 Relay @ 2026-05-15 9:54 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, linux-kernel, cgroups, Kairui Song,
Lorenzo Stoakes, Yosry Ahmed, Qi Zheng
In-Reply-To: <20260515-swap-table-p4-v4-0-f1b49e845a8d@tencent.com>
From: Kairui Song <kasong@tencent.com>
Now all swap cgroup records are stored in the swap cluster directly,
the static array is no longer needed.
Acked-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
MAINTAINERS | 1 -
include/linux/swap_cgroup.h | 47 ------------
mm/Makefile | 3 -
mm/internal.h | 1 -
mm/memcontrol-v1.c | 1 -
mm/memcontrol.c | 1 -
mm/swap_cgroup.c | 174 --------------------------------------------
mm/swapfile.c | 8 --
8 files changed, 236 deletions(-)
diff --git a/MAINTAINERS b/MAINTAINERS
index 0116eb99b708..9be179722d42 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6564,7 +6564,6 @@ F: mm/memcontrol.c
F: mm/memcontrol-v1.c
F: mm/memcontrol-v1.h
F: mm/page_counter.c
-F: mm/swap_cgroup.c
F: samples/cgroup/*
F: tools/testing/selftests/cgroup/memcg_protection.m
F: tools/testing/selftests/cgroup/test_hugetlb_memcg.c
diff --git a/include/linux/swap_cgroup.h b/include/linux/swap_cgroup.h
deleted file mode 100644
index 91cdf12190a0..000000000000
--- a/include/linux/swap_cgroup.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_SWAP_CGROUP_H
-#define __LINUX_SWAP_CGROUP_H
-
-#include <linux/swap.h>
-
-#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
-
-extern void swap_cgroup_record(struct folio *folio, unsigned short id, swp_entry_t ent);
-extern unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents);
-extern unsigned short lookup_swap_cgroup_id(swp_entry_t ent);
-extern int swap_cgroup_swapon(int type, unsigned long max_pages);
-extern void swap_cgroup_swapoff(int type);
-
-#else
-
-static inline
-void swap_cgroup_record(struct folio *folio, unsigned short id, swp_entry_t ent)
-{
-}
-
-static inline
-unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents)
-{
- return 0;
-}
-
-static inline
-unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
-{
- return 0;
-}
-
-static inline int
-swap_cgroup_swapon(int type, unsigned long max_pages)
-{
- return 0;
-}
-
-static inline void swap_cgroup_swapoff(int type)
-{
- return;
-}
-
-#endif
-
-#endif /* __LINUX_SWAP_CGROUP_H */
diff --git a/mm/Makefile b/mm/Makefile
index 8ad2ab08244e..eff9f9e7e061 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -103,9 +103,6 @@ obj-$(CONFIG_PAGE_COUNTER) += page_counter.o
obj-$(CONFIG_LIVEUPDATE_MEMFD) += memfd_luo.o
obj-$(CONFIG_MEMCG_V1) += memcontrol-v1.o
obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o
-ifdef CONFIG_SWAP
-obj-$(CONFIG_MEMCG) += swap_cgroup.o
-endif
ifdef CONFIG_BPF_SYSCALL
obj-$(CONFIG_MEMCG) += bpf_memcontrol.o
endif
diff --git a/mm/internal.h b/mm/internal.h
index 9d2fec696bd6..7646ecb9d621 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -17,7 +17,6 @@
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/leafops.h>
-#include <linux/swap_cgroup.h>
#include <linux/tracepoint-defs.h>
/* Internal core VMA manipulation functions. */
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 494e7b9adc60..08be1a752c2e 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -5,7 +5,6 @@
#include <linux/mm_inline.h>
#include <linux/pagewalk.h>
#include <linux/backing-dev.h>
-#include <linux/swap_cgroup.h>
#include <linux/eventfd.h>
#include <linux/poll.h>
#include <linux/sort.h>
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index b5c267a061a9..039e9bc8971c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -54,7 +54,6 @@
#include <linux/vmpressure.h>
#include <linux/memremap.h>
#include <linux/mm_inline.h>
-#include <linux/swap_cgroup.h>
#include <linux/cpu.h>
#include <linux/oom.h>
#include <linux/lockdep.h>
diff --git a/mm/swap_cgroup.c b/mm/swap_cgroup.c
deleted file mode 100644
index 95c38e54dd58..000000000000
--- a/mm/swap_cgroup.c
+++ /dev/null
@@ -1,174 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/swap_cgroup.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-
-#include <linux/swapops.h> /* depends on mm.h include */
-
-static DEFINE_MUTEX(swap_cgroup_mutex);
-
-/* Pack two cgroup id (short) of two entries in one swap_cgroup (atomic_t) */
-#define ID_PER_SC (sizeof(struct swap_cgroup) / sizeof(unsigned short))
-#define ID_SHIFT (BITS_PER_TYPE(unsigned short))
-#define ID_MASK (BIT(ID_SHIFT) - 1)
-struct swap_cgroup {
- atomic_t ids;
-};
-
-struct swap_cgroup_ctrl {
- struct swap_cgroup *map;
-};
-
-static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
-
-static unsigned short __swap_cgroup_id_lookup(struct swap_cgroup *map,
- pgoff_t offset)
-{
- unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT;
- unsigned int old_ids = atomic_read(&map[offset / ID_PER_SC].ids);
-
- BUILD_BUG_ON(!is_power_of_2(ID_PER_SC));
- BUILD_BUG_ON(sizeof(struct swap_cgroup) != sizeof(atomic_t));
-
- return (old_ids >> shift) & ID_MASK;
-}
-
-static unsigned short __swap_cgroup_id_xchg(struct swap_cgroup *map,
- pgoff_t offset,
- unsigned short new_id)
-{
- unsigned short old_id;
- struct swap_cgroup *sc = &map[offset / ID_PER_SC];
- unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT;
- unsigned int new_ids, old_ids = atomic_read(&sc->ids);
-
- do {
- old_id = (old_ids >> shift) & ID_MASK;
- new_ids = (old_ids & ~(ID_MASK << shift));
- new_ids |= ((unsigned int)new_id) << shift;
- } while (!atomic_try_cmpxchg(&sc->ids, &old_ids, new_ids));
-
- return old_id;
-}
-
-/**
- * swap_cgroup_record - record mem_cgroup for a set of swap entries.
- * These entries must belong to one single folio, and that folio
- * must be being charged for swap space (swap out), and these
- * entries must not have been charged
- *
- * @folio: the folio that the swap entry belongs to
- * @id: mem_cgroup ID to be recorded
- * @ent: the first swap entry to be recorded
- */
-void swap_cgroup_record(struct folio *folio, unsigned short id,
- swp_entry_t ent)
-{
- unsigned int nr_ents = folio_nr_pages(folio);
- struct swap_cgroup *map;
- pgoff_t offset, end;
- unsigned short old;
-
- offset = swp_offset(ent);
- end = offset + nr_ents;
- map = swap_cgroup_ctrl[swp_type(ent)].map;
-
- do {
- old = __swap_cgroup_id_xchg(map, offset, id);
- VM_BUG_ON(old);
- } while (++offset != end);
-}
-
-/**
- * swap_cgroup_clear - clear mem_cgroup for a set of swap entries.
- * These entries must be being uncharged from swap. They either
- * belongs to one single folio in the swap cache (swap in for
- * cgroup v1), or no longer have any users (slot freeing).
- *
- * @ent: the first swap entry to be recorded into
- * @nr_ents: number of swap entries to be recorded
- *
- * Returns the existing old value.
- */
-unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents)
-{
- pgoff_t offset, end;
- struct swap_cgroup *map;
- unsigned short old, iter = 0;
-
- offset = swp_offset(ent);
- end = offset + nr_ents;
- map = swap_cgroup_ctrl[swp_type(ent)].map;
-
- do {
- old = __swap_cgroup_id_xchg(map, offset, 0);
- if (!iter)
- iter = old;
- VM_BUG_ON(iter != old);
- } while (++offset != end);
-
- return old;
-}
-
-/**
- * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
- * @ent: swap entry to be looked up.
- *
- * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
- */
-unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
-{
- struct swap_cgroup_ctrl *ctrl;
-
- if (mem_cgroup_disabled())
- return 0;
-
- ctrl = &swap_cgroup_ctrl[swp_type(ent)];
- if (unlikely(!ctrl->map))
- return 0;
- return __swap_cgroup_id_lookup(ctrl->map, swp_offset(ent));
-}
-
-int swap_cgroup_swapon(int type, unsigned long max_pages)
-{
- struct swap_cgroup *map;
- struct swap_cgroup_ctrl *ctrl;
-
- if (mem_cgroup_disabled())
- return 0;
-
- BUILD_BUG_ON(sizeof(unsigned short) * ID_PER_SC !=
- sizeof(struct swap_cgroup));
- map = vzalloc(DIV_ROUND_UP(max_pages, ID_PER_SC) *
- sizeof(struct swap_cgroup));
- if (!map)
- goto nomem;
-
- ctrl = &swap_cgroup_ctrl[type];
- mutex_lock(&swap_cgroup_mutex);
- ctrl->map = map;
- mutex_unlock(&swap_cgroup_mutex);
-
- return 0;
-nomem:
- pr_info("couldn't allocate enough memory for swap_cgroup\n");
- pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
- return -ENOMEM;
-}
-
-void swap_cgroup_swapoff(int type)
-{
- struct swap_cgroup *map;
- struct swap_cgroup_ctrl *ctrl;
-
- if (mem_cgroup_disabled())
- return;
-
- mutex_lock(&swap_cgroup_mutex);
- ctrl = &swap_cgroup_ctrl[type];
- map = ctrl->map;
- ctrl->map = NULL;
- mutex_unlock(&swap_cgroup_mutex);
-
- vfree(map);
-}
diff --git a/mm/swapfile.c b/mm/swapfile.c
index ae14d4049e4b..095e9c953e49 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -45,7 +45,6 @@
#include <asm/tlbflush.h>
#include <linux/leafops.h>
-#include <linux/swap_cgroup.h>
#include "swap_table.h"
#include "internal.h"
#include "swap.h"
@@ -3200,8 +3199,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
p->global_cluster = NULL;
kvfree(zeromap);
free_swap_cluster_info(cluster_info, maxpages);
- /* Destroy swap account information */
- swap_cgroup_swapoff(p->type);
inode = mapping->host;
@@ -3732,10 +3729,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
if (error)
goto bad_swap_unlock_inode;
- error = swap_cgroup_swapon(si->type, maxpages);
- if (error)
- goto bad_swap_unlock_inode;
-
/*
* Use kvmalloc_array instead of bitmap_zalloc as the allocation order might
* be above MAX_PAGE_ORDER incase of a large swap file.
@@ -3846,7 +3839,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
si->global_cluster = NULL;
inode = NULL;
destroy_swap_extents(si, swap_file);
- swap_cgroup_swapoff(si->type);
free_swap_cluster_info(si->cluster_info, si->max);
si->cluster_info = NULL;
kvfree(si->zeromap);
--
2.54.0
^ permalink raw reply related
* [PATCH v4 07/12] mm, swap: support flexible batch freeing of slots in different memcgs
From: Kairui Song via B4 Relay @ 2026-05-15 9:54 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, linux-kernel, cgroups, Kairui Song,
Lorenzo Stoakes, Yosry Ahmed, Qi Zheng
In-Reply-To: <20260515-swap-table-p4-v4-0-f1b49e845a8d@tencent.com>
From: Kairui Song <kasong@tencent.com>
Instead of requiring the caller to ensure all slots are in the same
memcg, make the function handle different memcgs at once.
This is both a micro optimization and required for removing the memcg
lookup in the page table layer, so it can be unified at the swap layer.
We are not removing the memcg lookup in the page table in this commit.
It has to be done after the memcg lookup is deferred to the swap layer.
Acked-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
mm/swapfile.c | 33 +++++++++++++++++++++++++++++----
1 file changed, 29 insertions(+), 4 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 5c8bb15719bf..c9c80ba9252b 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1873,21 +1873,46 @@ void __swap_cluster_free_entries(struct swap_info_struct *si,
unsigned int ci_start, unsigned int nr_pages)
{
unsigned long old_tb;
+ unsigned int type = si->type;
+ unsigned short batch_id = 0, id_cur;
unsigned int ci_off = ci_start, ci_end = ci_start + nr_pages;
- unsigned long offset = cluster_offset(si, ci) + ci_start;
+ unsigned long ci_head = cluster_offset(si, ci);
+ unsigned int batch_off = ci_off;
+ swp_entry_t entry;
VM_WARN_ON(ci->count < nr_pages);
ci->count -= nr_pages;
do {
old_tb = __swap_table_get(ci, ci_off);
- /* Release the last ref, or after swap cache is dropped */
+ /*
+ * Freeing is done after release of the last swap count
+ * ref, or after swap cache is dropped
+ */
VM_WARN_ON(!swp_tb_is_shadow(old_tb) || __swp_tb_get_count(old_tb) > 1);
__swap_table_set(ci, ci_off, null_to_swp_tb());
+
+ /*
+ * Uncharge swap slots by memcg in batches. Consecutive
+ * slots with the same cgroup id are uncharged together.
+ */
+ entry = swp_entry(type, ci_head + ci_off);
+ id_cur = lookup_swap_cgroup_id(entry);
+ if (batch_id != id_cur) {
+ if (batch_id)
+ mem_cgroup_uncharge_swap(swp_entry(type, ci_head + batch_off),
+ ci_off - batch_off);
+ batch_id = id_cur;
+ batch_off = ci_off;
+ }
} while (++ci_off < ci_end);
- mem_cgroup_uncharge_swap(swp_entry(si->type, offset), nr_pages);
- swap_range_free(si, offset, nr_pages);
+ if (batch_id) {
+ mem_cgroup_uncharge_swap(swp_entry(type, ci_head + batch_off),
+ ci_off - batch_off);
+ }
+
+ swap_range_free(si, ci_head + ci_start, nr_pages);
swap_cluster_assert_empty(ci, ci_start, nr_pages, false);
if (!ci->count)
--
2.54.0
^ permalink raw reply related
* [PATCH v4 09/12] mm, swap: consolidate cluster allocation helpers
From: Kairui Song via B4 Relay @ 2026-05-15 9:54 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, linux-kernel, cgroups, Kairui Song,
Lorenzo Stoakes, Yosry Ahmed, Qi Zheng
In-Reply-To: <20260515-swap-table-p4-v4-0-f1b49e845a8d@tencent.com>
From: Kairui Song <kasong@tencent.com>
Swap cluster table management is spread across several narrow
helpers. As a result, the allocation and fallback sequences are
open-coded in multiple places.
A few more per-cluster tables will be added soon, so avoid
duplicating these sequences per table type. Fold the existing
pairs into cluster-oriented helpers, and rename for consistency.
No functional change, only a few sanity checks are slightly adjusted.
Acked-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
mm/swapfile.c | 110 ++++++++++++++++++++++++++--------------------------------
1 file changed, 49 insertions(+), 61 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index c9c80ba9252b..7740ba99f87e 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -411,20 +411,7 @@ static inline unsigned int cluster_offset(struct swap_info_struct *si,
return cluster_index(si, ci) * SWAPFILE_CLUSTER;
}
-static struct swap_table *swap_table_alloc(gfp_t gfp)
-{
- struct folio *folio;
-
- if (!SWP_TABLE_USE_PAGE)
- return kmem_cache_zalloc(swap_table_cachep, gfp);
-
- folio = folio_alloc(gfp | __GFP_ZERO, 0);
- if (folio)
- return folio_address(folio);
- return NULL;
-}
-
-static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
+static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head)
{
struct folio *folio;
@@ -432,15 +419,46 @@ static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
folio_put(folio);
}
-static void swap_table_free(struct swap_table *table)
+static void swap_cluster_free_table(struct swap_cluster_info *ci)
{
+ struct swap_table *table;
+
+ table = (struct swap_table *)rcu_dereference_protected(ci->table, true);
+ if (!table)
+ return;
+
+ rcu_assign_pointer(ci->table, NULL);
if (!SWP_TABLE_USE_PAGE) {
kmem_cache_free(swap_table_cachep, table);
return;
}
call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
- swap_table_free_folio_rcu_cb);
+ swap_cluster_free_table_folio_rcu_cb);
+}
+
+static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
+{
+ struct swap_table *table = NULL;
+ struct folio *folio;
+
+ /* The cluster must be empty and not on any list during allocation. */
+ VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
+ if (rcu_access_pointer(ci->table))
+ return 0;
+
+ if (SWP_TABLE_USE_PAGE) {
+ folio = folio_alloc(gfp | __GFP_ZERO, 0);
+ if (folio)
+ table = folio_address(folio);
+ } else {
+ table = kmem_cache_zalloc(swap_table_cachep, gfp);
+ }
+ if (!table)
+ return -ENOMEM;
+
+ rcu_assign_pointer(ci->table, table);
+ return 0;
}
/*
@@ -471,27 +489,15 @@ static void swap_cluster_assert_empty(struct swap_cluster_info *ci,
WARN_ON_ONCE(nr == SWAPFILE_CLUSTER && ci->extend_table);
}
-static void swap_cluster_free_table(struct swap_cluster_info *ci)
-{
- struct swap_table *table;
-
- /* Only empty cluster's table is allow to be freed */
- lockdep_assert_held(&ci->lock);
- table = (void *)rcu_dereference_protected(ci->table, true);
- rcu_assign_pointer(ci->table, NULL);
-
- swap_table_free(table);
-}
-
/*
* Allocate swap table for one cluster. Attempt an atomic allocation first,
* then fallback to sleeping allocation.
*/
static struct swap_cluster_info *
-swap_cluster_alloc_table(struct swap_info_struct *si,
+swap_cluster_populate(struct swap_info_struct *si,
struct swap_cluster_info *ci)
{
- struct swap_table *table;
+ int ret;
/*
* Only cluster isolation from the allocator does table allocation.
@@ -502,14 +508,9 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
lockdep_assert_held(&si->global_cluster_lock);
lockdep_assert_held(&ci->lock);
- /* The cluster must be free and was just isolated from the free list. */
- VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
-
- table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
- if (table) {
- rcu_assign_pointer(ci->table, table);
+ if (!swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
+ __GFP_NOWARN))
return ci;
- }
/*
* Try a sleep allocation. Each isolated free cluster may cause
@@ -521,7 +522,8 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
spin_unlock(&si->global_cluster_lock);
local_unlock(&percpu_swap_cluster.lock);
- table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | GFP_KERNEL);
+ ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
+ GFP_KERNEL);
/*
* Back to atomic context. We might have migrated to a new CPU with a
@@ -536,20 +538,11 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
spin_lock(&si->global_cluster_lock);
spin_lock(&ci->lock);
- /* Nothing except this helper should touch a dangling empty cluster. */
- if (WARN_ON_ONCE(cluster_table_is_alloced(ci))) {
- if (table)
- swap_table_free(table);
- return ci;
- }
-
- if (!table) {
+ if (ret) {
move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
spin_unlock(&ci->lock);
return NULL;
}
-
- rcu_assign_pointer(ci->table, table);
return ci;
}
@@ -621,12 +614,11 @@ static struct swap_cluster_info *isolate_lock_cluster(
}
spin_unlock(&si->lock);
- if (found && !cluster_table_is_alloced(found)) {
- /* Only an empty free cluster's swap table can be freed. */
- VM_WARN_ON_ONCE(flags != CLUSTER_FLAG_FREE);
+ /* Cluster's table is freed when and only when it's on the free list. */
+ if (found && flags == CLUSTER_FLAG_FREE) {
VM_WARN_ON_ONCE(list != &si->free_clusters);
- VM_WARN_ON_ONCE(!cluster_is_empty(found));
- return swap_cluster_alloc_table(si, found);
+ VM_WARN_ON_ONCE(cluster_table_is_alloced(found));
+ return swap_cluster_populate(si, found);
}
return found;
@@ -769,7 +761,6 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
unsigned int ci_off = offset % SWAPFILE_CLUSTER;
unsigned long idx = offset / SWAPFILE_CLUSTER;
struct swap_cluster_info *ci;
- struct swap_table *table;
int ret = 0;
/* si->max may got shrunk by swap swap_activate() */
@@ -790,12 +781,9 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
}
ci = cluster_info + idx;
- if (!ci->table) {
- table = swap_table_alloc(GFP_KERNEL);
- if (!table)
- return -ENOMEM;
- rcu_assign_pointer(ci->table, table);
- }
+ /* Need to allocate swap table first for initial bad slot marking. */
+ if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL))
+ return -ENOMEM;
spin_lock(&ci->lock);
/* Check for duplicated bad swap slots. */
if (__swap_table_xchg(ci, ci_off, SWP_TB_BAD) != SWP_TB_NULL) {
@@ -3054,7 +3042,7 @@ static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
ci = cluster_info + i;
/* Cluster with bad marks count will have a remaining table */
spin_lock(&ci->lock);
- if (rcu_dereference_protected(ci->table, true)) {
+ if (cluster_table_is_alloced(ci)) {
swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, true);
swap_cluster_free_table(ci);
}
--
2.54.0
^ permalink raw reply related
* [PATCH v4 06/12] mm/memcg, swap: tidy up cgroup v1 memsw swap helpers
From: Kairui Song via B4 Relay @ 2026-05-15 9:54 UTC (permalink / raw)
To: linux-mm
Cc: Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, linux-kernel, cgroups, Kairui Song,
Lorenzo Stoakes, Yosry Ahmed, Qi Zheng
In-Reply-To: <20260515-swap-table-p4-v4-0-f1b49e845a8d@tencent.com>
From: Kairui Song <kasong@tencent.com>
The cgroup v1 swap helpers always operate on swap cache folios whose
swap entry is stable: the folio is locked and in the swap cache. There
is no need to pass the swap entry or page count as separate parameters
when they can be derived from the folio itself.
Simplify the redundant parameters and add sanity checks to document
the required preconditions.
Also rename memcg1_swapout to __memcg1_swapout to indicate it requires
special calling context: the folio must be isolated and dying, and the
call must be made with interrupts disabled.
No functional change.
Acked-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/memcontrol.h | 8 ++++----
include/linux/swap.h | 10 ++++------
mm/huge_memory.c | 2 +-
mm/memcontrol-v1.c | 33 ++++++++++++++++++++-------------
mm/memcontrol.c | 9 ++++-----
mm/swap_state.c | 4 ++--
mm/swapfile.c | 2 +-
mm/vmscan.c | 2 +-
8 files changed, 37 insertions(+), 33 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index dc3fa687759b..7d08128de1fd 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -1899,8 +1899,8 @@ static inline void mem_cgroup_exit_user_fault(void)
current->in_user_fault = 0;
}
-void memcg1_swapout(struct folio *folio, swp_entry_t entry);
-void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages);
+void __memcg1_swapout(struct folio *folio);
+void memcg1_swapin(struct folio *folio);
#else /* CONFIG_MEMCG_V1 */
static inline
@@ -1929,11 +1929,11 @@ static inline void mem_cgroup_exit_user_fault(void)
{
}
-static inline void memcg1_swapout(struct folio *folio, swp_entry_t entry)
+static inline void __memcg1_swapout(struct folio *folio)
{
}
-static inline void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages)
+static inline void memcg1_swapin(struct folio *folio)
{
}
diff --git a/include/linux/swap.h b/include/linux/swap.h
index aa89e1d30a77..6b3acdf9bdd4 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -576,13 +576,12 @@ static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
#endif
#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
-int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry);
-static inline int mem_cgroup_try_charge_swap(struct folio *folio,
- swp_entry_t entry)
+int __mem_cgroup_try_charge_swap(struct folio *folio);
+static inline int mem_cgroup_try_charge_swap(struct folio *folio)
{
if (mem_cgroup_disabled())
return 0;
- return __mem_cgroup_try_charge_swap(folio, entry);
+ return __mem_cgroup_try_charge_swap(folio);
}
extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
@@ -596,8 +595,7 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p
extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
extern bool mem_cgroup_swap_full(struct folio *folio);
#else
-static inline int mem_cgroup_try_charge_swap(struct folio *folio,
- swp_entry_t entry)
+static inline int mem_cgroup_try_charge_swap(struct folio *folio)
{
return 0;
}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index c565b2a651e0..42b86e8ab7c0 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -4430,7 +4430,7 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped)
/*
* Exclude swapcache: originally to avoid a corrupt deferred split
- * queue. Nowadays that is fully prevented by memcg1_swapout();
+ * queue. Nowadays that is fully prevented by __memcg1_swapout();
* but if page reclaim is already handling the same folio, it is
* unnecessary to handle it again in the shrinker, so excluding
* swapcache here may still be a useful optimization.
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 433bba9dfe71..36c507d81dc5 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -604,18 +604,23 @@ void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
}
/**
- * memcg1_swapout - transfer a memsw charge to swap
+ * __memcg1_swapout - transfer a memsw charge to swap
* @folio: folio whose memsw charge to transfer
- * @entry: swap entry to move the charge to
*
- * Transfer the memsw charge of @folio to @entry.
+ * Transfer the memsw charge of @folio to the swap entry stored in
+ * folio->swap.
+ *
+ * Context: folio must be isolated, unmapped, locked and is just about
+ * to be freed, and caller must disable IRQs.
*/
-void memcg1_swapout(struct folio *folio, swp_entry_t entry)
+void __memcg1_swapout(struct folio *folio)
{
struct mem_cgroup *memcg, *swap_memcg;
struct obj_cgroup *objcg;
unsigned int nr_entries;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
@@ -641,7 +646,7 @@ void memcg1_swapout(struct folio *folio, swp_entry_t entry)
swap_memcg = mem_cgroup_private_id_get_online(memcg, nr_entries);
mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
- swap_cgroup_record(folio, mem_cgroup_private_id(swap_memcg), entry);
+ swap_cgroup_record(folio, mem_cgroup_private_id(swap_memcg), folio->swap);
folio_unqueue_deferred_split(folio);
folio->memcg_data = 0;
@@ -671,18 +676,20 @@ void memcg1_swapout(struct folio *folio, swp_entry_t entry)
obj_cgroup_put(objcg);
}
-/*
+/**
* memcg1_swapin - uncharge swap slot
- * @entry: the first swap entry for which the pages are charged
- * @nr_pages: number of pages which will be uncharged
+ * @folio: folio being swapped in
*
- * Call this function after successfully adding the charged page to swapcache.
+ * Call this function after successfully adding the charged
+ * folio to swapcache.
*
- * Note: This function assumes the page for which swap slot is being uncharged
- * is order 0 page.
+ * Context: The folio has to be in swap cache and locked.
*/
-void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages)
+void memcg1_swapin(struct folio *folio)
{
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
/*
* Cgroup1's unified memory+swap counter has been charged with the
* new swapcache page, finish the transfer by uncharging the swap
@@ -701,7 +708,7 @@ void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages)
* let's not wait for it. The page already received a
* memory+swap charge, drop the swap entry duplicate.
*/
- mem_cgroup_uncharge_swap(entry, nr_pages);
+ mem_cgroup_uncharge_swap(folio->swap, folio_nr_pages(folio));
}
}
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index d978e18b9b2d..a28a68eed7ba 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -5464,13 +5464,12 @@ int __init mem_cgroup_init(void)
/**
* __mem_cgroup_try_charge_swap - try charging swap space for a folio
* @folio: folio being added to swap
- * @entry: swap entry to charge
*
- * Try to charge @folio's memcg for the swap space at @entry.
+ * Try to charge @folio's memcg for the swap space at folio->swap.
*
* Returns 0 on success, -ENOMEM on failure.
*/
-int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)
+int __mem_cgroup_try_charge_swap(struct folio *folio)
{
unsigned int nr_pages = folio_nr_pages(folio);
struct page_counter *counter;
@@ -5487,7 +5486,7 @@ int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)
rcu_read_lock();
memcg = obj_cgroup_memcg(objcg);
- if (!entry.val) {
+ if (!folio_test_swapcache(folio)) {
memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
rcu_read_unlock();
return 0;
@@ -5506,7 +5505,7 @@ int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)
}
mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
- swap_cgroup_record(folio, mem_cgroup_private_id(memcg), entry);
+ swap_cgroup_record(folio, mem_cgroup_private_id(memcg), folio->swap);
return 0;
}
diff --git a/mm/swap_state.c b/mm/swap_state.c
index f177c4b3ea7a..cdb7859eb502 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -455,8 +455,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
return ERR_PTR(-ENOMEM);
}
- /* For memsw accounting, swap is uncharged when folio is added to swap cache */
- memcg1_swapin(entry, 1 << order);
+ /* memsw uncharges swap when folio is added to swap cache */
+ memcg1_swapin(folio);
if (shadow)
workingset_refault(folio, shadow);
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 4e5a54769e81..5c8bb15719bf 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1731,7 +1731,7 @@ int folio_alloc_swap(struct folio *folio)
}
/* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */
- if (unlikely(mem_cgroup_try_charge_swap(folio, folio->swap)))
+ if (unlikely(mem_cgroup_try_charge_swap(folio)))
swap_cache_del_folio(folio);
if (unlikely(!folio_test_swapcache(folio)))
diff --git a/mm/vmscan.c b/mm/vmscan.c
index b3e555561417..924c84326551 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -737,7 +737,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
if (reclaimed && !mapping_exiting(mapping))
shadow = workingset_eviction(folio, target_memcg);
- memcg1_swapout(folio, swap);
+ __memcg1_swapout(folio);
__swap_cache_del_folio(ci, folio, swap, shadow);
swap_cluster_unlock_irq(ci);
} else {
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox