* [PATCH v7 2/2] mm/swap: remove redundant swap device reference in alloc/free
From: Youngjun Park @ 2026-03-21 10:33 UTC (permalink / raw)
To: rafael, akpm
Cc: chrisl, kasong, pavel, shikemeng, nphamcs, bhe, baohua,
youngjun.park, usama.arif, linux-pm, linux-mm
In-Reply-To: <20260321103309.439265-1-youngjun.park@lge.com>
In the previous commit, uswsusp was modified to pin the swap device
when the swap type is determined, ensuring the device remains valid
throughout the hibernation I/O path.
Therefore, it is no longer necessary to repeatedly get and put the swap
device reference for each swap slot allocation and free operation.
For hibernation via the sysfs interface, user-space tasks are frozen
before swap allocation begins, so swapoff cannot race with allocation.
After resume, tasks remain frozen while swap slots are freed, so
additional reference management is not required there either.
Remove the redundant swap device get/put operations from the
hibernation swap allocation and free paths.
Also remove the SWP_WRITEOK check before allocation, as the cluster
allocation logic already validates the swap device state.
Signed-off-by: Youngjun Park <youngjun.park@lge.com>
---
mm/swapfile.c | 43 ++++++++++++++++++++-----------------------
1 file changed, 20 insertions(+), 23 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index ac1574acade7..dd9631658808 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1923,7 +1923,12 @@ void swap_put_entries_direct(swp_entry_t entry, int nr)
}
#ifdef CONFIG_HIBERNATION
-/* Allocate a slot for hibernation */
+/*
+ * Allocate a slot for hibernation.
+ *
+ * Note: The caller must ensure the swap device is stable, either by
+ * holding a reference or by freezing user-space before calling this.
+ */
swp_entry_t swap_alloc_hibernation_slot(int type)
{
struct swap_info_struct *si = swap_type_to_info(type);
@@ -1933,43 +1938,35 @@ swp_entry_t swap_alloc_hibernation_slot(int type)
if (!si)
goto fail;
- /* This is called for allocating swap entry, not cache */
- if (get_swap_device_info(si)) {
- if (si->flags & SWP_WRITEOK) {
- /*
- * Grab the local lock to be compliant
- * with swap table allocation.
- */
- local_lock(&percpu_swap_cluster.lock);
- offset = cluster_alloc_swap_entry(si, NULL);
- local_unlock(&percpu_swap_cluster.lock);
- if (offset)
- entry = swp_entry(si->type, offset);
- }
- put_swap_device(si);
- }
+ /*
+ * Grab the local lock to be compliant
+ * with swap table allocation.
+ */
+ local_lock(&percpu_swap_cluster.lock);
+ offset = cluster_alloc_swap_entry(si, NULL);
+ local_unlock(&percpu_swap_cluster.lock);
+ if (offset)
+ entry = swp_entry(si->type, offset);
fail:
return entry;
}
-/* Free a slot allocated by swap_alloc_hibernation_slot */
+/*
+ * Free a slot allocated by swap_alloc_hibernation_slot.
+ * As with allocation, the caller must ensure the swap device is stable.
+ */
void swap_free_hibernation_slot(swp_entry_t entry)
{
- struct swap_info_struct *si;
+ struct swap_info_struct *si = __swap_entry_to_info(entry);
struct swap_cluster_info *ci;
pgoff_t offset = swp_offset(entry);
- si = get_swap_device(entry);
- if (WARN_ON(!si))
- return;
-
ci = swap_cluster_lock(si, offset);
swap_put_entry_locked(si, ci, offset);
swap_cluster_unlock(ci);
/* In theory readahead might add it to the swap cache by accident */
__try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
- put_swap_device(si);
}
static int __find_hibernation_swap_type(dev_t device, sector_t offset)
--
2.34.1
^ permalink raw reply related
* [PATCH v7 1/2] mm/swap, PM: hibernate: fix swapoff race in uswsusp by pinning swap device
From: Youngjun Park @ 2026-03-21 10:33 UTC (permalink / raw)
To: rafael, akpm
Cc: chrisl, kasong, pavel, shikemeng, nphamcs, bhe, baohua,
youngjun.park, usama.arif, linux-pm, linux-mm
In-Reply-To: <20260321103309.439265-1-youngjun.park@lge.com>
Hibernation via uswsusp (/dev/snapshot ioctls) has a race window:
after selecting the resume swap area but before user space is frozen,
swapoff may run and invalidate the selected swap device.
Fix this by pinning the swap device with SWP_HIBERNATION while it is
in use. The pin is exclusive, which is sufficient since
hibernate_acquire() already prevents concurrent hibernation sessions.
The kernel swsusp path (sysfs-based hibernate/resume) uses
find_hibernation_swap_type() which is not affected by the pin. It
freezes user space before touching swap, so swapoff cannot race.
Introduce dedicated helpers:
- pin_hibernation_swap_type(): Look up and pin the swap device.
Used by the uswsusp path.
- find_hibernation_swap_type(): Lookup without pinning.
Used by the kernel swsusp path.
- unpin_hibernation_swap_type(): Clear the hibernation pin.
While a swap device is pinned, swapoff is prevented from proceeding.
Signed-off-by: Youngjun Park <youngjun.park@lge.com>
---
include/linux/swap.h | 5 +-
kernel/power/swap.c | 2 +-
kernel/power/user.c | 15 ++++-
mm/swapfile.c | 135 ++++++++++++++++++++++++++++++++++++++-----
4 files changed, 136 insertions(+), 21 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 62fc7499b408..82bfc965c3f8 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -216,6 +216,7 @@ enum {
SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */
+ SWP_HIBERNATION = (1 << 13), /* pinned for hibernation */
/* add others here before... */
};
@@ -452,7 +453,9 @@ static inline long get_nr_swap_pages(void)
extern void si_swapinfo(struct sysinfo *);
extern int add_swap_count_continuation(swp_entry_t, gfp_t);
-int swap_type_of(dev_t device, sector_t offset);
+int pin_hibernation_swap_type(dev_t device, sector_t offset);
+void unpin_hibernation_swap_type(int type);
+int find_hibernation_swap_type(dev_t device, sector_t offset);
int find_first_swap(dev_t *device);
extern unsigned int count_swap_pages(int, int);
extern sector_t swapdev_block(int, pgoff_t);
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 2e64869bb5a0..cc4764149e8f 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -341,7 +341,7 @@ static int swsusp_swap_check(void)
* This is called before saving the image.
*/
if (swsusp_resume_device)
- res = swap_type_of(swsusp_resume_device, swsusp_resume_block);
+ res = find_hibernation_swap_type(swsusp_resume_device, swsusp_resume_block);
else
res = find_first_swap(&swsusp_resume_device);
if (res < 0)
diff --git a/kernel/power/user.c b/kernel/power/user.c
index 4401cfe26e5c..aab9aece1009 100644
--- a/kernel/power/user.c
+++ b/kernel/power/user.c
@@ -71,7 +71,7 @@ static int snapshot_open(struct inode *inode, struct file *filp)
memset(&data->handle, 0, sizeof(struct snapshot_handle));
if ((filp->f_flags & O_ACCMODE) == O_RDONLY) {
/* Hibernating. The image device should be accessible. */
- data->swap = swap_type_of(swsusp_resume_device, 0);
+ data->swap = pin_hibernation_swap_type(swsusp_resume_device, 0);
data->mode = O_RDONLY;
data->free_bitmaps = false;
error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
@@ -90,8 +90,10 @@ static int snapshot_open(struct inode *inode, struct file *filp)
data->free_bitmaps = !error;
}
}
- if (error)
+ if (error) {
+ unpin_hibernation_swap_type(data->swap);
hibernate_release();
+ }
data->frozen = false;
data->ready = false;
@@ -115,6 +117,7 @@ static int snapshot_release(struct inode *inode, struct file *filp)
data = filp->private_data;
data->dev = 0;
free_all_swap_pages(data->swap);
+ unpin_hibernation_swap_type(data->swap);
if (data->frozen) {
pm_restore_gfp_mask();
free_basic_memory_bitmaps();
@@ -235,11 +238,17 @@ static int snapshot_set_swap_area(struct snapshot_data *data,
offset = swap_area.offset;
}
+ /*
+ * Pin the swap device if a swap area was already
+ * set by SNAPSHOT_SET_SWAP_AREA.
+ */
+ unpin_hibernation_swap_type(data->swap);
+
/*
* User space encodes device types as two-byte values,
* so we need to recode them
*/
- data->swap = swap_type_of(swdev, offset);
+ data->swap = pin_hibernation_swap_type(swdev, offset);
if (data->swap < 0)
return swdev ? -ENODEV : -EINVAL;
data->dev = swdev;
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 94af29d1de88..ac1574acade7 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -133,7 +133,7 @@ static DEFINE_PER_CPU(struct percpu_swap_cluster, percpu_swap_cluster) = {
/* May return NULL on invalid type, caller must check for NULL return */
static struct swap_info_struct *swap_type_to_info(int type)
{
- if (type >= MAX_SWAPFILES)
+ if (type < 0 || type >= MAX_SWAPFILES)
return NULL;
return READ_ONCE(swap_info[type]); /* rcu_dereference() */
}
@@ -1972,22 +1972,15 @@ void swap_free_hibernation_slot(swp_entry_t entry)
put_swap_device(si);
}
-/*
- * Find the swap type that corresponds to given device (if any).
- *
- * @offset - number of the PAGE_SIZE-sized block of the device, starting
- * from 0, in which the swap header is expected to be located.
- *
- * This is needed for the suspend to disk (aka swsusp).
- */
-int swap_type_of(dev_t device, sector_t offset)
+static int __find_hibernation_swap_type(dev_t device, sector_t offset)
{
int type;
+ lockdep_assert_held(&swap_lock);
+
if (!device)
- return -1;
+ return -EINVAL;
- spin_lock(&swap_lock);
for (type = 0; type < nr_swapfiles; type++) {
struct swap_info_struct *sis = swap_info[type];
@@ -1997,16 +1990,118 @@ int swap_type_of(dev_t device, sector_t offset)
if (device == sis->bdev->bd_dev) {
struct swap_extent *se = first_se(sis);
- if (se->start_block == offset) {
- spin_unlock(&swap_lock);
+ if (se->start_block == offset)
return type;
- }
}
}
- spin_unlock(&swap_lock);
return -ENODEV;
}
+/**
+ * pin_hibernation_swap_type - Pin the swap device for hibernation
+ * @device: Block device containing the resume image
+ * @offset: Offset identifying the swap area
+ *
+ * Locate the swap device for @device/@offset and mark it as pinned
+ * for hibernation. While pinned, swapoff() is prevented.
+ *
+ * Only one uswsusp context may pin a swap device at a time.
+ * If already pinned, this function returns -EBUSY.
+ *
+ * Return:
+ * >= 0 on success (swap type).
+ * -EINVAL if @device is invalid.
+ * -ENODEV if the swap device is not found.
+ * -EBUSY if the device is already pinned for hibernation.
+ */
+int pin_hibernation_swap_type(dev_t device, sector_t offset)
+{
+ int type;
+ struct swap_info_struct *si;
+
+ spin_lock(&swap_lock);
+
+ type = __find_hibernation_swap_type(device, offset);
+ if (type < 0) {
+ spin_unlock(&swap_lock);
+ return type;
+ }
+
+ si = swap_type_to_info(type);
+ if (WARN_ON_ONCE(!si)) {
+ spin_unlock(&swap_lock);
+ return -ENODEV;
+ }
+
+ /*
+ * hibernate_acquire() prevents concurrent hibernation sessions.
+ * This check additionally guards against double-pinning within
+ * the same session.
+ */
+ if (WARN_ON_ONCE(si->flags & SWP_HIBERNATION)) {
+ spin_unlock(&swap_lock);
+ return -EBUSY;
+ }
+
+ si->flags |= SWP_HIBERNATION;
+
+ spin_unlock(&swap_lock);
+ return type;
+}
+
+/**
+ * unpin_hibernation_swap_type - Unpin the swap device for hibernation
+ * @type: Swap type previously returned by pin_hibernation_swap_type()
+ *
+ * Clear the hibernation pin on the given swap device, allowing
+ * swapoff() to proceed normally.
+ *
+ * If @type does not refer to a valid swap device, this function
+ * does nothing.
+ */
+void unpin_hibernation_swap_type(int type)
+{
+ struct swap_info_struct *si;
+
+ spin_lock(&swap_lock);
+ si = swap_type_to_info(type);
+ if (!si) {
+ spin_unlock(&swap_lock);
+ return;
+ }
+ si->flags &= ~SWP_HIBERNATION;
+ spin_unlock(&swap_lock);
+}
+
+/**
+ * find_hibernation_swap_type - Find swap type for hibernation
+ * @device: Block device containing the resume image
+ * @offset: Offset within the device identifying the swap area
+ *
+ * Locate the swap device corresponding to @device and @offset.
+ *
+ * Unlike pin_hibernation_swap_type(), this function only performs a
+ * lookup and does not mark the swap device as pinned for hibernation.
+ *
+ * This is safe in the sysfs-based hibernation path where user space
+ * is already frozen and swapoff() cannot run concurrently.
+ *
+ * Return:
+ * A non-negative swap type on success.
+ * -EINVAL if @device is invalid.
+ * -ENODEV if no matching swap device is found.
+ */
+int find_hibernation_swap_type(dev_t device, sector_t offset)
+{
+ int type;
+
+ spin_lock(&swap_lock);
+ type = __find_hibernation_swap_type(device, offset);
+ spin_unlock(&swap_lock);
+
+ return type;
+}
+
int find_first_swap(dev_t *device)
{
int type;
@@ -2803,6 +2898,14 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
spin_unlock(&swap_lock);
goto out_dput;
}
+
+ /* Refuse swapoff while the device is pinned for hibernation */
+ if (p->flags & SWP_HIBERNATION) {
+ err = -EBUSY;
+ spin_unlock(&swap_lock);
+ goto out_dput;
+ }
+
if (!security_vm_enough_memory_mm(current->mm, p->pages))
vm_unacct_memory(p->pages);
else {
--
2.34.1
^ permalink raw reply related
* [PATCH v7 0/2] mm/swap, PM: hibernate: fix swapoff race and optimize swap
From: Youngjun Park @ 2026-03-21 10:33 UTC (permalink / raw)
To: rafael, akpm
Cc: chrisl, kasong, pavel, shikemeng, nphamcs, bhe, baohua,
youngjun.park, usama.arif, linux-pm, linux-mm
Apologies for the frequent revisions. Hopefully this version is close to final.
Currently, in the uswsusp path, only the swap type value is retrieved at
lookup time without holding a reference. If swapoff races after the type
is acquired, subsequent slot allocations operate on a stale swap device.
Additionally, grabbing and releasing the swap device reference on every
slot allocation is inefficient across the entire hibernation swap path.
This patch series addresses these issues:
- Patch 1: Fixes the swapoff race in uswsusp by pinning the swap device
from the point it is looked up until the session completes.
- Patch 2: Removes the overhead of per-slot reference counting in alloc/free
paths and cleans up the redundant SWP_WRITEOK check.
This series is based on v7.0-rc4(Refael's request for PM's modification)
. Happy to rebase onto mm-new if needed.
Links:
RFC v1: https://lore.kernel.org/linux-mm/20260305202413.1888499-1-usama.arif@linux.dev/T/#m3693d45180f14f441b6951984f4b4bfd90ec0c9d
RFC v2: https://lore.kernel.org/linux-mm/20260306024608.1720991-1-youngjun.park@lge.com/
RFC v3: https://lore.kernel.org/linux-mm/20260312112511.3596781-1-youngjun.park@lge.com/
v4: https://lore.kernel.org/linux-mm/abv+rjgyArqZ2uym@yjaykim-PowerEdge-T330/T/#m924fa3e58d0f0da488300653163ee8db7e870e4a
v5: https://lore.kernel.org/linux-mm/ab0YEn+Fd41q6LM7@yjaykim-PowerEdge-T330/T/#m8409d470c68cb152b0849940759bff7d7806f397
v6: https://lore.kernel.org/linux-mm/20260320182227.896f9ab62d62961b2caab5f7@linux-foundation.org/T/#m10ee3346cd8dcd052749105d9a8e2052dbf3bc80
Testing:
- Hibernate/resume via sysfs
(echo reboot > /sys/power/disk && echo disk > /sys/power/state)
- Hibernate with suspend via sysfs
(echo suspend > /sys/power/disk && echo disk > /sys/power/state)
- Hibernate/resume via uswsusp (suspend-utils s2disk/resume on QEMU)
- Verified swap I/O works correctly after resume.
- Verified swapoff succeeds after snapshot resume completes.
- swapoff during active uswsusp session:
- Verified swapoff returns -EBUSY while swap device is pinned (Patch 1).
- Verified swapoff succeeds after uswsusp process terminates.
Changelog:
v6 -> v7:
- Dropped Patch 3 (pm_restore_gfp_mask fix) from series as it has
no dependency on Patches 1-2. Will be sent separately.
(Rafael J. Wysocki feedback)
- Andrew Morton's AI review findings applied only to Patch 3;
Patches 1-2 are unchanged. (no problem on AI's review)
v5 -> v6:
- Replaced get/put reference approach with SWP_HIBERNATION
pinning to prevent swapoff, per Kairui's feedback. Renamed helpers
from get/find/put_hibernation_swap_type() to
pin/find/unpin_hibernation_swap_type().
- Renamed swap_type_of() to __find_hibernation_swap_type() since
it is now an internal helper with no external callers.
(Kairui's feedback)
- Removed swapoff waiting on hibernation reference.
swapoff now returns -EBUSY immediately when the swap device is
pinned.
- Updated function comments per Kairui's review.
- Updated commit message.
v4 -> v5:
- Rebased onto v7.0-rc4 (Rafael J. Wysocki comment)
- No functional changes. rebase conflict fix.
rfc v3 -> v4:
- Introduced get/find/put_hibernation_swap_type() helpers per Kairui's
feedback. find_ for lookup-only, get/put for reference management.
- Switched to swap_type_to_info() and added type < 0 check per
Kairui's suggestion.
- Fixed get_hibernation_swap_type() return when ref == false (Reviewed by Kairui)
- Made swapoff wait interruptible to prevent hang when uswsusp
holds a swap reference.
- Rebased onto latest mm-new tree.
rfc v2 -> rfc v3:
- Split into 2 patches per Chris Li's feedback.
- Simplified by not holding reference in normal hibernation path
per Chris Li's suggestion.
- Removed redundant SWP_WRITEOK check.
- Rebased onto f543926f9d0c3f6dfb354adfe7fbaeedd1277c6b.
rfc v1 -> rfc v2:
- Squashed into single patch per Usama Arif's feedback.
Youngjun Park (2):
mm/swap, PM: hibernate: fix swapoff race in uswsusp by pinning swap
device
mm/swap: remove redundant swap device reference in alloc/free
include/linux/swap.h | 5 +-
kernel/power/swap.c | 2 +-
kernel/power/user.c | 15 +++-
mm/swapfile.c | 178 +++++++++++++++++++++++++++++++++----------
4 files changed, 156 insertions(+), 44 deletions(-)
base-commit: f338e77383789c0cae23ca3d48adcc5e9e137e3c
--
2.34.1
^ permalink raw reply
* Re: [PATCH v2 2/8] dt-bindings: thermal: Add qcom,qmi-cooling yaml bindings
From: Daniel Lezcano @ 2026-03-21 9:00 UTC (permalink / raw)
To: Konrad Dybcio, Gaurav Kohli, Krzysztof Kozlowski
Cc: andersson, mathieu.poirier, robh, krzk+dt, conor+dt, rui.zhang,
lukasz.luba, konradybcio, mani, casey.connolly, amit.kucheria,
linux-arm-msm, devicetree, linux-kernel, linux-pm,
manaf.pallikunhi
In-Reply-To: <74f59ef0-ead7-483f-a80e-a3da2f6ebcdb@oss.qualcomm.com>
Hi Konrad,
On 3/19/26 10:51, Konrad Dybcio wrote:
> On 3/18/26 11:17 AM, Gaurav Kohli wrote:
>>
>>
>> On 3/17/2026 1:27 AM, Daniel Lezcano wrote:
>>> On Tue, Feb 24, 2026 at 01:17:22PM +0100, Krzysztof Kozlowski wrote:
>>>> On 24/02/2026 13:09, Gaurav Kohli wrote:
>>>
>>> [ ... ]
>>>
>>>>>>> As a result, each core requires its own cooling device, which must be
>>>>>>> linked to its TSENS thermal zone. Because of this, we introduced
>>>>>>> multiple child nodes—one for each cooling device.
>>>>>>
>>>>>> So you have one device with cooling cells=1+2, no?
>>>>>>
>>>>>
>>>>> This will be a bigger framework change which is not supported, i can see
>>>>
>>>> I don't think that changing open source frameworks is "not supported". I
>>>> am pretty sure that changing is not only supported, but actually desired.
>>>
>>> Yes, IMO it could make sense. There are the thermal zones with phandle
>>> to a sensor and a sensor id. We can have the same with a phandle to a
>>> cooling device and a cooling device id.
>>>
>>> (... or several ids because the thermal sensor can also have multiple
>>> ids ?)
>>>
>>> May be an array of names corresponding to the TMD names at the 'id'
>>> position ?
>>>
>>
>> I am using dt node like below to use with cooling-cells = <3> approach, will post new patches with that.
>>
>> cdsp_tmd: cdsp-tmd {
>> compatible = "qcom,qmi-cooling-cdsp";
>> tmd-names = "cdsp_sw", "cdsp_hw";
>> #cooling-cells = <3>;
>> };
>>
>> please let me know, if you are expecting something like this only.
>
> My question about the need of a separate node still remains, i.e.
> why can't this be:
>
> remoteproc_cdsp: remoteproc@cafebabe {
> compatible = "qcom,foo-cdsp"
>
> ...
>
> tmd-names = "abc", "xyz";
> #cooling-cells = <3>;
> };
>
>
>
> foo-thermal {
> cooling-maps {
> map0 {
> cooling-device = <&remoteproc_cdsp CDSP_COOLING_XYZ
> THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
> };
> };
> };
>
> where you'd presumably call something like qmi_cooling_register(...) from
> the remoteproc driver, making your added code essentially a library, not a
> separate platform device
I'm not sure to get your question. My understanding of the 3
cooling-cells is exactly what you described. The second argument of the
cooling-device map is an index corresponding to the id of the TMD. BTW I
prefer also the compatible name like 'qcom,foo-cdsp'
^ permalink raw reply
* [PATCH] PM: hibernate: call preallocate_image after freeze prepare
From: Matthew Leach @ 2026-03-21 8:51 UTC (permalink / raw)
To: Rafael J. Wysocki, Pavel Machek, Len Brown, Mario Limonciello
Cc: linux-pm, linux-kernel, kernel, Matthew Leach
Certain drivers release resources (pinned pages, etc.) into system
memory during the prepare freeze PM op, making them swappable.
Currently, hibernate_preallocate_memory is called before prepare freeze,
so those drivers have no opportunity to release resources first. If a
driver is holding a large amount of unswappable system RAM, this can
cause hibernate_preallocate_memory to fail.
Move the call to hibernate_preallocate_memory after prepare freeze.
According to the documentation for the prepare callback, devices should
be left in a usable state, so storage drivers should still be able to
service I/O requests. This allows drivers to release unswappable
resources prior to preallocation, so they can be swapped out through
hibernate_preallocate_memory's reclaim path.
Signed-off-by: Matthew Leach <matthew.leach@collabora.com>
---
kernel/power/hibernate.c | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index af8d07bafe02..ff62e220ded1 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -425,14 +425,9 @@ int hibernation_snapshot(int platform_mode)
if (error)
goto Close;
- /* Preallocate image memory before shutting down devices. */
- error = hibernate_preallocate_memory();
- if (error)
- goto Close;
-
error = freeze_kernel_threads();
if (error)
- goto Cleanup;
+ goto Close;
if (hibernation_test(TEST_FREEZER)) {
@@ -450,6 +445,11 @@ int hibernation_snapshot(int platform_mode)
goto Thaw;
}
+ /* Preallocate image memory before shutting down devices. */
+ error = hibernate_preallocate_memory();
+ if (error)
+ goto Thaw;
+
/*
* Device drivers may move lots of data to shmem in dpm_prepare(). The shmem
* pages will use lots of system memory, causing hibernation image creation
@@ -494,8 +494,6 @@ int hibernation_snapshot(int platform_mode)
Thaw:
thaw_kernel_threads();
- Cleanup:
- swsusp_free();
goto Close;
}
---
base-commit: f338e77383789c0cae23ca3d48adcc5e9e137e3c
change-id: 20260321-hibernation-fixes-69bca2ee5b65
Best regards,
--
Matt
^ permalink raw reply related
* Re: [RFC PATCH 1/2] thermal/cpufreq_cooling: remove unused cpu_idx in get_load()
From: Xuewen Yan @ 2026-03-21 8:48 UTC (permalink / raw)
To: Lukasz Luba
Cc: Xuewen Yan, rui.zhang, rafael, linux-pm, viresh.kumar,
amit.kachhap, daniel.lezcano, linux-kernel, ke.wang, di.shen,
jeson.gao
In-Reply-To: <031562ee-b88f-49b9-8b1e-dbbbe1a508c6@arm.com>
On Fri, Mar 20, 2026 at 8:32 PM Lukasz Luba <lukasz.luba@arm.com> wrote:
>
> Hi Xuewen,
>
> On 3/20/26 11:31, Xuewen Yan wrote:
> > From: Di Shen <di.shen@unisoc.com>
> >
> > The cpu_idx variable in the get_load function is now
> > unused and can be safely removed.
> >
> > No code logic is affected.
> >
> > Signed-off-by: Di Shen <di.shen@unisoc.com>
> > ---
> > drivers/thermal/cpufreq_cooling.c | 13 +++++--------
> > 1 file changed, 5 insertions(+), 8 deletions(-)
> >
> > diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c
> > index 32bf5ab44f4a..d030dbeb2973 100644
> > --- a/drivers/thermal/cpufreq_cooling.c
> > +++ b/drivers/thermal/cpufreq_cooling.c
> > @@ -151,26 +151,23 @@ static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev,
> > * get_load() - get load for a cpu
> > * @cpufreq_cdev: struct cpufreq_cooling_device for the cpu
> > * @cpu: cpu number
> > - * @cpu_idx: index of the cpu in time_in_idle array
> > *
> > * Return: The average load of cpu @cpu in percentage since this
> > * function was last called.
> > */
> > #ifdef CONFIG_SMP
> > -static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu,
> > - int cpu_idx)
> > +static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu)
> > {
> > unsigned long util = sched_cpu_util(cpu);
> >
> > return (util * 100) / arch_scale_cpu_capacity(cpu);
> > }
> > #else /* !CONFIG_SMP */
> > -static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu,
> > - int cpu_idx)
> > +static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu)
> > {
> > u32 load;
> > u64 now, now_idle, delta_time, delta_idle;
> > - struct time_in_idle *idle_time = &cpufreq_cdev->idle_time[cpu_idx];
> > + struct time_in_idle *idle_time = &cpufreq_cdev->idle_time[cpu];
>
> This is a bug. We allocate 'num_cpus' size of array based on
> number of CPU in the cpumask for a given cpufreq policy.
> If there are 4 cpus in the CPU cluster but CPUs have ids:
> CPU4-7 then accessing it with this change would explode.
Sorry, it was our oversight. However, this is fine without the other patch.
For patch-v2, maybe we can add back i++ to work around this issue.
Thanks!
>
> Please re-design this patch set slightly and I will have a look
> on the 2nd version (and particularly the part in current patch 2/2).
>
> Regards,
> Lukasz
^ permalink raw reply
* Re: [PATCH 6.12.y 1/6] cpuidle: menu: Drop a redundant local variable
From: Greg KH @ 2026-03-21 8:24 UTC (permalink / raw)
To: Ionut Nechita
Cc: stable, rafael.j.wysocki, linux-pm, christian.loehle,
artem.bityutskiy, quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-2-ionut.nechita@windriver.com>
On Fri, Mar 20, 2026 at 10:29:03PM +0200, Ionut Nechita wrote:
> From: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
>
> Local variable min in get_typical_interval() is updated, but never
> accessed later, so drop it.
>
> No functional impact.
>
> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
> Reviewed-by: Christian Loehle <christian.loehle@arm.com>
> Tested-by: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
> Tested-by: Christian Loehle <christian.loehle@arm.com>
> Tested-by: Aboorva Devarajan <aboorvad@linux.ibm.com>
> Link: https://patch.msgid.link/13699686.uLZWGnKmhe@rjwysocki.net
> ---
> drivers/cpuidle/governors/menu.c | 6 +-----
> 1 file changed, 1 insertion(+), 5 deletions(-)
Again, you forgot the original git id. Please redo the series with that
information.
And you all know this, I went through "how to properly backport patches
to stable" a lot with your team, this needs to be reviewed by someone
else at windriver before it can be accepted as you also messed up on
something else here that you should have gotten correct :(
thanks,
greg k-h
^ permalink raw reply
* Re: [PATCH v1 1/1] devfreq: tegra30-devfreq: add support for Tegra114
From: Chanwoo Choi @ 2026-03-21 7:10 UTC (permalink / raw)
To: myungjoo.ham
Cc: Dmitry Osipenko, Svyatoslav Ryhel, Kyungmin Park, Chanwoo Choi,
Thierry Reding, Jonathan Hunter, Mikko Perttunen, linux-pm,
linux-tegra, linux-kernel
In-Reply-To: <CAJ0PZbS6JhPOMNP3fB4biiu3b32jF-A6AMQ6M7usOEG0B88JJw@mail.gmail.com>
Hi,
I'm sorry for late reply.
Applied it. Thanks.
On Sun, Mar 8, 2026 at 5:02 PM MyungJoo Ham <myungjoo.ham@samsung.com> wrote:
>
> 2026년 3월 3일 (화) AM 8:24, Dmitry Osipenko <digetx@gmail.com>님이 작성:
> >
> > 26.01.2026 21:54, Svyatoslav Ryhel пишет:
> > > Lets add Tegra114 support to activity monitor device as a preparation to
> > > upcoming EMC controller support.
> > >
> > > Signed-off-by: Svyatoslav Ryhel <clamor95@gmail.com>
> > > Reviewed-by: Mikko Perttunen <mperttunen@nvidia.com>
> > > ---
> > > drivers/devfreq/tegra30-devfreq.c | 17 ++++++++++++-----
> > > 1 file changed, 12 insertions(+), 5 deletions(-)
> > >
> > > diff --git a/drivers/devfreq/tegra30-devfreq.c b/drivers/devfreq/tegra30-devfreq.c
> > > index 8ea5b482bfb3..fa83480a923f 100644
> > > --- a/drivers/devfreq/tegra30-devfreq.c
> > > +++ b/drivers/devfreq/tegra30-devfreq.c
> > > @@ -963,16 +963,22 @@ static int tegra_devfreq_probe(struct platform_device *pdev)
> > > return 0;
> > > }
> > >
> > > +/*
> > > + * The activity counter is incremented every 256 memory transactions. However,
> > > + * the number of clock cycles required for each transaction varies across
> > > + * different SoC generations. For instance, a single transaction takes 2 EMC
> > > + * clocks on Tegra30, 1 EMC clock on Tegra114, and 4 EMC clocks on Tegra124.
> > > + */
> > > static const struct tegra_devfreq_soc_data tegra124_soc = {
> > > .configs = tegra124_device_configs,
> > > -
> > > - /*
> > > - * Activity counter is incremented every 256 memory transactions,
> > > - * and each transaction takes 4 EMC clocks.
> > > - */
> > > .count_weight = 4 * 256,
> > > };
> > >
> > > +static const struct tegra_devfreq_soc_data tegra114_soc = {
> > > + .configs = tegra124_device_configs,
> > > + .count_weight = 256,
> > > +};
> > > +
> > > static const struct tegra_devfreq_soc_data tegra30_soc = {
> > > .configs = tegra30_device_configs,
> > > .count_weight = 2 * 256,
> > > @@ -980,6 +986,7 @@ static const struct tegra_devfreq_soc_data tegra30_soc = {
> > >
> > > static const struct of_device_id tegra_devfreq_of_match[] = {
> > > { .compatible = "nvidia,tegra30-actmon", .data = &tegra30_soc, },
> > > + { .compatible = "nvidia,tegra114-actmon", .data = &tegra114_soc, },
> > > { .compatible = "nvidia,tegra124-actmon", .data = &tegra124_soc, },
> > > { },
> > > };
> >
> > Acked-by: Dmitry Osipenko <digetx@gmail.com>
>
> Acked-by: MyungJoo Ham <myungjoo.ham@gmail.com>
>
> --
> MyungJoo Ham, Ph.D.
> Samsung Electronics
>
--
Best Regards,
Chanwoo Choi
Samsung Electronics
^ permalink raw reply
* Re: [PATCH v6 0/3] mm/swap, PM: hibernate: fix swapoff race and optimize swap
From: Andrew Morton @ 2026-03-21 1:22 UTC (permalink / raw)
To: Youngjun Park
Cc: rafael, chrisl, kasong, pavel, shikemeng, nphamcs, bhe, baohua,
usama.arif, linux-pm, linux-mm
In-Reply-To: <20260320170313.163386-1-youngjun.park@lge.com>
On Sat, 21 Mar 2026 02:03:10 +0900 Youngjun Park <youngjun.park@lge.com> wrote:
> Currently, in the uswsusp path, only the swap type value is retrieved at
> lookup time without holding a reference. If swapoff races after the type
> is acquired, subsequent slot allocations operate on a stale swap device.
>
> Additionally, grabbing and releasing the swap device reference on every
> slot allocation is inefficient across the entire hibernation swap path.
AI review has a couple of questions:
https://sashiko.dev/#/patchset/20260320170313.163386-1-youngjun.park@lge.com
^ permalink raw reply
* Re: [PATCH v1 2/2] PM: Add config flag to gate sysfs wakeup_sources
From: Samuel Wu @ 2026-03-21 0:46 UTC (permalink / raw)
To: Rafael J. Wysocki
Cc: Pavel Machek, Len Brown, Greg Kroah-Hartman, Danilo Krummrich,
andrii, memxor, bpf, kernel-team, linux-pm, driver-core,
linux-kernel
In-Reply-To: <CAJZ5v0j-DWOe63meNY0eOCuT3US7Oqd+0M_hStmBO_nPnkA-FA@mail.gmail.com>
On Fri, Mar 20, 2026 at 9:09 AM Rafael J. Wysocki <rafael@kernel.org> wrote:
>
[ ... ]
> >
> > +config PM_WAKEUP_STATS_SYSFS
> > + bool "Sysfs wakeup statistics"
> > + depends on PM_SLEEP
> > + default y
> > + help
> > + Enable this for wakeup statistics in sysfs under /sys/class/wakeup/
> > +
> > + Disabling this option eliminates the work of creating the wakeup
> > + sources and each of their attributes in sysfs. Depending on the
> > + number of wakeup sources, this can also have a non-negligible memory
> > + impact. Regardless of this config option's value, wakeup statistics
> > + are still available via debugfs and BPF.
>
> Well, except that otherwise it may be hard to figure out which device
> the given wakeup source belongs to, so I'd rather not do that.
>
Since the config flag defaults to yes, this wouldn't affect current
users right? And then the idea is that other users who don't need this
sysfs node can turn the flag off to save some cycles and memory.
[ ... ]
^ permalink raw reply
* Re: [PATCH v3 2/4] serdev: add rust private data to serdev_device
From: Markus Probst @ 2026-03-20 21:08 UTC (permalink / raw)
To: Danilo Krummrich
Cc: Greg Kroah-Hartman, Rob Herring, Jiri Slaby, Miguel Ojeda,
Gary Guo, Björn Roy Baron, Benno Lossin, Andreas Hindborg,
Alice Ryhl, Trevor Gross, Kari Argillander, Rafael J. Wysocki,
Viresh Kumar, Boqun Feng, David Airlie, Simona Vetter,
linux-serial, linux-kernel, rust-for-linux, linux-pm, driver-core,
dri-devel
In-Reply-To: <b8465b11-7a4a-4842-b292-6e3659bd95b3@kernel.org>
[-- Attachment #1: Type: text/plain, Size: 1356 bytes --]
On Fri, 2026-03-20 at 20:59 +0100, Danilo Krummrich wrote:
> On 3/20/26 6:13 PM, Markus Probst wrote:
> > This is unique for a bus device.
> >
> > A class device has its own Data (e. g. PwmOps), i. e. it will only be
> > registered after this Data has been initialized by the driver.
> >
> > The receive_buf callback on the serdev device on the other hand must
> > happen on the drvdata, as there is no place to store its own Data.
> >
> > The drvdata is only available at the end of the probe in Rust.
>
> In other words, devm_serdev_device_open() relies on the driver having its
> private data set before this is called, since once devm_serdev_device_open() has
> been called, the driver's receive_buf() callback may be called.
>
> So, in C it is the driver's responsibility to do things in the correct order, in
> Rust we want to make sure the driver can't do it out of order in the first place.
>
> I'm still not sure whether the current approach is the best option. For
> instance, we could also solve this with a separate callback.
>
> Wasn't this even
> what you had in a previous version?
Yes, but Kari pointed out that changing the ops while it is in use is
unsafe [1].
Thanks
- Markus Probst
[1]
https://lore.kernel.org/rust-for-linux/CAC=eVgR2WYdDTW3kOeemyQPP-H0aAUsrzn5Gk5zCe2hQEB709w@mail.gmail.com/
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 870 bytes --]
^ permalink raw reply
* Re: [PATCH v7 3/7] dax/cxl, hmem: Initialize hmem early and defer dax_cxl binding
From: Koralahalli Channabasappa, Smita @ 2026-03-20 20:42 UTC (permalink / raw)
To: Dan Williams, Alison Schofield, Smita Koralahalli,
Jonathan Cameron
Cc: linux-cxl, linux-kernel, nvdimm, linux-fsdevel, linux-pm,
Ard Biesheuvel, Vishal Verma, Ira Weiny, Yazen Ghannam,
Dave Jiang, Davidlohr Bueso, Matthew Wilcox, Jan Kara,
Rafael J . Wysocki, Len Brown, Pavel Machek, Li Ming,
Jeff Johnson, Ying Huang, Yao Xingtao, Peter Zijlstra,
Greg Kroah-Hartman, Nathan Fontenot, Terry Bowman, Robert Richter,
Benjamin Cheatham, Zhijian Li, Borislav Petkov, Tomasz Wolski
In-Reply-To: <69bc81bfa9baa_7ee310093@dwillia2-mobl4.notmuch>
Hi Dan,
On 3/19/2026 4:07 PM, Dan Williams wrote:
> Koralahalli Channabasappa, Smita wrote:
> [..]
>>> I agree with Jonathan's comments in Patch 6, using __WORK_INITIALIZER or
>>> initializing work in dax_hmem_init() and gating flush on pdev will fix
>>> the WARN — I will add both for v8. But I think the WARN is likely
>>> indicating an ordering issue here..
>
> Yes, Jonathan is right, static initialization is also my expecation.
>
>>> On initial boot, the Makefile ordering ensures dax_hmem_init() runs
>>> before cxl_dax_region_init(), so both work items land on system_long_wq
>>> in the right order and dax_hmem's deferred work is queued before
>>> dax_cxl's driver registration work.
>
> There is nothing that guarantees that 2 work items in system_long_wq run
> in submission order. Unlikely that matters given the explicit flushing.
>
>>> On module reload which Alison is trying here I dont think, modules are
>>> loaded by Makefile order. I think dax_cxl's workqueue is calling
>>> dax_hmem_flush_work() before dax_hmem probe has had a chance to queue
>>> its work, so flush_work() flushes nothing and dax_cxl registers its
>>> driver without waiting.
>
> Module load order does not matter after initial probe completion.
>
> ...and dax_hmem is guaranteed to always load before dax_cxl due to the
> symbol dependency of dax_hmem_flush_work().
>
>>> __WORK_INITIALIZER fixes the WARN, but doesn't fix the race I guess if
>>> we are hitting that here..
>>>
>>> [ 34.673051] initcall dax_hmem_init+0x0/0xff0 [dax_hmem] returned 0
>>> after 2225 usecs
>>> [ 34.676011] calling cxl_dax_region_init+0x0/0xff0 [dax_cxl] @ 1059
>>>
>>> These two lines indicate cxl_dax started after dax_hmem_init() returns
>>> but I dont think that guarantees dax_hmem_platform_probe() has actually
>>> run..
>>>
>>> I dont know if wait_for_device_probe() in cxl_dax_region_driver_register
>>> might help..
>>>
>>> Thanks
>>> Smita
>>
>> Actually, thinking about this more..
>>
>> dax_hmem_initial_probe lives in device.c (built-in) so it survives
>> module reload. On reload it's still true from the first boot. This means
>> hmem_register_device() skips the deferral path entirely..
>
> Yes, that is the expectation.
>
>> The problem is this bypasses the cxl_region_contains_resource() check
>> that the deferred work normally does. On first boot,
>> process_defer_work() walks each range and decides per-range: if CXL
>> covers it, skip. If not, register with HMEM. On reload, that check never
>> happens — whoever registers first via alloc_dax_region() wins,
>> regardless of whether CXL actually covers the range.
>
> Yes, I think you have hit on a real issue. There is no point in having
> dax_hmem auto-attach on driver reload. If userspace unloads the driver
> it gets to keep the pieces. So that means something like this:
>
> diff --git a/drivers/dax/hmem/hmem.c b/drivers/dax/hmem/hmem.c
> index 15e462589b92..7478bc78a698 100644
> --- a/drivers/dax/hmem/hmem.c
> +++ b/drivers/dax/hmem/hmem.c
> @@ -112,10 +112,12 @@ static int hmem_register_device(struct device *host, int target_nid,
> region_intersects(res->start, resource_size(res), IORESOURCE_MEM,
> IORES_DESC_CXL) != REGION_DISJOINT) {
> if (!dax_hmem_initial_probe) {
> - dev_dbg(host, "deferring range to CXL: %pr\n", res);
> + dev_dbg(host, "await CXL initial probe: %pr\n", res);
> queue_work(system_long_wq, &dax_hmem_work.work);
> return 0;
> }
> + dev_dbg(host, "deferring range to CXL: %pr\n", res);
> + return 0;
> }
One issue with the reload fix - At boot, hmem_register_cxl_device()
calls hmem_register_device() to register ranges that aren't claimed by
CXL. But hmem_register_device() now always returns 0 for those ranges at
boot.
I was thinking factoring out registration logic into
__hmem_register_device() and have hmem_register_cxl_device() call that
directly, bypassing the CXL gating. Something like:
+static int __hmem_register_device(...)
+{
+ /* Remaining in hmem_register_device after the CXL check */
+}
static int hmem_register_device(..)
{
if (IS_ENABLED(CONFIG_DEV_DAX_CXL) && .. {
+ if (!dax_hmem_initial_probe_done) {
+ queue_work(system_long_wq, &dax_hmem_work);
+ return 0;
+ }
+ return 0;
+ }
+ return __hmem_register_device(host, target_nid, res);
}
+static int hmem_register_cxl_device(...)
+{
...
+ return __hmem_register_device(host, target_nid, res);
+}
+static void process_defer_work(...)
+{
+ ...
- dax_hmem_initial_probe = true;
- walk_hmem_resources(&pdev->dev, hmem_register_cxl_device);
+ if (!dax_hmem_initial_probe) {
+ dax_hmem_initial_probe = true;
+ walk_hmem_resources(.., hmem_register_cxl_device);
+ }
..
+}
Tracing it
At boot:
probe -> walk(hmem_register_device)
CXL range, !dax_hmem_initial_probe -> queue_work, return 0
non-CXL ranges -> __hmem_register_device -> registers
process_defer_work:
!dax_hmem_initial_probe
dax_hmem_initial_probe = true
walk(hmem_register_cxl_device)
CXL covers -> return 0
CXL doesn't cover -> __hmem_register_device()
no CXL check again, straight to registration..
On reload:
probe -> walk(hmem_register_device)
CXL range, dax_hmem_initial_probe = true, "your return 0" -> skips
non-CXL ranges -> __hmem_register_device -> registers
process_defer_work:
dax_hmem_initial_probe = true -> skip the walk entirely..
Or do you think this can be simplified better and the above approach has
some caveats?
Thanks
Smita
>
> rc = region_intersects_soft_reserve(res->start, resource_size(res));
>
> ---
>
> ...because if userspace wants to reload the dax_hmem driver, then it
> needs to pick what happens with the CXL intersection. Userspace can
> always unload cxl_acpi to force everything back to dax_hmem.
>
> Now, you might say, "but this means that if the initial probe results in
> a partial result of some regions in dax_hmem and others in dax_cxl, that
> state can not be recovered outside of a reboot". I think that is ok.
> This mechanism is automatic best-effort workaround for bugs / missing
> capabilities in the CXL driver. Module reload fidelity is out of scope.
>
>> So if dax_cxl registers first on reload, it could claim a range that CXL
>> doesn't actually cover, and dax_hmem would lose a range it should own..
>
> With the above change, dax_cxl always wins in the "reload" scenario iff
> cxl_acpi is loaded. Otherwise dax_hmem owns all the Soft Reserved.
>
>> I dont know if Im thinking through this right..
>
> You definitely identified the need for that fixup above.
^ permalink raw reply
* [PATCH 6.12.y 5/6] cpuidle: menu: Update documentation after get_typical_interval() changes
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-1-ionut.nechita@windriver.com>
From: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
The documentation of the menu cpuidle governor needs to be updated
to match the code behavior after some changes made recently.
No functional impact.
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Link: https://patch.msgid.link/4998484.31r3eYUQgx@rjwysocki.net
[ rjw: More specific subject, two typos fixed in the changelog ]
---
Documentation/admin-guide/pm/cpuidle.rst | 56 +++++++++++++++---------
| 29 +++++-------
2 files changed, 45 insertions(+), 40 deletions(-)
diff --git a/Documentation/admin-guide/pm/cpuidle.rst b/Documentation/admin-guide/pm/cpuidle.rst
index 19754beb5a4e6..9fcc35498fb0e 100644
--- a/Documentation/admin-guide/pm/cpuidle.rst
+++ b/Documentation/admin-guide/pm/cpuidle.rst
@@ -295,30 +295,44 @@ values and, when predicting the idle duration next time, it computes the average
and variance of them. If the variance is small (smaller than 400 square
milliseconds) or it is small relative to the average (the average is greater
that 6 times the standard deviation), the average is regarded as the "typical
-interval" value. Otherwise, the longest of the saved observed idle duration
+interval" value. Otherwise, either the longest or the shortest (depending on
+which one is farther from the average) of the saved observed idle duration
values is discarded and the computation is repeated for the remaining ones.
+
Again, if the variance of them is small (in the above sense), the average is
taken as the "typical interval" value and so on, until either the "typical
-interval" is determined or too many data points are disregarded, in which case
-the "typical interval" is assumed to equal "infinity" (the maximum unsigned
-integer value). The "typical interval" computed this way is compared with the
-sleep length multiplied by the correction factor and the minimum of the two is
-taken as the predicted idle duration.
-
-Then, the governor computes an extra latency limit to help "interactive"
-workloads. It uses the observation that if the exit latency of the selected
-idle state is comparable with the predicted idle duration, the total time spent
-in that state probably will be very short and the amount of energy to save by
-entering it will be relatively small, so likely it is better to avoid the
-overhead related to entering that state and exiting it. Thus selecting a
-shallower state is likely to be a better option then. The first approximation
-of the extra latency limit is the predicted idle duration itself which
-additionally is divided by a value depending on the number of tasks that
-previously ran on the given CPU and now they are waiting for I/O operations to
-complete. The result of that division is compared with the latency limit coming
-from the power management quality of service, or `PM QoS <cpu-pm-qos_>`_,
-framework and the minimum of the two is taken as the limit for the idle states'
-exit latency.
+interval" is determined or too many data points are disregarded. In the latter
+case, if the size of the set of data points still under consideration is
+sufficiently large, the next idle duration is not likely to be above the largest
+idle duration value still in that set, so that value is taken as the predicted
+next idle duration. Finally, if the set of data points still under
+consideration is too small, no prediction is made.
+
+If the preliminary prediction of the next idle duration computed this way is
+long enough, the governor obtains the time until the closest timer event with
+the assumption that the scheduler tick will be stopped. That time, referred to
+as the *sleep length* in what follows, is the upper bound on the time before the
+next CPU wakeup. It is used to determine the sleep length range, which in turn
+is needed to get the sleep length correction factor.
+
+The ``menu`` governor maintains an array containing several correction factor
+values that correspond to different sleep length ranges organized so that each
+range represented in the array is approximately 10 times wider than the previous
+one.
+
+The correction factor for the given sleep length range (determined before
+selecting the idle state for the CPU) is updated after the CPU has been woken
+up and the closer the sleep length is to the observed idle duration, the closer
+to 1 the correction factor becomes (it must fall between 0 and 1 inclusive).
+The sleep length is multiplied by the correction factor for the range that it
+falls into to obtain an approximation of the predicted idle duration that is
+compared to the "typical interval" determined previously and the minimum of
+the two is taken as the final idle duration prediction.
+
+If the "typical interval" value is small, which means that the CPU is likely
+to be woken up soon enough, the sleep length computation is skipped as it may
+be costly and the idle duration is simply predicted to equal the "typical
+interval" value.
Now, the governor is ready to walk the list of idle states and choose one of
them. For this purpose, it compares the target residency of each state with
--git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 8ab5123c81040..a18477ecce433 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -41,7 +41,7 @@
* the C state is required to actually break even on this cost. CPUIDLE
* provides us this duration in the "target_residency" field. So all that we
* need is a good prediction of how long we'll be idle. Like the traditional
- * menu governor, we start with the actual known "next timer event" time.
+ * menu governor, we take the actual known "next timer event" time.
*
* Since there are other source of wakeups (interrupts for example) than
* the next timer event, this estimation is rather optimistic. To get a
@@ -50,30 +50,21 @@
* duration always was 50% of the next timer tick, the correction factor will
* be 0.5.
*
- * menu uses a running average for this correction factor, however it uses a
- * set of factors, not just a single factor. This stems from the realization
- * that the ratio is dependent on the order of magnitude of the expected
- * duration; if we expect 500 milliseconds of idle time the likelihood of
- * getting an interrupt very early is much higher than if we expect 50 micro
- * seconds of idle time. A second independent factor that has big impact on
- * the actual factor is if there is (disk) IO outstanding or not.
- * (as a special twist, we consider every sleep longer than 50 milliseconds
- * as perfect; there are no power gains for sleeping longer than this)
- *
- * For these two reasons we keep an array of 12 independent factors, that gets
- * indexed based on the magnitude of the expected duration as well as the
- * "is IO outstanding" property.
+ * menu uses a running average for this correction factor, but it uses a set of
+ * factors, not just a single factor. This stems from the realization that the
+ * ratio is dependent on the order of magnitude of the expected duration; if we
+ * expect 500 milliseconds of idle time the likelihood of getting an interrupt
+ * very early is much higher than if we expect 50 micro seconds of idle time.
+ * For this reason, menu keeps an array of 6 independent factors, that gets
+ * indexed based on the magnitude of the expected duration.
*
* Repeatable-interval-detector
* ----------------------------
* There are some cases where "next timer" is a completely unusable predictor:
* Those cases where the interval is fixed, for example due to hardware
- * interrupt mitigation, but also due to fixed transfer rate devices such as
- * mice.
+ * interrupt mitigation, but also due to fixed transfer rate devices like mice.
* For this, we use a different predictor: We track the duration of the last 8
- * intervals and if the stand deviation of these 8 intervals is below a
- * threshold value, we use the average of these intervals as prediction.
- *
+ * intervals and use them to estimate the duration of the next one.
*/
struct menu_device {
--
2.53.0
^ permalink raw reply related
* [PATCH 6.12.y 0/6] cpuidle: menu: Backport get_typical_interval() improvements
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad, Ionut Nechita
From: Ionut Nechita <ionut_n2001@yahoo.com>
This series backports 6 upstream commits that improve the menu
governor's get_typical_interval() function to linux-6.12.y stable.
These patches are already present in linux-6.18.y but were not picked
up for 6.12.y because they lack Cc: stable tags.
The key improvement is in patch 2/6 which merges the two separate loops
for average and variance computation into a single pass, reducing the
latency of menu_select() on isolated (nohz_full) cores. The remaining
patches refactor outlier detection to cover both ends of the sample set,
update documentation to match the new code, and add a minor bucket
assignment optimization.
After applying this series, drivers/cpuidle/governors/menu.c matches
linux-6.18.y exactly.
All patches are clean cherry-picks from mainline with one trivial
conflict resolution in Documentation/admin-guide/pm/cpuidle.rst
(patch 5/6).
Upstream commits:
d2cd195b57cf ("cpuidle: menu: Drop a redundant local variable")
13982929fb08 ("cpuidle: menu: Use one loop for average and variance computations")
60256e458e1c ("cpuidle: menu: Tweak threshold use in get_typical_interval()")
8de7606f0fe2 ("cpuidle: menu: Eliminate outliers on both ends of the sample set")
5c35041099965 ("cpuidle: menu: Update documentation after get_typical_interval() changes")
d4a7882f93bf ("cpuidle: menu: Optimize bucket assignment when next_timer_ns equals KTIME_MAX")
Rafael J. Wysocki (5):
cpuidle: menu: Drop a redundant local variable
cpuidle: menu: Use one loop for average and variance computations
cpuidle: menu: Tweak threshold use in get_typical_interval()
cpuidle: menu: Eliminate outliers on both ends of the sample set
cpuidle: menu: Update documentation after get_typical_interval()
changes
Zhongqiu Han (1):
cpuidle: menu: Optimize bucket assignment when next_timer_ns equals
KTIME_MAX
Documentation/admin-guide/pm/cpuidle.rst | 56 +++++++----
drivers/cpuidle/governors/menu.c | 118 +++++++++++------------
2 files changed, 91 insertions(+), 83 deletions(-)
--
2.53.0
^ permalink raw reply
* [PATCH 6.12.y 1/6] cpuidle: menu: Drop a redundant local variable
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-1-ionut.nechita@windriver.com>
From: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Local variable min in get_typical_interval() is updated, but never
accessed later, so drop it.
No functional impact.
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
Tested-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Aboorva Devarajan <aboorvad@linux.ibm.com>
Link: https://patch.msgid.link/13699686.uLZWGnKmhe@rjwysocki.net
---
| 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
--git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 0ce7323450011..dd7e2a965878e 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -125,7 +125,7 @@ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev);
static unsigned int get_typical_interval(struct menu_device *data)
{
int i, divisor;
- unsigned int min, max, thresh, avg;
+ unsigned int max, thresh, avg;
uint64_t sum, variance;
thresh = INT_MAX; /* Discard outliers above this value */
@@ -133,7 +133,6 @@ static unsigned int get_typical_interval(struct menu_device *data)
again:
/* First calculate the average of past intervals */
- min = UINT_MAX;
max = 0;
sum = 0;
divisor = 0;
@@ -144,9 +143,6 @@ static unsigned int get_typical_interval(struct menu_device *data)
divisor++;
if (value > max)
max = value;
-
- if (value < min)
- min = value;
}
}
--
2.53.0
^ permalink raw reply related
* [PATCH 6.12.y 3/6] cpuidle: menu: Tweak threshold use in get_typical_interval()
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-1-ionut.nechita@windriver.com>
From: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
To prepare get_typical_interval() for subsequent changes, rearrange
the use of the data point threshold in it a bit and initialize that
threshold to UINT_MAX which is more consistent with its data type.
No intentional functional impact.
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Tested-by: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Aboorva Devarajan <aboorvad@linux.ibm.com>
Link: https://patch.msgid.link/8490144.T7Z3S40VBb@rjwysocki.net
---
| 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
--git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 8943bb8f19190..96bee77b8354f 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -124,7 +124,7 @@ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev);
*/
static unsigned int get_typical_interval(struct menu_device *data)
{
- unsigned int max, divisor, thresh = INT_MAX;
+ unsigned int max, divisor, thresh = UINT_MAX;
u64 avg, variance, avg_sq;
int i;
@@ -137,8 +137,8 @@ static unsigned int get_typical_interval(struct menu_device *data)
for (i = 0; i < INTERVALS; i++) {
unsigned int value = data->intervals[i];
- /* Discard data points above the threshold. */
- if (value > thresh)
+ /* Discard data points above or at the threshold. */
+ if (value >= thresh)
continue;
divisor++;
@@ -202,7 +202,7 @@ static unsigned int get_typical_interval(struct menu_device *data)
if (divisor * 4 <= INTERVALS * 3)
return UINT_MAX;
- thresh = max - 1;
+ thresh = max;
goto again;
}
--
2.53.0
^ permalink raw reply related
* [PATCH 6.12.y 4/6] cpuidle: menu: Eliminate outliers on both ends of the sample set
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-1-ionut.nechita@windriver.com>
From: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Currently, get_typical_interval() attempts to eliminate outliers at the
high end of the sample set only (probably in order to bias the prediction
toward lower values), but this it problematic because if the outliers are
present at the low end of the sample set, discarding the highest values
will not help to reduce the variance.
Since the presence of outliers at the low end of the sample set is
generally as likely as their presence at the high end of the sample
set, modify get_typical_interval() to treat samples at the largest
distances from the average (on both ends of the sample set) as outliers.
This should increase the likelihood of making a meaningful prediction
in some cases.
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Reported-by: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
Tested-by: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Aboorva Devarajan <aboorvad@linux.ibm.com>
Link: https://patch.msgid.link/2301940.iZASKD2KPV@rjwysocki.net
---
| 32 ++++++++++++++++++++++----------
1 file changed, 22 insertions(+), 10 deletions(-)
--git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 96bee77b8354f..8ab5123c81040 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -124,30 +124,37 @@ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev);
*/
static unsigned int get_typical_interval(struct menu_device *data)
{
- unsigned int max, divisor, thresh = UINT_MAX;
+ s64 value, min_thresh = -1, max_thresh = UINT_MAX;
+ unsigned int max, min, divisor;
u64 avg, variance, avg_sq;
int i;
again:
/* Compute the average and variance of past intervals. */
max = 0;
+ min = UINT_MAX;
avg = 0;
variance = 0;
divisor = 0;
for (i = 0; i < INTERVALS; i++) {
- unsigned int value = data->intervals[i];
-
- /* Discard data points above or at the threshold. */
- if (value >= thresh)
+ value = data->intervals[i];
+ /*
+ * Discard the samples outside the interval between the min and
+ * max thresholds.
+ */
+ if (value <= min_thresh || value >= max_thresh)
continue;
divisor++;
avg += value;
- variance += (u64)value * value;
+ variance += value * value;
if (value > max)
max = value;
+
+ if (value < min)
+ min = value;
}
if (!max)
@@ -183,10 +190,10 @@ static unsigned int get_typical_interval(struct menu_device *data)
}
/*
- * If we have outliers to the upside in our distribution, discard
- * those by setting the threshold to exclude these outliers, then
+ * If there are outliers, discard them by setting thresholds to exclude
+ * data points at a large enough distance from the average, then
* calculate the average and standard deviation again. Once we get
- * down to the bottom 3/4 of our samples, stop excluding samples.
+ * down to the last 3/4 of our samples, stop excluding samples.
*
* This can deal with workloads that have long pauses interspersed
* with sporadic activity with a bunch of short pauses.
@@ -202,7 +209,12 @@ static unsigned int get_typical_interval(struct menu_device *data)
if (divisor * 4 <= INTERVALS * 3)
return UINT_MAX;
- thresh = max;
+ /* Update the thresholds for the next round. */
+ if (avg - min > max - avg)
+ min_thresh = min;
+ else
+ max_thresh = max;
+
goto again;
}
--
2.53.0
^ permalink raw reply related
* [PATCH 6.12.y 2/6] cpuidle: menu: Use one loop for average and variance computations
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-1-ionut.nechita@windriver.com>
From: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Use the observation that one loop is sufficient to compute the average
of an array of values and their variance to eliminate one of the loops
from get_typical_interval().
While at it, make get_typical_interval() consistently use u64 as the
64-bit unsigned integer data type and rearrange some white space and the
declarations of local variables in it (to make them follow the reverse
X-mas tree pattern).
No intentional functional impact.
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Tested-by: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Christian Loehle <christian.loehle@arm.com>
Tested-by: Aboorva Devarajan <aboorvad@linux.ibm.com>
Link: https://patch.msgid.link/3339073.aeNJFYEL58@rjwysocki.net
---
| 61 +++++++++++++++-----------------
1 file changed, 28 insertions(+), 33 deletions(-)
--git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index dd7e2a965878e..8943bb8f19190 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -124,49 +124,45 @@ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev);
*/
static unsigned int get_typical_interval(struct menu_device *data)
{
- int i, divisor;
- unsigned int max, thresh, avg;
- uint64_t sum, variance;
-
- thresh = INT_MAX; /* Discard outliers above this value */
+ unsigned int max, divisor, thresh = INT_MAX;
+ u64 avg, variance, avg_sq;
+ int i;
again:
-
- /* First calculate the average of past intervals */
+ /* Compute the average and variance of past intervals. */
max = 0;
- sum = 0;
+ avg = 0;
+ variance = 0;
divisor = 0;
for (i = 0; i < INTERVALS; i++) {
unsigned int value = data->intervals[i];
- if (value <= thresh) {
- sum += value;
- divisor++;
- if (value > max)
- max = value;
- }
+
+ /* Discard data points above the threshold. */
+ if (value > thresh)
+ continue;
+
+ divisor++;
+
+ avg += value;
+ variance += (u64)value * value;
+
+ if (value > max)
+ max = value;
}
if (!max)
return UINT_MAX;
- if (divisor == INTERVALS)
- avg = sum >> INTERVAL_SHIFT;
- else
- avg = div_u64(sum, divisor);
-
- /* Then try to determine variance */
- variance = 0;
- for (i = 0; i < INTERVALS; i++) {
- unsigned int value = data->intervals[i];
- if (value <= thresh) {
- int64_t diff = (int64_t)value - avg;
- variance += diff * diff;
- }
- }
- if (divisor == INTERVALS)
+ if (divisor == INTERVALS) {
+ avg >>= INTERVAL_SHIFT;
variance >>= INTERVAL_SHIFT;
- else
+ } else {
+ do_div(avg, divisor);
do_div(variance, divisor);
+ }
+
+ avg_sq = avg * avg;
+ variance -= avg_sq;
/*
* The typical interval is obtained when standard deviation is
@@ -181,10 +177,9 @@ static unsigned int get_typical_interval(struct menu_device *data)
* Use this result only if there is no timer to wake us up sooner.
*/
if (likely(variance <= U64_MAX/36)) {
- if ((((u64)avg*avg > variance*36) && (divisor * 4 >= INTERVALS * 3))
- || variance <= 400) {
+ if ((avg_sq > variance * 36 && divisor * 4 >= INTERVALS * 3) ||
+ variance <= 400)
return avg;
- }
}
/*
--
2.53.0
^ permalink raw reply related
* [PATCH 6.12.y 6/6] cpuidle: menu: Optimize bucket assignment when next_timer_ns equals KTIME_MAX
From: Ionut Nechita @ 2026-03-20 20:29 UTC (permalink / raw)
To: stable
Cc: rafael.j.wysocki, linux-pm, christian.loehle, artem.bityutskiy,
quic_zhonhan, aboorvad
In-Reply-To: <20260320202908.24377-1-ionut.nechita@windriver.com>
From: Zhongqiu Han <quic_zhonhan@quicinc.com>
Directly assign the last bucket value instead of calling which_bucket()
when next_timer_ns equals KTIME_MAX, the largest possible value that
always falls into the last bucket.
This avoids unnecessary calculations and enhances performance.
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Signed-off-by: Zhongqiu Han <quic_zhonhan@quicinc.com>
Link: https://patch.msgid.link/20250405135308.1854342-1-quic_zhonhan@quicinc.com
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
---
| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index a18477ecce433..ca863ba03d454 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -278,7 +278,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
*/
data->next_timer_ns = KTIME_MAX;
delta_tick = TICK_NSEC / 2;
- data->bucket = which_bucket(KTIME_MAX);
+ data->bucket = BUCKETS - 1;
}
if (unlikely(drv->state_count <= 1 || latency_req == 0) ||
--
2.53.0
^ permalink raw reply related
* Re: [PATCH v3 2/4] serdev: add rust private data to serdev_device
From: Danilo Krummrich @ 2026-03-20 19:59 UTC (permalink / raw)
To: Markus Probst
Cc: Greg Kroah-Hartman, Rob Herring, Jiri Slaby, Miguel Ojeda,
Gary Guo, Björn Roy Baron, Benno Lossin, Andreas Hindborg,
Alice Ryhl, Trevor Gross, Kari Argillander, Rafael J. Wysocki,
Viresh Kumar, Boqun Feng, David Airlie, Simona Vetter,
linux-serial, linux-kernel, rust-for-linux, linux-pm, driver-core,
dri-devel
In-Reply-To: <4f4c09d683023df9daef89ca634b17b5c16625f7.camel@posteo.de>
On 3/20/26 6:13 PM, Markus Probst wrote:
> This is unique for a bus device.
>
> A class device has its own Data (e. g. PwmOps), i. e. it will only be
> registered after this Data has been initialized by the driver.
>
> The receive_buf callback on the serdev device on the other hand must
> happen on the drvdata, as there is no place to store its own Data.
>
> The drvdata is only available at the end of the probe in Rust.
In other words, devm_serdev_device_open() relies on the driver having its
private data set before this is called, since once devm_serdev_device_open() has
been called, the driver's receive_buf() callback may be called.
So, in C it is the driver's responsibility to do things in the correct order, in
Rust we want to make sure the driver can't do it out of order in the first place.
I'm still not sure whether the current approach is the best option. For
instance, we could also solve this with a separate callback. (Wasn't this even
what you had in a previous version?)
^ permalink raw reply
* Re: [PATCH 4/6] mfd: sprd-sc27xx: Switch to devm_mfd_add_devices()
From: Otto Pflüger @ 2026-03-20 19:41 UTC (permalink / raw)
To: Lee Jones
Cc: Alexandre Belloni, Rob Herring, Krzysztof Kozlowski, Conor Dooley,
Orson Zhai, Baolin Wang, Chunyan Zhang, Pavel Machek,
Liam Girdwood, Mark Brown, Sebastian Reichel, linux-rtc,
devicetree, linux-kernel, linux-leds, linux-pm
In-Reply-To: <20260309185856.GZ183676@google.com>
On Mon, Mar 09, 2026 at 06:58:56PM +0000, Lee Jones wrote:
> On Sun, 22 Feb 2026, Otto Pflüger wrote:
>
> > To allow instantiating subdevices such as the regulator and poweroff
> > devices that do not have corresponding device tree nodes with a
> > "compatible" property, use devm_mfd_add_devices() with MFD cells instead
> > of devm_of_platform_populate(). Since different PMICs in the SC27xx
> > series contain different components, use separate MFD cell tables for
> > each PMIC model. Define cells for all components that have upstream
> > drivers at this point.
>
> We're not passing one device registration API's data (MFD)
> through another (Device Tree).
>
> Pass an identifier through and match on that instead.
>
> Look at how all of the other drivers in MFD do it.
>
> [...]
> > +static const struct mfd_cell sc2730_devices[] = {
> > + MFD_CELL_OF("sc2730-adc", NULL, NULL, 0, 0, "sprd,sc2730-adc"),
> > + MFD_CELL_OF("sc2730-bltc", NULL, NULL, 0, 0, "sprd,sc2730-bltc"),
> > + MFD_CELL_OF("sc2730-efuse", NULL, NULL, 0, 0, "sprd,sc2730-efuse"),
> > + MFD_CELL_OF("sc2730-eic", NULL, NULL, 0, 0, "sprd,sc2730-eic"),
> > + MFD_CELL_OF("sc2730-fgu", NULL, NULL, 0, 0, "sprd,sc2730-fgu"),
> > + MFD_CELL_OF("sc2730-rtc", NULL, NULL, 0, 0, "sprd,sc2730-rtc"),
> > + MFD_CELL_OF("sc2730-vibrator", NULL, NULL, 0, 0, "sprd,sc2730-vibrator"),
> > +};
> > +
> > +static const struct mfd_cell sc2731_devices[] = {
> > + MFD_CELL_OF("sc2731-adc", NULL, NULL, 0, 0, "sprd,sc2731-adc"),
> > + MFD_CELL_OF("sc2731-bltc", NULL, NULL, 0, 0, "sprd,sc2731-bltc"),
> > + MFD_CELL_OF("sc2731-charger", NULL, NULL, 0, 0, "sprd,sc2731-charger"),
> > + MFD_CELL_OF("sc2731-efuse", NULL, NULL, 0, 0, "sprd,sc2731-efuse"),
> > + MFD_CELL_OF("sc2731-eic", NULL, NULL, 0, 0, "sprd,sc2731-eic"),
> > + MFD_CELL_OF("sc2731-fgu", NULL, NULL, 0, 0, "sprd,sc2731-fgu"),
> > + MFD_CELL_NAME("sc2731-poweroff"),
> > + MFD_CELL_NAME("sc2731-regulator"),
> > + MFD_CELL_OF("sc2731-rtc", NULL, NULL, 0, 0, "sprd,sc2731-rtc"),
> > + MFD_CELL_OF("sc2731-vibrator", NULL, NULL, 0, 0, "sprd,sc2731-vibrator"),
> > };
Assuming that these tables are the "registration API's data", I don't
see where it is being passed through the device tree. The device tree
contains nodes for some of these MFD components, and I've listed their
compatibles here so that the MFD core finds these nodes and registers
them with the corresponding devices (which was previously done
automatically by devm_of_platform_populate).
> >
> > /*
> > @@ -59,12 +84,16 @@ static const struct sprd_pmic_data sc2730_data = {
> > .irq_base = SPRD_SC2730_IRQ_BASE,
> > .num_irqs = SPRD_SC2730_IRQ_NUMS,
> > .charger_det = SPRD_SC2730_CHG_DET,
> > + .cells = sc2730_devices,
> > + .num_cells = ARRAY_SIZE(sc2730_devices),
> > };
> >
> > static const struct sprd_pmic_data sc2731_data = {
> > .irq_base = SPRD_SC2731_IRQ_BASE,
> > .num_irqs = SPRD_SC2731_IRQ_NUMS,
> > .charger_det = SPRD_SC2731_CHG_DET,
> > + .cells = sc2731_devices,
> > + .num_cells = ARRAY_SIZE(sc2731_devices),
> > };
Here I am simply referencing the tables above in the device-specific
MFD data. These structs containing device-specific data already exist,
they are private to the MFD driver, and I wouldn't consider them part
of the device tree.
I've looked at mt6397-core.c and it seems to be doing the exact same
thing with its "struct chip_data". Some other drivers use a numeric ID
for this purpose, but how would that be different from a pointer as long
as it identifies the same data within the MFD driver?
Could you clarify what should be changed?
^ permalink raw reply
* [PATCH v5 21/21] vswap: batch contiguous vswap free calls
From: Nhat Pham @ 2026-03-20 19:27 UTC (permalink / raw)
To: kasong
Cc: Liam.Howlett, akpm, apopple, axelrasmussen, baohua, baolin.wang,
bhe, byungchul, cgroups, chengming.zhou, chrisl, corbet, david,
dev.jain, gourry, hannes, hughd, jannh, joshua.hahnjy, lance.yang,
lenb, linux-doc, linux-kernel, linux-mm, linux-pm,
lorenzo.stoakes, matthew.brost, mhocko, muchun.song, npache,
nphamcs, pavel, peterx, peterz, pfalcato, rafael, rakie.kim,
roman.gushchin, rppt, ryan.roberts, shakeel.butt, shikemeng,
surenb, tglx, vbabka, weixugc, ying.huang, yosry.ahmed, yuanchu,
zhengqi.arch, ziy, kernel-team, riel
In-Reply-To: <20260320192735.748051-1-nphamcs@gmail.com>
In vswap_free(), we release and reacquire the cluster lock for every
single entry, even for non-disk-swap backends where the lock drop is
unnecessary. Batch consecutive free operations to avoid this overhead.
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
include/linux/memcontrol.h | 6 ++
mm/memcontrol.c | 2 +-
mm/vswap.c | 185 ++++++++++++++++++++++++-------------
3 files changed, 126 insertions(+), 67 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 0651865a4564f..0f7f5489e1675 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -827,6 +827,7 @@ static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
return memcg->id.id;
}
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
+void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n);
#ifdef CONFIG_SHRINKER_DEBUG
static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)
@@ -1289,6 +1290,11 @@ static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
return NULL;
}
+static inline void mem_cgroup_id_put_many(struct mem_cgroup *memcg,
+ unsigned int n)
+{
+}
+
#ifdef CONFIG_SHRINKER_DEBUG
static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)
{
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 4525c21754e7f..c6d307b8127a8 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3597,7 +3597,7 @@ void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
refcount_add(n, &memcg->id.ref);
}
-static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
+void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
{
if (refcount_sub_and_test(n, &memcg->id.ref)) {
mem_cgroup_id_remove(memcg);
diff --git a/mm/vswap.c b/mm/vswap.c
index fa37165cb10d0..82092502130a6 100644
--- a/mm/vswap.c
+++ b/mm/vswap.c
@@ -482,18 +482,18 @@ static void vswap_cluster_free(struct vswap_cluster *cluster)
kvfree_rcu(cluster, rcu);
}
-static inline void release_vswap_slot(struct vswap_cluster *cluster,
- unsigned long index)
+static inline void release_vswap_slot_nr(struct vswap_cluster *cluster,
+ unsigned long index, int nr)
{
unsigned long slot_index = VSWAP_IDX_WITHIN_CLUSTER_VAL(index);
lockdep_assert_held(&cluster->lock);
- cluster->count--;
+ cluster->count -= nr;
- bitmap_clear(cluster->bitmap, slot_index, 1);
+ bitmap_clear(cluster->bitmap, slot_index, nr);
/* we only free uncached empty clusters */
- if (refcount_dec_and_test(&cluster->refcnt))
+ if (refcount_sub_and_test(nr, &cluster->refcnt))
vswap_cluster_free(cluster);
else if (cluster->full && cluster_is_alloc_candidate(cluster)) {
cluster->full = false;
@@ -506,7 +506,7 @@ static inline void release_vswap_slot(struct vswap_cluster *cluster,
}
}
- atomic_dec(&vswap_used);
+ atomic_sub(nr, &vswap_used);
}
/*
@@ -528,23 +528,33 @@ void vswap_rmap_set(struct swap_cluster_info *ci, swp_slot_t slot,
}
/*
- * Caller needs to handle races with other operations themselves.
+ * release_backing - release the backend storage for a given range of virtual
+ * swap slots.
*
- * Specifically, this function is safe to be called in contexts where the swap
- * entry has been added to the swap cache and the associated folio is locked.
- * We cannot race with other accessors, and the swap entry is guaranteed to be
- * valid the whole time (since swap cache implies one refcount).
+ * Entered with the cluster locked, but might drop the lock in between.
+ * This is because several operations, such as releasing physical swap slots
+ * (i.e swap_slot_free_nr()) require the cluster to be unlocked to avoid
+ * deadlocks.
*
- * We cannot assume that the backends will be of the same type,
- * contiguous, etc. We might have a large folio coalesced from subpages with
- * mixed backend, which is only rectified when it is reclaimed.
+ * This is safe, because:
+ *
+ * 1. Callers ensure no concurrent modification of the swap entry's internal
+ * state can occur. This is guaranteed by one of the following:
+ * - For vswap_free_nr() callers: the swap entry's refcnt (swap count and
+ * swapcache pin) is down to 0.
+ * - For vswap_store_folio(), swap_zeromap_folio_set(), and zswap_entry_store()
+ * callers: the folio is locked and in the swap cache.
+ *
+ * 2. The swap entry still holds a refcnt to the cluster, keeping the cluster
+ * itself valid.
+ *
+ * We will exit the function with the cluster re-locked.
*/
-static void release_backing(swp_entry_t entry, int nr)
+static void release_backing(struct vswap_cluster *cluster, swp_entry_t entry,
+ int nr)
{
- struct vswap_cluster *cluster = NULL;
struct swp_desc *desc;
unsigned long flush_nr, phys_swap_start = 0, phys_swap_end = 0;
- unsigned long phys_swap_released = 0;
unsigned int phys_swap_type = 0;
bool need_flushing_phys_swap = false;
swp_slot_t flush_slot;
@@ -552,9 +562,8 @@ static void release_backing(swp_entry_t entry, int nr)
VM_WARN_ON(!entry.val);
- rcu_read_lock();
for (i = 0; i < nr; i++) {
- desc = vswap_iter(&cluster, entry.val + i);
+ desc = __vswap_iter(cluster, entry.val + i);
VM_WARN_ON(!desc);
/*
@@ -574,7 +583,6 @@ static void release_backing(swp_entry_t entry, int nr)
if (desc->type == VSWAP_ZSWAP && desc->zswap_entry) {
zswap_entry_free(desc->zswap_entry);
} else if (desc->type == VSWAP_SWAPFILE) {
- phys_swap_released++;
if (!phys_swap_start) {
/* start a new contiguous range of phys swap */
phys_swap_start = swp_slot_offset(desc->slot);
@@ -590,56 +598,49 @@ static void release_backing(swp_entry_t entry, int nr)
if (need_flushing_phys_swap) {
spin_unlock(&cluster->lock);
- cluster = NULL;
swap_slot_free_nr(flush_slot, flush_nr);
+ mem_cgroup_uncharge_swap(entry, flush_nr);
+ spin_lock(&cluster->lock);
need_flushing_phys_swap = false;
}
}
- if (cluster)
- spin_unlock(&cluster->lock);
- rcu_read_unlock();
/* Flush any remaining physical swap range */
if (phys_swap_start) {
flush_slot = swp_slot(phys_swap_type, phys_swap_start);
flush_nr = phys_swap_end - phys_swap_start;
+ spin_unlock(&cluster->lock);
swap_slot_free_nr(flush_slot, flush_nr);
+ mem_cgroup_uncharge_swap(entry, flush_nr);
+ spin_lock(&cluster->lock);
}
-
- if (phys_swap_released)
- mem_cgroup_uncharge_swap(entry, phys_swap_released);
}
+static void __vswap_swap_cgroup_clear(struct vswap_cluster *cluster,
+ swp_entry_t entry, unsigned int nr_ents);
+
/*
- * Entered with the cluster locked, but might unlock the cluster.
- * This is because several operations, such as releasing physical swap slots
- * (i.e swap_slot_free_nr()) require the cluster to be unlocked to avoid
- * deadlocks.
- *
- * This is safe, because:
- *
- * 1. The swap entry to be freed has refcnt (swap count and swapcache pin)
- * down to 0, so no one can change its internal state
- *
- * 2. The swap entry to be freed still holds a refcnt to the cluster, keeping
- * the cluster itself valid.
- *
- * We will exit the function with the cluster re-locked.
+ * Entered with the cluster locked. We will exit the function with the cluster
+ * still locked.
*/
-static void vswap_free(struct vswap_cluster *cluster, struct swp_desc *desc,
- swp_entry_t entry)
+static void vswap_free_nr(struct vswap_cluster *cluster, swp_entry_t entry,
+ int nr)
{
- /* Clear shadow if present */
- if (xa_is_value(desc->shadow))
- desc->shadow = NULL;
- spin_unlock(&cluster->lock);
+ struct swp_desc *desc;
+ int i;
- release_backing(entry, 1);
- mem_cgroup_clear_swap(entry, 1);
+ for (i = 0; i < nr; i++) {
+ desc = __vswap_iter(cluster, entry.val + i);
+ /* Clear shadow if present */
+ if (xa_is_value(desc->shadow))
+ desc->shadow = NULL;
+ }
- /* erase forward mapping and release the virtual slot for reallocation */
- spin_lock(&cluster->lock);
- release_vswap_slot(cluster, entry.val);
+ release_backing(cluster, entry, nr);
+ __vswap_swap_cgroup_clear(cluster, entry, nr);
+
+ /* erase forward mapping and release the virtual slots for reallocation */
+ release_vswap_slot_nr(cluster, entry.val, nr);
}
/**
@@ -818,18 +819,32 @@ static bool vswap_free_nr_any_cache_only(swp_entry_t entry, int nr)
struct vswap_cluster *cluster = NULL;
struct swp_desc *desc;
bool ret = false;
- int i;
+ swp_entry_t free_start;
+ int i, free_nr = 0;
+ free_start.val = 0;
rcu_read_lock();
for (i = 0; i < nr; i++) {
+ /* flush pending free batch at cluster boundary */
+ if (free_nr && !VSWAP_IDX_WITHIN_CLUSTER_VAL(entry.val)) {
+ vswap_free_nr(cluster, free_start, free_nr);
+ free_nr = 0;
+ }
desc = vswap_iter(&cluster, entry.val);
VM_WARN_ON(!desc);
ret |= (desc->swap_count == 1 && desc->in_swapcache);
desc->swap_count--;
- if (!desc->swap_count && !desc->in_swapcache)
- vswap_free(cluster, desc, entry);
+ if (!desc->swap_count && !desc->in_swapcache) {
+ if (!free_nr++)
+ free_start = entry;
+ } else if (free_nr) {
+ vswap_free_nr(cluster, free_start, free_nr);
+ free_nr = 0;
+ }
entry.val++;
}
+ if (free_nr)
+ vswap_free_nr(cluster, free_start, free_nr);
if (cluster)
spin_unlock(&cluster->lock);
rcu_read_unlock();
@@ -952,19 +967,33 @@ void swapcache_clear(swp_entry_t entry, int nr)
{
struct vswap_cluster *cluster = NULL;
struct swp_desc *desc;
- int i;
+ swp_entry_t free_start;
+ int i, free_nr = 0;
if (!nr)
return;
+ free_start.val = 0;
rcu_read_lock();
for (i = 0; i < nr; i++) {
+ /* flush pending free batch at cluster boundary */
+ if (free_nr && !VSWAP_IDX_WITHIN_CLUSTER_VAL(entry.val)) {
+ vswap_free_nr(cluster, free_start, free_nr);
+ free_nr = 0;
+ }
desc = vswap_iter(&cluster, entry.val);
desc->in_swapcache = false;
- if (!desc->swap_count)
- vswap_free(cluster, desc, entry);
+ if (!desc->swap_count) {
+ if (!free_nr++)
+ free_start = entry;
+ } else if (free_nr) {
+ vswap_free_nr(cluster, free_start, free_nr);
+ free_nr = 0;
+ }
entry.val++;
}
+ if (free_nr)
+ vswap_free_nr(cluster, free_start, free_nr);
if (cluster)
spin_unlock(&cluster->lock);
rcu_read_unlock();
@@ -1105,11 +1134,13 @@ void vswap_store_folio(swp_entry_t entry, struct folio *folio)
VM_BUG_ON(!folio_test_locked(folio));
VM_BUG_ON(folio->swap.val != entry.val);
- release_backing(entry, nr);
-
rcu_read_lock();
+ desc = vswap_iter(&cluster, entry.val);
+ VM_WARN_ON(!desc);
+ release_backing(cluster, entry, nr);
+
for (i = 0; i < nr; i++) {
- desc = vswap_iter(&cluster, entry.val + i);
+ desc = __vswap_iter(cluster, entry.val + i);
VM_WARN_ON(!desc);
desc->type = VSWAP_FOLIO;
desc->swap_cache = folio;
@@ -1134,11 +1165,13 @@ void swap_zeromap_folio_set(struct folio *folio)
VM_BUG_ON(!folio_test_locked(folio));
VM_BUG_ON(!entry.val);
- release_backing(entry, nr);
-
rcu_read_lock();
+ desc = vswap_iter(&cluster, entry.val);
+ VM_WARN_ON(!desc);
+ release_backing(cluster, entry, nr);
+
for (i = 0; i < nr; i++) {
- desc = vswap_iter(&cluster, entry.val + i);
+ desc = __vswap_iter(cluster, entry.val + i);
VM_WARN_ON(!desc);
desc->type = VSWAP_ZERO;
}
@@ -1772,11 +1805,10 @@ void zswap_entry_store(swp_entry_t swpentry, struct zswap_entry *entry)
struct vswap_cluster *cluster = NULL;
struct swp_desc *desc;
- release_backing(swpentry, 1);
-
rcu_read_lock();
desc = vswap_iter(&cluster, swpentry.val);
VM_WARN_ON(!desc);
+ release_backing(cluster, swpentry, 1);
desc->zswap_entry = entry;
desc->type = VSWAP_ZSWAP;
spin_unlock(&cluster->lock);
@@ -1850,6 +1882,22 @@ static unsigned short __vswap_cgroup_record(struct vswap_cluster *cluster,
return oldid;
}
+/*
+ * Clear swap cgroup for a range of swap entries.
+ * Entered with the cluster locked. Caller must be under rcu_read_lock().
+ */
+static void __vswap_swap_cgroup_clear(struct vswap_cluster *cluster,
+ swp_entry_t entry, unsigned int nr_ents)
+{
+ unsigned short id;
+ struct mem_cgroup *memcg;
+
+ id = __vswap_cgroup_record(cluster, entry, 0, nr_ents);
+ memcg = mem_cgroup_from_id(id);
+ if (memcg)
+ mem_cgroup_id_put_many(memcg, nr_ents);
+}
+
static unsigned short vswap_cgroup_record(swp_entry_t entry,
unsigned short memcgid, unsigned int nr_ents)
{
@@ -1955,6 +2003,11 @@ unsigned short lookup_swap_cgroup_id(swp_entry_t entry)
rcu_read_unlock();
return ret;
}
+#else /* !CONFIG_MEMCG */
+static void __vswap_swap_cgroup_clear(struct vswap_cluster *cluster,
+ swp_entry_t entry, unsigned int nr_ents)
+{
+}
#endif /* CONFIG_MEMCG */
int vswap_init(void)
--
2.52.0
^ permalink raw reply related
* [PATCH v5 20/21] swapfile: replace the swap map with bitmaps
From: Nhat Pham @ 2026-03-20 19:27 UTC (permalink / raw)
To: kasong
Cc: Liam.Howlett, akpm, apopple, axelrasmussen, baohua, baolin.wang,
bhe, byungchul, cgroups, chengming.zhou, chrisl, corbet, david,
dev.jain, gourry, hannes, hughd, jannh, joshua.hahnjy, lance.yang,
lenb, linux-doc, linux-kernel, linux-mm, linux-pm,
lorenzo.stoakes, matthew.brost, mhocko, muchun.song, npache,
nphamcs, pavel, peterx, peterz, pfalcato, rafael, rakie.kim,
roman.gushchin, rppt, ryan.roberts, shakeel.butt, shikemeng,
surenb, tglx, vbabka, weixugc, ying.huang, yosry.ahmed, yuanchu,
zhengqi.arch, ziy, kernel-team, riel
In-Reply-To: <20260320192735.748051-1-nphamcs@gmail.com>
Now that we have moved the swap count state to virtual swap layer, each
swap map entry only has 3 possible states: free, allocated, and bad.
Replace the swap map with 2 bitmaps (one for allocated state and one for
bad state), saving 6 bits per swap entry.
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
include/linux/swap.h | 3 +-
mm/swapfile.c | 81 +++++++++++++++++++++++---------------------
2 files changed, 44 insertions(+), 40 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 21e528d8d3480..3c789149996c5 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -259,7 +259,8 @@ struct swap_info_struct {
struct plist_node list; /* entry in swap_active_head */
signed char type; /* strange name for an index */
unsigned int max; /* extent of the swap_map */
- unsigned char *swap_map; /* vmalloc'ed array of usage counts */
+ unsigned long *swap_map; /* bitmap for allocated state */
+ unsigned long *bad_map; /* bitmap for bad state */
struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
diff --git a/mm/swapfile.c b/mm/swapfile.c
index b553652125d11..3e2bfcf1aa789 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -760,25 +760,19 @@ static bool cluster_reclaim_range(struct swap_info_struct *si,
struct swap_cluster_info *ci,
unsigned long start, unsigned long end)
{
- unsigned char *map = si->swap_map;
unsigned long offset = start;
int nr_reclaim;
spin_unlock(&ci->lock);
do {
- switch (READ_ONCE(map[offset])) {
- case 0:
+ if (!test_bit(offset, si->swap_map)) {
offset++;
- break;
- case SWAP_MAP_ALLOCATED:
+ } else {
nr_reclaim = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
if (nr_reclaim > 0)
offset += nr_reclaim;
else
goto out;
- break;
- default:
- goto out;
}
} while (offset < end);
out:
@@ -787,11 +781,7 @@ static bool cluster_reclaim_range(struct swap_info_struct *si,
* Recheck the range no matter reclaim succeeded or not, the slot
* could have been be freed while we are not holding the lock.
*/
- for (offset = start; offset < end; offset++)
- if (READ_ONCE(map[offset]))
- return false;
-
- return true;
+ return find_next_bit(si->swap_map, end, start) >= end;
}
static bool cluster_scan_range(struct swap_info_struct *si,
@@ -800,15 +790,16 @@ static bool cluster_scan_range(struct swap_info_struct *si,
bool *need_reclaim)
{
unsigned long offset, end = start + nr_pages;
- unsigned char *map = si->swap_map;
- unsigned char count;
if (cluster_is_empty(ci))
return true;
for (offset = start; offset < end; offset++) {
- count = READ_ONCE(map[offset]);
- if (!count)
+ /* Bad slots cannot be used for allocation */
+ if (test_bit(offset, si->bad_map))
+ return false;
+
+ if (!test_bit(offset, si->swap_map))
continue;
if (swap_cache_only(si, offset)) {
@@ -841,7 +832,7 @@ static bool cluster_alloc_range(struct swap_info_struct *si, struct swap_cluster
if (cluster_is_empty(ci))
ci->order = order;
- memset(si->swap_map + start, usage, nr_pages);
+ bitmap_set(si->swap_map, start, nr_pages);
swap_range_alloc(si, nr_pages);
ci->count += nr_pages;
@@ -1407,7 +1398,7 @@ static struct swap_info_struct *_swap_info_get(swp_slot_t slot)
offset = swp_slot_offset(slot);
if (offset >= si->max)
goto bad_offset;
- if (data_race(!si->swap_map[swp_slot_offset(slot)]))
+ if (data_race(!test_bit(offset, si->swap_map)))
goto bad_free;
return si;
@@ -1521,8 +1512,7 @@ static void swap_slots_free(struct swap_info_struct *si,
swp_slot_t slot, unsigned int nr_pages)
{
unsigned long offset = swp_slot_offset(slot);
- unsigned char *map = si->swap_map + offset;
- unsigned char *map_end = map + nr_pages;
+ unsigned long end = offset + nr_pages;
/* It should never free entries across different clusters */
VM_BUG_ON(ci != __swap_offset_to_cluster(si, offset + nr_pages - 1));
@@ -1530,10 +1520,8 @@ static void swap_slots_free(struct swap_info_struct *si,
VM_BUG_ON(ci->count < nr_pages);
ci->count -= nr_pages;
- do {
- VM_BUG_ON(!swap_is_last_ref(*map));
- *map = 0;
- } while (++map < map_end);
+ VM_BUG_ON(find_next_zero_bit(si->swap_map, end, offset) < end);
+ bitmap_clear(si->swap_map, offset, nr_pages);
swap_range_free(si, offset, nr_pages);
@@ -1744,9 +1732,7 @@ unsigned int count_swap_pages(int type, int free)
static bool swap_slot_allocated(struct swap_info_struct *si,
unsigned long offset)
{
- unsigned char count = READ_ONCE(si->swap_map[offset]);
-
- return count && swap_count(count) != SWAP_MAP_BAD;
+ return test_bit(offset, si->swap_map);
}
/*
@@ -2067,7 +2053,7 @@ static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
}
static void setup_swap_info(struct swap_info_struct *si, int prio,
- unsigned char *swap_map,
+ unsigned long *swap_map,
struct swap_cluster_info *cluster_info)
{
si->prio = prio;
@@ -2095,7 +2081,7 @@ static void _enable_swap_info(struct swap_info_struct *si)
}
static void enable_swap_info(struct swap_info_struct *si, int prio,
- unsigned char *swap_map,
+ unsigned long *swap_map,
struct swap_cluster_info *cluster_info)
{
spin_lock(&swap_lock);
@@ -2188,7 +2174,8 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
{
struct swap_info_struct *p = NULL;
- unsigned char *swap_map;
+ unsigned long *swap_map;
+ unsigned long *bad_map;
struct swap_cluster_info *cluster_info;
struct file *swap_file, *victim;
struct address_space *mapping;
@@ -2283,6 +2270,8 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
p->swap_file = NULL;
swap_map = p->swap_map;
p->swap_map = NULL;
+ bad_map = p->bad_map;
+ p->bad_map = NULL;
maxpages = p->max;
cluster_info = p->cluster_info;
p->max = 0;
@@ -2293,7 +2282,8 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
mutex_unlock(&swapon_mutex);
kfree(p->global_cluster);
p->global_cluster = NULL;
- vfree(swap_map);
+ kvfree(swap_map);
+ kvfree(bad_map);
free_cluster_info(cluster_info, maxpages);
inode = mapping->host;
@@ -2641,18 +2631,20 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
static int setup_swap_map(struct swap_info_struct *si,
union swap_header *swap_header,
- unsigned char *swap_map,
+ unsigned long *swap_map,
+ unsigned long *bad_map,
unsigned long maxpages)
{
unsigned long i;
- swap_map[0] = SWAP_MAP_BAD; /* omit header page */
+ set_bit(0, bad_map); /* omit header page */
+
for (i = 0; i < swap_header->info.nr_badpages; i++) {
unsigned int page_nr = swap_header->info.badpages[i];
if (page_nr == 0 || page_nr > swap_header->info.last_page)
return -EINVAL;
if (page_nr < maxpages) {
- swap_map[page_nr] = SWAP_MAP_BAD;
+ set_bit(page_nr, bad_map);
si->pages--;
}
}
@@ -2756,7 +2748,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
int nr_extents;
sector_t span;
unsigned long maxpages;
- unsigned char *swap_map = NULL;
+ unsigned long *swap_map = NULL, *bad_map = NULL;
struct swap_cluster_info *cluster_info = NULL;
struct folio *folio = NULL;
struct inode *inode = NULL;
@@ -2852,16 +2844,24 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
maxpages = si->max;
/* OK, set up the swap map and apply the bad block list */
- swap_map = vzalloc(maxpages);
+ swap_map = kvcalloc(BITS_TO_LONGS(maxpages), sizeof(long), GFP_KERNEL);
if (!swap_map) {
error = -ENOMEM;
goto bad_swap_unlock_inode;
}
- error = setup_swap_map(si, swap_header, swap_map, maxpages);
+ bad_map = kvcalloc(BITS_TO_LONGS(maxpages), sizeof(long), GFP_KERNEL);
+ if (!bad_map) {
+ error = -ENOMEM;
+ goto bad_swap_unlock_inode;
+ }
+
+ error = setup_swap_map(si, swap_header, swap_map, bad_map, maxpages);
if (error)
goto bad_swap_unlock_inode;
+ si->bad_map = bad_map;
+
if (si->bdev && bdev_stable_writes(si->bdev))
si->flags |= SWP_STABLE_WRITES;
@@ -2955,7 +2955,10 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
si->swap_file = NULL;
si->flags = 0;
spin_unlock(&swap_lock);
- vfree(swap_map);
+ if (swap_map)
+ kvfree(swap_map);
+ if (bad_map)
+ kvfree(bad_map);
if (cluster_info)
free_cluster_info(cluster_info, maxpages);
if (inced_nr_rotate_swap)
--
2.52.0
^ permalink raw reply related
* [PATCH v5 19/21] swap: simplify swapoff using virtual swap
From: Nhat Pham @ 2026-03-20 19:27 UTC (permalink / raw)
To: kasong
Cc: Liam.Howlett, akpm, apopple, axelrasmussen, baohua, baolin.wang,
bhe, byungchul, cgroups, chengming.zhou, chrisl, corbet, david,
dev.jain, gourry, hannes, hughd, jannh, joshua.hahnjy, lance.yang,
lenb, linux-doc, linux-kernel, linux-mm, linux-pm,
lorenzo.stoakes, matthew.brost, mhocko, muchun.song, npache,
nphamcs, pavel, peterx, peterz, pfalcato, rafael, rakie.kim,
roman.gushchin, rppt, ryan.roberts, shakeel.butt, shikemeng,
surenb, tglx, vbabka, weixugc, ying.huang, yosry.ahmed, yuanchu,
zhengqi.arch, ziy, kernel-team, riel
In-Reply-To: <20260320192735.748051-1-nphamcs@gmail.com>
This patch presents the second applications of virtual swap design -
simplifying and optimizing swapoff.
With virtual swap slots stored at page table entries and used as indices
to various swap-related data structures, we no longer have to perform a
page table walk in swapoff. Simply iterate through all the allocated
swap slots on the swapfile, find their corresponding virtual swap slots,
and fault them in.
This is significantly cleaner, as well as slightly more performant,
especially when there are a lot of unrelated VMAs (since the old swapoff
code would have to traverse through all of them).
In a simple benchmark, in which we swapoff a 32 GB swapfile that is 50%
full, and in which there is a process that maps a 128GB file into
memory:
Baseline:
sys: 11.48s
New Design:
sys: 9.96s
Disregarding the real time reduction (which is mostly due to more IO
asynchrony), the new design reduces the kernel CPU time by about 13%.
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
include/linux/shmem_fs.h | 7 +-
mm/filemap.c | 14 +-
mm/shmem.c | 196 +---------------
mm/swapfile.c | 474 +++++++++------------------------------
4 files changed, 126 insertions(+), 565 deletions(-)
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index e2069b3179c41..bac6b6cafe89c 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -41,17 +41,13 @@ struct shmem_inode_info {
unsigned long swapped; /* subtotal assigned to swap */
union {
struct offset_ctx dir_offsets; /* stable directory offsets */
- struct {
- struct list_head shrinklist; /* shrinkable hpage inodes */
- struct list_head swaplist; /* chain of maybes on swap */
- };
+ struct list_head shrinklist; /* shrinkable hpage inodes */
};
struct timespec64 i_crtime; /* file creation time */
struct shared_policy policy; /* NUMA memory alloc policy */
struct simple_xattrs xattrs; /* list of xattrs */
pgoff_t fallocend; /* highest fallocate endindex */
unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */
- atomic_t stop_eviction; /* hold when working on inode */
#ifdef CONFIG_TMPFS_QUOTA
struct dquot __rcu *i_dquot[MAXQUOTAS];
#endif
@@ -127,7 +123,6 @@ struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
struct list_head *folio_list);
void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
-int shmem_unuse(unsigned int type);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
unsigned long shmem_allowable_huge_orders(struct inode *inode,
diff --git a/mm/filemap.c b/mm/filemap.c
index ebd75684cb0a7..53aad273ea2f1 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -4614,13 +4614,13 @@ static void filemap_cachestat(struct address_space *mapping,
/*
* Getting a swap entry from the shmem
- * inode means we beat
- * shmem_unuse(). rcu_read_lock()
- * ensures swapoff waits for us before
- * freeing the swapper space. However,
- * we can race with swapping and
- * invalidation, so there might not be
- * a shadow in the swapcache (yet).
+ * inode means we beat swapoff.
+ * rcu_read_lock() ensures swapoff waits
+ * for us before freeing the swapper
+ * space. However, we can race with
+ * swapping and invalidation, so there
+ * might not be a shadow in the swapcache
+ * (yet).
*/
shadow = swap_cache_get_shadow(swp);
if (!shadow)
diff --git a/mm/shmem.c b/mm/shmem.c
index 3a346cca114ab..984e01ea88d3c 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -290,9 +290,6 @@ bool vma_is_shmem(const struct vm_area_struct *vma)
return vma_is_anon_shmem(vma) || vma->vm_ops == &shmem_vm_ops;
}
-static LIST_HEAD(shmem_swaplist);
-static DEFINE_SPINLOCK(shmem_swaplist_lock);
-
#ifdef CONFIG_TMPFS_QUOTA
static int shmem_enable_quotas(struct super_block *sb,
@@ -1413,16 +1410,6 @@ static void shmem_evict_inode(struct inode *inode)
}
spin_unlock(&sbinfo->shrinklist_lock);
}
- while (!list_empty(&info->swaplist)) {
- /* Wait while shmem_unuse() is scanning this inode... */
- wait_var_event(&info->stop_eviction,
- !atomic_read(&info->stop_eviction));
- spin_lock(&shmem_swaplist_lock);
- /* ...but beware of the race if we peeked too early */
- if (!atomic_read(&info->stop_eviction))
- list_del_init(&info->swaplist);
- spin_unlock(&shmem_swaplist_lock);
- }
}
simple_xattrs_free(&info->xattrs, sbinfo->max_inodes ? &freed : NULL);
@@ -1435,153 +1422,6 @@ static void shmem_evict_inode(struct inode *inode)
#endif
}
-static unsigned int shmem_find_swap_entries(struct address_space *mapping,
- pgoff_t start, struct folio_batch *fbatch,
- pgoff_t *indices, unsigned int type)
-{
- XA_STATE(xas, &mapping->i_pages, start);
- struct folio *folio;
- swp_entry_t entry;
- swp_slot_t slot;
-
- rcu_read_lock();
- xas_for_each(&xas, folio, ULONG_MAX) {
- if (xas_retry(&xas, folio))
- continue;
-
- if (!xa_is_value(folio))
- continue;
-
- entry = radix_to_swp_entry(folio);
- slot = swp_entry_to_swp_slot(entry);
-
- /*
- * swapin error entries can be found in the mapping. But they're
- * deliberately ignored here as we've done everything we can do.
- */
- if (!slot.val || swp_slot_type(slot) != type)
- continue;
-
- indices[folio_batch_count(fbatch)] = xas.xa_index;
- if (!folio_batch_add(fbatch, folio))
- break;
-
- if (need_resched()) {
- xas_pause(&xas);
- cond_resched_rcu();
- }
- }
- rcu_read_unlock();
-
- return folio_batch_count(fbatch);
-}
-
-/*
- * Move the swapped pages for an inode to page cache. Returns the count
- * of pages swapped in, or the error in case of failure.
- */
-static int shmem_unuse_swap_entries(struct inode *inode,
- struct folio_batch *fbatch, pgoff_t *indices)
-{
- int i = 0;
- int ret = 0;
- int error = 0;
- struct address_space *mapping = inode->i_mapping;
-
- for (i = 0; i < folio_batch_count(fbatch); i++) {
- struct folio *folio = fbatch->folios[i];
-
- error = shmem_swapin_folio(inode, indices[i], &folio, SGP_CACHE,
- mapping_gfp_mask(mapping), NULL, NULL);
- if (error == 0) {
- folio_unlock(folio);
- folio_put(folio);
- ret++;
- }
- if (error == -ENOMEM)
- break;
- error = 0;
- }
- return error ? error : ret;
-}
-
-/*
- * If swap found in inode, free it and move page from swapcache to filecache.
- */
-static int shmem_unuse_inode(struct inode *inode, unsigned int type)
-{
- struct address_space *mapping = inode->i_mapping;
- pgoff_t start = 0;
- struct folio_batch fbatch;
- pgoff_t indices[PAGEVEC_SIZE];
- int ret = 0;
-
- do {
- folio_batch_init(&fbatch);
- if (!shmem_find_swap_entries(mapping, start, &fbatch,
- indices, type)) {
- ret = 0;
- break;
- }
-
- ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
- if (ret < 0)
- break;
-
- start = indices[folio_batch_count(&fbatch) - 1];
- } while (true);
-
- return ret;
-}
-
-/*
- * Read all the shared memory data that resides in the swap
- * device 'type' back into memory, so the swap device can be
- * unused.
- */
-int shmem_unuse(unsigned int type)
-{
- struct shmem_inode_info *info, *next;
- int error = 0;
-
- if (list_empty(&shmem_swaplist))
- return 0;
-
- spin_lock(&shmem_swaplist_lock);
-start_over:
- list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
- if (!info->swapped) {
- list_del_init(&info->swaplist);
- continue;
- }
- /*
- * Drop the swaplist mutex while searching the inode for swap;
- * but before doing so, make sure shmem_evict_inode() will not
- * remove placeholder inode from swaplist, nor let it be freed
- * (igrab() would protect from unlink, but not from unmount).
- */
- atomic_inc(&info->stop_eviction);
- spin_unlock(&shmem_swaplist_lock);
-
- error = shmem_unuse_inode(&info->vfs_inode, type);
- cond_resched();
-
- spin_lock(&shmem_swaplist_lock);
- if (atomic_dec_and_test(&info->stop_eviction))
- wake_up_var(&info->stop_eviction);
- if (error)
- break;
- if (list_empty(&info->swaplist))
- goto start_over;
- next = list_next_entry(info, swaplist);
- if (!info->swapped)
- list_del_init(&info->swaplist);
- }
- spin_unlock(&shmem_swaplist_lock);
-
- return error;
-}
-
/**
* shmem_writeout - Write the folio to swap
* @folio: The folio to write
@@ -1668,24 +1508,9 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
}
if (!folio_alloc_swap(folio)) {
- bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages);
int error;
- /*
- * Add inode to shmem_unuse()'s list of swapped-out inodes,
- * if it's not already there. Do it now before the folio is
- * removed from page cache, when its pagelock no longer
- * protects the inode from eviction. And do it now, after
- * we've incremented swapped, because shmem_unuse() will
- * prune a !swapped inode from the swaplist.
- */
- if (first_swapped) {
- spin_lock(&shmem_swaplist_lock);
- if (list_empty(&info->swaplist))
- list_add(&info->swaplist, &shmem_swaplist);
- spin_unlock(&shmem_swaplist_lock);
- }
-
+ shmem_recalc_inode(inode, 0, nr_pages);
swap_shmem_alloc(folio->swap, nr_pages);
shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));
@@ -2116,12 +1941,12 @@ static struct folio *shmem_swap_alloc_folio(struct inode *inode,
}
/*
- * When a page is moved from swapcache to shmem filecache (either by the
- * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
- * shmem_unuse_inode()), it may have been read in earlier from swap, in
- * ignorance of the mapping it belongs to. If that mapping has special
- * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
- * we may need to copy to a suitable page before moving to filecache.
+ * When a page is moved from swapcache to shmem filecache (by the usual
+ * swapin of shmem_get_folio_gfp()), it may have been read in earlier from
+ * swap, in ignorance of the mapping it belongs to. If that mapping has
+ * special constraints (like the gma500 GEM driver, which requires RAM
+ * below 4GB), we may need to copy to a suitable page before moving to
+ * filecache.
*
* In a future release, this may well be extended to respect cpuset and
* NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
@@ -3106,7 +2931,6 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
info = SHMEM_I(inode);
memset(info, 0, (char *)inode - (char *)info);
spin_lock_init(&info->lock);
- atomic_set(&info->stop_eviction, 0);
info->seals = F_SEAL_SEAL;
info->flags = (flags & VM_NORESERVE) ? SHMEM_F_NORESERVE : 0;
info->i_crtime = inode_get_mtime(inode);
@@ -3115,7 +2939,6 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
if (info->fsflags)
shmem_set_inode_flags(inode, info->fsflags, NULL);
INIT_LIST_HEAD(&info->shrinklist);
- INIT_LIST_HEAD(&info->swaplist);
simple_xattrs_init(&info->xattrs);
cache_no_acl(inode);
if (sbinfo->noswap)
@@ -5785,11 +5608,6 @@ void __init shmem_init(void)
BUG_ON(IS_ERR(shm_mnt));
}
-int shmem_unuse(unsigned int type)
-{
- return 0;
-}
-
int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
{
return 0;
diff --git a/mm/swapfile.c b/mm/swapfile.c
index aeb3575df8a0b..b553652125d11 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1741,300 +1741,12 @@ unsigned int count_swap_pages(int type, int free)
}
#endif /* CONFIG_HIBERNATION */
-static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
+static bool swap_slot_allocated(struct swap_info_struct *si,
+ unsigned long offset)
{
- return pte_same(pte_swp_clear_flags(pte), swp_pte);
-}
-
-/*
- * No need to decide whether this PTE shares the swap entry with others,
- * just let do_wp_page work it out if a write is requested later - to
- * force COW, vm_page_prot omits write permission from any private vma.
- */
-static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long addr, swp_entry_t entry, struct folio *folio)
-{
- struct page *page;
- struct folio *swapcache;
- spinlock_t *ptl;
- pte_t *pte, new_pte, old_pte;
- bool hwpoisoned = false;
- int ret = 1;
-
- /*
- * If the folio is removed from swap cache by others, continue to
- * unuse other PTEs. try_to_unuse may try again if we missed this one.
- */
- if (!folio_matches_swap_entry(folio, entry))
- return 0;
-
- swapcache = folio;
- folio = ksm_might_need_to_copy(folio, vma, addr);
- if (unlikely(!folio))
- return -ENOMEM;
- else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
- hwpoisoned = true;
- folio = swapcache;
- }
-
- page = folio_file_page(folio, swp_offset(entry));
- if (PageHWPoison(page))
- hwpoisoned = true;
-
- pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
- if (unlikely(!pte || !pte_same_as_swp(ptep_get(pte),
- swp_entry_to_pte(entry)))) {
- ret = 0;
- goto out;
- }
-
- old_pte = ptep_get(pte);
-
- if (unlikely(hwpoisoned || !folio_test_uptodate(folio))) {
- swp_entry_t swp_entry;
-
- dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
- if (hwpoisoned) {
- swp_entry = make_hwpoison_entry(page);
- } else {
- swp_entry = make_poisoned_swp_entry();
- }
- new_pte = swp_entry_to_pte(swp_entry);
- ret = 0;
- goto setpte;
- }
-
- /*
- * Some architectures may have to restore extra metadata to the page
- * when reading from swap. This metadata may be indexed by swap entry
- * so this must be called before swap_free().
- */
- arch_swap_restore(folio_swap(entry, folio), folio);
-
- dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
- inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
- folio_get(folio);
- if (folio == swapcache) {
- rmap_t rmap_flags = RMAP_NONE;
-
- /*
- * See do_swap_page(): writeback would be problematic.
- * However, we do a folio_wait_writeback() just before this
- * call and have the folio locked.
- */
- VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
- if (pte_swp_exclusive(old_pte))
- rmap_flags |= RMAP_EXCLUSIVE;
- /*
- * We currently only expect small !anon folios, which are either
- * fully exclusive or fully shared. If we ever get large folios
- * here, we have to be careful.
- */
- if (!folio_test_anon(folio)) {
- VM_WARN_ON_ONCE(folio_test_large(folio));
- VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
- folio_add_new_anon_rmap(folio, vma, addr, rmap_flags);
- } else {
- folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags);
- }
- } else { /* ksm created a completely new copy */
- folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
- folio_add_lru_vma(folio, vma);
- }
- new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot));
- if (pte_swp_soft_dirty(old_pte))
- new_pte = pte_mksoft_dirty(new_pte);
- if (pte_swp_uffd_wp(old_pte))
- new_pte = pte_mkuffd_wp(new_pte);
-setpte:
- set_pte_at(vma->vm_mm, addr, pte, new_pte);
- swap_free(entry);
-out:
- if (pte)
- pte_unmap_unlock(pte, ptl);
- if (folio != swapcache) {
- folio_unlock(folio);
- folio_put(folio);
- }
- return ret;
-}
-
-static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long addr, unsigned long end,
- unsigned int type)
-{
- pte_t *pte = NULL;
- struct swap_info_struct *si;
-
- si = swap_info[type];
- do {
- struct folio *folio;
- unsigned long offset;
- unsigned char swp_count;
- softleaf_t entry;
- swp_slot_t slot;
- int ret;
- pte_t ptent;
-
- if (!pte++) {
- pte = pte_offset_map(pmd, addr);
- if (!pte)
- break;
- }
-
- ptent = ptep_get_lockless(pte);
- entry = softleaf_from_pte(ptent);
-
- if (!softleaf_is_swap(entry))
- continue;
-
- slot = swp_entry_to_swp_slot(entry);
- if (swp_slot_type(slot) != type)
- continue;
-
- offset = swp_slot_offset(slot);
- pte_unmap(pte);
- pte = NULL;
-
- folio = swap_cache_get_folio(entry);
- if (!folio) {
- struct vm_fault vmf = {
- .vma = vma,
- .address = addr,
- .real_address = addr,
- .pmd = pmd,
- };
-
- folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
- &vmf);
- }
- if (!folio) {
- swp_count = READ_ONCE(si->swap_map[offset]);
- if (swp_count == 0 || swp_count == SWAP_MAP_BAD)
- continue;
- return -ENOMEM;
- }
-
- folio_lock(folio);
- folio_wait_writeback(folio);
- ret = unuse_pte(vma, pmd, addr, entry, folio);
- if (ret < 0) {
- folio_unlock(folio);
- folio_put(folio);
- return ret;
- }
-
- folio_free_swap(folio);
- folio_unlock(folio);
- folio_put(folio);
- } while (addr += PAGE_SIZE, addr != end);
-
- if (pte)
- pte_unmap(pte);
- return 0;
-}
-
-static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
- unsigned long addr, unsigned long end,
- unsigned int type)
-{
- pmd_t *pmd;
- unsigned long next;
- int ret;
-
- pmd = pmd_offset(pud, addr);
- do {
- cond_resched();
- next = pmd_addr_end(addr, end);
- ret = unuse_pte_range(vma, pmd, addr, next, type);
- if (ret)
- return ret;
- } while (pmd++, addr = next, addr != end);
- return 0;
-}
-
-static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
- unsigned long addr, unsigned long end,
- unsigned int type)
-{
- pud_t *pud;
- unsigned long next;
- int ret;
-
- pud = pud_offset(p4d, addr);
- do {
- next = pud_addr_end(addr, end);
- if (pud_none_or_clear_bad(pud))
- continue;
- ret = unuse_pmd_range(vma, pud, addr, next, type);
- if (ret)
- return ret;
- } while (pud++, addr = next, addr != end);
- return 0;
-}
-
-static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
- unsigned long addr, unsigned long end,
- unsigned int type)
-{
- p4d_t *p4d;
- unsigned long next;
- int ret;
-
- p4d = p4d_offset(pgd, addr);
- do {
- next = p4d_addr_end(addr, end);
- if (p4d_none_or_clear_bad(p4d))
- continue;
- ret = unuse_pud_range(vma, p4d, addr, next, type);
- if (ret)
- return ret;
- } while (p4d++, addr = next, addr != end);
- return 0;
-}
-
-static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
-{
- pgd_t *pgd;
- unsigned long addr, end, next;
- int ret;
-
- addr = vma->vm_start;
- end = vma->vm_end;
-
- pgd = pgd_offset(vma->vm_mm, addr);
- do {
- next = pgd_addr_end(addr, end);
- if (pgd_none_or_clear_bad(pgd))
- continue;
- ret = unuse_p4d_range(vma, pgd, addr, next, type);
- if (ret)
- return ret;
- } while (pgd++, addr = next, addr != end);
- return 0;
-}
+ unsigned char count = READ_ONCE(si->swap_map[offset]);
-static int unuse_mm(struct mm_struct *mm, unsigned int type)
-{
- struct vm_area_struct *vma;
- int ret = 0;
- VMA_ITERATOR(vmi, mm, 0);
-
- mmap_read_lock(mm);
- if (check_stable_address_space(mm))
- goto unlock;
- for_each_vma(vmi, vma) {
- if (vma->anon_vma && !is_vm_hugetlb_page(vma)) {
- ret = unuse_vma(vma, type);
- if (ret)
- break;
- }
-
- cond_resched();
- }
-unlock:
- mmap_read_unlock(mm);
- return ret;
+ return count && swap_count(count) != SWAP_MAP_BAD;
}
/*
@@ -2046,7 +1758,6 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
unsigned int prev)
{
unsigned int i;
- unsigned char count;
/*
* No need for swap_lock here: we're just looking
@@ -2055,8 +1766,7 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
* allocations from this area (while holding swap_lock).
*/
for (i = prev + 1; i < si->max; i++) {
- count = READ_ONCE(si->swap_map[i]);
- if (count && swap_count(count) != SWAP_MAP_BAD)
+ if (swap_slot_allocated(si, i))
break;
if ((i % LATENCY_LIMIT) == 0)
cond_resched();
@@ -2068,101 +1778,139 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
return i;
}
+#define for_each_allocated_offset(si, offset) \
+ while (swap_usage_in_pages(si) && \
+ !signal_pending(current) && \
+ (offset = find_next_to_unuse(si, offset)) != 0)
+
+static struct folio *pagein(swp_entry_t entry, struct swap_iocb **splug,
+ struct mempolicy *mpol)
+{
+ bool folio_was_allocated;
+ struct folio *folio = __read_swap_cache_async(entry, GFP_KERNEL, mpol,
+ NO_INTERLEAVE_INDEX, &folio_was_allocated, false);
+
+ if (folio_was_allocated)
+ swap_read_folio(folio, splug);
+ return folio;
+}
+
static int try_to_unuse(unsigned int type)
{
- struct mm_struct *prev_mm;
- struct mm_struct *mm;
- struct list_head *p;
- int retval = 0;
struct swap_info_struct *si = swap_info[type];
+ struct swap_iocb *splug = NULL;
+ struct mempolicy *mpol;
+ struct blk_plug plug;
+ unsigned long offset;
struct folio *folio;
swp_entry_t entry;
swp_slot_t slot;
- unsigned int i;
+ int ret = 0;
if (!swap_usage_in_pages(si))
goto success;
-retry:
- retval = shmem_unuse(type);
- if (retval)
- return retval;
-
- prev_mm = &init_mm;
- mmget(prev_mm);
-
- spin_lock(&mmlist_lock);
- p = &init_mm.mmlist;
- while (swap_usage_in_pages(si) &&
- !signal_pending(current) &&
- (p = p->next) != &init_mm.mmlist) {
+ mpol = get_task_policy(current);
+ blk_start_plug(&plug);
- mm = list_entry(p, struct mm_struct, mmlist);
- if (!mmget_not_zero(mm))
+ /* first round - submit the reads */
+ offset = 0;
+ for_each_allocated_offset(si, offset) {
+ slot = swp_slot(type, offset);
+ entry = swp_slot_to_swp_entry(slot);
+ if (!entry.val)
continue;
- spin_unlock(&mmlist_lock);
- mmput(prev_mm);
- prev_mm = mm;
- retval = unuse_mm(mm, type);
- if (retval) {
- mmput(prev_mm);
- return retval;
- }
- /*
- * Make sure that we aren't completely killing
- * interactive performance.
- */
- cond_resched();
- spin_lock(&mmlist_lock);
+ folio = pagein(entry, &splug, mpol);
+ if (folio)
+ folio_put(folio);
}
- spin_unlock(&mmlist_lock);
+ blk_finish_plug(&plug);
+ swap_read_unplug(splug);
+ splug = NULL;
+ lru_add_drain();
+
+ /* second round - updating the virtual swap slots' backing state */
+ offset = 0;
+ for_each_allocated_offset(si, offset) {
+ slot = swp_slot(type, offset);
+retry:
+ entry = swp_slot_to_swp_entry(slot);
+ if (!entry.val) {
+ if (!swap_slot_allocated(si, offset))
+ continue;
- mmput(prev_mm);
+ if (signal_pending(current)) {
+ ret = -EINTR;
+ goto out;
+ }
- i = 0;
- while (swap_usage_in_pages(si) &&
- !signal_pending(current) &&
- (i = find_next_to_unuse(si, i)) != 0) {
+ /* we might be racing with zswap writeback or disk swapout */
+ schedule_timeout_uninterruptible(1);
+ goto retry;
+ }
- slot = swp_slot(type, i);
- entry = swp_slot_to_swp_entry(slot);
- folio = swap_cache_get_folio(entry);
- if (!folio)
- continue;
+ /* try to allocate swap cache folio */
+ folio = pagein(entry, &splug, mpol);
+ if (!folio) {
+ if (!swp_slot_to_swp_entry(swp_slot(type, offset)).val)
+ continue;
+ ret = -ENOMEM;
+ pr_err("swapoff: unable to allocate swap cache folio for %lu\n",
+ entry.val);
+ goto out;
+ }
+
+ folio_lock(folio);
/*
- * It is conceivable that a racing task removed this folio from
- * swap cache just before we acquired the page lock. The folio
- * might even be back in swap cache on another swap area. But
- * that is okay, folio_free_swap() only removes stale folios.
+ * We need to check if the folio is still in swap cache, and is still
+ * backed by the physical swap slot we are trying to release.
+ *
+ * We can, for instance, race with zswap writeback, obtaining the
+ * temporary folio it allocated for decompression and writeback, which
+ * would be promptly deleted from swap cache. By the time we lock that
+ * folio, it might have already contained stale data.
+ *
+ * Concurrent swap operations might have also come in before we
+ * reobtain the folio's lock, deleting the folio from swap cache,
+ * invalidating the virtual swap slot, then swapping out the folio
+ * again to a different swap backends.
+ *
+ * In all of these cases, we must retry the physical -> virtual lookup.
*/
- folio_lock(folio);
+ if (!folio_matches_swap_slot(folio, entry, slot)) {
+ folio_unlock(folio);
+ folio_put(folio);
+ if (signal_pending(current)) {
+ ret = -EINTR;
+ goto out;
+ }
+ schedule_timeout_uninterruptible(1);
+ goto retry;
+ }
+
folio_wait_writeback(folio);
- folio_free_swap(folio);
+ vswap_store_folio(entry, folio);
+ folio_mark_dirty(folio);
folio_unlock(folio);
folio_put(folio);
}
- /*
- * Lets check again to see if there are still swap entries in the map.
- * If yes, we would need to do retry the unuse logic again.
- * Under global memory pressure, swap entries can be reinserted back
- * into process space after the mmlist loop above passes over them.
- *
- * Limit the number of retries? No: when mmget_not_zero()
- * above fails, that mm is likely to be freeing swap from
- * exit_mmap(), which proceeds at its own independent pace;
- * and even shmem_writeout() could have been preempted after
- * folio_alloc_swap(), temporarily hiding that swap. It's easy
- * and robust (though cpu-intensive) just to keep retrying.
- */
- if (swap_usage_in_pages(si)) {
- if (!signal_pending(current))
- goto retry;
- return -EINTR;
+ /* concurrent swappers might still be releasing physical swap slots... */
+ while (swap_usage_in_pages(si)) {
+ if (signal_pending(current)) {
+ ret = -EINTR;
+ goto out;
+ }
+ schedule_timeout_uninterruptible(1);
}
+out:
+ swap_read_unplug(splug);
+ if (ret)
+ return ret;
+
success:
/*
* Make sure that further cleanups after try_to_unuse() returns happen
--
2.52.0
^ permalink raw reply related
* [PATCH v5 18/21] memcg: swap: only charge physical swap slots
From: Nhat Pham @ 2026-03-20 19:27 UTC (permalink / raw)
To: kasong
Cc: Liam.Howlett, akpm, apopple, axelrasmussen, baohua, baolin.wang,
bhe, byungchul, cgroups, chengming.zhou, chrisl, corbet, david,
dev.jain, gourry, hannes, hughd, jannh, joshua.hahnjy, lance.yang,
lenb, linux-doc, linux-kernel, linux-mm, linux-pm,
lorenzo.stoakes, matthew.brost, mhocko, muchun.song, npache,
nphamcs, pavel, peterx, peterz, pfalcato, rafael, rakie.kim,
roman.gushchin, rppt, ryan.roberts, shakeel.butt, shikemeng,
surenb, tglx, vbabka, weixugc, ying.huang, yosry.ahmed, yuanchu,
zhengqi.arch, ziy, kernel-team, riel
In-Reply-To: <20260320192735.748051-1-nphamcs@gmail.com>
Now that zswap and the zero-filled swap page optimization no longer
takes up any physical swap space, we should not charge towards the swap
usage and limits of the memcg in these case. We will only record the
memcg id on virtual swap slot allocation, and defer physical swap
charging (i.e towards memory.swap.current) until the virtual swap slot
is backed by an actual physical swap slot (on zswap store failure
fallback or zswap writeback).
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
include/linux/swap.h | 26 ++++++++++++++
mm/memcontrol-v1.c | 6 ++++
mm/memcontrol.c | 83 ++++++++++++++++++++++++++++++++------------
mm/vswap.c | 39 +++++++++------------
4 files changed, 108 insertions(+), 46 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index cc1ca4ac2946d..21e528d8d3480 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -676,6 +676,22 @@ static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
#endif
#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
+void __mem_cgroup_record_swap(struct folio *folio, swp_entry_t entry);
+static inline void mem_cgroup_record_swap(struct folio *folio,
+ swp_entry_t entry)
+{
+ if (!mem_cgroup_disabled())
+ __mem_cgroup_record_swap(folio, entry);
+}
+
+void __mem_cgroup_clear_swap(swp_entry_t entry, unsigned int nr_pages);
+static inline void mem_cgroup_clear_swap(swp_entry_t entry,
+ unsigned int nr_pages)
+{
+ if (!mem_cgroup_disabled())
+ __mem_cgroup_clear_swap(entry, nr_pages);
+}
+
int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry);
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
swp_entry_t entry)
@@ -696,6 +712,16 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p
extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
extern bool mem_cgroup_swap_full(struct folio *folio);
#else
+static inline void mem_cgroup_record_swap(struct folio *folio,
+ swp_entry_t entry)
+{
+}
+
+static inline void mem_cgroup_clear_swap(swp_entry_t entry,
+ unsigned int nr_pages)
+{
+}
+
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
swp_entry_t entry)
{
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 7b010e165e1ba..12bc5c680b03a 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -680,6 +680,12 @@ void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages)
* memory+swap charge, drop the swap entry duplicate.
*/
mem_cgroup_uncharge_swap(entry, nr_pages);
+
+ /*
+ * Clear the cgroup association now to prevent double memsw
+ * uncharging when the backends are released later.
+ */
+ mem_cgroup_clear_swap(entry, nr_pages);
}
}
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 2ba5811e7edba..4525c21754e7f 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -5172,6 +5172,49 @@ int __init mem_cgroup_init(void)
}
#ifdef CONFIG_SWAP
+/**
+ * __mem_cgroup_record_swap - record the folio's cgroup for the swap entries.
+ * @folio: folio being swapped out.
+ * @entry: the first swap entry in the range.
+ */
+void __mem_cgroup_record_swap(struct folio *folio, swp_entry_t entry)
+{
+ unsigned int nr_pages = folio_nr_pages(folio);
+ struct mem_cgroup *memcg;
+
+ /* Recording will be done by memcg1_swapout(). */
+ if (do_memsw_account())
+ return;
+
+ memcg = folio_memcg(folio);
+
+ VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
+ if (!memcg)
+ return;
+
+ memcg = mem_cgroup_id_get_online(memcg);
+ if (nr_pages > 1)
+ mem_cgroup_id_get_many(memcg, nr_pages - 1);
+ swap_cgroup_record(folio, mem_cgroup_id(memcg), entry);
+}
+
+/**
+ * __mem_cgroup_clear_swap - clear cgroup information of the swap entries.
+ * @entry: the first swap entry in the range.
+ * @nr_pages: the number of pages in the range.
+ */
+void __mem_cgroup_clear_swap(swp_entry_t entry, unsigned int nr_pages)
+{
+ unsigned short id = swap_cgroup_clear(entry, nr_pages);
+ struct mem_cgroup *memcg;
+
+ rcu_read_lock();
+ memcg = mem_cgroup_from_id(id);
+ if (memcg)
+ mem_cgroup_id_put_many(memcg, nr_pages);
+ rcu_read_unlock();
+}
+
/**
* __mem_cgroup_try_charge_swap - try charging swap space for a folio
* @folio: folio being added to swap
@@ -5190,34 +5233,24 @@ int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)
if (do_memsw_account())
return 0;
- memcg = folio_memcg(folio);
-
- VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
- if (!memcg)
- return 0;
-
- if (!entry.val) {
- memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
- return 0;
- }
-
- memcg = mem_cgroup_id_get_online(memcg);
+ /*
+ * We already record the cgroup on virtual swap allocation.
+ * Note that the virtual swap slot holds a reference to memcg,
+ * so this lookup should be safe.
+ */
+ rcu_read_lock();
+ memcg = mem_cgroup_from_id(lookup_swap_cgroup_id(entry));
+ rcu_read_unlock();
if (!mem_cgroup_is_root(memcg) &&
!page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
memcg_memory_event(memcg, MEMCG_SWAP_MAX);
memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
- mem_cgroup_id_put(memcg);
return -ENOMEM;
}
- /* Get references for the tail pages, too */
- if (nr_pages > 1)
- mem_cgroup_id_get_many(memcg, nr_pages - 1);
mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
- swap_cgroup_record(folio, mem_cgroup_id(memcg), entry);
-
return 0;
}
@@ -5231,7 +5264,8 @@ void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
struct mem_cgroup *memcg;
unsigned short id;
- id = swap_cgroup_clear(entry, nr_pages);
+ id = lookup_swap_cgroup_id(entry);
+
rcu_read_lock();
memcg = mem_cgroup_from_id(id);
if (memcg) {
@@ -5242,7 +5276,6 @@ void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
page_counter_uncharge(&memcg->swap, nr_pages);
}
mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
- mem_cgroup_id_put_many(memcg, nr_pages);
}
rcu_read_unlock();
}
@@ -5251,14 +5284,18 @@ static bool mem_cgroup_may_zswap(struct mem_cgroup *original_memcg);
long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
{
- long nr_swap_pages, nr_zswap_pages = 0;
+ long nr_swap_pages;
if (zswap_is_enabled() && (mem_cgroup_disabled() || do_memsw_account() ||
mem_cgroup_may_zswap(memcg))) {
- nr_zswap_pages = PAGE_COUNTER_MAX;
+ /*
+ * No need to check swap cgroup limits, since zswap is not charged
+ * towards swap consumption.
+ */
+ return PAGE_COUNTER_MAX;
}
- nr_swap_pages = max_t(long, nr_zswap_pages, get_nr_swap_pages());
+ nr_swap_pages = get_nr_swap_pages();
if (mem_cgroup_disabled() || do_memsw_account())
return nr_swap_pages;
for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg))
diff --git a/mm/vswap.c b/mm/vswap.c
index 1040bb8a9f320..fa37165cb10d0 100644
--- a/mm/vswap.c
+++ b/mm/vswap.c
@@ -544,6 +544,7 @@ static void release_backing(swp_entry_t entry, int nr)
struct vswap_cluster *cluster = NULL;
struct swp_desc *desc;
unsigned long flush_nr, phys_swap_start = 0, phys_swap_end = 0;
+ unsigned long phys_swap_released = 0;
unsigned int phys_swap_type = 0;
bool need_flushing_phys_swap = false;
swp_slot_t flush_slot;
@@ -573,6 +574,7 @@ static void release_backing(swp_entry_t entry, int nr)
if (desc->type == VSWAP_ZSWAP && desc->zswap_entry) {
zswap_entry_free(desc->zswap_entry);
} else if (desc->type == VSWAP_SWAPFILE) {
+ phys_swap_released++;
if (!phys_swap_start) {
/* start a new contiguous range of phys swap */
phys_swap_start = swp_slot_offset(desc->slot);
@@ -603,6 +605,9 @@ static void release_backing(swp_entry_t entry, int nr)
flush_nr = phys_swap_end - phys_swap_start;
swap_slot_free_nr(flush_slot, flush_nr);
}
+
+ if (phys_swap_released)
+ mem_cgroup_uncharge_swap(entry, phys_swap_released);
}
/*
@@ -630,7 +635,7 @@ static void vswap_free(struct vswap_cluster *cluster, struct swp_desc *desc,
spin_unlock(&cluster->lock);
release_backing(entry, 1);
- mem_cgroup_uncharge_swap(entry, 1);
+ mem_cgroup_clear_swap(entry, 1);
/* erase forward mapping and release the virtual slot for reallocation */
spin_lock(&cluster->lock);
@@ -645,9 +650,6 @@ static void vswap_free(struct vswap_cluster *cluster, struct swp_desc *desc,
*/
int folio_alloc_swap(struct folio *folio)
{
- struct vswap_cluster *cluster = NULL;
- int i, nr = folio_nr_pages(folio);
- struct swp_desc *desc;
swp_entry_t entry;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
@@ -657,25 +659,7 @@ int folio_alloc_swap(struct folio *folio)
if (!entry.val)
return -ENOMEM;
- /*
- * XXX: for now, we charge towards the memory cgroup's swap limit on virtual
- * swap slots allocation. This will be changed soon - we will only charge on
- * physical swap slots allocation.
- */
- if (mem_cgroup_try_charge_swap(folio, entry)) {
- rcu_read_lock();
- for (i = 0; i < nr; i++) {
- desc = vswap_iter(&cluster, entry.val + i);
- VM_WARN_ON(!desc);
- vswap_free(cluster, desc, (swp_entry_t){ entry.val + i });
- }
- spin_unlock(&cluster->lock);
- rcu_read_unlock();
- atomic_add(nr, &vswap_alloc_reject);
- entry.val = 0;
- return -ENOMEM;
- }
-
+ mem_cgroup_record_swap(folio, entry);
swap_cache_add_folio(folio, entry, NULL);
return 0;
@@ -717,6 +701,15 @@ bool vswap_alloc_swap_slot(struct folio *folio)
if (!slot.val)
return false;
+ if (mem_cgroup_try_charge_swap(folio, entry)) {
+ /*
+ * We have not updated the backing type of the virtual swap slot.
+ * Simply free up the physical swap slots here!
+ */
+ swap_slot_free_nr(slot, nr);
+ return false;
+ }
+
/* establish the vrtual <-> physical swap slots linkages. */
si = __swap_slot_to_info(slot);
ci = swap_cluster_lock(si, swp_slot_offset(slot));
--
2.52.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox