* Re: [PATCH v6 3/4] mm: memcontrol: add interfaces for swap tier selection
From: Baoquan He @ 2026-05-26 15:33 UTC (permalink / raw)
To: Youngjun Park
Cc: akpm, chrisl, linux-mm, cgroups, linux-kernel, kasong, hannes,
mhocko, roman.gushchin, shakeel.butt, muchun.song, shikemeng,
nphamcs, bhe, baohua, gunho.lee, taejoon.song, hyungjun.cho,
mkoutny, baver.bae, matia.kim
In-Reply-To: <20260421055323.940344-4-youngjun.park@lge.com>
On 04/21/26 at 02:53pm, Youngjun Park wrote:
...snip...
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index c3d98ab41f1f..0f67572e5e3e 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -68,6 +68,7 @@
> #include <net/ip.h>
> #include "slab.h"
> #include "memcontrol-v1.h"
> +#include "swap_tier.h"
>
> #include <linux/uaccess.h>
>
> @@ -4130,6 +4131,8 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
> refcount_set(&memcg->id.ref, 1);
> css_get(css);
>
> + swap_tiers_memcg_inherit_mask(memcg);
> +
> /*
> * Ensure mem_cgroup_from_private_id() works once we're fully online.
> *
> @@ -5667,6 +5670,88 @@ static int swap_events_show(struct seq_file *m, void *v)
> return 0;
> }
>
> +static int swap_tier_show(struct seq_file *m, void *v)
> +{
> + struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
> +
> + swap_tiers_mask_show(m, READ_ONCE(memcg->tier_mask));
> + return 0;
> +}
> +
> +static ssize_t swap_tier_write(struct kernfs_open_file *of,
> + char *buf, size_t nbytes, loff_t off)
> +{
> + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
> + char *pos, *token;
> + int ret = 0;
> + int original_mask = 0;
> +
> + pos = strstrip(buf);
> +
> + spin_lock(&swap_tier_lock);
> + if (!*pos) {
> + WRITE_ONCE(memcg->tier_mask, TIER_ALL_MASK);
> + goto sync;
> + }
> +
> + original_mask = memcg->tier_mask;
> +
> + while ((token = strsep(&pos, " \t\n")) != NULL) {
> + int mask;
> +
> + if (!*token)
> + continue;
> +
> + if (token[0] != '-' && token[0] != '+') {
> + ret = -EINVAL;
> + goto err;
> + }
> +
> + mask = swap_tiers_mask_lookup(token+1);
> + if (!mask) {
> + ret = -EINVAL;
> + goto err;
> + }
> +
> + /*
> + * if child already set, cannot add that tiers for hierarch mismatching.
> + * parent compatible, child must respect parent selected swap device.
> + */
This paragraph of code comment sounds a little unnatural. We are writing
it into memcg, the child memcg is handled in
swap_tiers_memcg_sync_mask(), isn't it? I don't get the 2nd sentence.
Could you help explain?
> + switch (token[0]) {
> + case '-':
> + WRITE_ONCE(memcg->tier_mask,
> + memcg->tier_mask & ~mask);
> + break;
> + case '+':
> + WRITE_ONCE(memcg->tier_mask,
> + memcg->tier_mask | mask);
> + break;
> + default:
> + ret = -EINVAL;
> + break;
> + }
> +
> + if (ret)
> + goto err;
> + }
> +
> +sync:
> + swap_tiers_memcg_sync_mask(memcg);
> +err:
> + if (ret)
> + WRITE_ONCE(memcg->tier_mask, original_mask);
> + spin_unlock(&swap_tier_lock);
> + return ret ? ret : nbytes;
> +}
> +
...snip...
^ permalink raw reply
* Re: [PATCH v6 2/6] cgroup,cgroup/dmem: Add (dmem_)cgroup_common_ancestor helper
From: Thadeu Lima de Souza Cascardo @ 2026-05-26 15:28 UTC (permalink / raw)
To: Natalie Vock
Cc: Maarten Lankhorst, Maxime Ripard, Tejun Heo, Johannes Weiner,
Michal Koutný, Christian Koenig, Huang Rui, Matthew Auld,
Matthew Brost, Maarten Lankhorst, Thomas Zimmermann, David Airlie,
Simona Vetter, Tvrtko Ursulin, cgroups, dri-devel
In-Reply-To: <20260313-dmemcg-aggressive-protect-v6-2-7c71cc1492db@gmx.de>
On Fri, Mar 13, 2026 at 12:40:01PM +0100, Natalie Vock wrote:
> This helps to find a common subtree of two resources, which is important
> when determining whether it's helpful to evict one resource in favor of
> another.
>
> To facilitate this, add a common helper to find the ancestor of two
> cgroups using each cgroup's ancestor array.
>
> Signed-off-by: Natalie Vock <natalie.vock@gmx.de>
> ---
> include/linux/cgroup.h | 21 +++++++++++++++++++++
> include/linux/cgroup_dmem.h | 9 +++++++++
> kernel/cgroup/dmem.c | 28 ++++++++++++++++++++++++++++
> 3 files changed, 58 insertions(+)
>
> diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
> index bc892e3b37eea..560ae995e3a54 100644
> --- a/include/linux/cgroup.h
> +++ b/include/linux/cgroup.h
> @@ -561,6 +561,27 @@ static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp,
> return cgrp->ancestors[ancestor_level];
> }
>
> +/**
> + * cgroup_common_ancestor - find common ancestor of two cgroups
> + * @a: first cgroup to find common ancestor of
> + * @b: second cgroup to find common ancestor of
> + *
> + * Find the first cgroup that is an ancestor of both @a and @b, if it exists
> + * and return a pointer to it. If such a cgroup doesn't exist, return NULL.
> + *
> + * This function is safe to call as long as both @a and @b are accessible.
> + */
> +static inline struct cgroup *cgroup_common_ancestor(struct cgroup *a,
> + struct cgroup *b)
> +{
> + int level;
> +
> + for (level = min(a->level, b->level); level >= 0; level--)
> + if (a->ancestors[level] == b->ancestors[level])
> + return a->ancestors[level];
> + return NULL;
> +}
> +
> /**
> * task_under_cgroup_hierarchy - test task's membership of cgroup ancestry
> * @task: the task to be tested
> diff --git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h
> index 1a88cd0c9eb00..9d72457c4cb9d 100644
> --- a/include/linux/cgroup_dmem.h
> +++ b/include/linux/cgroup_dmem.h
> @@ -28,6 +28,8 @@ bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root,
> struct dmem_cgroup_pool_state *test);
> bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root,
> struct dmem_cgroup_pool_state *test);
> +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a,
> + struct dmem_cgroup_pool_state *b);
>
> void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool);
> #else
> @@ -75,6 +77,13 @@ static inline bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root,
> return false;
> }
>
> +static inline
> +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a,
> + struct dmem_cgroup_pool_state *b)
> +{
> + return NULL;
> +}
> +
> static inline void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
> { }
>
> diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
> index 28227405f7cfe..9ae085a7fcb73 100644
> --- a/kernel/cgroup/dmem.c
> +++ b/kernel/cgroup/dmem.c
> @@ -756,6 +756,34 @@ bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root,
> }
> EXPORT_SYMBOL_GPL(dmem_cgroup_below_low);
>
> +/**
> + * dmem_cgroup_get_common_ancestor(): Find the first common ancestor of two pools.
> + * @a: First pool to find the common ancestor of.
> + * @b: First pool to find the common ancestor of.
> + *
> + * Return: The first pool that is a parent of both @a and @b, or NULL if either @a or @b are NULL,
> + * or if such a pool does not exist. A reference to the returned pool is grabbed and must be
> + * released by the caller when it is done using the pool.
> + */
> +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a,
> + struct dmem_cgroup_pool_state *b)
> +{
> + struct cgroup *ancestor_cgroup;
> + struct cgroup_subsys_state *ancestor_css;
> +
> + if (!a || !b)
> + return NULL;
> +
> + ancestor_cgroup = cgroup_common_ancestor(a->cs->css.cgroup, b->cs->css.cgroup);
> + if (!ancestor_cgroup)
> + return NULL;
> +
> + ancestor_css = cgroup_e_css(ancestor_cgroup, &dmem_cgrp_subsys);
cgroup_e_css must be called in RCU read context. Besides, a reference to
ancestor_css must be got as later on, dmem_cgroup_pool_state_put will call
css_put.
Here is my fixup, which I tested and did not cause RCU or reference
warnings whereas the original patch caused such issues.
Feel free to use it with my sign-off.
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@igalia.com>
Regards.
Cascardo.
diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
index 72ee8f1d69ef..28adb042baca 100644
--- a/kernel/cgroup/dmem.c
+++ b/kernel/cgroup/dmem.c
@@ -773,6 +773,7 @@ struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgrou
{
struct cgroup *ancestor_cgroup;
struct cgroup_subsys_state *ancestor_css;
+ struct dmem_cgroup_pool_state *pool = NULL;
if (!a || !b)
return NULL;
@@ -781,9 +782,15 @@ struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgrou
if (!ancestor_cgroup)
return NULL;
+ rcu_read_lock();
ancestor_css = cgroup_e_css(ancestor_cgroup, &dmem_cgrp_subsys);
+ if (css_tryget(ancestor_css))
+ pool = get_cg_pool_unlocked(css_to_dmemcs(ancestor_css), a->region);
+ if (!pool)
+ css_put(ancestor_css);
+ rcu_read_unlock();
- return get_cg_pool_unlocked(css_to_dmemcs(ancestor_css), a->region);
+ return pool;
}
EXPORT_SYMBOL_GPL(dmem_cgroup_get_common_ancestor);
--
2.47.3
> +
> + return get_cg_pool_unlocked(css_to_dmemcs(ancestor_css), a->region);
> +}
> +EXPORT_SYMBOL_GPL(dmem_cgroup_get_common_ancestor);
> +
> static int dmem_cgroup_region_capacity_show(struct seq_file *sf, void *v)
> {
> struct dmem_cgroup_region *region;
>
> --
> 2.53.0
>
^ permalink raw reply related
* Re: [PATCH 0/7 v3] mm/memcontrol, page_counter: move stock from mem_cgroup to page_counter
From: Joshua Hahn @ 2026-05-26 15:16 UTC (permalink / raw)
To: Joshua Hahn
Cc: Johannes Weiner, Michal Hocko, Roman Gushchin, Shakeel Butt,
Muchun Song, Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, cgroups, linux-mm, linux-kernel, kernel-team
In-Reply-To: <20260525190455.2843786-1-joshua.hahnjy@gmail.com>
On Mon, 25 May 2026 12:04:47 -0700 Joshua Hahn <joshua.hahnjy@gmail.com> wrote:
> Memcg currently keeps a "stock" of 64 pages per-cpu to cache pre-charged
> allocations, allowing small allocations to avoid walking the expensive
> mem_cgroup hierarchy traversal and atomic operations on each charge.
> This design introduces a fastpath, but there is room for improvement:
Hello everyone,
Sashiko has left some great comments on the series, some of which are
things I need to address for the next iteration, while others are false
positives. There aren't a bunch, so I'll go over all of them below.
Note that the same warning brought up in 1/7 is duplicated for the rest
of the series; I've addressed my thoughts for that in [1].
> Joshua Hahn (7):
> mm/page_counter: introduce per-page_counter stock
Sashiko raised a concern about how zeroing per-cpu stock during the
nolock drain could race with in-flight charges that read the value
before it gets zeroed, leading to duplicate drains. I think this is
solvable by just changing the order in the callsites (disable, then drain).
More details can be found in [1].
> mm/page_counter: use page_counter_stock in page_counter_try_charge
Sashiko raises the same concern as [1].
> mm/page_counter: introduce stock drain APIs
Same concern as [1].
> mm/memcontrol: convert memcg to use page_counter_stock
Sashiko raises 4 concerns, of which I think 3 of them are false positives
(or not as serious as Sashiko makes it out to be).
(1) Sashiko asks whether the synchronous draining with the percpu_charge_mutex
lock held could lead to more time spent holding the lock, which means
more callers of drain_all_stock would fail the trylock and just skip draining.
To clarify, even in the original code, two tasks simultaneously calling
drain_all_stock would serialize and only one of them would schedule the
drain work, so this problem definitely existed before as well. It's just that
the window for this race is a bit longer now.
I do think that there is actually a behavior change here (for the better).
Previously, drain_all_stock had no guarantees on whether the stock was
drained before retrying. Now, if the caller can get the trylock, they have
a stronger guarantee that the stock is drained before retrying the drain.
On the note of premature OOMs, each retry loop takes much longer than the
draining itself; I would imagine that by the time the next retry loop happens,
there's a better chance that the trylock succeeds in the next iteration.
(2) Sashiko also raises another concern about a potential ABBA deadlock with
the mmap_lock. I think this concern is not really true, the synchronous
work being done (drain_stock_on_cpu) only takes a local lock. Hopefully I'm
not missing anything here.
(3) I think Sashiko's concerns about NOHZ / CPU isolation is real. But it shouldn't
be too bad, all I need is a cpu_is_isolated() check in the for_each_online_cpu
iterator. Again, not draining a CPU is not fatal here, so it shouldn't be too
big of a problem to skip some of them.
I also just wanted to note here explicitly that we don't need the
migrate_disable() for the memcg stock drain, since we don't differentiate
between local drain work & remote drain scheduling (like objcg_stock).
(4) Finally Sashiko asks if we should enable the memcg->memsw stock here.
That's included in the very next patch : -) I separated them so that they
can be reviewed separately, since they are separate ideas.
> mm/memcontrol: optimize memsw stock for cgroup v1
Both concerns here are addressed in the previous section.
> mm/memcontrol: optimize stock usage for cgroup v2
Sashiko raises 3 concerns, of which I think all of them are actually OK.
(1) If we drain the parent memcg stock on first child creation, then this would
mean that there will be additional synchronous work being done with the
cgroup_mutex lock held. I personally think this is fine, since it happens
once per parent cgroup, and the draining work is pretty cheap. But I would
appreciate it if other reviewers could chime in here.
(2) Sashiko also asks whether we need cpus_read_lock during the iteration.
I think it's fine without it; if a CPU happens to go offline during the
iteration, then that work will be scheduled on another CPU. That's fine,
duplicate draining work on 1 CPU isn't the end of the world (and preferable
to taking a cpus_read_lock here). As for the dying CPU, it will drain its
own stock during the destruction path anyways, so no stock is lost.
(3) This one is not related to this series, so I'll move on.
> mm/memcontrol: remove unused memcg_stock code
No comments for this patch.
I think that's all the comments that Sashiko raised for this patch. Most of them
had to do with performance tradeoffs, for which I hope that my testing results
in the cover letter were able to instill some confidence that a lot of these
tradeoffs aren't as bad as they seem. Regardless, I would really appreciate
reviewer feedback on whether they think it is acceptable.
There are definitely some real bugs that I want to address, so a v4 will be
incoming to address those (in a week or so).
Thank you Sashiko!
Joshua
[1] https://lore.kernel.org/linux-mm/20260525194506.3414995-1-joshua.hahnjy@gmail.com/
^ permalink raw reply
* [PATCH v3] security: Expand task_setscheduler LSM hook to include CPU affinity mask
From: Aaron Tomlin @ 2026-05-26 14:28 UTC (permalink / raw)
To: tsbogend, paul, jmorris, serge, mingo, peterz, juri.lelli,
vincent.guittot, stephen.smalley.work, casey, longman, tj, hannes,
mkoutny
Cc: chenridong, dietmar.eggemann, rostedt, bsegall, mgorman, vschneid,
kprateek.nayak, omosnace, kees, atomlin, neelx, sean, chjohnst,
steve, mproche, nick.lange, cgroups, linux-mips, linux-fsdevel,
linux-security-module, selinux, linux-kernel
At present, the task_setscheduler LSM hook provides security modules
with the opportunity to mediate changes to a task's scheduling policy.
However, when invoked via sched_setaffinity(), the hook lacks
visibility into the actual CPU affinity mask being requested.
Consequently, BPF-based security modules are entirely blind to the
target CPUs and cannot make granular access control decisions based on
spatial isolation.
In modern multi-tenant and real-time environments, CPU isolation is a
critical boundary. The inability to audit or restrict specific CPU
pinning requests limits the effectiveness of eBPF-driven security
policies, particularly when attempting to shield isolated or
cryptographic cores from unprivileged or compromised tasks.
This patch expands the security_task_setscheduler() hook signature to
include a pointer to the requested cpumask. Because this is a shared
hook used for multiple scheduling attribute changes, call sites that do
not modify CPU affinity are updated to safely pass NULL.
To protect against unverified dereferences, the parameter is annotated
with __nullable in the LSM hook definition, ensuring the BPF verifier
mandates explicit NULL checks for attached eBPF programs.
This change updates all in-tree security modules (SELinux and Smack) to
accommodate the new parameter mechanically, whilst providing BPF LSMs
with the necessary context to enforce strict affinity policies.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
This patch is strictly dependent on the prior acceptance of "mips:
sched: Fix CPUMASK_OFFSTACK memory corruption in MT fpaff" (Message-ID:
20260526141651.773306-1-atomlin@atomlin.com), as expanding the LSM hook
signature requires passing the mask pointer from
mipsmt_sys_sched_setaffinity().
Changes since v2 [1]:
- Dropped patch 1. This is to be addressed by the cgroup cpuset
maintainer (Waiman Long)
- Dropped patch 3. Will be submitted as a separate patch (Paul Moore)
Changes since v1 [2]:
- Reordered the allocation and user-copy of new_mask in the MIPS
architecture's mipsmt_sys_sched_setaffinity() to occur before the
LSM hook is invoked. This ensures the security modules evaluate a fully
populated mask rather than uninitialised memory, while cleanly handling
error unwinding
- Updated cpuset_can_fork() to pass the destination cpuset's effective CPU
mask instead of NULL
[1]: https://lore.kernel.org/lkml/20260509213803.968464-1-atomlin@atomlin.com/
[2]: https://lore.kernel.org/lkml/20260509164847.939294-1-atomlin@atomlin.com/
---
arch/mips/kernel/mips-mt-fpaff.c | 2 +-
fs/proc/base.c | 2 +-
include/linux/lsm_hook_defs.h | 3 ++-
include/linux/security.h | 11 +++++++----
kernel/cgroup/cpuset.c | 4 ++--
kernel/sched/syscalls.c | 4 ++--
security/commoncap.c | 7 +++++--
security/security.c | 11 ++++++-----
security/selinux/hooks.c | 3 ++-
security/smack/smack_lsm.c | 11 +++++++++--
10 files changed, 37 insertions(+), 21 deletions(-)
diff --git a/arch/mips/kernel/mips-mt-fpaff.c b/arch/mips/kernel/mips-mt-fpaff.c
index 4fead87d2f43..c68d1676350e 100644
--- a/arch/mips/kernel/mips-mt-fpaff.c
+++ b/arch/mips/kernel/mips-mt-fpaff.c
@@ -110,7 +110,7 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len,
goto out_unlock;
}
- retval = security_task_setscheduler(p);
+ retval = security_task_setscheduler(p, new_mask);
if (retval)
goto out_unlock;
diff --git a/fs/proc/base.c b/fs/proc/base.c
index d9acfa89c894..ac4096958a00 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -2619,7 +2619,7 @@ static ssize_t timerslack_ns_write(struct file *file, const char __user *buf,
}
rcu_read_unlock();
- err = security_task_setscheduler(p);
+ err = security_task_setscheduler(p, NULL);
if (err) {
count = err;
goto out;
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index 2b8dfb35caed..6ec7bc04a1b7 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -255,7 +255,8 @@ LSM_HOOK(int, 0, task_prlimit, const struct cred *cred,
const struct cred *tcred, unsigned int flags)
LSM_HOOK(int, 0, task_setrlimit, struct task_struct *p, unsigned int resource,
struct rlimit *new_rlim)
-LSM_HOOK(int, 0, task_setscheduler, struct task_struct *p)
+LSM_HOOK(int, 0, task_setscheduler, struct task_struct *p,
+ const struct cpumask *in_mask__nullable)
LSM_HOOK(int, 0, task_getscheduler, struct task_struct *p)
LSM_HOOK(int, 0, task_movememory, struct task_struct *p)
LSM_HOOK(int, 0, task_kill, struct task_struct *p, struct kernel_siginfo *info,
diff --git a/include/linux/security.h b/include/linux/security.h
index 41d7367cf403..8b74153daa43 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -196,7 +196,8 @@ extern int cap_mmap_addr(unsigned long addr);
extern int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags);
extern int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
unsigned long arg4, unsigned long arg5);
-extern int cap_task_setscheduler(struct task_struct *p);
+extern int cap_task_setscheduler(struct task_struct *p,
+ const struct cpumask *in_mask);
extern int cap_task_setioprio(struct task_struct *p, int ioprio);
extern int cap_task_setnice(struct task_struct *p, int nice);
extern int cap_vm_enough_memory(struct mm_struct *mm, long pages);
@@ -531,7 +532,8 @@ int security_task_prlimit(const struct cred *cred, const struct cred *tcred,
unsigned int flags);
int security_task_setrlimit(struct task_struct *p, unsigned int resource,
struct rlimit *new_rlim);
-int security_task_setscheduler(struct task_struct *p);
+int security_task_setscheduler(struct task_struct *p,
+ const struct cpumask *in_mask);
int security_task_getscheduler(struct task_struct *p);
int security_task_movememory(struct task_struct *p);
int security_task_kill(struct task_struct *p, struct kernel_siginfo *info,
@@ -1392,9 +1394,10 @@ static inline int security_task_setrlimit(struct task_struct *p,
return 0;
}
-static inline int security_task_setscheduler(struct task_struct *p)
+static inline int security_task_setscheduler(struct task_struct *p,
+ const struct cpumask *in_mask)
{
- return cap_task_setscheduler(p);
+ return cap_task_setscheduler(p, in_mask);
}
static inline int security_task_getscheduler(struct task_struct *p)
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 5c33ab20cc20..7b3dfccb77d8 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -3033,7 +3033,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
goto out_unlock;
if (setsched_check) {
- ret = security_task_setscheduler(task);
+ ret = security_task_setscheduler(task, cs->effective_cpus);
if (ret)
goto out_unlock;
}
@@ -3591,7 +3591,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
if (ret)
goto out_unlock;
- ret = security_task_setscheduler(task);
+ ret = security_task_setscheduler(task, cs->effective_cpus);
if (ret)
goto out_unlock;
diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c
index b215b0ead9a6..68bc7e466fb1 100644
--- a/kernel/sched/syscalls.c
+++ b/kernel/sched/syscalls.c
@@ -540,7 +540,7 @@ int __sched_setscheduler(struct task_struct *p,
if (attr->sched_flags & SCHED_FLAG_SUGOV)
return -EINVAL;
- retval = security_task_setscheduler(p);
+ retval = security_task_setscheduler(p, NULL);
if (retval)
return retval;
}
@@ -1213,7 +1213,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
return -EPERM;
}
- retval = security_task_setscheduler(p);
+ retval = security_task_setscheduler(p, in_mask);
if (retval)
return retval;
diff --git a/security/commoncap.c b/security/commoncap.c
index 3399535808fe..d86f1c2b9210 100644
--- a/security/commoncap.c
+++ b/security/commoncap.c
@@ -1222,13 +1222,16 @@ static int cap_safe_nice(struct task_struct *p)
/**
* cap_task_setscheduler - Determine if scheduler policy change is permitted
* @p: The task to affect
+ * @in_mask: Requested CPU affinity mask (ignored)
*
* Determine if the requested scheduler policy change is permitted for the
- * specified task.
+ * specified task. The capabilities security module does not evaluate the
+ * @in_mask parameter, relying solely on cap_safe_nice().
*
* Return: 0 if permission is granted, -ve if denied.
*/
-int cap_task_setscheduler(struct task_struct *p)
+int cap_task_setscheduler(struct task_struct *p,
+ const struct cpumask *in_mask __always_unused)
{
return cap_safe_nice(p);
}
diff --git a/security/security.c b/security/security.c
index 4e999f023651..53804ee40df5 100644
--- a/security/security.c
+++ b/security/security.c
@@ -3240,17 +3240,18 @@ int security_task_setrlimit(struct task_struct *p, unsigned int resource,
}
/**
- * security_task_setscheduler() - Check if setting sched policy/param is allowed
+ * security_task_setscheduler() - Check if setting sched policy/param/affinity is allowed
* @p: target task
+ * @in_mask: requested CPU affinity mask, or NULL if not changing affinity
*
- * Check permission before setting scheduling policy and/or parameters of
- * process @p.
+ * Check permission before setting the scheduling policy, parameters, and/or
+ * CPU affinity of process @p.
*
* Return: Returns 0 if permission is granted.
*/
-int security_task_setscheduler(struct task_struct *p)
+int security_task_setscheduler(struct task_struct *p, const struct cpumask *in_mask)
{
- return call_int_hook(task_setscheduler, p);
+ return call_int_hook(task_setscheduler, p, in_mask);
}
/**
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 0f704380a8c8..5f0914db23f6 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -4557,7 +4557,8 @@ static int selinux_task_setrlimit(struct task_struct *p, unsigned int resource,
return 0;
}
-static int selinux_task_setscheduler(struct task_struct *p)
+static int selinux_task_setscheduler(struct task_struct *p,
+ const struct cpumask *in_mask __always_unused)
{
return avc_has_perm(current_sid(), task_sid_obj(p), SECCLASS_PROCESS,
PROCESS__SETSCHED, NULL);
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 3f9ae05039a2..a77143beff44 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -2343,10 +2343,17 @@ static int smack_task_getioprio(struct task_struct *p)
/**
* smack_task_setscheduler - Smack check on setting scheduler
* @p: the task object
+ * @in_mask: Requested CPU affinity mask (ignored)
*
- * Return 0 if read access is permitted
+ * Evaluate whether the current task has write access to the target task @p
+ * to change its scheduling policy. The Smack security module relies
+ * strictly on label-based access control and does not evaluate CPU
+ * affinity masks.
+ *
+ * Return: 0 if write access is permitted
*/
-static int smack_task_setscheduler(struct task_struct *p)
+static int smack_task_setscheduler(struct task_struct *p,
+ const struct cpumask *in_mask __always_unused)
{
return smk_curacc_on_task(p, MAY_WRITE, __func__);
}
base-commit: 5200f5f493f79f14bbdc349e402a40dfb32f23c8
prerequisite-patch-id: f9200d420002c9fd0663d0ec00c83db866889c19
--
2.51.0
^ permalink raw reply related
* Re: [RFC PATCH bpf-next v7 00/11] mm: BPF struct_ops for dynamic memory protection and async reclaim
From: Usama Arif @ 2026-05-26 13:41 UTC (permalink / raw)
To: Hui Zhu
Cc: Usama Arif, Daniel Borkmann, John Fastabend, Andrii Nakryiko,
Martin KaFai Lau, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Song Liu, Yonghong Song, Jiri Olsa, Johannes Weiner, Michal Hocko,
Roman Gushchin, Shakeel Butt, Muchun Song, JP Kobryn,
Andrew Morton, Shuah Khan, davem, Jakub Kicinski,
Jesper Dangaard Brouer, Stanislav Fomichev, KP Singh, Tao Chen,
Mykyta Yatsenko, Leon Hwang, Anton Protopopov, Amery Hung,
Tobias Klauser, Eyal Birger, Rong Tao, Hao Luo, Peter Zijlstra,
Miguel Ojeda, Nathan Chancellor, Kees Cook, Tejun Heo, Jeff Xu,
mkoutny, Jan Hendrik Farr, Christian Brauner, Randy Dunlap,
Brian Gerst, Masahiro Yamada, Willem de Bruijn, Jason Xing,
Paul Chaignon, Chen Ridong, Lance Yang, Jiayuan Chen,
linux-kernel, bpf, cgroups, linux-mm, netdev, linux-kselftest,
geliang, baohua, Hui Zhu
In-Reply-To: <cover.1779760876.git.zhuhui@kylinos.cn>
On Tue, 26 May 2026 10:20:00 +0800 Hui Zhu <hui.zhu@linux.dev> wrote:
> From: Hui Zhu <zhuhui@kylinos.cn>
>
> Overview:
> This series introduces BPF struct_ops support for the memory controller,
> enabling userspace BPF programs to implement custom, dynamic memory
> management policies per cgroup. The feature allows BPF programs to hook
> into the core reclaim and charge paths without requiring kernel
> modifications, providing a flexible alternative to static knobs such as
> memory.low and memory.min.
>
> The series enables two complementary use cases.
>
> Dynamic memory protection: static memory protection thresholds
> (memory.low, memory.min) are poor fits for workloads whose actual memory
> activity varies over time. A high-priority cgroup holding a large working
> set but temporarily idle will still suppress reclaim on its siblings,
> wasting available memory. A BPF-driven approach can observe real workload
> activity -- page faults, charge/uncharge events -- and activate or
> withdraw protection dynamically. The test results at the end of this
> letter quantify the difference: in a scenario where the high-priority
> cgroup is idle, the BPF-controlled low-priority cgroup achieves roughly
> 37x higher throughput than with static memory.low.
>
> Asynchronous proactive reclaim: the memcg_charged and memcg_uncharged
> hooks, combined with the BPF workqueue mechanism and the new
> bpf_try_to_free_mem_cgroup_pages() kfunc, enable BPF programs to perform
> proactive background reclaim without blocking the charge path. The
> pattern works as follows: the memcg_charged callback tracks accumulated
> memory usage; when usage crosses a configurable threshold, it enqueues an
> asynchronous work item via bpf_wq_start() and returns immediately without
> throttling the charging task. The workqueue callback then invokes
> bpf_try_to_free_mem_cgroup_pages() to reclaim pages from the target
> cgroup; if usage remains elevated after reclaim, the callback re-enqueues
> itself to continue. This allows a BPF program to keep a cgroup's
> footprint below its hard limit (memory.max) entirely in the background,
> avoiding the OOM killer or direct-reclaim stalls that would otherwise
> occur. The selftest for this feature (patch 10/11) validates the
> mechanism concretely: a workload that writes and mmaps a 64 MB file inside
> a 32 MB cgroup reliably triggers memory.events "max" events without BPF;
> with the async reclaim program attached, the "max" counter does not
> increase at all across the same workload.
>
Hi Hui,
Thanks for the series.
Would it not be simpler to just have another memcg knob, something like
memory.high_async.
When memory usage > memory.high_async, queue a per-memcg work item that calls
try_to_free_mem_cgroup_pages() until usage drops back below some threshold.
I am not sure I see what programability aspect from bpf you need here.
Thanks
>
> 08/11 selftests/bpf: Add tests for memcg_bpf_ops
> Adds prog_tests/memcg_ops.c covering three scenarios:
> memcg_charged-only throttling, below_low + memcg_charged
> interaction, and below_min + memcg_charged interaction. A
> tracepoint on memcg:count_memcg_events (PGFAULT) is used to
> detect memory pressure and trigger hooks accordingly.
>
> 09/11 selftests/bpf: Add test for memcg_bpf_ops hierarchies
> Validates BPF_F_ALLOW_OVERRIDE attachment semantics across a
> three-level cgroup hierarchy: attach with ALLOW_OVERRIDE at the
> root, override at the middle level without the flag, then assert
> that attaching to the leaf correctly fails with -EBUSY.
>
> 10/11 selftests/bpf: Add selftest for memcg async reclaim via BPF
> Demonstrates and validates asynchronous memory reclaim: a BPF
> program uses the memcg_charged/memcg_uncharged hooks to track
> accumulated usage and, when a threshold is exceeded, enqueues a
> bpf_wq_start() workqueue item that calls
> bpf_try_to_free_mem_cgroup_pages() without blocking the charge
> path. The test asserts that with the BPF program active,
> memory.events "max" events do not increase under a workload
> that would otherwise exceed the hard limit.
>
> 11/11 samples/bpf: Add memcg priority control and async reclaim example
> Adds a complete sample (samples/bpf/memcg.bpf.c + memcg.c)
> demonstrating both features. The BPF side monitors PGFAULT
> events on a high-priority cgroup; when the per-second fault
> count crosses a configurable threshold, it activates below_low
> or below_min protection for the high-priority cgroup and/or
> applies a charge delay to the low-priority cgroup. Six
> struct_ops variants are exported so userspace can attach only
> the hooks needed. Async reclaim is optionally combined with
> priority throttling via a shared low-cgroup ops map.
>
> Test Environment:
> The following examples run on x86_64 QEMU (10 CPUs, 2 GB RAM), using
> a tmpfs-backed file on the host as a swap device to reduce I/O impact.
> Two cgroups are created -- high (high-priority) and low (low-priority)
> -- and each test runs two concurrent stress-ng workloads, one per
> cgroup, each requesting 3 GB of memory.
>
> # mkdir /sys/fs/cgroup/high /sys/fs/cgroup/low
> # free -h
> total used free shared buff/cache available
> Mem: 1.9Gi 317Mi 1.6Gi 1.0Mi 144Mi 1.6Gi
> Swap: 4.0Gi 0B 4.0Gi
>
> Baseline: no memory priority policy:
> Both cgroups run without any reclaim protection. Results are roughly
> equal, as expected:
>
> cgroup bogo ops/s
> high 4,979
> low 4,927
>
> Test 1: memory.low protection:
> Setting memory.low on the high-priority cgroup protects it from
> reclaim, at the cost of pushing reclaim pressure onto the low-priority
> cgroup:
>
> # echo $((3 * 1024 * 1024 * 1024)) > /sys/fs/cgroup/high/memory.low
>
> cgroup bogo ops/s
> high 450,290
> low 11,307
>
> The high-priority cgroup benefits significantly, but memory.low relies
> on static usage thresholds and cannot adapt to actual workload
> behavior.
>
> Test 2: memory.low with an idle high-priority task:
> Here the high-priority cgroup runs a Python script that allocates 3 GB
> and then sleeps, simulating a low-activity but memory-holding workload.
> Because the process is idle, it generates no page faults and does not
> actively use its memory. Yet memory.low still protects it, continuing
> to suppress the low-priority cgroup's performance:
>
> cgroup bogo ops/s
> low 14,757
>
> The low-priority cgroup remains significantly throttled despite the
> high-priority cgroup being effectively idle -- a clear limitation of
> static memory.low control.
>
> Test 3: memcg eBPF -- dynamic priority control:
> memcg is a sample program introduced in this patch series
> (samples/bpf/memcg.c + memcg.bpf.c). It loads a BPF program that
> monitors PGFAULT events in the high-priority cgroup. When the
> per-second fault count exceeds a configured threshold, the hook
> activates below_min protection for one second; otherwise the cgroup
> receives no special treatment.
>
> # ./memcg --low_path=/sys/fs/cgroup/low \
> --high_path=/sys/fs/cgroup/high \
> --threshold=1 --use_below_min
> Successfully attached!
>
> 3a. Both cgroups under active memory pressure:
>
> When both cgroups run stress-ng, the high-priority cgroup generates
> frequent page faults and the BPF hook activates protection, matching
> the behavior of memory.low:
>
> cgroup bogo ops/s
> high 404,392
> low 11,404
>
> 3b. High-priority cgroup is idle (Python + sleep):
>
> Because the sleeping Python process generates no page faults, the BPF
> hook never activates, and the low-priority cgroup is free to reclaim
> memory normally:
>
> cgroup bogo ops/s
> low 551,083
>
> This is a ~37x improvement over the equivalent memory.low scenario
> (Test 2), demonstrating that eBPF-driven dynamic control can
> accurately reflect actual workload activity and avoid unnecessary
> protection of idle high-priority tasks.
>
> Summary:
> Scenario low-cgroup bogo ops/s
> Baseline (no policy) ~4,927
> memory.low, both active ~11,307
> memory.low, high idle ~14,757
> memcg eBPF, both active ~11,404
> memcg eBPF, high idle ~551,083
>
> References:
> [1] https://patchew.org/linux/20260127024421.494929-1-roman.gushchin@linux.dev/
>
> Changelog:
> v7:
> Change base commits of "mm: BPF OOM" to v3.
> Some fixes according to the comments of bpf-ci.
> Rename get_high_delay_ms hook to memcg_charged; add memcg_uncharged
> hook for tracking uncharge events.
> Update below_low and below_min hooks to receive elow/emin and usage
> as explicit arguments.
> Add bpf_try_to_free_mem_cgroup_pages kfunc to expose cgroup reclaim
> to BPF programs.
> Add selftest for BPF-driven asynchronous page reclaim.
> Extend samples/bpf/memcg to support async reclaim in addition to
> priority throttling.
> v6:
> Based on the bot+bof-ci comments, fixed the following issues.
> Added fast-path check with unlikely() before SRCU lock acquisition to
> optimize the no-BPF case in BPF_MEMCG_CALL.
> Add missing newline in pr_warn message to bpf_memcontrol_init.
> Added comprehensive child process exit status checking with WIFEXITED()
> and WEXITSTATUS(), and added zombie process prevention in
> real_test_memcg_ops.
> Changed malloc() to calloc() for BSS data allocation in all test
> functions and samples main function.
> Change srcu_read_lock(&memcg_bpf_srcu) to
> lockdep_assert_held(&cgroup_mutex) in function memcontrol_bpf_online
> and memcontrol_bpf_offline.
> v5:
> Based on the bot+bof-ci comments, fixed the following issues.
> Fixed issues in memcg_ops.c and memcg.bpf.c by moving variable
> declaration to the beginning of need_threshold() function.
> The 'u64 current_ts' variable must be declared before any
> executable statements
> Improved input validation in samples/bpf/memcg.c by adding a new
> parse_u64() helper function. This function properly handles errors
> from strtoull() and provides better error messages when parsing
> threshold and over_high_ms command-line arguments.
> Move check for prog->sleepable after validating member offsets in
> mm/bpf_memcontrol.c bpf_memcg_ops_check_member.
> Fixed sscanf return value checking in prog_tests/memcg_ops.c.
> Changed the condition from 'sscanf() < 0' to 'sscanf() != 1' because
> sscanf returns the number of successfully matched items, not a negative
> value on error. This makes the test more reliable when reading timing
> data from temporary files.
> v4:
> Fix the issues according to the comments from bot+bof-ci.
> According to JP Kobryn's comments, move exit(0) from
> real_test_memcg_ops_child_work to real_test_memcg_ops.
> Fix issues in the bpf_memcg_ops_reg function.
> v3:
> According to the comments from Michal Koutný and Chen Ridong, update hooks
> to get_high_delay_ms, below_low, below_min, handle_cgroup_online, and
> handle_cgroup_offline.
> According to Michal Koutný's comments, add BPF_F_ALLOW_OVERRIDE
> support to memcg_bpf_ops.
> v2:
> According to Tejun Heo's comments, rebased on Roman Gushchin's BPF
> OOM patch series [1] and added hierarchical delegation support.
> According to the comments from Roman Gushchin and Michal Hocko, designed
> concrete use case scenarios and provided test results.
>
> Hui Zhu (7):
> bpf: Pass flags in bpf_link_create for struct_ops
> mm: memcontrol: Add BPF struct_ops for memory controller
> mm/bpf: Add bpf_try_to_free_mem_cgroup_pages kfunc
> selftests/bpf: Add tests for memcg_bpf_ops
> selftests/bpf: Add test for memcg_bpf_ops hierarchies
> selftests/bpf: Add selftest for memcg async reclaim via BPF
> samples/bpf: Add memcg priority control and async reclaim example
>
> Roman Gushchin (4):
> bpf: move bpf_struct_ops_link into bpf.h
> bpf: allow attaching struct_ops to cgroups
> libbpf: fix return value on memory allocation failure
> libbpf: introduce bpf_map__attach_struct_ops_opts()
>
> MAINTAINERS | 6 +
> include/linux/bpf-cgroup-defs.h | 3 +
> include/linux/bpf-cgroup.h | 16 +
> include/linux/bpf.h | 10 +
> include/linux/memcontrol.h | 250 ++++++-
> include/uapi/linux/bpf.h | 5 +-
> kernel/bpf/bpf_struct_ops.c | 67 +-
> kernel/bpf/cgroup.c | 46 ++
> mm/bpf_memcontrol.c | 355 +++++++++-
> mm/memcontrol.c | 43 +-
> samples/bpf/.gitignore | 1 +
> samples/bpf/Makefile | 8 +-
> samples/bpf/memcg.bpf.c | 380 +++++++++++
> samples/bpf/memcg.c | 411 ++++++++++++
> tools/include/uapi/linux/bpf.h | 3 +-
> tools/lib/bpf/libbpf.c | 22 +-
> tools/lib/bpf/libbpf.h | 14 +
> tools/lib/bpf/libbpf.map | 1 +
> tools/testing/selftests/bpf/cgroup_helpers.c | 41 ++
> tools/testing/selftests/bpf/cgroup_helpers.h | 2 +
> .../bpf/prog_tests/memcg_async_reclaim.c | 333 +++++++++
> .../selftests/bpf/prog_tests/memcg_ops.c | 634 ++++++++++++++++++
> .../selftests/bpf/progs/memcg_async_reclaim.c | 203 ++++++
> tools/testing/selftests/bpf/progs/memcg_ops.c | 132 ++++
> 24 files changed, 2952 insertions(+), 34 deletions(-)
> create mode 100644 samples/bpf/memcg.bpf.c
> create mode 100644 samples/bpf/memcg.c
> create mode 100644 tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c
> create mode 100644 tools/testing/selftests/bpf/prog_tests/memcg_ops.c
> create mode 100644 tools/testing/selftests/bpf/progs/memcg_async_reclaim.c
> create mode 100644 tools/testing/selftests/bpf/progs/memcg_ops.c
>
> --
> 2.43.0
>
>
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: Peter Zijlstra @ 2026-05-26 12:40 UTC (permalink / raw)
To: K Prateek Nayak
Cc: Zhang Qiao, mingo, longman, chenridong, juri.lelli,
vincent.guittot, dietmar.eggemann, rostedt, bsegall, mgorman,
vschneid, tj, hannes, mkoutny, cgroups, linux-kernel, jstultz,
qyousef, Hui Tang
In-Reply-To: <20260526110709.GF4149641@noisy.programming.kicks-ass.net>
On Tue, May 26, 2026 at 01:07:09PM +0200, Peter Zijlstra wrote:
> On Tue, May 26, 2026 at 04:24:32PM +0530, K Prateek Nayak wrote:
> > Hello Peter,
> >
> > On 5/26/2026 3:22 PM, Peter Zijlstra wrote:
> > > On Tue, May 26, 2026 at 02:45:45PM +0530, K Prateek Nayak wrote:
> > >
> > >> The suggested diff above solves the crash in my case but your
> > >> mileage may vary. Peter can comment if this is the right thing
> > >> to do or not :-)
> > >
> > > Is this a different issue than the one you raised before?
> >
> > Yes, this is different. Essentially, this is what is happening:
> >
> > throttle_cfs_rq_work()
> > task_rq_lock()
> >
> > dequeue_task_fair(current) /* Task is dequeued on cfs side */
> > __dequeue_task(current)
> > dequeue_hierarchy(current);
> > current->se.on_rq = 0;
> > /* update_load_sub() */
> > resched_curr(); /* Initiates a resched */
> >
> > task_rq_unlock()
> > local_irq_enable();
> >
> > =====> sched_tick()
> > task_tick_fair()
> > __calc_prop_weight()
> > /*
> > * Oops: update_load_sub() above has
> > * 0ed the weight of cfs_rq.
> > */
> > <====
> >
> > preempt_schedule_irq()
> > next = ...
> > put_prev_set_next_task() /* The runtime context is switched here */
> >
>
> Ah, right. OK, I'll go have a poke once I get these proxy patches I've
> been spending too much time on posted.
Yes, your solution seems reasonable. I'll fold that and push out a new
version a little later today.
^ permalink raw reply
* Re: [PATCH v4 7/8] mm/memory: flatten folio allocation retry loops
From: Usama Arif @ 2026-05-26 12:12 UTC (permalink / raw)
To: Johannes Weiner
Cc: Usama Arif, Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Shakeel Butt, Michal Hocko, Dave Chinner, Roman Gushchin,
Muchun Song, Qi Zheng, Yosry Ahmed, Zi Yan, Liam R . Howlett,
Kiryl Shutsemau, Vlastimil Babka, Kairui Song, Mikhail Zaslonko,
Vasily Gorbik, Baolin Wang, Barry Song, Dev Jain, Lance Yang,
Nico Pache, Ryan Roberts, cgroups, linux-mm, linux-kernel
In-Reply-To: <20260521150330.1955924-8-hannes@cmpxchg.org>
On Thu, 21 May 2026 11:02:13 -0400 Johannes Weiner <hannes@cmpxchg.org> wrote:
> alloc_swap_folio() and alloc_anon_folio() use a top-level if (folio)
> that buries the success path four levels deep. This makes for awkward
> long lines and wrapping. The next patch will add more code here, so
> flatten this now to keep things clean and simple.
>
> alloc_anon_folio() already has a next label, use it for !folio. Add
> the equivalent to alloc_swap_folio().
>
> No functional change intended.
>
> Suggested-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
> Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Usama Arif <usama.arif@linux.dev>
^ permalink raw reply
* Re: [PATCH v4 8/8] mm: switch deferred split shrinker to list_lru
From: Usama Arif @ 2026-05-26 12:09 UTC (permalink / raw)
To: Johannes Weiner
Cc: Usama Arif, Andrew Morton, David Hildenbrand, Lorenzo Stoakes,
Shakeel Butt, Michal Hocko, Dave Chinner, Roman Gushchin,
Muchun Song, Qi Zheng, Yosry Ahmed, Zi Yan, Liam R . Howlett,
Kiryl Shutsemau, Vlastimil Babka, Kairui Song, Mikhail Zaslonko,
Vasily Gorbik, Baolin Wang, Barry Song, Dev Jain, Lance Yang,
Nico Pache, Ryan Roberts, cgroups, linux-mm, linux-kernel
In-Reply-To: <20260521150330.1955924-9-hannes@cmpxchg.org>
On Thu, 21 May 2026 11:02:14 -0400 Johannes Weiner <hannes@cmpxchg.org> wrote:
> The deferred split queue handles cgroups in a suboptimal fashion. The
> queue is per-NUMA node or per-cgroup, not the intersection. That means
> on a cgrouped system, a node-restricted allocation entering reclaim
> can end up splitting large pages on other nodes:
>
> alloc/unmap
> deferred_split_folio()
> list_add_tail(memcg->split_queue)
> set_shrinker_bit(memcg, node, deferred_shrinker_id)
>
> for_each_zone_zonelist_nodemask(restricted_nodes)
> mem_cgroup_iter()
> shrink_slab(node, memcg)
> shrink_slab_memcg(node, memcg)
> if test_shrinker_bit(memcg, node, deferred_shrinker_id)
> deferred_split_scan()
> walks memcg->split_queue
>
> The shrinker bit adds an imperfect guard rail. As soon as the cgroup
> has a single large page on the node of interest, all large pages owned
> by that memcg, including those on other nodes, will be split.
>
> list_lru properly sets up per-node, per-cgroup lists. As a bonus, it
> streamlines a lot of the list operations and reclaim walks. It's used
> widely by other major shrinkers already. Convert the deferred split
> queue as well.
>
> The list_lru per-memcg heads are instantiated on demand when the first
> object of interest is allocated for a cgroup, by calling
> folio_memcg_alloc_deferred(). Add calls to where splittable pages are
> created: anon faults, swapin faults, khugepaged collapse.
>
> These calls create all possible node heads for the cgroup at once, so
> the migration code (between nodes) doesn't need any special care.
>
> Reported-by: Mikhail Zaslonko <zaslonko@linux.ibm.com>
> Tested-by: Mikhail Zaslonko <zaslonko@linux.ibm.com>
> Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
> Reviewed-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
> Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
> ---
> include/linux/huge_mm.h | 7 +-
> include/linux/memcontrol.h | 4 -
> include/linux/mmzone.h | 12 --
> mm/huge_memory.c | 355 ++++++++++++-------------------------
> mm/internal.h | 2 +-
> mm/khugepaged.c | 3 +
> mm/memcontrol.c | 12 +-
> mm/memory.c | 8 +
> mm/mm_init.c | 15 --
> 9 files changed, 133 insertions(+), 285 deletions(-)
>
> diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
> index 127f9e1e7604..dc939873f5df 100644
> --- a/include/linux/huge_mm.h
> +++ b/include/linux/huge_mm.h
> @@ -398,10 +398,10 @@ static inline int split_huge_page(struct page *page)
> {
> return split_huge_page_to_list_to_order(page, NULL, 0);
> }
> +
> +int folio_memcg_alloc_deferred(struct folio *folio);
> +
> void deferred_split_folio(struct folio *folio, bool partially_mapped);
> -#ifdef CONFIG_MEMCG
> -void reparent_deferred_split_queue(struct mem_cgroup *memcg);
> -#endif
>
> void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
> unsigned long address, bool freeze);
> @@ -634,7 +634,6 @@ static inline int folio_split(struct folio *folio, unsigned int new_order,
> }
>
> static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {}
> -static inline void reparent_deferred_split_queue(struct mem_cgroup *memcg) {}
> #define split_huge_pmd(__vma, __pmd, __address) \
> do { } while (0)
>
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index dc3fa687759b..4a7d8c4f55b4 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -277,10 +277,6 @@ struct mem_cgroup {
> struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
> #endif
>
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> - struct deferred_split deferred_split_queue;
> -#endif
> -
> #ifdef CONFIG_LRU_GEN_WALKS_MMU
> /* per-memcg mm_struct list */
> struct lru_gen_mm_list mm_list;
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 1331a7b93f33..8e449f524f26 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -1431,14 +1431,6 @@ struct zonelist {
> */
> extern struct page *mem_map;
>
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> -struct deferred_split {
> - spinlock_t split_queue_lock;
> - struct list_head split_queue;
> - unsigned long split_queue_len;
> -};
> -#endif
> -
> #ifdef CONFIG_MEMORY_FAILURE
> /*
> * Per NUMA node memory failure handling statistics.
> @@ -1564,10 +1556,6 @@ typedef struct pglist_data {
> unsigned long first_deferred_pfn;
> #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
>
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> - struct deferred_split deferred_split_queue;
> -#endif
> -
> #ifdef CONFIG_NUMA_BALANCING
> /* start time in ms of current promote rate limit period */
> unsigned int nbp_rl_start;
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index c565b2a651e0..67be09a58d5a 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -14,6 +14,7 @@
> #include <linux/mmu_notifier.h>
> #include <linux/rmap.h>
> #include <linux/swap.h>
> +#include <linux/list_lru.h>
> #include <linux/shrinker.h>
> #include <linux/mm_inline.h>
> #include <linux/swapops.h>
> @@ -67,6 +68,8 @@ unsigned long transparent_hugepage_flags __read_mostly =
> (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
> (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
>
> +static struct lock_class_key deferred_split_key;
> +static struct list_lru deferred_split_lru;
> static struct shrinker *deferred_split_shrinker;
> static unsigned long deferred_split_count(struct shrinker *shrink,
> struct shrink_control *sc);
> @@ -943,6 +946,13 @@ static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
> }
> #endif /* CONFIG_SYSFS */
>
> +int folio_memcg_alloc_deferred(struct folio *folio)
> +{
> + if (mem_cgroup_disabled())
> + return 0;
> + return folio_memcg_list_lru_alloc(folio, &deferred_split_lru, GFP_KERNEL);
> +}
> +
> static int __init thp_shrinker_init(void)
> {
> deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE |
> @@ -952,6 +962,13 @@ static int __init thp_shrinker_init(void)
> if (!deferred_split_shrinker)
> return -ENOMEM;
>
> + if (list_lru_init_memcg_key(&deferred_split_lru,
> + deferred_split_shrinker,
> + &deferred_split_key)) {
> + shrinker_free(deferred_split_shrinker);
> + return -ENOMEM;
> + }
> +
> deferred_split_shrinker->count_objects = deferred_split_count;
> deferred_split_shrinker->scan_objects = deferred_split_scan;
> shrinker_register(deferred_split_shrinker);
> @@ -973,6 +990,7 @@ static int __init thp_shrinker_init(void)
> huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero");
> if (!huge_zero_folio_shrinker) {
> shrinker_free(deferred_split_shrinker);
> + list_lru_destroy(&deferred_split_lru);
> return -ENOMEM;
> }
>
> @@ -987,6 +1005,7 @@ static void __init thp_shrinker_exit(void)
> {
> shrinker_free(huge_zero_folio_shrinker);
> shrinker_free(deferred_split_shrinker);
> + list_lru_destroy(&deferred_split_lru);
> }
>
> static int __init hugepage_init(void)
> @@ -1166,119 +1185,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
> return pmd;
> }
>
> -static struct deferred_split *split_queue_node(int nid)
> -{
> - struct pglist_data *pgdata = NODE_DATA(nid);
> -
> - return &pgdata->deferred_split_queue;
> -}
> -
> -#ifdef CONFIG_MEMCG
> -static inline
> -struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
> - struct deferred_split *queue)
> -{
> - if (mem_cgroup_disabled())
> - return NULL;
> - if (split_queue_node(folio_nid(folio)) == queue)
> - return NULL;
> - return container_of(queue, struct mem_cgroup, deferred_split_queue);
> -}
> -
> -static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
> -{
> - return memcg ? &memcg->deferred_split_queue : split_queue_node(nid);
> -}
> -#else
> -static inline
> -struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
> - struct deferred_split *queue)
> -{
> - return NULL;
> -}
> -
> -static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
> -{
> - return split_queue_node(nid);
> -}
> -#endif
> -
> -static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg)
> -{
> - struct deferred_split *queue;
> -
> -retry:
> - queue = memcg_split_queue(nid, memcg);
> - spin_lock(&queue->split_queue_lock);
> - /*
> - * There is a period between setting memcg to dying and reparenting
> - * deferred split queue, and during this period the THPs in the deferred
> - * split queue will be hidden from the shrinker side.
> - */
> - if (unlikely(memcg_is_dying(memcg))) {
> - spin_unlock(&queue->split_queue_lock);
> - memcg = parent_mem_cgroup(memcg);
> - goto retry;
> - }
> -
> - return queue;
> -}
> -
> -static struct deferred_split *
> -split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags)
> -{
> - struct deferred_split *queue;
> -
> -retry:
> - queue = memcg_split_queue(nid, memcg);
> - spin_lock_irqsave(&queue->split_queue_lock, *flags);
> - if (unlikely(memcg_is_dying(memcg))) {
> - spin_unlock_irqrestore(&queue->split_queue_lock, *flags);
> - memcg = parent_mem_cgroup(memcg);
> - goto retry;
> - }
> -
> - return queue;
> -}
> -
> -static struct deferred_split *folio_split_queue_lock(struct folio *folio)
> -{
> - struct deferred_split *queue;
> -
> - rcu_read_lock();
> - queue = split_queue_lock(folio_nid(folio), folio_memcg(folio));
> - /*
> - * The memcg destruction path is acquiring the split queue lock for
> - * reparenting. Once you have it locked, it's safe to drop the rcu lock.
> - */
> - rcu_read_unlock();
> -
> - return queue;
> -}
> -
> -static struct deferred_split *
> -folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags)
> -{
> - struct deferred_split *queue;
> -
> - rcu_read_lock();
> - queue = split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags);
> - rcu_read_unlock();
> -
> - return queue;
> -}
> -
> -static inline void split_queue_unlock(struct deferred_split *queue)
> -{
> - spin_unlock(&queue->split_queue_lock);
> -}
> -
> -static inline void split_queue_unlock_irqrestore(struct deferred_split *queue,
> - unsigned long flags)
> -{
> - spin_unlock_irqrestore(&queue->split_queue_lock, flags);
> -}
> -
> static inline bool is_transparent_hugepage(const struct folio *folio)
> {
> if (!folio_test_large(folio))
> @@ -1379,6 +1285,14 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma,
> count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
> return NULL;
> }
> +
> + if (folio_memcg_alloc_deferred(folio)) {
> + folio_put(folio);
> + count_vm_event(THP_FAULT_FALLBACK);
> + count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
> + return NULL;
> + }
> +
> folio_throttle_swaprate(folio, gfp);
>
> /*
> @@ -3888,34 +3802,40 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
> struct folio *end_folio = folio_next(folio);
> struct folio *new_folio, *next;
> int old_order = folio_order(folio);
> + struct list_lru_one *lru;
> + bool dequeue_deferred;
> int ret = 0;
> - struct deferred_split *ds_queue;
>
> VM_WARN_ON_ONCE(!mapping && end);
> - /* Prevent deferred_split_scan() touching ->_refcount */
> - ds_queue = folio_split_queue_lock(folio);
> + /*
> + * If this folio can be on the deferred split queue, lock out
> + * the shrinker before freezing the ref. If the shrinker sees
> + * a 0-ref folio, it assumes it beat folio_put() to the list
> + * lock and must clean up the LRU state - the same dequeue we
> + * will do below as part of the split.
> + */
> + dequeue_deferred = folio_test_anon(folio) && old_order > 1;
> + if (dequeue_deferred) {
> + rcu_read_lock();
> + lru = list_lru_lock(&deferred_split_lru,
> + folio_nid(folio), folio_memcg(folio));
> + }
> if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) {
> struct swap_cluster_info *ci = NULL;
> struct lruvec *lruvec;
>
> - if (old_order > 1) {
> - if (!list_empty(&folio->_deferred_list)) {
> - ds_queue->split_queue_len--;
> - /*
> - * Reinitialize page_deferred_list after removing the
> - * page from the split_queue, otherwise a subsequent
> - * split will see list corruption when checking the
> - * page_deferred_list.
> - */
> - list_del_init(&folio->_deferred_list);
> - }
> + if (dequeue_deferred) {
> + __list_lru_del(&deferred_split_lru, lru,
> + &folio->_deferred_list, folio_nid(folio));
> if (folio_test_partially_mapped(folio)) {
> folio_clear_partially_mapped(folio);
> mod_mthp_stat(old_order,
> MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
> }
> + list_lru_unlock(lru);
> + rcu_read_unlock();
> }
> - split_queue_unlock(ds_queue);
> +
> if (mapping) {
> int nr = folio_nr_pages(folio);
>
> @@ -4015,7 +3935,10 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
> if (ci)
> swap_cluster_unlock(ci);
> } else {
> - split_queue_unlock(ds_queue);
> + if (dequeue_deferred) {
> + list_lru_unlock(lru);
> + rcu_read_unlock();
> + }
> return -EAGAIN;
> }
>
> @@ -4381,33 +4304,35 @@ int split_folio_to_list(struct folio *folio, struct list_head *list)
> * queueing THP splits, and that list is (racily observed to be) non-empty.
> *
> * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is
> - * zero: because even when split_queue_lock is held, a non-empty _deferred_list
> - * might be in use on deferred_split_scan()'s unlocked on-stack list.
> + * zero: because even when the list_lru lock is held, a non-empty
> + * _deferred_list might be in use on deferred_split_scan()'s unlocked
> + * on-stack list.
> *
> - * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is
> - * therefore important to unqueue deferred split before changing folio memcg.
> + * The list_lru sublist is determined by folio's memcg: it is therefore
> + * important to unqueue deferred split before changing folio memcg.
> */
> bool __folio_unqueue_deferred_split(struct folio *folio)
> {
> - struct deferred_split *ds_queue;
> + struct list_lru_one *lru;
> + int nid = folio_nid(folio);
> unsigned long flags;
> bool unqueued = false;
>
> WARN_ON_ONCE(folio_ref_count(folio));
> WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio));
>
> - ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
> - if (!list_empty(&folio->_deferred_list)) {
> - ds_queue->split_queue_len--;
> + rcu_read_lock();
> + lru = list_lru_lock_irqsave(&deferred_split_lru, nid, folio_memcg(folio), &flags);
> + if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) {
> if (folio_test_partially_mapped(folio)) {
> folio_clear_partially_mapped(folio);
> mod_mthp_stat(folio_order(folio),
> MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
> }
> - list_del_init(&folio->_deferred_list);
> unqueued = true;
> }
> - split_queue_unlock_irqrestore(ds_queue, flags);
> + list_lru_unlock_irqrestore(lru, &flags);
> + rcu_read_unlock();
>
> return unqueued; /* useful for debug warnings */
> }
> @@ -4415,7 +4340,9 @@ bool __folio_unqueue_deferred_split(struct folio *folio)
> /* partially_mapped=false won't clear PG_partially_mapped folio flag */
> void deferred_split_folio(struct folio *folio, bool partially_mapped)
> {
> - struct deferred_split *ds_queue;
> + struct list_lru_one *lru;
> + int nid;
> + struct mem_cgroup *memcg;
> unsigned long flags;
>
> /*
> @@ -4438,7 +4365,11 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped)
> if (folio_test_swapcache(folio))
> return;
>
> - ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
> + nid = folio_nid(folio);
> +
> + rcu_read_lock();
> + memcg = folio_memcg(folio);
> + lru = list_lru_lock_irqsave(&deferred_split_lru, nid, memcg, &flags);
> if (partially_mapped) {
> if (!folio_test_partially_mapped(folio)) {
> folio_set_partially_mapped(folio);
> @@ -4446,36 +4377,20 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped)
> count_vm_event(THP_DEFERRED_SPLIT_PAGE);
> count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED);
> mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1);
> -
> }
> } else {
> /* partially mapped folios cannot become non-partially mapped */
> VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio);
> }
> - if (list_empty(&folio->_deferred_list)) {
> - struct mem_cgroup *memcg;
> -
> - memcg = folio_split_queue_memcg(folio, ds_queue);
> - list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
> - ds_queue->split_queue_len++;
> - if (memcg)
> - set_shrinker_bit(memcg, folio_nid(folio),
> - shrinker_id(deferred_split_shrinker));
> - }
> - split_queue_unlock_irqrestore(ds_queue, flags);
> + __list_lru_add(&deferred_split_lru, lru, &folio->_deferred_list, nid, memcg);
> + list_lru_unlock_irqrestore(lru, &flags);
> + rcu_read_unlock();
Can the shrinker bit end up on the wrong memcg here?
deferred_split_folio() takes the lock via list_lru_lock_irqsave() with
the original folio_memcg() of the folio. If that memcg is dying and
already reparented, lock_list_lru_of_memcg() walks up parent_mem_cgroup()
until it finds a live sublist and locks it (lru), but the memcg local
variable still points at the dying child.
__list_lru_add() then calls set_shrinker_bit(memcg, ...) with the
original (dying / reparented) memcg, not the parent that actually owns
the locked sublist where the folio was queued.
> }
>
> static unsigned long deferred_split_count(struct shrinker *shrink,
> struct shrink_control *sc)
> {
> - struct pglist_data *pgdata = NODE_DATA(sc->nid);
> - struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
> -
> -#ifdef CONFIG_MEMCG
> - if (sc->memcg)
> - ds_queue = &sc->memcg->deferred_split_queue;
> -#endif
> - return READ_ONCE(ds_queue->split_queue_len);
> + return list_lru_shrink_count(&deferred_split_lru, sc);
> }
>
> static bool thp_underused(struct folio *folio)
> @@ -4505,45 +4420,45 @@ static bool thp_underused(struct folio *folio)
> return false;
> }
>
> +static enum lru_status deferred_split_isolate(struct list_head *item,
> + struct list_lru_one *lru,
> + void *cb_arg)
> +{
> + struct folio *folio = container_of(item, struct folio, _deferred_list);
> + struct list_head *freeable = cb_arg;
> +
> + if (folio_try_get(folio)) {
> + list_lru_isolate_move(lru, item, freeable);
> + return LRU_REMOVED;
> + }
> +
> + /* We lost race with folio_put() */
> + list_lru_isolate(lru, item);
> + if (folio_test_partially_mapped(folio)) {
> + folio_clear_partially_mapped(folio);
> + mod_mthp_stat(folio_order(folio),
> + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
> + }
> + return LRU_REMOVED;
> +}
> +
> static unsigned long deferred_split_scan(struct shrinker *shrink,
> struct shrink_control *sc)
> {
> - struct deferred_split *ds_queue;
> - unsigned long flags;
> + LIST_HEAD(dispose);
> struct folio *folio, *next;
> - int split = 0, i;
> - struct folio_batch fbatch;
> + int split = 0;
> + unsigned long isolated;
>
> - folio_batch_init(&fbatch);
> + isolated = list_lru_shrink_walk_irq(&deferred_split_lru, sc,
> + deferred_split_isolate, &dispose);
>
> -retry:
> - ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags);
> - /* Take pin on all head pages to avoid freeing them under us */
> - list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
> - _deferred_list) {
> - if (folio_try_get(folio)) {
> - folio_batch_add(&fbatch, folio);
> - } else if (folio_test_partially_mapped(folio)) {
> - /* We lost race with folio_put() */
> - folio_clear_partially_mapped(folio);
> - mod_mthp_stat(folio_order(folio),
> - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
> - }
> - list_del_init(&folio->_deferred_list);
> - ds_queue->split_queue_len--;
> - if (!--sc->nr_to_scan)
> - break;
> - if (!folio_batch_space(&fbatch))
> - break;
> - }
> - split_queue_unlock_irqrestore(ds_queue, flags);
> -
> - for (i = 0; i < folio_batch_count(&fbatch); i++) {
> + list_for_each_entry_safe(folio, next, &dispose, _deferred_list) {
> bool did_split = false;
> bool underused = false;
> - struct deferred_split *fqueue;
>
> - folio = fbatch.folios[i];
> + list_del_init(&folio->_deferred_list);
> +
> if (!folio_test_partially_mapped(folio)) {
> /*
> * See try_to_map_unused_to_zeropage(): we cannot
> @@ -4572,63 +4487,23 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
> * underused, then consider it used and don't add it back to
> * split_queue.
> */
> - if (did_split || !folio_test_partially_mapped(folio))
> - continue;
> + if (!did_split && folio_test_partially_mapped(folio)) {
> requeue:
> - /*
> - * Add back partially mapped folios, or underused folios that
> - * we could not lock this round.
> - */
> - fqueue = folio_split_queue_lock_irqsave(folio, &flags);
> - if (list_empty(&folio->_deferred_list)) {
> - list_add_tail(&folio->_deferred_list, &fqueue->split_queue);
> - fqueue->split_queue_len++;
> + rcu_read_lock();
> + list_lru_add_irq(&deferred_split_lru,
> + &folio->_deferred_list,
> + folio_nid(folio),
> + folio_memcg(folio));
> + rcu_read_unlock();
> }
> - split_queue_unlock_irqrestore(fqueue, flags);
> - }
> - folios_put(&fbatch);
> -
> - if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) {
> - cond_resched();
> - goto retry;
> + folio_put(folio);
> }
>
> - /*
> - * Stop shrinker if we didn't split any page, but the queue is empty.
> - * This can happen if pages were freed under us.
> - */
> - if (!split && list_empty(&ds_queue->split_queue))
> + if (!split && !isolated)
> return SHRINK_STOP;
> return split;
> }
>
> -#ifdef CONFIG_MEMCG
> -void reparent_deferred_split_queue(struct mem_cgroup *memcg)
> -{
> - struct mem_cgroup *parent = parent_mem_cgroup(memcg);
> - struct deferred_split *ds_queue = &memcg->deferred_split_queue;
> - struct deferred_split *parent_ds_queue = &parent->deferred_split_queue;
> - int nid;
> -
> - spin_lock_irq(&ds_queue->split_queue_lock);
> - spin_lock_nested(&parent_ds_queue->split_queue_lock, SINGLE_DEPTH_NESTING);
> -
> - if (!ds_queue->split_queue_len)
> - goto unlock;
> -
> - list_splice_tail_init(&ds_queue->split_queue, &parent_ds_queue->split_queue);
> - parent_ds_queue->split_queue_len += ds_queue->split_queue_len;
> - ds_queue->split_queue_len = 0;
> -
> - for_each_node(nid)
> - set_shrinker_bit(parent, nid, shrinker_id(deferred_split_shrinker));
> -
> -unlock:
> - spin_unlock(&parent_ds_queue->split_queue_lock);
> - spin_unlock_irq(&ds_queue->split_queue_lock);
> -}
> -#endif
> -
> #ifdef CONFIG_DEBUG_FS
> static void split_huge_pages_all(void)
> {
> diff --git a/mm/internal.h b/mm/internal.h
> index 09931b1e535f..e5bf549ad78e 100644
> --- a/mm/internal.h
> +++ b/mm/internal.h
> @@ -861,7 +861,7 @@ static inline bool folio_unqueue_deferred_split(struct folio *folio)
> /*
> * At this point, there is no one trying to add the folio to
> * deferred_list. If folio is not in deferred_list, it's safe
> - * to check without acquiring the split_queue_lock.
> + * to check without acquiring the list_lru lock.
> */
> if (data_race(list_empty(&folio->_deferred_list)))
> return false;
> diff --git a/mm/khugepaged.c b/mm/khugepaged.c
> index 1a25af3d6d0f..644de0410c12 100644
> --- a/mm/khugepaged.c
> +++ b/mm/khugepaged.c
> @@ -1301,6 +1301,9 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
> if (result != SCAN_SUCCEED)
> goto out_nolock;
>
> + if (folio_memcg_alloc_deferred(folio))
> + goto out_nolock;
> +
Over here, you will end up reporting success on allocation failure.
Maybe set result to SCAN_ALLOC_HUGE_PAGE_FAIL?
> mmap_read_lock(mm);
> result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
> &vma, cc, order);
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index aa7361b0d2da..fe896824078b 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -4065,11 +4065,6 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
> for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
> memcg->cgwb_frn[i].done =
> __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
> -#endif
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> - spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
> - INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
> - memcg->deferred_split_queue.split_queue_len = 0;
> #endif
> lru_gen_init_memcg(memcg);
> return memcg;
> @@ -4221,11 +4216,10 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
> zswap_memcg_offline_cleanup(memcg);
>
> memcg_offline_kmem(memcg);
> - reparent_deferred_split_queue(memcg);
> /*
> - * The reparenting of objcg must be after the reparenting of the
> - * list_lru and deferred_split_queue above, which ensures that they will
> - * not mistakenly get the parent list_lru and deferred_split_queue.
> + * The reparenting of objcg must be after the reparenting of
> + * the list_lru in memcg_offline_kmem(), which ensures that
> + * they will not mistakenly get the parent list_lru.
> */
> memcg_reparent_objcgs(memcg);
> reparent_shrinker_deferred(memcg);
> diff --git a/mm/memory.c b/mm/memory.c
> index 552fe26a042a..b97e3145982a 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -4759,6 +4759,10 @@ static struct folio *alloc_swap_folio(struct vm_fault *vmf)
> folio_put(folio);
> goto next;
> }
> + if (order > 1 && folio_memcg_alloc_deferred(folio)) {
> + folio_put(folio);
> + goto fallback;
> + }
> return folio;
> next:
> count_mthp_stat(order, MTHP_STAT_SWPIN_FALLBACK);
> @@ -5279,6 +5283,10 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf)
> folio_put(folio);
> goto next;
> }
> + if (order > 1 && folio_memcg_alloc_deferred(folio)) {
> + folio_put(folio);
> + goto fallback;
> + }
> folio_throttle_swaprate(folio, gfp);
> /*
> * When a folio is not zeroed during allocation
> diff --git a/mm/mm_init.c b/mm/mm_init.c
> index db5568cf36e1..c0a7f1cf6fef 100644
> --- a/mm/mm_init.c
> +++ b/mm/mm_init.c
> @@ -1373,19 +1373,6 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat,
> pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
> }
>
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> -static void pgdat_init_split_queue(struct pglist_data *pgdat)
> -{
> - struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
> -
> - spin_lock_init(&ds_queue->split_queue_lock);
> - INIT_LIST_HEAD(&ds_queue->split_queue);
> - ds_queue->split_queue_len = 0;
> -}
> -#else
> -static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
> -#endif
> -
> #ifdef CONFIG_COMPACTION
> static void pgdat_init_kcompactd(struct pglist_data *pgdat)
> {
> @@ -1401,8 +1388,6 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
>
> pgdat_resize_init(pgdat);
> pgdat_kswapd_lock_init(pgdat);
> -
> - pgdat_init_split_queue(pgdat);
> pgdat_init_kcompactd(pgdat);
>
> init_waitqueue_head(&pgdat->kswapd_wait);
> --
> 2.54.0
>
>
^ permalink raw reply
* Re: [PATCH v2 0/4] mm/zswap: Implement per-cgroup proactive writeback
From: Hao Jia @ 2026-05-26 11:56 UTC (permalink / raw)
To: Andrew Morton
Cc: tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin, cgroups, linux-mm,
linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260525122424.3b2818f06832d9d55da8d69b@linux-foundation.org>
On 2026/5/26 03:24, Andrew Morton wrote:
> On Mon, 25 May 2026 20:22:38 +0800 Hao Jia <jiahao.kernel@gmail.com> wrote:
>
>> Zswap currently writes back pages to backing swap reactively, triggered
>> either by the shrinker or by the pool reaching its size limit. Although
>> proactive memory reclaim can automatically write back a portion of zswap
>> pages via the shrinker, it cannot explicitly control the amount of
>> writeback for a specific memory cgroup. Moreover, proactive memory reclaim
>> may not always be triggered during a steady state.
>>
>> In certain scenarios, it is desirable to trigger writeback in advance to
>> free up memory. For example, users may want to prepare for an upcoming
>> memory-intensive workload by flushing cold memory to the backing storage
>> when the system is relatively idle.
>>
>> This patch series introduces a "zswap_writeback_only" key to memory.reclaim
>> cgroup interface, allowing users to proactively write back cold compressed
>> pages from zswap to the backing swap device. When specified, this key
>> bypasses standard memory reclaim and exclusively performs proactive zswap
>> writeback up to the requested budget. If omitted, the default reclaim
>> behavior remains unchanged.
>
> Thanks. AI review found a few things to complain about, one of them
> described as "preexisting".
>
Thanks Andrew. I have replied to the AI's review comments in a separate
email and posted v3.
https://lore.kernel.org/all/20260526114601.67041-1-jiahao.kernel@gmail.com
Thanks,
Hao
^ permalink raw reply
* [PATCH v3 4/4] selftests/cgroup: Add tests for zswap proactive writeback
From: Hao Jia @ 2026-05-26 11:46 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: cgroups, linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260526114601.67041-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Add test_zswap_proactive_writeback() to cover the new memory.reclaim
"zswap_writeback_only" key. The test populates a memory cgroup zswap
pool, triggers proactive writeback, and verifies the behavior by
observing the change in zswpwb_proactive. Invalid input combinations
are also covered.
Extend test_zswap_writeback_one() to assert that the existing
non-proactive writeback path leaves zswpwb_proactive at zero.
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
tools/testing/selftests/cgroup/test_zswap.c | 155 +++++++++++++++++++-
1 file changed, 154 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c
index 49b36ee79160..6ab9394a37cc 100644
--- a/tools/testing/selftests/cgroup/test_zswap.c
+++ b/tools/testing/selftests/cgroup/test_zswap.c
@@ -60,7 +60,12 @@ static int get_zswap_stored_pages(size_t *value)
static long get_cg_wb_count(const char *cg)
{
- return cg_read_key_long(cg, "memory.stat", "zswpwb");
+ return cg_read_key_long(cg, "memory.stat", "zswpwb ");
+}
+
+static long get_cg_pwb_count(const char *cg)
+{
+ return cg_read_key_long(cg, "memory.stat", "zswpwb_proactive ");
}
static long get_zswpout(const char *cgroup)
@@ -355,6 +360,7 @@ static int attempt_writeback(const char *cgroup, void *arg)
static int test_zswap_writeback_one(const char *cgroup, bool wb)
{
long zswpwb_before, zswpwb_after;
+ long pwb_cnt;
zswpwb_before = get_cg_wb_count(cgroup);
if (zswpwb_before != 0) {
@@ -362,6 +368,12 @@ static int test_zswap_writeback_one(const char *cgroup, bool wb)
return -1;
}
+ pwb_cnt = get_cg_pwb_count(cgroup);
+ if (pwb_cnt != 0) {
+ ksft_print_msg("zswpwb_proactive_before = %ld instead of 0\n", pwb_cnt);
+ return -1;
+ }
+
if (cg_run(cgroup, attempt_writeback, (void *) &wb))
return -1;
@@ -379,6 +391,17 @@ static int test_zswap_writeback_one(const char *cgroup, bool wb)
return -1;
}
+ /*
+ * attempt_writeback() does not use the proactive writeback path, so
+ * zswpwb_proactive must stay at zero regardless of whether writeback
+ * was enabled.
+ */
+ pwb_cnt = get_cg_pwb_count(cgroup);
+ if (pwb_cnt != 0) {
+ ksft_print_msg("zswpwb_proactive_after is %ld, expected 0\n", pwb_cnt);
+ return -1;
+ }
+
return 0;
}
@@ -770,6 +793,135 @@ static int test_zswap_incompressible(const char *root)
return ret;
}
+/*
+ * Trigger proactive zswap writeback with the following steps:
+ * 1. Allocate memory.
+ * 2. Push allocated memory into zswap.
+ * 3. Proactively write back zswap pages to swap
+ * using "zswap_writeback_only".
+ */
+static int proactive_writeback_workload(const char *cgroup, void *arg)
+{
+ size_t memsize = page_size * 1024;
+ char reclaim_cmd[64];
+ char buf[page_size];
+ long zswap_usage;
+ int ret = -1;
+ char *mem;
+
+ mem = (char *)malloc(memsize);
+ if (!mem)
+ return ret;
+
+ for (int i = 0; i < page_size; i++)
+ buf[i] = i < page_size / 2 ? (char)i : 0;
+ for (int i = 0; i < memsize; i += page_size)
+ memcpy(&mem[i], buf, page_size);
+
+ /* Evict allocated memory into zswap. */
+ if (cg_write_numeric(cgroup, "memory.reclaim", memsize)) {
+ ksft_print_msg("Failed to push pages into zswap\n");
+ goto out;
+ }
+
+ zswap_usage = cg_read_long(cgroup, "memory.zswap.current");
+ if (zswap_usage <= 0) {
+ ksft_print_msg("no zswap pool to write back\n");
+ goto out;
+ }
+
+ /* Trigger proactive zswap writeback. */
+ snprintf(reclaim_cmd, sizeof(reclaim_cmd), "%zu zswap_writeback_only", memsize);
+ int rc = cg_write(cgroup, "memory.reclaim", reclaim_cmd);
+ if (rc && rc != -EAGAIN) {
+ ksft_print_msg("proactive zswap writeback failed: %d\n", rc);
+ goto out;
+ }
+
+ ret = 0;
+out:
+ free(mem);
+ return ret;
+}
+
+static int check_writeback_invalid_inputs(const char *cgroup)
+{
+ static char * const bad_inputs[] = {
+ "zswap_writeback_only",
+ "1M zswap_writeback_only swappiness=60",
+ "1M swappiness=60 zswap_writeback_only",
+ "1M zswap_writeback_only swappiness=max",
+ "1M swappiness=max zswap_writeback_only",
+ };
+ int i, rc;
+
+ for (i = 0; i < ARRAY_SIZE(bad_inputs); i++) {
+ rc = cg_write(cgroup, "memory.reclaim", bad_inputs[i]);
+ if (rc != -EINVAL) {
+ ksft_print_msg("memory.reclaim '%s': returned %d, expected %d\n",
+ bad_inputs[i], rc, -EINVAL);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int test_zswap_proactive_writeback(const char *root)
+{
+ long pwb_before, wb_before, pwb_after, wb_after;
+ long pwb_delta, wb_delta;
+ int ret = KSFT_FAIL;
+ char *test_group;
+
+ if (cg_read_strcmp(root, "memory.zswap.writeback", "1"))
+ return KSFT_SKIP;
+
+ test_group = cg_name(root, "zswap_proactive_test");
+ if (!test_group)
+ return KSFT_FAIL;
+ if (cg_create(test_group))
+ goto out;
+ if (check_writeback_invalid_inputs(test_group))
+ goto out;
+
+ pwb_before = get_cg_pwb_count(test_group);
+ wb_before = get_cg_wb_count(test_group);
+ if (pwb_before < 0 || wb_before < 0)
+ goto out;
+
+ if (cg_run(test_group, proactive_writeback_workload, NULL))
+ goto out;
+
+ pwb_after = get_cg_pwb_count(test_group);
+ wb_after = get_cg_wb_count(test_group);
+ if (pwb_after < 0 || wb_after < 0)
+ goto out;
+
+ pwb_delta = pwb_after - pwb_before;
+ wb_delta = wb_after - wb_before;
+
+ if (pwb_delta <= 0) {
+ ksft_print_msg("zswpwb_proactive did not increase: delta=%ld\n",
+ pwb_delta);
+ goto out;
+ }
+ if (wb_delta <= 0) {
+ ksft_print_msg("zswpwb did not increase: delta=%ld\n", wb_delta);
+ goto out;
+ }
+ if (pwb_delta > wb_delta) {
+ ksft_print_msg("zswpwb_proactive delta (%ld) > zswpwb delta (%ld)\n",
+ pwb_delta, wb_delta);
+ goto out;
+ }
+
+ ret = KSFT_PASS;
+out:
+ cg_destroy(test_group);
+ free(test_group);
+ return ret;
+}
+
#define T(x) { x, #x }
struct zswap_test {
int (*fn)(const char *root);
@@ -783,6 +935,7 @@ struct zswap_test {
T(test_no_kmem_bypass),
T(test_no_invasive_cgroup_shrink),
T(test_zswap_incompressible),
+ T(test_zswap_proactive_writeback),
};
#undef T
--
2.34.1
^ permalink raw reply related
* [PATCH v3 3/4] mm/zswap: Add per-memcg stat for proactive writeback
From: Hao Jia @ 2026-05-26 11:46 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: cgroups, linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260526114601.67041-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Currently, zswap writeback can be triggered by either the pool limit
being hit or by the proactive writeback mechanism. However, the
existing 'zswpwb' metric in memory.stat and /proc/vmstat counts all
written back pages, making it difficult to distinguish between pages
written back due to the pool limit and those written back proactively.
Add a new statistic 'zswpwb_proactive' to memory.stat and /proc/vmstat.
This counter tracks the number of pages written back due to proactive
writeback. This allows users to better monitor and tune the proactive
writeback mechanism.
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
Documentation/admin-guide/cgroup-v2.rst | 4 +++
include/linux/vm_event_item.h | 1 +
mm/memcontrol.c | 1 +
mm/vmstat.c | 1 +
mm/zswap.c | 41 ++++++++++++++++++-------
5 files changed, 37 insertions(+), 11 deletions(-)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 6564abf0dec5..7d65aef83f7b 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1748,6 +1748,10 @@ The following nested keys are defined.
zswpwb
Number of pages written from zswap to swap.
+ zswpwb_proactive
+ Number of pages written from zswap to swap by proactive
+ writeback. This is a subset of zswpwb.
+
zswap_incomp
Number of incompressible pages currently stored in zswap
without compression. These pages could not be compressed to
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 03fe95f5a020..7a5bee0a20b6 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -138,6 +138,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
ZSWPIN,
ZSWPOUT,
ZSWPWB,
+ ZSWPWB_PROACTIVE,
#endif
#ifdef CONFIG_X86
DIRECT_MAP_LEVEL2_SPLIT,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index e205e5de193d..7648b3fd940e 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -571,6 +571,7 @@ static const unsigned int memcg_vm_event_stat[] = {
ZSWPIN,
ZSWPOUT,
ZSWPWB,
+ ZSWPWB_PROACTIVE,
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
THP_FAULT_ALLOC,
diff --git a/mm/vmstat.c b/mm/vmstat.c
index f534972f517d..66fd06d1bb01 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1452,6 +1452,7 @@ const char * const vmstat_text[] = {
[I(ZSWPIN)] = "zswpin",
[I(ZSWPOUT)] = "zswpout",
[I(ZSWPWB)] = "zswpwb",
+ [I(ZSWPWB_PROACTIVE)] = "zswpwb_proactive",
#endif
#ifdef CONFIG_X86
[I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits",
diff --git a/mm/zswap.c b/mm/zswap.c
index 7bcbf788f634..b45d094f532a 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -160,6 +160,11 @@ struct zswap_pool {
char tfm_name[CRYPTO_MAX_ALG_NAME];
};
+struct zswap_shrink_walk_arg {
+ bool proactive;
+ bool encountered_page_in_swapcache;
+};
+
/* Global LRU lists shared by all zswap pools. */
static struct list_lru zswap_list_lru;
@@ -1042,7 +1047,8 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio)
* freed.
*/
static int zswap_writeback_entry(struct zswap_entry *entry,
- swp_entry_t swpentry)
+ swp_entry_t swpentry,
+ bool proactive)
{
struct xarray *tree;
pgoff_t offset = swp_offset(swpentry);
@@ -1097,6 +1103,12 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
if (entry->objcg)
count_objcg_events(entry->objcg, ZSWPWB, 1);
+ if (proactive) {
+ count_vm_event(ZSWPWB_PROACTIVE);
+ if (entry->objcg)
+ count_objcg_events(entry->objcg, ZSWPWB_PROACTIVE, 1);
+ }
+
zswap_entry_free(entry);
/* folio is up to date */
@@ -1146,7 +1158,8 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
void *arg)
{
struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
- bool *encountered_page_in_swapcache = (bool *)arg;
+ struct zswap_shrink_walk_arg *walk_arg = arg;
+ bool proactive_wb = walk_arg && walk_arg->proactive;
swp_entry_t swpentry;
enum lru_status ret = LRU_REMOVED_RETRY;
int writeback_result;
@@ -1201,7 +1214,7 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
*/
spin_unlock(&l->lock);
- writeback_result = zswap_writeback_entry(entry, swpentry);
+ writeback_result = zswap_writeback_entry(entry, swpentry, proactive_wb);
if (writeback_result) {
zswap_reject_reclaim_fail++;
@@ -1212,9 +1225,9 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
* into the warmer region. We should terminate shrinking (if we're in the dynamic
* shrinker context).
*/
- if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
+ if (writeback_result == -EEXIST && walk_arg) {
ret = LRU_STOP;
- *encountered_page_in_swapcache = true;
+ walk_arg->encountered_page_in_swapcache = true;
}
} else {
zswap_written_back_pages++;
@@ -1226,8 +1239,11 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
struct shrink_control *sc)
{
+ struct zswap_shrink_walk_arg walk_arg = {
+ .proactive = false,
+ .encountered_page_in_swapcache = false,
+ };
unsigned long shrink_ret;
- bool encountered_page_in_swapcache = false;
if (!zswap_shrinker_enabled ||
!mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
@@ -1236,9 +1252,9 @@ static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
}
shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
- &encountered_page_in_swapcache);
+ &walk_arg);
- if (encountered_page_in_swapcache)
+ if (walk_arg.encountered_page_in_swapcache)
return SHRINK_STOP;
return shrink_ret ? shrink_ret : SHRINK_STOP;
@@ -1709,7 +1725,10 @@ static long zswap_proactive_shrink_memcg(struct mem_cgroup *memcg,
return -ENOENT;
for_each_node_state(nid, N_NORMAL_MEMORY) {
- bool encountered_page_in_swapcache = false;
+ struct zswap_shrink_walk_arg walk_arg = {
+ .proactive = true,
+ .encountered_page_in_swapcache = false,
+ };
unsigned long nr_to_scan, nr_scanned = 0;
/*
@@ -1743,12 +1762,12 @@ static long zswap_proactive_shrink_memcg(struct mem_cgroup *memcg,
nr_written += list_lru_walk_one(&zswap_list_lru, nid, memcg,
&shrink_memcg_cb,
- &encountered_page_in_swapcache,
+ &walk_arg,
&nr_to_walk);
if (nr_written >= nr_to_write)
return nr_written;
- if (encountered_page_in_swapcache)
+ if (walk_arg.encountered_page_in_swapcache)
break;
cond_resched();
--
2.34.1
^ permalink raw reply related
* [PATCH v3 2/4] mm/zswap: Implement proactive writeback
From: Hao Jia @ 2026-05-26 11:45 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: cgroups, linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260526114601.67041-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Zswap currently writes back pages to backing swap reactively, triggered
either by the shrinker or when the pool reaches its size limit. There is
no mechanism to control the amount of writeback for a specific memory
cgroup. However, users may want to proactively write back zswap pages,
e.g., to free up memory for other applications or to prepare for
memory-intensive workloads.
Introduce a "zswap_writeback_only" key to the memory.reclaim cgroup
interface. When specified, this key bypasses standard memory reclaim
and exclusively performs proactive zswap writeback up to the requested
budget. If omitted, the default reclaim behavior remains unchanged.
Example usage:
# Write back 100MB of pages from zswap to the backing swap
echo "100M zswap_writeback_only" > memory.reclaim
Note that the actual amount written back may be less than requested due
to the zswap second-chance algorithm: referenced entries are rotated on
the LRU on the first encounter and only written back on a second pass.
If fewer bytes are written back than requested, -EAGAIN is returned,
matching the existing memory.reclaim semantics.
Internally, extend user_proactive_reclaim() to parse the new
"zswap_writeback_only" token and invoke the dedicated handler. Add
zswap_proactive_writeback() to walk the target memcg subtree via the
per-memcg writeback cursor, draining per-node zswap LRUs through
list_lru_walk_one() with the shrink_memcg_cb() callback.
Suggested-by: Yosry Ahmed <yosry@kernel.org>
Suggested-by: Nhat Pham <nphamcs@gmail.com>
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
Documentation/admin-guide/cgroup-v2.rst | 18 +++-
Documentation/admin-guide/mm/zswap.rst | 11 +-
include/linux/zswap.h | 7 ++
mm/vmscan.c | 14 +++
mm/zswap.c | 138 ++++++++++++++++++++++++
5 files changed, 185 insertions(+), 3 deletions(-)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 6efd0095ed99..6564abf0dec5 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1425,9 +1425,10 @@ PAGE_SIZE multiple when read back.
The following nested keys are defined.
- ========== ================================
+ ==================== ==================================================
swappiness Swappiness value to reclaim with
- ========== ================================
+ zswap_writeback_only Only perform proactive zswap writeback
+ ==================== ==================================================
Specifying a swappiness value instructs the kernel to perform
the reclaim with that swappiness value. Note that this has the
@@ -1437,6 +1438,19 @@ The following nested keys are defined.
The valid range for swappiness is [0-200, max], setting
swappiness=max exclusively reclaims anonymous memory.
+ The zswap_writeback_only key skips ordinary memory reclaim and
+ writes back pages from zswap to the backing swap device until
+ the requested amount has been written or no further candidates
+ are found. This is useful to proactively offload cold pages from
+ the zswap pool to the swap device. It is only available if
+ zswap writeback is enabled. zswap_writeback_only cannot be combined
+ with swappiness; specifying both returns -EINVAL.
+
+ Example::
+
+ # Write back up to 100MB of pages from zswap to the backing swap
+ echo "100M zswap_writeback_only" > memory.reclaim
+
memory.peak
A read-write single value file which exists on non-root cgroups.
diff --git a/Documentation/admin-guide/mm/zswap.rst b/Documentation/admin-guide/mm/zswap.rst
index 2464425c783d..1c0598e77958 100644
--- a/Documentation/admin-guide/mm/zswap.rst
+++ b/Documentation/admin-guide/mm/zswap.rst
@@ -131,7 +131,16 @@ User can enable it as follows::
echo Y > /sys/module/zswap/parameters/shrinker_enabled
This can be enabled at the boot time if ``CONFIG_ZSWAP_SHRINKER_DEFAULT_ON`` is
-selected.
+selected. Once enabled, the shrinker automatically writes back zswap pages to
+backing swap during memory reclaim.
+
+If users want to explicitly trigger proactive zswap writeback for a specific
+memory cgroup without invoking standard page reclaim, it can be done as follows::
+
+ echo "100M zswap_writeback_only" > /sys/fs/cgroup/<cgroup-name>/memory.reclaim
+
+Both of the methods mentioned above are subject to the ``memory.zswap.writeback``
+control. This means that ``memory.zswap.writeback`` can reject all zswap writeback.
A debugfs interface is provided for various statistic about pool size, number
of pages stored, same-value filled pages and various counters for the reasons
diff --git a/include/linux/zswap.h b/include/linux/zswap.h
index efa6b551217e..98434d39339a 100644
--- a/include/linux/zswap.h
+++ b/include/linux/zswap.h
@@ -44,6 +44,7 @@ void zswap_lruvec_state_init(struct lruvec *lruvec);
void zswap_folio_swapin(struct folio *folio);
bool zswap_is_enabled(void);
bool zswap_never_enabled(void);
+int zswap_proactive_writeback(struct mem_cgroup *memcg, unsigned long nr_to_writeback);
#else
struct zswap_lruvec_state {};
@@ -78,6 +79,12 @@ static inline bool zswap_never_enabled(void)
return true;
}
+static inline int zswap_proactive_writeback(struct mem_cgroup *memcg,
+ unsigned long nr_to_writeback)
+{
+ return -EOPNOTSUPP;
+}
+
#endif
#endif /* _LINUX_ZSWAP_H */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index ca4533eba701..63fa4341b823 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -64,6 +64,7 @@
#include <linux/swapops.h>
#include <linux/sched/sysctl.h>
+#include <linux/zswap.h>
#include "internal.h"
#include "swap.h"
@@ -7856,11 +7857,13 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask,
enum {
MEMORY_RECLAIM_SWAPPINESS = 0,
MEMORY_RECLAIM_SWAPPINESS_MAX,
+ MEMORY_RECLAIM_ZSWAP_WRITEBACK_ONLY,
MEMORY_RECLAIM_NULL,
};
static const match_table_t tokens = {
{ MEMORY_RECLAIM_SWAPPINESS, "swappiness=%d"},
{ MEMORY_RECLAIM_SWAPPINESS_MAX, "swappiness=max"},
+ { MEMORY_RECLAIM_ZSWAP_WRITEBACK_ONLY, "zswap_writeback_only"},
{ MEMORY_RECLAIM_NULL, NULL },
};
@@ -7870,6 +7873,7 @@ int user_proactive_reclaim(char *buf,
unsigned int nr_retries = MAX_RECLAIM_RETRIES;
unsigned long nr_to_reclaim, nr_reclaimed = 0;
int swappiness = -1;
+ bool zswap_writeback_only = false;
char *old_buf, *start;
substring_t args[MAX_OPT_ARGS];
gfp_t gfp_mask = GFP_KERNEL;
@@ -7900,11 +7904,21 @@ int user_proactive_reclaim(char *buf,
case MEMORY_RECLAIM_SWAPPINESS_MAX:
swappiness = SWAPPINESS_ANON_ONLY;
break;
+ case MEMORY_RECLAIM_ZSWAP_WRITEBACK_ONLY:
+ zswap_writeback_only = true;
+ break;
default:
return -EINVAL;
}
}
+ if (zswap_writeback_only) {
+ /* zswap_writeback_only and swappiness are mutually exclusive. */
+ if (swappiness != -1)
+ return -EINVAL;
+ return zswap_proactive_writeback(memcg, nr_to_reclaim);
+ }
+
while (nr_reclaimed < nr_to_reclaim) {
/* Will converge on zero, but reclaim enforces a minimum */
unsigned long batch_size = (nr_to_reclaim - nr_reclaimed) / 4;
diff --git a/mm/zswap.c b/mm/zswap.c
index 73e64a635690..7bcbf788f634 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1679,6 +1679,144 @@ int zswap_load(struct folio *folio)
return 0;
}
+/*
+ * Maximum LRU scan limit:
+ * number of entries to scan per page of remaining budget.
+ */
+#define ZSWAP_PROACTIVE_WB_SCAN_RATIO 16UL
+/*
+ * Batch size for proactive writeback:
+ * - As the per-memcg writeback target in the outer memcg loop.
+ * - As the per-walk budget passed to list_lru_walk_one().
+ */
+#define ZSWAP_PROACTIVE_WB_BATCH 128UL
+
+/*
+ * Walk the per-node LRUs of @memcg to write back up to @nr_to_write pages.
+ * Returns the number of pages written back, or -ENOENT if @memcg is a
+ * zombie or has writeback disabled.
+ */
+static long zswap_proactive_shrink_memcg(struct mem_cgroup *memcg,
+ unsigned long nr_to_write)
+{
+ unsigned long nr_written = 0;
+ int nid;
+
+ if (!mem_cgroup_zswap_writeback_enabled(memcg))
+ return -ENOENT;
+
+ if (!mem_cgroup_online(memcg))
+ return -ENOENT;
+
+ for_each_node_state(nid, N_NORMAL_MEMORY) {
+ bool encountered_page_in_swapcache = false;
+ unsigned long nr_to_scan, nr_scanned = 0;
+
+ /*
+ * Cap by LRU length: bounds rewalks when referenced
+ * entries keep rotating to the tail.
+ */
+ nr_to_scan = list_lru_count_one(&zswap_list_lru, nid, memcg);
+ if (!nr_to_scan)
+ continue;
+
+ /*
+ * Cap by SCAN_RATIO * remaining budget: bounds scan cost
+ * to the remaining writeback budget.
+ */
+ nr_to_scan = min(nr_to_scan,
+ (nr_to_write - nr_written) * ZSWAP_PROACTIVE_WB_SCAN_RATIO);
+
+ while (nr_scanned < nr_to_scan) {
+ unsigned long nr_to_walk = min(ZSWAP_PROACTIVE_WB_BATCH,
+ nr_to_scan - nr_scanned);
+
+ if (signal_pending(current))
+ return nr_written;
+
+ /*
+ * Account for the committed budget rather than the walker's
+ * actual delta. If the list is emptied concurrently, the
+ * walker visits nothing and nr_scanned would never advance.
+ */
+ nr_scanned += nr_to_walk;
+
+ nr_written += list_lru_walk_one(&zswap_list_lru, nid, memcg,
+ &shrink_memcg_cb,
+ &encountered_page_in_swapcache,
+ &nr_to_walk);
+
+ if (nr_written >= nr_to_write)
+ return nr_written;
+ if (encountered_page_in_swapcache)
+ break;
+
+ cond_resched();
+ }
+ }
+
+ return nr_written;
+}
+
+int zswap_proactive_writeback(struct mem_cgroup *memcg,
+ unsigned long nr_to_writeback)
+{
+ struct mem_cgroup *iter_memcg;
+ unsigned long nr_written = 0;
+ int failures = 0, attempts = 0;
+
+ if (!memcg)
+ return -EINVAL;
+ if (!nr_to_writeback)
+ return 0;
+
+ /*
+ * Writeback will be aborted with -EAGAIN if we encounter
+ * the following MAX_RECLAIM_RETRIES times:
+ * - No writeback-candidate memcgs found in a subtree walk.
+ * - A writeback-candidate memcg wrote back zero pages.
+ */
+ while (nr_written < nr_to_writeback) {
+ unsigned long batch_size;
+ long shrunk;
+
+ if (signal_pending(current))
+ return -EINTR;
+
+ iter_memcg = zswap_mem_cgroup_iter(memcg);
+
+ if (!iter_memcg) {
+ /*
+ * Continue without incrementing failures if we found
+ * candidate memcgs in the last subtree walk.
+ */
+ if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
+ return -EAGAIN;
+ attempts = 0;
+ continue;
+ }
+
+ batch_size = min(nr_to_writeback - nr_written,
+ ZSWAP_PROACTIVE_WB_BATCH);
+ shrunk = zswap_proactive_shrink_memcg(iter_memcg, batch_size);
+ mem_cgroup_put(iter_memcg);
+
+ /* Writeback-disabled or offline: skip without counting. */
+ if (shrunk == -ENOENT)
+ continue;
+
+ ++attempts;
+ if (shrunk > 0)
+ nr_written += shrunk;
+ else if (++failures == MAX_RECLAIM_RETRIES)
+ return -EAGAIN;
+
+ cond_resched();
+ }
+
+ return 0;
+}
+
void zswap_invalidate(swp_entry_t swp)
{
pgoff_t offset = swp_offset(swp);
--
2.34.1
^ permalink raw reply related
* [PATCH v3 1/4] mm/zswap: Make shrink_worker writeback cursor per-memcg
From: Hao Jia @ 2026-05-26 11:45 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: cgroups, linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260526114601.67041-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
The zswap background writeback worker shrink_worker() uses a global
cursor zswap_next_shrink, protected by zswap_shrink_lock, to round-robin
across the online memcgs under root_mem_cgroup.
Proactive writeback also wants a similar per-memcg cursor that is
scoped to the specified memcg, so that repeated invocations against
the same memcg make forward progress across its descendant memcgs
instead of restarting from the first child memcg each time.
Naturally, group the cursor and its protecting spinlock into a
zswap_wb_iter struct, and make it a member of struct mem_cgroup to
realize per-memcg cursor management. Accordingly, shrink_worker() now
uses the lock and cursor in root_mem_cgroup->zswap_wb_iter.
Because the cursor is now per-memcg, the offline cleanup must visit
every ancestor that could be holding a reference to the dying memcg.
Factor out __zswap_memcg_offline_cleanup() and walk from dead_memcg up
to the root.
No functional change intended for shrink_worker().
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
include/linux/memcontrol.h | 3 +
include/linux/zswap.h | 9 +++
mm/memcontrol.c | 3 +
mm/zswap.c | 119 ++++++++++++++++++++++++++-----------
4 files changed, 98 insertions(+), 36 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index bf1a6e131eca..5e29c2b7e376 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -229,6 +229,9 @@ struct mem_cgroup {
* swap, and from being swapped out on zswap store failures.
*/
bool zswap_writeback;
+
+ /* Per-memcg writeback cursor */
+ struct zswap_wb_iter zswap_wb_iter;
#endif
/* vmpressure notifications */
diff --git a/include/linux/zswap.h b/include/linux/zswap.h
index 30c193a1207e..efa6b551217e 100644
--- a/include/linux/zswap.h
+++ b/include/linux/zswap.h
@@ -11,6 +11,15 @@ extern atomic_long_t zswap_stored_pages;
#ifdef CONFIG_ZSWAP
+/* Iteration cursor for zswap writeback over a memcg's subtree. */
+struct zswap_wb_iter {
+ /* protects @pos against concurrent advances */
+ spinlock_t lock;
+ struct mem_cgroup *pos;
+};
+
+void zswap_wb_iter_init(struct zswap_wb_iter *iter);
+
struct zswap_lruvec_state {
/*
* Number of swapped in pages from disk, i.e not found in the zswap pool.
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 13f5d4b2a78e..e205e5de193d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4024,6 +4024,9 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
INIT_LIST_HEAD(&memcg->memory_peaks);
INIT_LIST_HEAD(&memcg->swap_peaks);
spin_lock_init(&memcg->peaks_lock);
+#ifdef CONFIG_ZSWAP
+ zswap_wb_iter_init(&memcg->zswap_wb_iter);
+#endif
memcg->socket_pressure = get_jiffies_64();
#if BITS_PER_LONG < 64
seqlock_init(&memcg->socket_pressure_seqlock);
diff --git a/mm/zswap.c b/mm/zswap.c
index 761cd699e0a3..73e64a635690 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -163,9 +163,6 @@ struct zswap_pool {
/* Global LRU lists shared by all zswap pools. */
static struct list_lru zswap_list_lru;
-/* The lock protects zswap_next_shrink updates. */
-static DEFINE_SPINLOCK(zswap_shrink_lock);
-static struct mem_cgroup *zswap_next_shrink;
static struct work_struct zswap_shrink_work;
static struct shrinker *zswap_shrinker;
@@ -717,28 +714,88 @@ void zswap_folio_swapin(struct folio *folio)
}
}
-/*
- * This function should be called when a memcg is being offlined.
+void zswap_wb_iter_init(struct zswap_wb_iter *iter)
+{
+ spin_lock_init(&iter->lock);
+}
+
+#ifdef CONFIG_MEMCG
+/**
+ * zswap_mem_cgroup_iter - advance the writeback cursor
+ * @root: subtree root whose cursor to advance
+ *
+ * Advance @root->zswap_wb_iter.pos to @root itself or the next online
+ * descendant. Passing root_mem_cgroup yields a global walk.
*
- * Since the global shrinker shrink_worker() may hold a reference
- * of the memcg, we must check and release the reference in
- * zswap_next_shrink.
+ * The cursor is retained across invocations, so successive calls walk
+ * @root's subtree cyclically in pre-order and, after %NULL, restart
+ * from the beginning.
*
- * shrink_worker() must handle the case where this function releases
- * the reference of memcg being shrunk.
+ * The returned memcg carries an extra reference; release it with
+ * mem_cgroup_put().
+ *
+ * Return: the next online memcg in @root's subtree, or @root itself,
+ * with an extra reference, or %NULL after a full round-trip.
*/
-void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
+static struct mem_cgroup *zswap_mem_cgroup_iter(struct mem_cgroup *root)
{
- /* lock out zswap shrinker walking memcg tree */
- spin_lock(&zswap_shrink_lock);
- if (zswap_next_shrink == memcg) {
+ struct mem_cgroup *memcg;
+
+ if (mem_cgroup_disabled())
+ return NULL;
+
+ spin_lock(&root->zswap_wb_iter.lock);
+ do {
+ memcg = mem_cgroup_iter(root, root->zswap_wb_iter.pos, NULL);
+ root->zswap_wb_iter.pos = memcg;
+ } while (memcg && !mem_cgroup_tryget_online(memcg));
+ spin_unlock(&root->zswap_wb_iter.lock);
+
+ return memcg;
+}
+
+/*
+ * If @root's cursor currently points at @dead_memcg, advance it to the
+ * next online descendant so @dead_memcg can be freed.
+ */
+static void __zswap_memcg_offline_cleanup(struct mem_cgroup *root,
+ struct mem_cgroup *dead_memcg)
+{
+ spin_lock(&root->zswap_wb_iter.lock);
+ if (root->zswap_wb_iter.pos == dead_memcg) {
do {
- zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
- } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink));
+ root->zswap_wb_iter.pos =
+ mem_cgroup_iter(root,
+ root->zswap_wb_iter.pos, NULL);
+ } while (root->zswap_wb_iter.pos &&
+ !mem_cgroup_online(root->zswap_wb_iter.pos));
}
- spin_unlock(&zswap_shrink_lock);
+ spin_unlock(&root->zswap_wb_iter.lock);
+}
+
+/*
+ * Called when a memcg is being offlined. If @memcg or any of its
+ * ancestors has a cursor pointing at @memcg, it must be advanced
+ * past @memcg before @memcg can be freed. Walk the chain and
+ * release such references.
+ */
+void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
+{
+ struct mem_cgroup *parent = memcg;
+
+ do {
+ __zswap_memcg_offline_cleanup(parent, memcg);
+ } while ((parent = parent_mem_cgroup(parent)));
+}
+#else /* !CONFIG_MEMCG */
+static struct mem_cgroup *zswap_mem_cgroup_iter(struct mem_cgroup *root)
+{
+ return NULL;
}
+void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) { }
+#endif /* CONFIG_MEMCG */
+
/*********************************
* zswap entry functions
**********************************/
@@ -1323,38 +1380,28 @@ static void shrink_worker(struct work_struct *w)
* - No writeback-candidate memcgs found in a memcg tree walk.
* - Shrinking a writeback-candidate memcg failed.
*
- * We save iteration cursor memcg into zswap_next_shrink,
+ * We save the iteration cursor in root_mem_cgroup->zswap_wb_iter.pos,
* which can be modified by the offline memcg cleaner
* zswap_memcg_offline_cleanup().
*
* Since the offline cleaner is called only once, we cannot leave an
- * offline memcg reference in zswap_next_shrink.
+ * offline memcg reference in root_mem_cgroup->zswap_wb_iter.pos.
* We can rely on the cleaner only if we get online memcg under lock.
*
* If we get an offline memcg, we cannot determine if the cleaner has
* already been called or will be called later. We must put back the
* reference before returning from this function. Otherwise, the
- * offline memcg left in zswap_next_shrink will hold the reference
- * until the next run of shrink_worker().
+ * offline memcg left in root_mem_cgroup->zswap_wb_iter.pos will hold
+ * the reference until the next run of shrink_worker().
*/
do {
/*
- * Start shrinking from the next memcg after zswap_next_shrink.
- * When the offline cleaner has already advanced the cursor,
- * advancing the cursor here overlooks one memcg, but this
- * should be negligibly rare.
- *
- * If we get an online memcg, keep the extra reference in case
- * the original one obtained by mem_cgroup_iter() is dropped by
- * zswap_memcg_offline_cleanup() while we are shrinking the
- * memcg.
+ * Start shrinking from the next memcg after
+ * root_mem_cgroup->zswap_wb_iter.pos. When the offline cleaner
+ * has already advanced the cursor, advancing the cursor here
+ * overlooks one memcg, but this should be negligibly rare.
*/
- spin_lock(&zswap_shrink_lock);
- do {
- memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
- zswap_next_shrink = memcg;
- } while (memcg && !mem_cgroup_tryget_online(memcg));
- spin_unlock(&zswap_shrink_lock);
+ memcg = zswap_mem_cgroup_iter(root_mem_cgroup);
if (!memcg) {
/*
--
2.34.1
^ permalink raw reply related
* [PATCH v3 0/4] mm/zswap: Implement per-cgroup proactive writeback
From: Hao Jia @ 2026-05-26 11:45 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: cgroups, linux-mm, linux-kernel, linux-doc, Hao Jia
From: Hao Jia <jiahao1@lixiang.com>
Zswap currently writes back pages to backing swap reactively, triggered
either by the shrinker or by the pool reaching its size limit. Although
proactive memory reclaim can automatically write back a portion of zswap
pages via the shrinker, it cannot explicitly control the amount of
writeback for a specific memory cgroup. Moreover, proactive memory reclaim
may not always be triggered during a steady state.
In certain scenarios, it is desirable to trigger writeback in advance to
free up memory. For example, users may want to prepare for an upcoming
memory-intensive workload by flushing cold memory to the backing storage
when the system is relatively idle.
This patch series introduces a "zswap_writeback_only" key to memory.reclaim
cgroup interface, allowing users to proactively write back cold compressed
pages from zswap to the backing swap device. When specified, this key
bypasses standard memory reclaim and exclusively performs proactive zswap
writeback up to the requested budget. If omitted, the default reclaim
behavior remains unchanged.
Example usage:
# Write back 100MB of pages from zswap to the backing swap
echo "100M zswap_writeback_only" > memory.reclaim
Patch 1: Move the global zswap shrink cursor into struct mem_cgroup as a
per-memcg zswap_wb_iter, so patch 2 can scope writeback to a given memcg
and make forward progress across its subtree on repeated invocations.
Patch 2: Extend the memory.reclaim cgroup v2 interface with a new
"zswap_writeback_only" key, allowing users to trigger proactive zswap
writeback up to a requested budget.
Patch 3: Add a zswpwb_proactive counter to memory.stat and /proc/vmstat
to track the number of writebacks triggered by proactive writeback.
Patch 4: Add tests for zswap proactive writeback.
v2->v3:
- Align the return value of zswap_proactive_writeback() with
memory.reclaim and update the corresponding documentation accordingly.
- Resolve conflicts in test_zswap.c on the mm-unstable branch.
- Enhance the zswap proactive writeback selftests to guard against potential
future regressions.
v1->v2:
- As suggested by Yosry and Nhat, extend the memory.reclaim cgroup v2
interface with a "zswap_writeback_only" key instead of adding a new
dedicated cgroup interface.
- Update the zswap documentation and add selftests for proactive writeback.
[v2] https://lore.kernel.org/all/20260525122242.36127-1-jiahao.kernel@gmail.com
[v1] https://lore.kernel.org/all/20260511105149.75584-1-jiahao.kernel@gmail.com
Hao Jia (4):
mm/zswap: Make shrink_worker writeback cursor per-memcg
mm/zswap: Implement proactive writeback
mm/zswap: Add per-memcg stat for proactive writeback
selftests/cgroup: Add tests for zswap proactive writeback
Documentation/admin-guide/cgroup-v2.rst | 22 +-
Documentation/admin-guide/mm/zswap.rst | 11 +-
include/linux/memcontrol.h | 3 +
include/linux/vm_event_item.h | 1 +
include/linux/zswap.h | 16 ++
mm/memcontrol.c | 4 +
mm/vmscan.c | 14 +
mm/vmstat.c | 1 +
mm/zswap.c | 292 +++++++++++++++++---
tools/testing/selftests/cgroup/test_zswap.c | 155 ++++++++++-
10 files changed, 471 insertions(+), 48 deletions(-)
--
2.34.1
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: Peter Zijlstra @ 2026-05-26 11:07 UTC (permalink / raw)
To: K Prateek Nayak
Cc: Zhang Qiao, mingo, longman, chenridong, juri.lelli,
vincent.guittot, dietmar.eggemann, rostedt, bsegall, mgorman,
vschneid, tj, hannes, mkoutny, cgroups, linux-kernel, jstultz,
qyousef, Hui Tang
In-Reply-To: <3f1fc681-a73b-4bd2-9a6a-e61b8fbd5826@amd.com>
On Tue, May 26, 2026 at 04:24:32PM +0530, K Prateek Nayak wrote:
> Hello Peter,
>
> On 5/26/2026 3:22 PM, Peter Zijlstra wrote:
> > On Tue, May 26, 2026 at 02:45:45PM +0530, K Prateek Nayak wrote:
> >
> >> The suggested diff above solves the crash in my case but your
> >> mileage may vary. Peter can comment if this is the right thing
> >> to do or not :-)
> >
> > Is this a different issue than the one you raised before?
>
> Yes, this is different. Essentially, this is what is happening:
>
> throttle_cfs_rq_work()
> task_rq_lock()
>
> dequeue_task_fair(current) /* Task is dequeued on cfs side */
> __dequeue_task(current)
> dequeue_hierarchy(current);
> current->se.on_rq = 0;
> /* update_load_sub() */
> resched_curr(); /* Initiates a resched */
>
> task_rq_unlock()
> local_irq_enable();
>
> =====> sched_tick()
> task_tick_fair()
> __calc_prop_weight()
> /*
> * Oops: update_load_sub() above has
> * 0ed the weight of cfs_rq.
> */
> <====
>
> preempt_schedule_irq()
> next = ...
> put_prev_set_next_task() /* The runtime context is switched here */
>
Ah, right. OK, I'll go have a poke once I get these proxy patches I've
been spending too much time on posted.
I think I've found a 'problem' with that PROXY_WAKING ==> '->is_blocked
&& !->blocked_on' scheme :-(
> > We talked about throtte, and you were going to make a proper patch of that cleanup
> > iirc.
>
> I had rebased your suggestion on tip and fixed a couple of splats but
> once it was functional, I noticed hackbench taking twice as long to
> complete compared to tip and I was chasing that before I fell sick.
>
> Let me go dig deeper to see where exactly it is all going sideways.
Sure, no worries. This happens; computers just never want to just DTRT
already. I lost a day and then some trying to figure out why my
seemingly 'trivial' proxy changes ended up trying to run a dead task
last week...
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: K Prateek Nayak @ 2026-05-26 10:54 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Zhang Qiao, mingo, longman, chenridong, juri.lelli,
vincent.guittot, dietmar.eggemann, rostedt, bsegall, mgorman,
vschneid, tj, hannes, mkoutny, cgroups, linux-kernel, jstultz,
qyousef, Hui Tang
In-Reply-To: <20260526095210.GC4149641@noisy.programming.kicks-ass.net>
Hello Peter,
On 5/26/2026 3:22 PM, Peter Zijlstra wrote:
> On Tue, May 26, 2026 at 02:45:45PM +0530, K Prateek Nayak wrote:
>
>> The suggested diff above solves the crash in my case but your
>> mileage may vary. Peter can comment if this is the right thing
>> to do or not :-)
>
> Is this a different issue than the one you raised before?
Yes, this is different. Essentially, this is what is happening:
throttle_cfs_rq_work()
task_rq_lock()
dequeue_task_fair(current) /* Task is dequeued on cfs side */
__dequeue_task(current)
dequeue_hierarchy(current);
current->se.on_rq = 0;
/* update_load_sub() */
resched_curr(); /* Initiates a resched */
task_rq_unlock()
local_irq_enable();
=====> sched_tick()
task_tick_fair()
__calc_prop_weight()
/*
* Oops: update_load_sub() above has
* 0ed the weight of cfs_rq.
*/
<====
preempt_schedule_irq()
next = ...
put_prev_set_next_task() /* The runtime context is switched here */
> We talked about throtte, and you were going to make a proper patch of that cleanup
> iirc.
I had rebased your suggestion on tip and fixed a couple of splats but
once it was functional, I noticed hackbench taking twice as long to
complete compared to tip and I was chasing that before I fell sick.
Let me go dig deeper to see where exactly it is all going sideways.
--
Thanks and Regards,
Prateek
^ permalink raw reply
* Re: [PATCH v6 1/4] mm: swap: introduce swap tier infrastructure
From: Baoquan He @ 2026-05-26 10:52 UTC (permalink / raw)
To: YoungJun Park
Cc: akpm, chrisl, linux-mm, cgroups, linux-kernel, kasong, hannes,
mhocko, roman.gushchin, shakeel.butt, muchun.song, shikemeng,
nphamcs, bhe, baohua, gunho.lee, taejoon.song, hyungjun.cho,
mkoutny, baver.bae, matia.kim
In-Reply-To: <ahU5N1Yi2Ww+vN21@yjaykim-PowerEdge-T330>
On 05/26/26 at 03:09pm, YoungJun Park wrote:
> On Tue, May 26, 2026 at 06:57:03AM +0800, Baoquan He wrote:
> > On 04/21/26 at 02:53pm, Youngjun Park wrote:
> > ...snip...
> > > +bool swap_tiers_validate(void)
> > > +{
> > > + struct swap_tier *tier;
> > > +
> > > + /*
> > > + * Initial setting might not cover DEF_SWAP_PRIO.
> > > + * Swap tier must cover the full range (DEF_SWAP_PRIO to SHRT_MAX).
> > > + */
> >
> > If so, do we need check if the upmost boundary SHRT_MAX is covered?
>
> Hello Baoquan
>
> It naturally covers SHRT_MAX.
> Because all swap_tier object prio represents start of priority
> and the prio value is assured on the range of DEF_SWAP_PRIO ~ SHAR_MAX
> on the privious routine.
>
> swap_tier_validate function is for checking the first tier cover DEF_SWAP_PRIO.
> if it is not, it breaks the assumtion "cover DEF_SWAP_PRIO to SHRT_MAX"
Thanks, I got it now. We only track the beginning of prio range via tier->prio,
while deduce the end of prio range from configured tiers. That checking
is reasonable. Sorry for the noise.
^ permalink raw reply
* Re: [PATCH v4 2/5] cgroup/dmem: Add reclaim callback for lowering max below current usage
From: Maarten Lankhorst @ 2026-05-26 9:53 UTC (permalink / raw)
To: Thomas Hellström, intel-xe
Cc: Natalie Vock, Johannes Weiner, Tejun Heo, Michal Koutný,
cgroups, Huang Rui, Matthew Brost, Matthew Auld, Maxime Ripard,
Thomas Zimmermann, Simona Vetter, David Airlie,
Christian König, Alex Deucher, Rodrigo Vivi, dri-devel,
amd-gfx, linux-kernel
In-Reply-To: <b797a2aaa1bcef239a2eba449043dd278b9fa51a.camel@linux.intel.com>
Hello Thomas,
Den 2026-05-26 kl. 11:24, skrev Thomas Hellström:
> Hi, Maarten,
>
> Thanks for reviewing.
>
> On Tue, 2026-05-26 at 10:27 +0200, Maarten Lankhorst wrote:
>> Hello,
>>
>> Den 2026-05-12 kl. 10:24, skrev Thomas Hellström:
>>> Add an optional reclaim callback to struct dmem_cgroup_region. When
>>> dmem.max is set below the current usage of a cgroup pool, the new
>>> limit
>>> is applied immediately (so that concurrent allocations are
>>> throttled
>>> while reclaim is in progress) and then the driver is asked to evict
>>> memory to bring usage back below the limit.
>>>
>>> Reclaim is attempted up to a bounded number of times. No error is
>>> returned to userspace if usage remains above the limit after
>>> reclaim,
>>> and a pending signal will abort the reclaim loop early. This
>>> matches
>>> the behavior of memory.max in the memory cgroup controller.
>>>
>>> Also honor O_NONBLOCK so that if that flag is set during the
>>> max value write, no reclaim is initiated. The idea is to avoid
>>> charging the reclaim cost to the writer of the max value.
>>>
>>> v2:
>>> - Write max before reclaim is attempted (Maarten)
>>> - Let signals abort the reclaim without error (Maarten)
>>> - If a new max value is written with the O_NONBLOCK flag,
>>> reclaim is not attempted (Maarten)
>>> - Extract region from the pool parameter rather than
>>> passing it explicitly to set_resource_xxx().
>>> v3:
>>> - Use an rwsem to protect reclaim callback registration and
>>> region unregister against concurrent reclaim invocations,
>>> ensuring reclaim_priv is visible when the callback is
>>> invoked. (Sashiko-bot)
>>>
>>> Assisted-by: GitHub_Copilot:claude-sonnet-4.6
>>> Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
>>> ---
>>> include/linux/cgroup_dmem.h | 24 ++++++++
>>> kernel/cgroup/dmem.c | 106
>>> +++++++++++++++++++++++++++++++++---
>>> 2 files changed, 121 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/include/linux/cgroup_dmem.h
>>> b/include/linux/cgroup_dmem.h
>>> index dd4869f1d736..c3bce21cbe80 100644
>>> --- a/include/linux/cgroup_dmem.h
>>> +++ b/include/linux/cgroup_dmem.h
>>> @@ -14,6 +14,21 @@ struct dmem_cgroup_pool_state;
>>> /* Opaque definition of a cgroup region, used internally */
>>> struct dmem_cgroup_region;
>>>
>>> +/**
>>> + * typedef dmem_cgroup_reclaim_fn_t - Reclaim callback for a dmem
>>> cgroup region.
>>> + * @pool: The cgroup pool that needs memory reclaimed.
>>> + * @target_bytes: Minimum number of bytes the driver should
>>> attempt to free.
>>> + * @priv: Private data registered with
>>> dmem_cgroup_region_set_reclaim().
>>> + *
>>> + * Called by the dmem cgroup controller when dmem.max is set below
>>> the current
>>> + * usage of @pool. The driver should evict at least @target_bytes
>>> of memory
>>> + * from @pool. May be called multiple times if usage remains above
>>> the limit.
>>> + *
>>> + * Return: 0 if progress was made, negative error code otherwise.
>>> + */
>>> +typedef int (*dmem_cgroup_reclaim_fn_t)(struct
>>> dmem_cgroup_pool_state *pool,
>>> + u64 target_bytes, void
>>> *priv);
>>> +
>>> #if IS_ENABLED(CONFIG_CGROUP_DMEM)
>>> struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size,
>>> const char *name_fmt, ...) __printf(2,3);
>>> void dmem_cgroup_unregister_region(struct dmem_cgroup_region
>>> *region);
>>> @@ -26,6 +41,9 @@ bool dmem_cgroup_state_evict_valuable(struct
>>> dmem_cgroup_pool_state *limit_pool,
>>> bool ignore_low, bool
>>> *ret_hit_low);
>>>
>>> void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state
>>> *pool);
>>> +void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region
>>> *region,
>>> + dmem_cgroup_reclaim_fn_t
>>> reclaim,
>>> + void *priv);
>>> #else
>>> static inline __printf(2,3) struct dmem_cgroup_region *
>>> dmem_cgroup_register_region(u64 size, const char *name_fmt, ...)
>>> @@ -62,5 +80,11 @@ bool dmem_cgroup_state_evict_valuable(struct
>>> dmem_cgroup_pool_state *limit_pool,
>>> static inline void dmem_cgroup_pool_state_put(struct
>>> dmem_cgroup_pool_state *pool)
>>> { }
>>>
>>> +static inline void
>>> +dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
>>> + dmem_cgroup_reclaim_fn_t reclaim,
>>> + void *priv)
>>> +{ }
>>> +
>>> #endif
>>> #endif /* _CGROUP_DMEM_H */
>>> diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
>>> index 1ab1fb47f271..5fd5a1634d21 100644
>>> --- a/kernel/cgroup/dmem.c
>>> +++ b/kernel/cgroup/dmem.c
>>> @@ -51,6 +51,20 @@ struct dmem_cgroup_region {
>>> * No new pools should be added to the region afterwards.
>>> */
>>> bool unregistered;
>>> +
>>> + /**
>>> + * @reclaim: Optional callback invoked when dmem.max is
>>> set below the
>>> + * current usage of a pool. The driver should attempt to
>>> free at least
>>> + * @target_bytes from @pool. May be called multiple times
>>> if usage
>>> + * remains above the limit after returning.
>>> + */
>>> + dmem_cgroup_reclaim_fn_t reclaim;
>>> +
>>> + /** @reclaim_priv: Private data passed to @reclaim. */
>>> + void *reclaim_priv;
>>> +
>>> + /** @unregister_sem: Protect @reclaim while it is running.
>>> */
>>> + struct rw_semaphore unregister_sem;
>>> };
>>>
>>> struct dmemcg_state {
>>> @@ -145,21 +159,58 @@ static void free_cg_pool(struct
>>> dmem_cgroup_pool_state *pool)
>>> }
>>>
>>> static void
>>> -set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val)
>>> +set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val,
>>> bool nonblock)
>>> {
>>> page_counter_set_min(&pool->cnt, val);
>>> }
>>>
>>> static void
>>> -set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val)
>>> +set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val,
>>> bool nonblock)
>>> {
>>> page_counter_set_low(&pool->cnt, val);
>>> }
>>>
>>> static void
>>> -set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val)
>>> +set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val,
>>> bool nonblock)
>>> {
>>> - page_counter_set_max(&pool->cnt, val);
>>> + struct dmem_cgroup_region *region = pool->region;
>>> +
>>> + /*
>>> + * Always update the limit, even if usage currently
>>> exceeds it.
>>> + * Concurrent allocations will be throttled against the
>>> new limit
>>> + * while reclaim is in progress.
>>> + */
>>> + xchg(&pool->cnt.max, (unsigned long)val);
>>> +
>>> + if (nonblock || !READ_ONCE(region->reclaim))
>>> + return;
>>> +
>>> + for (int retries = 5; retries > 0; retries--) {
>> Where does 5 come from? This code should retry until no longer above
>> limit, otherwise you'll get some hard to debug issues.
>
> The memcg controller uses MAX_RECLAIM_RETRIES, although if that fails,
> it will invoke the OOM killer, although if the reclaim callback makes
> progress, it will not consume a retry. Perhaps we should adopt the same
> behaviour except the OOM killer, at least for now. Note that if a
> signal is pending, the reclaim attempt is abandoned both here and in
> memcg.
Yeah, it would be good if we could re-use the same MAX_RECLAIM_RETRIES, so
we will at least know where it's coming from.
Perhaps move it to a slightly more public header. I'm thinking linux/mm.h,
but perhaps the mm maintainers will prefer another place.
It makes sense to eventually fail on no progress, especially if all remaining
memory is pinned. This can happen when we pin dma-buf for example.
Adding an entire DMEM OOM killer would be interesting, but probably far outside
the scope of dmemcg for the foreseeable future. It's only unfortunate that
we have no way of reporting or logging failure at all right now.
>>> <snip>
>
> Let me take a look at this.
> /Thomas
Thanks for looking into it.
Kind regards,
~Maarten Lankhorst
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: Peter Zijlstra @ 2026-05-26 9:52 UTC (permalink / raw)
To: K Prateek Nayak
Cc: Zhang Qiao, mingo, longman, chenridong, juri.lelli,
vincent.guittot, dietmar.eggemann, rostedt, bsegall, mgorman,
vschneid, tj, hannes, mkoutny, cgroups, linux-kernel, jstultz,
qyousef, Hui Tang
In-Reply-To: <85116808-8643-47d7-b4e7-2a11c3999b20@amd.com>
On Tue, May 26, 2026 at 02:45:45PM +0530, K Prateek Nayak wrote:
> The suggested diff above solves the crash in my case but your
> mileage may vary. Peter can comment if this is the right thing
> to do or not :-)
Is this a different issue than the one you raised before? We talked
about throtte, and you were going to make a proper patch of that cleanup
iirc.
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: Zhang Qiao @ 2026-05-26 9:36 UTC (permalink / raw)
To: K Prateek Nayak, Peter Zijlstra, mingo
Cc: longman, chenridong, juri.lelli, vincent.guittot,
dietmar.eggemann, rostedt, bsegall, mgorman, vschneid, tj, hannes,
mkoutny, cgroups, linux-kernel, jstultz, qyousef, Hui Tang
In-Reply-To: <85116808-8643-47d7-b4e7-2a11c3999b20@amd.com>
Hi Prateek,
在 2026/5/26 17:15, K Prateek Nayak 写道:
> Hello Zhang,
>
> On 5/26/2026 1:23 PM, Zhang Qiao wrote:
>> Testing sched/flat branch on AMD EPYC 9654 (384 CPUs, 8 NUMA nodes)
>> with a 2-level cgroup hierarchy and cfs_bandwidth quota enabled,
>> hackbench triggers a divide-by-zero oops:
>>
>> [ 142.308571] divide error: 0000 [#1] SMP NOPTI
>> [ 142.308582] RIP: 0010:task_tick_fair+0x19e/0x410
>> [ 142.308601] Call Trace:
>> [ 142.308604] <IRQ>
>> [ 142.308607] scheduler_tick+0x6a/0x110
>> [ 142.308609] update_process_times+0x6b/0x90
>> [ 142.308611] tick_sched_handle+0x2a/0x70
>> [ 142.308613] tick_sched_timer+0x57/0xb0
>
> More of this trace would have been helpful.
>
>>
>> faddr2line confirms:
>>
>> task_tick_fair+0x19e/0x410:
>> __calc_prop_weight at kernel/sched/fair.c:4085
>> (inlined by) task_tick_fair at kernel/sched/fair.c:13576
>
> Those line numbers don't match on the latest sched/flat but since you
> mention this happens with throttling, I believe it is tick hitting
> somewhere in between the task being dequeued by throttle_cfs_rq_work()
> and the CPU rescheduling and taking the task off the runqueue.
>
Sorry for the confusion on the line numbers — the mismatch was due
to some local debug code I had added on top of sched/flat,
not a difference in the base tree.
> Dequeue from throttle is slightly special since it keeps the task on
> runqueue but the sched entity goes off the cfs_rq changing the
> hierarchical weights.
> > Can you check if this helps:
>
> (Lightly tested with your reproducer)
>
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index b8bae794f063..d96e5915fb3e 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -14815,18 +14815,21 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
> static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
> {
> struct sched_entity *se = &curr->se;
> - unsigned long weight = NICE_0_LOAD;
> - struct cfs_rq *cfs_rq;
>
> - for_each_sched_entity(se) {
> - cfs_rq = cfs_rq_of(se);
> - entity_tick(cfs_rq, se, queued);
> + if (se->on_rq) {
> + unsigned long weight = NICE_0_LOAD;
> + struct cfs_rq *cfs_rq;
>
> - weight = __calc_prop_weight(cfs_rq, se, weight);
> - }
> + for_each_sched_entity(se) {
> + cfs_rq = cfs_rq_of(se);
> + entity_tick(cfs_rq, se, queued);
> +
> + weight = __calc_prop_weight(cfs_rq, se, weight);
> + }
>
> - se = &curr->se;
> - reweight_eevdf(cfs_rq, se, weight, se->on_rq);
> + se = &curr->se;
> + reweight_eevdf(cfs_rq, se, weight, se->on_rq);
> + }
>
throttle_cfs_rq_work() sets se->on_rq = 0 while the task is still running as
rq->curr, and the subsequent tick should not attempt to reweight an
already-dequeued entity. The unthrottle enqueue will handle the reweight anyway.
I've tested your suggested diff on my AMD EPYC 9654 (384 CPUs, 8 NUMA
nodes) and it resolves the crash. The reproducer no longer triggers the
divide error after running for several minutes.
Tested-by: Zhang Qiao <zhangqiao22@huawei.com>
Thanks,
Zhang Qiao
.
> if (queued)
> return;
> ---
>
> I don't think it makes too much sense to reweight an entity that
> has been dequeued. The enqueue at unthrottle will do it anyways.
>
>>
>> ===========================================================
>> Reproduction
>> ===========================================================
>>
>> Kernel: sched/flat branch (54d493980e00 and later)
>> Hardware: AMD EPYC 9654, 2S 384 logical CPUs
>>
>> # 2-level cgroup, quota = 50% of one period
>> cgcreate -g cpu:/bw/l1/l2
>> cgset -r cpu.cfs_quota_us=50000 /bw/l1/l2
>> cgset -r cpu.cfs_period_us=100000 /bw/l1/l2
>>
>> # high task count amplifies the throttle→tick race window
>> cgexec -g cpu:/bw/l1/l2 hackbench -g 48 -l 1000 -s 512 -T
>>
>> Typically crashes within 30 seconds on this machine. A single-CPU
>> kernel or a very loose quota (e.g. 90%) is unlikely to trigger it
>> because the race window is narrow.
>
> This was helpful! I see:
>
> [ 209.935597] Oops: divide error: 0000 [#1] SMP NOPTI
> [ 209.941061] CPU: 329 UID: 0 PID: 8247 Comm: sched-messaging Not tainted 7.1.0-rc2-test+ #73 PREEMPT(full)
> [ 209.951841] Hardware name: AMD Corporation Titanite_4G/Titanite_4G, BIOS RTI100CC 03/28/2024
> [ 209.961254] RIP: 0010:task_tick_fair+0x10d/0x850
> [ 209.966420] Code: dc 00 00 00 4c 89 f7 e8 f1 52 ff ff 45 85 e4 0f 85 ba 00 00 00 49 8b 06 4d 8b b6 b8 00 00 00 48 0f af c3 4d 85 f6 74 19 31 d2 <49> f7 37 ba 02 00 00 00 48 89 d3 48 39 d0 48 0f 43 d8 e9 20 ff ff
> [ 209.987382] RSP: 0018:ff581fd71e1fce58 EFLAGS: 00010046
> [ 209.993216] RAX: 0000010000000000 RBX: 0000000000100000 RCX: ff295dbfa9ad8080
> [ 210.001179] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ff295dbfa9ad8080
> [ 210.009141] RBP: 0000000000000000 R08: 0000000000000000 R09: 00000000000063eb
> [ 210.017104] R10: 0000000000000000 R11: ff581fd71e1fcff8 R12: 0000000000000000
> [ 210.025061] R13: ff295dbfa9ad8000 R14: ff295dc06c6eac00 R15: ff295dbfd9bc8600
> [ 210.033027] FS: 00007faef8c8b640(0000) GS:ff295e7c4acca000(0000) knlGS:0000000000000000
> [ 210.042060] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [ 210.048474] CR2: 00007f9884292d30 CR3: 000000011aa26001 CR4: 0000000000f71ef0
> [ 210.056430] PKRU: 55555554
> [ 210.059448] Call Trace:
> [ 210.062177] <IRQ>
> [ 210.064426] sched_tick+0x94/0x250
> [ 210.068229] update_process_times+0x99/0xc0
> [ 210.072903] tick_nohz_handler+0x95/0x1a0
> [ 210.077380] ? __pfx_tick_nohz_handler+0x10/0x10
> [ 210.082534] __hrtimer_run_queues+0xfe/0x260
> [ 210.087304] hrtimer_interrupt+0x122/0x1f0
> [ 210.091880] __sysvec_apic_timer_interrupt+0x55/0x130
> [ 210.097525] sysvec_apic_timer_interrupt+0x7a/0xb0
> [ 210.102873] </IRQ>
> [ 210.105203] <TASK>
> [ 210.107542] asm_sysvec_apic_timer_interrupt+0x1a/0x20
> [ 210.113284] RIP: 0010:_raw_spin_unlock_irqrestore+0x1d/0x40
> [ 210.119511] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 c6 07 00 0f 1f 00 f7 c6 00 02 00 00 74 06 fb 0f 1f 44 00 00 <65> ff 0d ec 20 fd 01 74 05 e9 c0 81 d4 fe e8 00 93 ec fe e9 b6 81
> [ 210.140469] RSP: 0018:ff581fd74032fe88 EFLAGS: 00000206
> [ 210.146308] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000004
> [ 210.154271] RDX: 0000000000000000 RSI: 0000000000000246 RDI: ff295dbfa9ad8d64
> [ 210.162235] RBP: ff295dbfa9ad8000 R08: 0000000000000000 R09: 0000000000000000
> [ 210.170196] R10: 0000000000000000 R11: 0000000000000000 R12: ff295dbfa9ad8d64
> [ 210.178159] R13: ff581fd74032ff48 R14: ff295dbfa9ad8000 R15: 00fffffffffff000
> [ 210.186139] task_work_run+0x5c/0x90
> [ 210.190137] exit_to_user_mode_loop+0x16e/0x550
> [ 210.195198] ? srso_alias_return_thunk+0x5/0xfbef5
> [ 210.200552] ? ksys_read+0xc5/0xe0
> [ 210.204352] do_syscall_64+0x26e/0x750
> [ 210.208540] ? do_syscall_64+0xaa/0x750
> [ 210.212823] ? srso_alias_return_thunk+0x5/0xfbef5
> [ 210.218174] entry_SYSCALL_64_after_hwframe+0x76/0x7e
> ---
>
> So the theory of throttle work causing this checks out.
>
> The suggested diff above solves the crash in my case but your
> mileage may vary. Peter can comment if this is the right thing
> to do or not :-)
>
^ permalink raw reply
* Re: [PATCH v4 2/5] cgroup/dmem: Add reclaim callback for lowering max below current usage
From: Thomas Hellström @ 2026-05-26 9:24 UTC (permalink / raw)
To: Maarten Lankhorst, intel-xe
Cc: Natalie Vock, Johannes Weiner, Tejun Heo, Michal Koutný,
cgroups, Huang Rui, Matthew Brost, Matthew Auld, Maxime Ripard,
Thomas Zimmermann, Simona Vetter, David Airlie,
Christian König, Alex Deucher, Rodrigo Vivi, dri-devel,
amd-gfx, linux-kernel
In-Reply-To: <9fe89d8e-9c32-4b03-ac2c-a634f5d4de0c@linux.intel.com>
Hi, Maarten,
Thanks for reviewing.
On Tue, 2026-05-26 at 10:27 +0200, Maarten Lankhorst wrote:
> Hello,
>
> Den 2026-05-12 kl. 10:24, skrev Thomas Hellström:
> > Add an optional reclaim callback to struct dmem_cgroup_region. When
> > dmem.max is set below the current usage of a cgroup pool, the new
> > limit
> > is applied immediately (so that concurrent allocations are
> > throttled
> > while reclaim is in progress) and then the driver is asked to evict
> > memory to bring usage back below the limit.
> >
> > Reclaim is attempted up to a bounded number of times. No error is
> > returned to userspace if usage remains above the limit after
> > reclaim,
> > and a pending signal will abort the reclaim loop early. This
> > matches
> > the behavior of memory.max in the memory cgroup controller.
> >
> > Also honor O_NONBLOCK so that if that flag is set during the
> > max value write, no reclaim is initiated. The idea is to avoid
> > charging the reclaim cost to the writer of the max value.
> >
> > v2:
> > - Write max before reclaim is attempted (Maarten)
> > - Let signals abort the reclaim without error (Maarten)
> > - If a new max value is written with the O_NONBLOCK flag,
> > reclaim is not attempted (Maarten)
> > - Extract region from the pool parameter rather than
> > passing it explicitly to set_resource_xxx().
> > v3:
> > - Use an rwsem to protect reclaim callback registration and
> > region unregister against concurrent reclaim invocations,
> > ensuring reclaim_priv is visible when the callback is
> > invoked. (Sashiko-bot)
> >
> > Assisted-by: GitHub_Copilot:claude-sonnet-4.6
> > Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> > ---
> > include/linux/cgroup_dmem.h | 24 ++++++++
> > kernel/cgroup/dmem.c | 106
> > +++++++++++++++++++++++++++++++++---
> > 2 files changed, 121 insertions(+), 9 deletions(-)
> >
> > diff --git a/include/linux/cgroup_dmem.h
> > b/include/linux/cgroup_dmem.h
> > index dd4869f1d736..c3bce21cbe80 100644
> > --- a/include/linux/cgroup_dmem.h
> > +++ b/include/linux/cgroup_dmem.h
> > @@ -14,6 +14,21 @@ struct dmem_cgroup_pool_state;
> > /* Opaque definition of a cgroup region, used internally */
> > struct dmem_cgroup_region;
> >
> > +/**
> > + * typedef dmem_cgroup_reclaim_fn_t - Reclaim callback for a dmem
> > cgroup region.
> > + * @pool: The cgroup pool that needs memory reclaimed.
> > + * @target_bytes: Minimum number of bytes the driver should
> > attempt to free.
> > + * @priv: Private data registered with
> > dmem_cgroup_region_set_reclaim().
> > + *
> > + * Called by the dmem cgroup controller when dmem.max is set below
> > the current
> > + * usage of @pool. The driver should evict at least @target_bytes
> > of memory
> > + * from @pool. May be called multiple times if usage remains above
> > the limit.
> > + *
> > + * Return: 0 if progress was made, negative error code otherwise.
> > + */
> > +typedef int (*dmem_cgroup_reclaim_fn_t)(struct
> > dmem_cgroup_pool_state *pool,
> > + u64 target_bytes, void
> > *priv);
> > +
> > #if IS_ENABLED(CONFIG_CGROUP_DMEM)
> > struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size,
> > const char *name_fmt, ...) __printf(2,3);
> > void dmem_cgroup_unregister_region(struct dmem_cgroup_region
> > *region);
> > @@ -26,6 +41,9 @@ bool dmem_cgroup_state_evict_valuable(struct
> > dmem_cgroup_pool_state *limit_pool,
> > bool ignore_low, bool
> > *ret_hit_low);
> >
> > void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state
> > *pool);
> > +void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region
> > *region,
> > + dmem_cgroup_reclaim_fn_t
> > reclaim,
> > + void *priv);
> > #else
> > static inline __printf(2,3) struct dmem_cgroup_region *
> > dmem_cgroup_register_region(u64 size, const char *name_fmt, ...)
> > @@ -62,5 +80,11 @@ bool dmem_cgroup_state_evict_valuable(struct
> > dmem_cgroup_pool_state *limit_pool,
> > static inline void dmem_cgroup_pool_state_put(struct
> > dmem_cgroup_pool_state *pool)
> > { }
> >
> > +static inline void
> > +dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
> > + dmem_cgroup_reclaim_fn_t reclaim,
> > + void *priv)
> > +{ }
> > +
> > #endif
> > #endif /* _CGROUP_DMEM_H */
> > diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
> > index 1ab1fb47f271..5fd5a1634d21 100644
> > --- a/kernel/cgroup/dmem.c
> > +++ b/kernel/cgroup/dmem.c
> > @@ -51,6 +51,20 @@ struct dmem_cgroup_region {
> > * No new pools should be added to the region afterwards.
> > */
> > bool unregistered;
> > +
> > + /**
> > + * @reclaim: Optional callback invoked when dmem.max is
> > set below the
> > + * current usage of a pool. The driver should attempt to
> > free at least
> > + * @target_bytes from @pool. May be called multiple times
> > if usage
> > + * remains above the limit after returning.
> > + */
> > + dmem_cgroup_reclaim_fn_t reclaim;
> > +
> > + /** @reclaim_priv: Private data passed to @reclaim. */
> > + void *reclaim_priv;
> > +
> > + /** @unregister_sem: Protect @reclaim while it is running.
> > */
> > + struct rw_semaphore unregister_sem;
> > };
> >
> > struct dmemcg_state {
> > @@ -145,21 +159,58 @@ static void free_cg_pool(struct
> > dmem_cgroup_pool_state *pool)
> > }
> >
> > static void
> > -set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val)
> > +set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val,
> > bool nonblock)
> > {
> > page_counter_set_min(&pool->cnt, val);
> > }
> >
> > static void
> > -set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val)
> > +set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val,
> > bool nonblock)
> > {
> > page_counter_set_low(&pool->cnt, val);
> > }
> >
> > static void
> > -set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val)
> > +set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val,
> > bool nonblock)
> > {
> > - page_counter_set_max(&pool->cnt, val);
> > + struct dmem_cgroup_region *region = pool->region;
> > +
> > + /*
> > + * Always update the limit, even if usage currently
> > exceeds it.
> > + * Concurrent allocations will be throttled against the
> > new limit
> > + * while reclaim is in progress.
> > + */
> > + xchg(&pool->cnt.max, (unsigned long)val);
> > +
> > + if (nonblock || !READ_ONCE(region->reclaim))
> > + return;
> > +
> > + for (int retries = 5; retries > 0; retries--) {
> Where does 5 come from? This code should retry until no longer above
> limit, otherwise you'll get some hard to debug issues.
The memcg controller uses MAX_RECLAIM_RETRIES, although if that fails,
it will invoke the OOM killer, although if the reclaim callback makes
progress, it will not consume a retry. Perhaps we should adopt the same
behaviour except the OOM killer, at least for now. Note that if a
signal is pending, the reclaim attempt is abandoned both here and in
memcg.
>
> > + u64 usage = page_counter_read(&pool->cnt);
> > + int ret;
> > +
> > + if (usage <= val)
> > + break;
> > +
> > + if (signal_pending(current))
> > + break;
> > +
> > + /* Block unregister until the reclaim callback
> > completes. */
> > + if (down_read_interruptible(®ion-
> > >unregister_sem))
> > + break;
> > +
> > + if (!region->reclaim) {
> > + up_read(®ion->unregister_sem);
> > + break;
> > + }
> > +
> > + ret = region->reclaim(pool, usage - val, region-
> > >reclaim_priv);
> > + up_read(®ion->unregister_sem);
> > + if (ret)
> > + break;
> > +
> > + cond_resched();
> > + }
> > }
> >
> > static u64 get_resource_low(struct dmem_cgroup_pool_state *pool)
> > @@ -184,9 +235,9 @@ static u64 get_resource_current(struct
> > dmem_cgroup_pool_state *pool)
> >
> > static void reset_all_resource_limits(struct
> > dmem_cgroup_pool_state *rpool)
> > {
> > - set_resource_min(rpool, 0);
> > - set_resource_low(rpool, 0);
> > - set_resource_max(rpool, PAGE_COUNTER_MAX);
> > + set_resource_min(rpool, 0, false);
> > + set_resource_low(rpool, 0, false);
> > + set_resource_max(rpool, PAGE_COUNTER_MAX, false);
> > }
> >
> > static void dmemcs_offline(struct cgroup_subsys_state *css)
> > @@ -491,6 +542,12 @@ void dmem_cgroup_unregister_region(struct
> > dmem_cgroup_region *region)
> > region->unregistered = true;
> > spin_unlock(&dmemcg_lock);
> >
> > + /* Ensure all reclaim() callbacks have finished. */
> > + down_write(®ion->unregister_sem);
> > + /* Pairs with READ_ONCE() in set_resource_max() */
> > + WRITE_ONCE(region->reclaim, NULL);
> > + up_write(®ion->unregister_sem);
> > +
> > kref_put(®ion->ref, dmemcg_free_region);
> > }
> I've thought about it some more, Can we do the same as dma-buf init?
>
> DEFINE_DMEMCG_REGION_INFO(info);
> info.size = size.
> info.ops = &drm_ttm_dmem_region_ops;
> info.region_priv = ttm_region;
> info.device_priv = drm_dev;
>
> dmem_region = dmem_cgroup_register_region(&info);
>
> This way we don't need to have a typedef for function pointers,
> no need for READ_ONCE() and/or additional locking, which was only
> added because it wasn't set at init.
>
> If we can push the responsibility for serialization against unload
> to the driver, we should also be able to use drm_dev_enter/exit here
> for the reclaim loop?
>
> Something like below:
>
> if (!ops->device_begin(device_priv, &cookie))
> return 0; // Device gone
>
> while (true) {
> ops->reclaim(region_priv, ...);
> }
>
> ops->device_end(device_priv, cookie);
>
> Although we will additionally need to ensure that the region holds a
> refcount on
> reclaim_priv until dmemcg_free_region is called, otherwise this
> breaks.
>
> So 4 ops needed:
> - device_begin
> - reclaim
> - device_end
> - free (called after region refcount drops to 0, called immediately
> on !CONFIG_DMEMCG, drops device refcount)
>
> Relatedly, I believe perhaps we should also convert from drmm managed
> to devm managed,
> as all memory is already freed after the device is physically
> detached.
>
> Hopefully this solves all lifetime issues, and this design allows for
> additional callbacks into the device or region later on if needed.
Let me take a look at this.
/Thomas
>
> Kind regards,
> ~Maarten Lankhorst
>
> > EXPORT_SYMBOL_GPL(dmem_cgroup_unregister_region);
> > @@ -530,6 +587,7 @@ struct dmem_cgroup_region
> > *dmem_cgroup_register_region(u64 size, const char *fmt
> > INIT_LIST_HEAD(&ret->pools);
> > ret->name = region_name;
> > ret->size = size;
> > + init_rwsem(&ret->unregister_sem);
> > kref_init(&ret->ref);
> >
> > spin_lock(&dmemcg_lock);
> > @@ -568,6 +626,34 @@ void dmem_cgroup_pool_state_put(struct
> > dmem_cgroup_pool_state *pool)
> > }
> > EXPORT_SYMBOL_GPL(dmem_cgroup_pool_state_put);
> >
> > +/**
> > + * dmem_cgroup_region_set_reclaim() - Register a reclaim callback
> > on a region.
> > + * @region: The region to register the callback for.
> > + * @reclaim: Callback to invoke when dmem.max is set below current
> > usage.
> > + * Called with the pool that needs reclaiming and the
> > number of
> > + * bytes to free. Returns 0 on progress, negative on
> > failure.
> > + * @priv: Opaque pointer passed back to @reclaim.
> > + *
> > + * When dmem.max is lowered below the current usage of a cgroup
> > pool, the
> > + * dmem controller will call @reclaim with a target number of
> > bytes to free.
> > + * After @reclaim returns the controller retries setting the
> > limit; if usage
> > + * is still too high it calls @reclaim again, up to a bounded
> > retry count.
> > + */
> > +void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region
> > *region,
> > + dmem_cgroup_reclaim_fn_t
> > reclaim,
> > + void *priv)
> > +{
> > + if (!region)
> > + return;
> > +
> > + down_write(®ion->unregister_sem);
> > + region->reclaim_priv = priv;
> > + /* Pairs with READ_ONCE() in set_resource_max() */
> > + WRITE_ONCE(region->reclaim, reclaim);
> > + up_write(®ion->unregister_sem);
> > +}
> > +EXPORT_SYMBOL_GPL(dmem_cgroup_region_set_reclaim);
> > +
> > static struct dmem_cgroup_pool_state *
> > get_cg_pool_unlocked(struct dmemcg_state *cg, struct
> > dmem_cgroup_region *region)
> > {
> > @@ -725,9 +811,10 @@ static int dmemcg_parse_limit(char *options,
> > u64 *new_limit)
> >
> > static ssize_t dmemcg_limit_write(struct kernfs_open_file *of,
> > char *buf, size_t nbytes, loff_t
> > off,
> > - void (*apply)(struct
> > dmem_cgroup_pool_state *, u64))
> > + void (*apply)(struct
> > dmem_cgroup_pool_state *, u64, bool))
> > {
> > struct dmemcg_state *dmemcs = css_to_dmemcs(of_css(of));
> > + bool nonblock = of->file->f_flags & O_NONBLOCK;
> > int err = 0;
> >
> > while (buf && !err) {
> > @@ -772,7 +859,8 @@ static ssize_t dmemcg_limit_write(struct
> > kernfs_open_file *of,
> > }
> >
> > /* And commit */
> > - apply(pool, new_limit);
> > + apply(pool, new_limit, nonblock);
> > +
> > dmemcg_pool_put(pool);
> >
> > out_put:
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: K Prateek Nayak @ 2026-05-26 9:15 UTC (permalink / raw)
To: Zhang Qiao, Peter Zijlstra, mingo
Cc: longman, chenridong, juri.lelli, vincent.guittot,
dietmar.eggemann, rostedt, bsegall, mgorman, vschneid, tj, hannes,
mkoutny, cgroups, linux-kernel, jstultz, qyousef, Hui Tang
In-Reply-To: <a06e4744-2393-724c-14ff-154f1caa22a6@huawei.com>
Hello Zhang,
On 5/26/2026 1:23 PM, Zhang Qiao wrote:
> Testing sched/flat branch on AMD EPYC 9654 (384 CPUs, 8 NUMA nodes)
> with a 2-level cgroup hierarchy and cfs_bandwidth quota enabled,
> hackbench triggers a divide-by-zero oops:
>
> [ 142.308571] divide error: 0000 [#1] SMP NOPTI
> [ 142.308582] RIP: 0010:task_tick_fair+0x19e/0x410
> [ 142.308601] Call Trace:
> [ 142.308604] <IRQ>
> [ 142.308607] scheduler_tick+0x6a/0x110
> [ 142.308609] update_process_times+0x6b/0x90
> [ 142.308611] tick_sched_handle+0x2a/0x70
> [ 142.308613] tick_sched_timer+0x57/0xb0
More of this trace would have been helpful.
>
> faddr2line confirms:
>
> task_tick_fair+0x19e/0x410:
> __calc_prop_weight at kernel/sched/fair.c:4085
> (inlined by) task_tick_fair at kernel/sched/fair.c:13576
Those line numbers don't match on the latest sched/flat but since you
mention this happens with throttling, I believe it is tick hitting
somewhere in between the task being dequeued by throttle_cfs_rq_work()
and the CPU rescheduling and taking the task off the runqueue.
Dequeue from throttle is slightly special since it keeps the task on
runqueue but the sched entity goes off the cfs_rq changing the
hierarchical weights.
Can you check if this helps:
(Lightly tested with your reproducer)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b8bae794f063..d96e5915fb3e 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -14815,18 +14815,21 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
{
struct sched_entity *se = &curr->se;
- unsigned long weight = NICE_0_LOAD;
- struct cfs_rq *cfs_rq;
- for_each_sched_entity(se) {
- cfs_rq = cfs_rq_of(se);
- entity_tick(cfs_rq, se, queued);
+ if (se->on_rq) {
+ unsigned long weight = NICE_0_LOAD;
+ struct cfs_rq *cfs_rq;
- weight = __calc_prop_weight(cfs_rq, se, weight);
- }
+ for_each_sched_entity(se) {
+ cfs_rq = cfs_rq_of(se);
+ entity_tick(cfs_rq, se, queued);
+
+ weight = __calc_prop_weight(cfs_rq, se, weight);
+ }
- se = &curr->se;
- reweight_eevdf(cfs_rq, se, weight, se->on_rq);
+ se = &curr->se;
+ reweight_eevdf(cfs_rq, se, weight, se->on_rq);
+ }
if (queued)
return;
---
I don't think it makes too much sense to reweight an entity that
has been dequeued. The enqueue at unthrottle will do it anyways.
>
> ===========================================================
> Reproduction
> ===========================================================
>
> Kernel: sched/flat branch (54d493980e00 and later)
> Hardware: AMD EPYC 9654, 2S 384 logical CPUs
>
> # 2-level cgroup, quota = 50% of one period
> cgcreate -g cpu:/bw/l1/l2
> cgset -r cpu.cfs_quota_us=50000 /bw/l1/l2
> cgset -r cpu.cfs_period_us=100000 /bw/l1/l2
>
> # high task count amplifies the throttle→tick race window
> cgexec -g cpu:/bw/l1/l2 hackbench -g 48 -l 1000 -s 512 -T
>
> Typically crashes within 30 seconds on this machine. A single-CPU
> kernel or a very loose quota (e.g. 90%) is unlikely to trigger it
> because the race window is narrow.
This was helpful! I see:
[ 209.935597] Oops: divide error: 0000 [#1] SMP NOPTI
[ 209.941061] CPU: 329 UID: 0 PID: 8247 Comm: sched-messaging Not tainted 7.1.0-rc2-test+ #73 PREEMPT(full)
[ 209.951841] Hardware name: AMD Corporation Titanite_4G/Titanite_4G, BIOS RTI100CC 03/28/2024
[ 209.961254] RIP: 0010:task_tick_fair+0x10d/0x850
[ 209.966420] Code: dc 00 00 00 4c 89 f7 e8 f1 52 ff ff 45 85 e4 0f 85 ba 00 00 00 49 8b 06 4d 8b b6 b8 00 00 00 48 0f af c3 4d 85 f6 74 19 31 d2 <49> f7 37 ba 02 00 00 00 48 89 d3 48 39 d0 48 0f 43 d8 e9 20 ff ff
[ 209.987382] RSP: 0018:ff581fd71e1fce58 EFLAGS: 00010046
[ 209.993216] RAX: 0000010000000000 RBX: 0000000000100000 RCX: ff295dbfa9ad8080
[ 210.001179] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ff295dbfa9ad8080
[ 210.009141] RBP: 0000000000000000 R08: 0000000000000000 R09: 00000000000063eb
[ 210.017104] R10: 0000000000000000 R11: ff581fd71e1fcff8 R12: 0000000000000000
[ 210.025061] R13: ff295dbfa9ad8000 R14: ff295dc06c6eac00 R15: ff295dbfd9bc8600
[ 210.033027] FS: 00007faef8c8b640(0000) GS:ff295e7c4acca000(0000) knlGS:0000000000000000
[ 210.042060] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 210.048474] CR2: 00007f9884292d30 CR3: 000000011aa26001 CR4: 0000000000f71ef0
[ 210.056430] PKRU: 55555554
[ 210.059448] Call Trace:
[ 210.062177] <IRQ>
[ 210.064426] sched_tick+0x94/0x250
[ 210.068229] update_process_times+0x99/0xc0
[ 210.072903] tick_nohz_handler+0x95/0x1a0
[ 210.077380] ? __pfx_tick_nohz_handler+0x10/0x10
[ 210.082534] __hrtimer_run_queues+0xfe/0x260
[ 210.087304] hrtimer_interrupt+0x122/0x1f0
[ 210.091880] __sysvec_apic_timer_interrupt+0x55/0x130
[ 210.097525] sysvec_apic_timer_interrupt+0x7a/0xb0
[ 210.102873] </IRQ>
[ 210.105203] <TASK>
[ 210.107542] asm_sysvec_apic_timer_interrupt+0x1a/0x20
[ 210.113284] RIP: 0010:_raw_spin_unlock_irqrestore+0x1d/0x40
[ 210.119511] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 c6 07 00 0f 1f 00 f7 c6 00 02 00 00 74 06 fb 0f 1f 44 00 00 <65> ff 0d ec 20 fd 01 74 05 e9 c0 81 d4 fe e8 00 93 ec fe e9 b6 81
[ 210.140469] RSP: 0018:ff581fd74032fe88 EFLAGS: 00000206
[ 210.146308] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000004
[ 210.154271] RDX: 0000000000000000 RSI: 0000000000000246 RDI: ff295dbfa9ad8d64
[ 210.162235] RBP: ff295dbfa9ad8000 R08: 0000000000000000 R09: 0000000000000000
[ 210.170196] R10: 0000000000000000 R11: 0000000000000000 R12: ff295dbfa9ad8d64
[ 210.178159] R13: ff581fd74032ff48 R14: ff295dbfa9ad8000 R15: 00fffffffffff000
[ 210.186139] task_work_run+0x5c/0x90
[ 210.190137] exit_to_user_mode_loop+0x16e/0x550
[ 210.195198] ? srso_alias_return_thunk+0x5/0xfbef5
[ 210.200552] ? ksys_read+0xc5/0xe0
[ 210.204352] do_syscall_64+0x26e/0x750
[ 210.208540] ? do_syscall_64+0xaa/0x750
[ 210.212823] ? srso_alias_return_thunk+0x5/0xfbef5
[ 210.218174] entry_SYSCALL_64_after_hwframe+0x76/0x7e
---
So the theory of throttle work causing this checks out.
The suggested diff above solves the crash in my case but your
mileage may vary. Peter can comment if this is the right thing
to do or not :-)
--
Thanks and Regards,
Prateek
^ permalink raw reply related
* Re: [PATCH 2/8] mm: percpu: charge obj_exts allocation with __GFP_ACCOUNT
From: Alexandre Ghiti @ 2026-05-26 8:35 UTC (permalink / raw)
To: Shakeel Butt
Cc: Andrew Morton, Johannes Weiner, Michal Hocko, Roman Gushchin,
Muchun Song, Dennis Zhou, Tejun Heo, Christoph Lameter,
Vlastimil Babka, Yosry Ahmed, Nhat Pham, Sergey Senozhatsky,
Chengming Zhou, Suren Baghdasaryan, Qi Zheng, David Hildenbrand,
Lorenzo Stoakes, Minchan Kim, Mike Rapoport, Axel Rasmussen,
Barry Song, Kairui Song, Wei Xu, Yuanchu Xie, Liam R . Howlett,
Joshua Hahn, linux-mm, linux-kernel, cgroups
In-Reply-To: <b434907e-2036-4260-bced-1adb8e26c917@ghiti.fr>
On 5/22/26 10:11, Alexandre Ghiti wrote:
> Hi Shakeel,
>
> On 5/21/26 19:25, Shakeel Butt wrote:
>> On Mon, May 11, 2026 at 10:20:37PM +0200, Alexandre Ghiti wrote:
>>> This is a preparatory patch for upcoming per-memcg-per-node kmem
>>> accounting.
>>>
>>> pcpu allocations are always fully charged at once using
>>> pcpu_obj_full_size(), which returns the size of the pcpu "metadata" +
>>> pcpu "payload". But metadata and payload may not be allocated on the
>>> same numa node, so charge the metadata independently from the payload.
>>>
>>> Do this by explicitly passing __GFP_ACCOUNT to the obj_exts allocation
>>> and remove its accounting in pcpu_memcg_pre_alloc_hook().
>> Will all the entries in obj_exts array be for the same memcg? If not
>> then why we
>> are charging the whole array to the one which happen to allocate the
>> array?
>
>
> Hmm, I overlooked the amount allocated, so that's my mistake: the
> chunk-allocating-memcg will be charged for all the metadata, although
> before the charge was distributed. And according to Claude, the
> metadata would represent 64kB, so not negligible.
>
I realize that I did not mention my setup: I have been testing this
series on a 176 core machine, and the 64KB that Claude gave me was based
on a 32K unit_size. But actually it's not right. Here is my understanding:
- unit_size is 512K on this machine, which means that each cpu gets a
region this size every time a new chunk is allocated => 176 * 512 = 88MB
per chunk
- obj_exts = unit_size / PCPU_MIN_ALLOC_SIZE * sizeof(pcpuobj_ext) =
512K * 2 = 1MB (obj_exts is one memcg pointer for each 4B)
Let me know what you think, but I don't think that's acceptable, I'm
looking into another solution.
Thanks
Alex
>
>>
>> Sorry I don't know the details of percpu allocator, so asking some dumb
>> questions:
>>
>> 1. Does the alloc_percpu() (& similar functions) allocate the
>> underlying on a
>> single node or does it allocate memory for each cpu on their
>> local node?
>> For slub, it is on the same node, so the situation is easier to
>> handle.
>
>
> To me, chunk metadata and actual pages are allocated differently:
>
> - pcpu_alloc_pages() tries to allocate the pages on the cpu local node
> https://elixir.bootlin.com/linux/v7.0.9/source/mm/percpu-vm.c#L95. But
> to me no guarantee it won't fallback to any other node. And I don't
> think that __GFP_THISNODE would be a good idea here.
>
> - pcpu_alloc_chunk() uses kmalloc or vmalloc depending on the size, so
> not attached to specific node, that's why I wanted GFP_ACCOUNT to do
> the job for us in the first place.
>
>
>>
>> 2. On a typical system how much memory is consumed by obj_exts for
>> the percpu
>> allocator chunks? I am wondering if we don't charge it, how much
>> will we
>> loose?
>
>
> So according to my previous answer, 64kB. I have just noticed that a
> bunch of dynamically allocated chunk fields are not accounted either,
> which again according to Claude represent 2.3kB. I don't have much
> experience in accounting but that's far from negligible right? Which
> amount are we keen to lose to make the code simpler (or for other
> reasons)?
>
>
>>
>> 3. What would be side effect on assuming that obj_exts is on the same
>> node as
>> the given chunk?
>
>
> Given the size of obj_exts, overcharging one node while undercharging
> others?
>
> To conclude, you're right, I did not dive deep enough into the
> metadata sizes, I'll fix that.
>
> Thanks,
>
> Alex
>
^ permalink raw reply
* Re: [PATCH v2] blk-throttle: schedule parent dispatch in tg_flush_bios()
From: Shin'ichiro Kawasaki @ 2026-05-26 8:32 UTC (permalink / raw)
To: Tao Cui; +Cc: tj, josef, axboe, cgroups, linux-block
In-Reply-To: <20260522091530.1901437-1-cuitao@kylinos.cn>
On May 22, 2026 / 17:15, Tao Cui wrote:
> tg_flush_bios() schedules pending_timer on the child tg's own
> service_queue, which causes throtl_pending_timer_fn() to dispatch from
> the child's pending_tree. For leaf cgroups this tree is empty, so the
> timer fires and exits without dispatching the throttled bio.
>
> The throttled bio sits in the parent's pending_tree with disptime set
> to jiffies (THROTL_TG_CANCELING zeroes all dispatch times), but the
> parent's timer is never explicitly rescheduled. The bio only gets
> dispatched when the parent timer eventually fires at its previously
> scheduled expiry.
>
> Fix by calling throtl_schedule_next_dispatch(sq->parent_sq, true)
> instead, matching what tg_set_limit() already does. This forces the
> parent's dispatch cycle to run immediately and flush all canceling
> bios without waiting for a stale timer.
>
> For the device deletion path (blk_throtl_cancel_bios), directly
> complete throttled bios with EIO via bio_io_error() instead of
> dispatching them through the timer -> work -> submission chain.
> This avoids a race with the SCSI state machine where bios can reach
> the SCSI layer while the device is in SDEV_CANCEL state, causing
> ENODEV instead of the expected EIO.
>
> Reported-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>
I reported that v1 patch fails with the blktests test case throtl/004,
but I did not report the problem that this patch addresses. Then I don't
think this Reported-by tag is valid. Please drop it.
I confirmed that the recent blktess CI test run with this v2 patch did not
fail at throtl/004. Thanks to your action for the failure.
^ permalink raw reply
* Re: [PATCH v4 2/5] cgroup/dmem: Add reclaim callback for lowering max below current usage
From: Maarten Lankhorst @ 2026-05-26 8:27 UTC (permalink / raw)
To: Thomas Hellström, intel-xe
Cc: Natalie Vock, Johannes Weiner, Tejun Heo, Michal Koutný,
cgroups, Huang Rui, Matthew Brost, Matthew Auld, Maxime Ripard,
Thomas Zimmermann, Simona Vetter, David Airlie,
Christian König, Alex Deucher, Rodrigo Vivi, dri-devel,
amd-gfx, linux-kernel
In-Reply-To: <20260512082406.44470-3-thomas.hellstrom@linux.intel.com>
Hello,
Den 2026-05-12 kl. 10:24, skrev Thomas Hellström:
> Add an optional reclaim callback to struct dmem_cgroup_region. When
> dmem.max is set below the current usage of a cgroup pool, the new limit
> is applied immediately (so that concurrent allocations are throttled
> while reclaim is in progress) and then the driver is asked to evict
> memory to bring usage back below the limit.
>
> Reclaim is attempted up to a bounded number of times. No error is
> returned to userspace if usage remains above the limit after reclaim,
> and a pending signal will abort the reclaim loop early. This matches
> the behavior of memory.max in the memory cgroup controller.
>
> Also honor O_NONBLOCK so that if that flag is set during the
> max value write, no reclaim is initiated. The idea is to avoid
> charging the reclaim cost to the writer of the max value.
>
> v2:
> - Write max before reclaim is attempted (Maarten)
> - Let signals abort the reclaim without error (Maarten)
> - If a new max value is written with the O_NONBLOCK flag,
> reclaim is not attempted (Maarten)
> - Extract region from the pool parameter rather than
> passing it explicitly to set_resource_xxx().
> v3:
> - Use an rwsem to protect reclaim callback registration and
> region unregister against concurrent reclaim invocations,
> ensuring reclaim_priv is visible when the callback is
> invoked. (Sashiko-bot)
>
> Assisted-by: GitHub_Copilot:claude-sonnet-4.6
> Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> ---
> include/linux/cgroup_dmem.h | 24 ++++++++
> kernel/cgroup/dmem.c | 106 +++++++++++++++++++++++++++++++++---
> 2 files changed, 121 insertions(+), 9 deletions(-)
>
> diff --git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h
> index dd4869f1d736..c3bce21cbe80 100644
> --- a/include/linux/cgroup_dmem.h
> +++ b/include/linux/cgroup_dmem.h
> @@ -14,6 +14,21 @@ struct dmem_cgroup_pool_state;
> /* Opaque definition of a cgroup region, used internally */
> struct dmem_cgroup_region;
>
> +/**
> + * typedef dmem_cgroup_reclaim_fn_t - Reclaim callback for a dmem cgroup region.
> + * @pool: The cgroup pool that needs memory reclaimed.
> + * @target_bytes: Minimum number of bytes the driver should attempt to free.
> + * @priv: Private data registered with dmem_cgroup_region_set_reclaim().
> + *
> + * Called by the dmem cgroup controller when dmem.max is set below the current
> + * usage of @pool. The driver should evict at least @target_bytes of memory
> + * from @pool. May be called multiple times if usage remains above the limit.
> + *
> + * Return: 0 if progress was made, negative error code otherwise.
> + */
> +typedef int (*dmem_cgroup_reclaim_fn_t)(struct dmem_cgroup_pool_state *pool,
> + u64 target_bytes, void *priv);
> +
> #if IS_ENABLED(CONFIG_CGROUP_DMEM)
> struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) __printf(2,3);
> void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region);
> @@ -26,6 +41,9 @@ bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
> bool ignore_low, bool *ret_hit_low);
>
> void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool);
> +void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
> + dmem_cgroup_reclaim_fn_t reclaim,
> + void *priv);
> #else
> static inline __printf(2,3) struct dmem_cgroup_region *
> dmem_cgroup_register_region(u64 size, const char *name_fmt, ...)
> @@ -62,5 +80,11 @@ bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
> static inline void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
> { }
>
> +static inline void
> +dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
> + dmem_cgroup_reclaim_fn_t reclaim,
> + void *priv)
> +{ }
> +
> #endif
> #endif /* _CGROUP_DMEM_H */
> diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
> index 1ab1fb47f271..5fd5a1634d21 100644
> --- a/kernel/cgroup/dmem.c
> +++ b/kernel/cgroup/dmem.c
> @@ -51,6 +51,20 @@ struct dmem_cgroup_region {
> * No new pools should be added to the region afterwards.
> */
> bool unregistered;
> +
> + /**
> + * @reclaim: Optional callback invoked when dmem.max is set below the
> + * current usage of a pool. The driver should attempt to free at least
> + * @target_bytes from @pool. May be called multiple times if usage
> + * remains above the limit after returning.
> + */
> + dmem_cgroup_reclaim_fn_t reclaim;
> +
> + /** @reclaim_priv: Private data passed to @reclaim. */
> + void *reclaim_priv;
> +
> + /** @unregister_sem: Protect @reclaim while it is running. */
> + struct rw_semaphore unregister_sem;
> };
>
> struct dmemcg_state {
> @@ -145,21 +159,58 @@ static void free_cg_pool(struct dmem_cgroup_pool_state *pool)
> }
>
> static void
> -set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val)
> +set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
> {
> page_counter_set_min(&pool->cnt, val);
> }
>
> static void
> -set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val)
> +set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
> {
> page_counter_set_low(&pool->cnt, val);
> }
>
> static void
> -set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val)
> +set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
> {
> - page_counter_set_max(&pool->cnt, val);
> + struct dmem_cgroup_region *region = pool->region;
> +
> + /*
> + * Always update the limit, even if usage currently exceeds it.
> + * Concurrent allocations will be throttled against the new limit
> + * while reclaim is in progress.
> + */
> + xchg(&pool->cnt.max, (unsigned long)val);
> +
> + if (nonblock || !READ_ONCE(region->reclaim))
> + return;
> +
> + for (int retries = 5; retries > 0; retries--) {
Where does 5 come from? This code should retry until no longer above limit, otherwise you'll get some hard to debug issues.
> + u64 usage = page_counter_read(&pool->cnt);
> + int ret;
> +
> + if (usage <= val)
> + break;
> +
> + if (signal_pending(current))
> + break;
> +
> + /* Block unregister until the reclaim callback completes. */
> + if (down_read_interruptible(®ion->unregister_sem))
> + break;
> +
> + if (!region->reclaim) {
> + up_read(®ion->unregister_sem);
> + break;
> + }
> +
> + ret = region->reclaim(pool, usage - val, region->reclaim_priv);
> + up_read(®ion->unregister_sem);
> + if (ret)
> + break;
> +
> + cond_resched();
> + }
> }
>
> static u64 get_resource_low(struct dmem_cgroup_pool_state *pool)
> @@ -184,9 +235,9 @@ static u64 get_resource_current(struct dmem_cgroup_pool_state *pool)
>
> static void reset_all_resource_limits(struct dmem_cgroup_pool_state *rpool)
> {
> - set_resource_min(rpool, 0);
> - set_resource_low(rpool, 0);
> - set_resource_max(rpool, PAGE_COUNTER_MAX);
> + set_resource_min(rpool, 0, false);
> + set_resource_low(rpool, 0, false);
> + set_resource_max(rpool, PAGE_COUNTER_MAX, false);
> }
>
> static void dmemcs_offline(struct cgroup_subsys_state *css)
> @@ -491,6 +542,12 @@ void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region)
> region->unregistered = true;
> spin_unlock(&dmemcg_lock);
>
> + /* Ensure all reclaim() callbacks have finished. */
> + down_write(®ion->unregister_sem);
> + /* Pairs with READ_ONCE() in set_resource_max() */
> + WRITE_ONCE(region->reclaim, NULL);
> + up_write(®ion->unregister_sem);
> +
> kref_put(®ion->ref, dmemcg_free_region);
> }
I've thought about it some more, Can we do the same as dma-buf init?
DEFINE_DMEMCG_REGION_INFO(info);
info.size = size.
info.ops = &drm_ttm_dmem_region_ops;
info.region_priv = ttm_region;
info.device_priv = drm_dev;
dmem_region = dmem_cgroup_register_region(&info);
This way we don't need to have a typedef for function pointers,
no need for READ_ONCE() and/or additional locking, which was only
added because it wasn't set at init.
If we can push the responsibility for serialization against unload
to the driver, we should also be able to use drm_dev_enter/exit here
for the reclaim loop?
Something like below:
if (!ops->device_begin(device_priv, &cookie))
return 0; // Device gone
while (true) {
ops->reclaim(region_priv, ...);
}
ops->device_end(device_priv, cookie);
Although we will additionally need to ensure that the region holds a refcount on
reclaim_priv until dmemcg_free_region is called, otherwise this breaks.
So 4 ops needed:
- device_begin
- reclaim
- device_end
- free (called after region refcount drops to 0, called immediately on !CONFIG_DMEMCG, drops device refcount)
Relatedly, I believe perhaps we should also convert from drmm managed to devm managed,
as all memory is already freed after the device is physically detached.
Hopefully this solves all lifetime issues, and this design allows for
additional callbacks into the device or region later on if needed.
Kind regards,
~Maarten Lankhorst
> EXPORT_SYMBOL_GPL(dmem_cgroup_unregister_region);
> @@ -530,6 +587,7 @@ struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *fmt
> INIT_LIST_HEAD(&ret->pools);
> ret->name = region_name;
> ret->size = size;
> + init_rwsem(&ret->unregister_sem);
> kref_init(&ret->ref);
>
> spin_lock(&dmemcg_lock);
> @@ -568,6 +626,34 @@ void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
> }
> EXPORT_SYMBOL_GPL(dmem_cgroup_pool_state_put);
>
> +/**
> + * dmem_cgroup_region_set_reclaim() - Register a reclaim callback on a region.
> + * @region: The region to register the callback for.
> + * @reclaim: Callback to invoke when dmem.max is set below current usage.
> + * Called with the pool that needs reclaiming and the number of
> + * bytes to free. Returns 0 on progress, negative on failure.
> + * @priv: Opaque pointer passed back to @reclaim.
> + *
> + * When dmem.max is lowered below the current usage of a cgroup pool, the
> + * dmem controller will call @reclaim with a target number of bytes to free.
> + * After @reclaim returns the controller retries setting the limit; if usage
> + * is still too high it calls @reclaim again, up to a bounded retry count.
> + */
> +void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
> + dmem_cgroup_reclaim_fn_t reclaim,
> + void *priv)
> +{
> + if (!region)
> + return;
> +
> + down_write(®ion->unregister_sem);
> + region->reclaim_priv = priv;
> + /* Pairs with READ_ONCE() in set_resource_max() */
> + WRITE_ONCE(region->reclaim, reclaim);
> + up_write(®ion->unregister_sem);
> +}
> +EXPORT_SYMBOL_GPL(dmem_cgroup_region_set_reclaim);
> +
> static struct dmem_cgroup_pool_state *
> get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region)
> {
> @@ -725,9 +811,10 @@ static int dmemcg_parse_limit(char *options, u64 *new_limit)
>
> static ssize_t dmemcg_limit_write(struct kernfs_open_file *of,
> char *buf, size_t nbytes, loff_t off,
> - void (*apply)(struct dmem_cgroup_pool_state *, u64))
> + void (*apply)(struct dmem_cgroup_pool_state *, u64, bool))
> {
> struct dmemcg_state *dmemcs = css_to_dmemcs(of_css(of));
> + bool nonblock = of->file->f_flags & O_NONBLOCK;
> int err = 0;
>
> while (buf && !err) {
> @@ -772,7 +859,8 @@ static ssize_t dmemcg_limit_write(struct kernfs_open_file *of,
> }
>
> /* And commit */
> - apply(pool, new_limit);
> + apply(pool, new_limit, nonblock);
> +
> dmemcg_pool_put(pool);
>
> out_put:
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox