* [PATCH 2/8] mm: percpu: charge obj_exts allocation with __GFP_ACCOUNT
From: Alexandre Ghiti @ 2026-05-11 20:20 UTC (permalink / raw)
To: Andrew Morton
Cc: Johannes Weiner, Michal Hocko, Roman Gushchin, Shakeel Butt,
Muchun Song, Dennis Zhou, Tejun Heo, Christoph Lameter,
Vlastimil Babka, Yosry Ahmed, Nhat Pham, Sergey Senozhatsky,
Chengming Zhou, Suren Baghdasaryan, Qi Zheng, David Hildenbrand,
Lorenzo Stoakes, Minchan Kim, Mike Rapoport, Axel Rasmussen,
Barry Song, Kairui Song, Wei Xu, Yuanchu Xie, Liam R . Howlett,
Joshua Hahn, linux-mm, linux-kernel, cgroups, Alexandre Ghiti
In-Reply-To: <20260511202136.330358-1-alex@ghiti.fr>
This is a preparatory patch for upcoming per-memcg-per-node kmem
accounting.
pcpu allocations are always fully charged at once using
pcpu_obj_full_size(), which returns the size of the pcpu "metadata" +
pcpu "payload". But metadata and payload may not be allocated on the
same numa node, so charge the metadata independently from the payload.
Do this by explicitly passing __GFP_ACCOUNT to the obj_exts allocation
and remove its accounting in pcpu_memcg_pre_alloc_hook().
Signed-off-by: Alexandre Ghiti <alex@ghiti.fr>
---
mm/percpu-internal.h | 16 +++-------------
mm/percpu.c | 15 ++++++++-------
2 files changed, 11 insertions(+), 20 deletions(-)
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
index 4b3d6ec43703..f01db026d213 100644
--- a/mm/percpu-internal.h
+++ b/mm/percpu-internal.h
@@ -144,22 +144,12 @@ static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
}
/**
- * pcpu_obj_full_size - helper to calculate size of each accounted object
+ * pcpu_obj_total_size - helper to calculate size of each accounted object
* @size: size of area to allocate in bytes
- *
- * For each accounted object there is an extra space which is used to store
- * obj_cgroup membership if kmemcg is not disabled. Charge it too.
*/
-static inline size_t pcpu_obj_full_size(size_t size)
+static inline size_t pcpu_obj_total_size(size_t size)
{
- size_t extra_size = 0;
-
-#ifdef CONFIG_MEMCG
- if (!mem_cgroup_kmem_disabled())
- extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *);
-#endif
-
- return size * num_possible_cpus() + extra_size;
+ return size * num_possible_cpus();
}
#ifdef CONFIG_PERCPU_STATS
diff --git a/mm/percpu.c b/mm/percpu.c
index b0676b8054ed..13de6e099d96 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1460,7 +1460,8 @@ static struct pcpu_chunk *pcpu_alloc_chunk(gfp_t gfp)
if (need_pcpuobj_ext()) {
chunk->obj_exts =
pcpu_mem_zalloc(pcpu_chunk_map_bits(chunk) *
- sizeof(struct pcpuobj_ext), gfp);
+ sizeof(struct pcpuobj_ext),
+ gfp | __GFP_ACCOUNT);
if (!chunk->obj_exts)
goto objcg_fail;
}
@@ -1625,7 +1626,7 @@ static bool pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp,
if (!objcg || obj_cgroup_is_root(objcg))
return true;
- if (obj_cgroup_charge(objcg, gfp, pcpu_obj_full_size(size)))
+ if (obj_cgroup_charge(objcg, gfp, pcpu_obj_total_size(size)))
return false;
*objcgp = objcg;
@@ -1645,10 +1646,10 @@ static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg,
rcu_read_lock();
mod_memcg_state(obj_cgroup_memcg(objcg), MEMCG_PERCPU_B,
- pcpu_obj_full_size(size));
+ pcpu_obj_total_size(size));
rcu_read_unlock();
} else {
- obj_cgroup_uncharge(objcg, pcpu_obj_full_size(size));
+ obj_cgroup_uncharge(objcg, pcpu_obj_total_size(size));
}
}
@@ -1664,11 +1665,11 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
return;
chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].cgroup = NULL;
- obj_cgroup_uncharge(objcg, pcpu_obj_full_size(size));
+ obj_cgroup_uncharge(objcg, pcpu_obj_total_size(size));
rcu_read_lock();
mod_memcg_state(obj_cgroup_memcg(objcg), MEMCG_PERCPU_B,
- -pcpu_obj_full_size(size));
+ -pcpu_obj_total_size(size));
rcu_read_unlock();
obj_cgroup_put(objcg);
@@ -1897,7 +1898,7 @@ void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved,
trace_percpu_alloc_percpu(_RET_IP_, reserved, is_atomic, size, align,
chunk->base_addr, off, ptr,
- pcpu_obj_full_size(size), gfp);
+ pcpu_obj_total_size(size), gfp);
pcpu_memcg_post_alloc_hook(objcg, chunk, off, size);
--
2.54.0
^ permalink raw reply related
* [PATCH 1/8] mm: memcontrol: propagate NMI slab stats to memcg vmstats
From: Alexandre Ghiti @ 2026-05-11 20:20 UTC (permalink / raw)
To: Andrew Morton
Cc: Johannes Weiner, Michal Hocko, Roman Gushchin, Shakeel Butt,
Muchun Song, Dennis Zhou, Tejun Heo, Christoph Lameter,
Vlastimil Babka, Yosry Ahmed, Nhat Pham, Sergey Senozhatsky,
Chengming Zhou, Suren Baghdasaryan, Qi Zheng, David Hildenbrand,
Lorenzo Stoakes, Minchan Kim, Mike Rapoport, Axel Rasmussen,
Barry Song, Kairui Song, Wei Xu, Yuanchu Xie, Liam R . Howlett,
Joshua Hahn, linux-mm, linux-kernel, cgroups, Alexandre Ghiti
In-Reply-To: <20260511202136.330358-1-alex@ghiti.fr>
flush_nmi_stats() drains per-node NMI slab atomics into the per-node
lruvec_stats, but does not propagate them to the memcg-level vmstats.
This is inconsistent with account_slab_nmi_safe() which updates both,
so fix this by propagating the NMI slab stats to the memcg-level vmstats.
Fixes: 940b01fc8dc1 ("memcg: nmi safe memcg stats for specific archs")
Signed-off-by: Alexandre Ghiti <alex@ghiti.fr>
---
mm/memcontrol.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c3d98ab41f1f..d81a76654b2c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4341,16 +4341,22 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B);
lstats->state[index] += slab;
+ memcg->vmstats->state[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
+ if (parent)
+ parent->vmstats->state_pending[index] += slab;
}
if (atomic_read(&pn->slab_unreclaimable)) {
int slab = atomic_xchg(&pn->slab_unreclaimable, 0);
int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B);
lstats->state[index] += slab;
+ memcg->vmstats->state[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
+ if (parent)
+ parent->vmstats->state_pending[index] += slab;
}
}
}
--
2.54.0
^ permalink raw reply related
* [PATCH 0/8] per-memcg-per-node kmem accounting
From: Alexandre Ghiti @ 2026-05-11 20:20 UTC (permalink / raw)
To: Andrew Morton
Cc: Johannes Weiner, Michal Hocko, Roman Gushchin, Shakeel Butt,
Muchun Song, Dennis Zhou, Tejun Heo, Christoph Lameter,
Vlastimil Babka, Yosry Ahmed, Nhat Pham, Sergey Senozhatsky,
Chengming Zhou, Suren Baghdasaryan, Qi Zheng, David Hildenbrand,
Lorenzo Stoakes, Minchan Kim, Mike Rapoport, Axel Rasmussen,
Barry Song, Kairui Song, Wei Xu, Yuanchu Xie, Liam R . Howlett,
Joshua Hahn, linux-mm, linux-kernel, cgroups, Alexandre Ghiti
This series pursues the work initiated by Joshua [1]. We need kernel
memory to be accounted on a per-node basis in order to be able to
know the memcg and physical memory association.
This series takes advantage of the recent introduction of per-node
obj_cgroup [2] and makes those obj_cgroup tied to their numa node.
The bulk of the series is percpu per-node accounting: percpu
"precharges" the memcg before we know the actual location of the pages
it uses, so charging and accounting had to be split. All other kmem
users (slab, zswap, __memcg_kmem_charge_page) are straightforward
conversions (zswap support is limited in this series because Joshua
is working on it in parallel [3]).
Thanks Joshua for your early feedbacks!
[1] https://lore.kernel.org/linux-mm/20260404033844.1892595-1-joshua.hahnjy@gmail.com/
[2] https://lore.kernel.org/linux-mm/56c04b1c5d54f75ccdc12896df6c1ca35403ecc3.1772711148.git.zhengqi.arch@bytedance.com/
[3] https://lore.kernel.org/linux-mm/20260311195153.4013476-1-joshua.hahnjy@gmail.com/
Alexandre Ghiti (8):
mm: memcontrol: propagate NMI slab stats to memcg vmstats
mm: percpu: charge obj_exts allocation with __GFP_ACCOUNT
mm: percpu: Split memcg charging and kmem accounting
mm: memcontrol: track MEMCG_KMEM per NUMA node
mm: memcontrol: per-node kmem accounting for page charges
mm: slab: per-node kmem accounting for slab
mm: percpu: per-node kmem accounting using local credit
mm: zswap: per-node kmem accounting for zswap/zsmalloc
include/linux/memcontrol.h | 27 +++++--
include/linux/mmzone.h | 1 +
include/linux/zsmalloc.h | 2 +
mm/memcontrol.c | 150 ++++++++++++++++++++++++++++---------
mm/percpu-internal.h | 16 +---
mm/percpu.c | 90 ++++++++++++++++++++--
mm/vmstat.c | 1 +
mm/zsmalloc.c | 11 +++
mm/zswap.c | 9 ++-
9 files changed, 242 insertions(+), 65 deletions(-)
--
2.54.0
^ permalink raw reply
* Re: [RFC PATCH 0/9 v2] mm/memcontrol: Make memory cgroup limits tier-aware
From: Joshua Hahn @ 2026-05-11 20:03 UTC (permalink / raw)
To: David Hildenbrand (Arm)
Cc: linux-mm, Tejun Heo, Johannes Weiner, Michal Koutny, Michal Hocko,
Roman Gushchin, Shakeel Butt, Andrew Morton, Chris Li,
Kairui Song, Muchun Song, Lorenzo Stoakes, Liam R. Howlett,
Vlastimil Babka, Mike Rapoport, Suren Baghdasaryan, Kemeng Shi,
Nhat Pham, Baoquan He, Barry Song, Youngjun Park, Qi Zheng,
Axel Rasmussen, Yuanchu Xie, Wei Xu, Kaiyang Zhao, David Rientjes,
Yiannis Nikolakopoulos, Rao, Bharata Bhasker, cgroups,
linux-kernel, kernel-team
In-Reply-To: <3ff8b23a-479d-46e8-b820-a23697587f01@kernel.org>
On Mon, May 11, 2026 at 5:56 PM David Hildenbrand (Arm)
<david@kernel.org> wrote:
>
> On 4/23/26 22:34, Joshua Hahn wrote:
> > INTRODUCTION
> > ============
> > Memory cgroups provide an interface that allow multiple works on a host to
> > co-exist via weak and strong memory isolation guarantees. This works, because
> > for the most part, all memory has equal utility. Isolating a cgroup’s memory
> > footprint restricts how much it can hurt other workloads competing for memory,
> > or protects it from other cgroups looking for more memory.
> >
> > However, on systems with tiered memory (e.g. CXL), memory utility is no longer
> > homogeneous; toptier and lowtier memory provide different performance
> > characteristics and have different scarcity, meaning memory footprint no longer
> > serves as an accurate representation of a cgroup’s consumption of the system’s
> > limited resources. As an extreme example, a cgroup with 10G of toptier
> > (e.g. DRAM) memory and a cgroup with 10G of lowtier (e.g. CXL) memory both
> > appear to be consuming the same amount of system resources from memcg’s
> > perspective, despite the performance asymmetry between the two workloads.
> >
> > Therefore on tiered systems, memory isolation cannot currently happen, as
> > workloads that are well-behaved within their memcg limits may still hurt the
> > performance of other well-behaving workloads by hogging more than its
> > “fair share” of toptier memory.
> >
> > Introduce tier-aware memcg limits, which establish independent toptier limits
> > that scale with the memory limits and the ratio of toptier:total memory
> > available on the system.
> >
> > INTERFACE
> > =========
> > This series introduces only one adjustable knob to userspace; a new cgroup mount
> > option “memory_tiered_limits” which toggles whether the cgroup mount will scale
> > toptier limits. It also introduces 4 new read-only sysfs entries per-cgroup:
> > memory.toptier_{min, low, high, max}.
> >
> > The new toptier memory limits are scaled according to the amount of toptier
> > memory and total memory available on the system as such:
> >
> > memory.toptier_high = (toptier_mem / total_mem) * memory.high
> >
> > For instance, on a host with 100GB memory, with 75G toptier and 25G CXL, the
> > “toptier ratio” would be 75 / 100 = 0.75. A cgroup with the following memcg
> > limits {min: 8G, low: 12G, high: 20G, max: 24G} might see toptier limits scaled
> > at {min: 6G, low: 9G, high: 15G, max: 18G}.
Hi David!!
It was great seeing you at LSFMMBPF. I didn't get a chance to have a
conversation with you at Zagreb but hopefully I will be less shy and say
hello next conference : -)
> Assume you have a bigger hierarchy (HBP, DRAM, CXL), or assume you have multiple
> NUMA nodes with a hierarchy each.
>
> Your proposal doesn't really seem to be very versatile, or am I wrong?
Let me address these comments separately!
First, for the multi-numa-per-tier case, I think this is already pretty well
handled by my series. Once we realize that a memcg is consuming too much memory
from a tier, we trigger reclaim from that memcg via
try_to_free_mem_cgroup_pages,
which as far as I can tell already handles the multi-numa per memcg case.
Other than restricting the scan_control's nodemask to target the nodes
from that tier, I don't think there's anything else to be done.
Next for the 3+ tier case, I think this is a lot more scalable than it seems
at first. This series depends on another RFC that I sent out [1],
which pushes the concept of "stock" from memcg to page_counter, which means
that it is more scalable to just add more page counters to each memcg.
This means that each tier would just need another page_counter to track its
memory usage, and we trigger selective reclaim on the tier that is being
targeted via the scan control nodemask we introduce in this series.
At my talk in LSFMMBPF, Usama noted that the user-visible API should probably
remain the same, no matter what. The way I have currently established the
memcg files aren't really scalable, so Usama suggested turning the
"memory.toptier_XXX" sysfs files to "memory.tiered_XXX", which would include
a newline-separated / space-separated list of per-tier limits. Something like:
$ cat memory.tiered_max
tier_0 20971520
tier_1 31457280
...
So we have a way to make both the user-facing side stable, and the internals
also more scalable.
With that said, I've opted to leave the internals to 2 tiers for now -- I think
it is not too late to add the generalization series when we start seeing
3+ tier systems out there in the wild. My goal was to introduce tieredness,
and we can work towards generalization in a future work.
On that note, it seems like in general mm is aware of 3+ tiers, but most of the
existing work revolves around distinguishing between toptier/everything else.
I got this impression from reading mm/memory-tiers.c -- but please feel free
to correct me if you feel like I have the wrong idea here : -)
So perhaps the generalization work would benefit from first introducing more
general tier awareness (not just toptier vs. rest) in memory-tiers.c.
What do you think? Does this approach of introducing toptier restriction for
now, and then generalizing in future work make sense to you?
Thanks again for your interest. Have a great day!
Joshua
[1] https://lore.kernel.org/all/20260410210742.550489-1-joshua.hahnjy@gmail.com/
^ permalink raw reply
* Re: [PATCH v3 2/2] cgroup/cpuset: reserve DL bandwidth only for root-domain moves
From: Waiman Long @ 2026-05-11 20:00 UTC (permalink / raw)
To: Guopeng Zhang, Tejun Heo, Michal Koutný, Ingo Molnar,
Peter Zijlstra, Juri Lelli, Chen Ridong
Cc: Johannes Weiner, Vincent Guittot, Dietmar Eggemann,
Steven Rostedt, Ben Segall, Mel Gorman, Valentin Schneider,
K Prateek Nayak, Gabriele Monaco, Will Deacon, linux-kernel,
cgroups
In-Reply-To: <20260509102031.97608-3-zhangguopeng@kylinos.cn>
On 5/9/26 6:20 AM, Guopeng Zhang wrote:
> cpuset_can_attach() currently adds the bandwidth of all migrating
> SCHED_DEADLINE tasks to sum_migrate_dl_bw. If the source and destination
> cpuset effective CPU masks do not overlap, the whole sum is then
> reserved in the destination root domain.
>
> set_cpus_allowed_dl(), however, subtracts bandwidth from the source
> root domain only when the affinity change really moves the task between
> root domains. A DL task can move between cpusets that are still in the
> same root domain, so including that task in sum_migrate_dl_bw can reserve
> destination bandwidth without a matching source-side subtraction.
>
> Share the root-domain move test with set_cpus_allowed_dl(). Keep
> nr_migrate_dl_tasks counting all migrating deadline tasks for cpuset DL
> task accounting, but add to sum_migrate_dl_bw only for tasks that need a
> root-domain bandwidth move. Keep using the destination cpuset effective
> CPU mask and leave the broader can_attach()/attach() transaction model
> unchanged.
>
> Fixes: 2ef269ef1ac0 ("cgroup/cpuset: Free DL BW in case can_attach() fails")
> Signed-off-by: Guopeng Zhang <zhangguopeng@kylinos.cn>
> ---
> include/linux/sched/deadline.h | 9 +++++++++
> kernel/cgroup/cpuset-internal.h | 1 +
> kernel/cgroup/cpuset.c | 33 ++++++++++++++++++---------------
> kernel/sched/deadline.c | 13 ++++++++++---
> 4 files changed, 38 insertions(+), 18 deletions(-)
>
> diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h
> index 1198138cb839..273538200a44 100644
> --- a/include/linux/sched/deadline.h
> +++ b/include/linux/sched/deadline.h
> @@ -33,6 +33,15 @@ struct root_domain;
> extern void dl_add_task_root_domain(struct task_struct *p);
> extern void dl_clear_root_domain(struct root_domain *rd);
> extern void dl_clear_root_domain_cpu(int cpu);
> +/*
> + * Return whether moving DL task @p to @new_mask requires moving DL
> + * bandwidth accounting between root domains. This helper is specific to
> + * DL bandwidth move accounting semantics and is shared by
> + * cpuset_can_attach() and set_cpus_allowed_dl() so both paths use the
> + * same source root-domain test.
> + */
> +extern bool dl_task_needs_bw_move(struct task_struct *p,
> + const struct cpumask *new_mask);
>
> extern u64 dl_cookie;
> extern bool dl_bw_visited(int cpu, u64 cookie);
> diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h
> index bb4e692bea30..f7aaf01f7cd5 100644
> --- a/kernel/cgroup/cpuset-internal.h
> +++ b/kernel/cgroup/cpuset-internal.h
> @@ -167,6 +167,7 @@ struct cpuset {
> */
> int nr_deadline_tasks;
> int nr_migrate_dl_tasks;
> + /* DL bandwidth that needs destination reservation for this attach. */
> u64 sum_migrate_dl_bw;
> /*
> * CPU used for temporary DL bandwidth allocation during attach;
> diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
> index b9c839538900..23abfbbb4686 100644
> --- a/kernel/cgroup/cpuset.c
> +++ b/kernel/cgroup/cpuset.c
> @@ -2993,7 +2993,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
> struct cpuset *cs, *oldcs;
> struct task_struct *task;
> bool setsched_check;
> - int ret;
> + int cpu, ret;
>
> /* used later by cpuset_attach() */
> cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
> @@ -3038,28 +3038,31 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
> }
>
> if (dl_task(task)) {
> + /*
> + * Count all migrating DL tasks for cpuset task accounting.
> + * Only tasks that need a root-domain bandwidth move
> + * contribute to sum_migrate_dl_bw.
> + */
> cs->nr_migrate_dl_tasks++;
> - cs->sum_migrate_dl_bw += task->dl.dl_bw;
> + if (dl_task_needs_bw_move(task, cs->effective_cpus))
> + cs->sum_migrate_dl_bw += task->dl.dl_bw;
> }
> }
>
> - if (!cs->nr_migrate_dl_tasks)
> + if (!cs->sum_migrate_dl_bw)
> goto out_success;
>
> - if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) {
> - int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
> -
> - if (unlikely(cpu >= nr_cpu_ids)) {
> - ret = -EINVAL;
> - goto out_unlock;
> - }
> + cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
> + if (unlikely(cpu >= nr_cpu_ids)) {
> + ret = -EINVAL;
> + goto out_unlock;
> + }
>
> - ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
> - if (ret)
> - goto out_unlock;
> + ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
> + if (ret)
> + goto out_unlock;
>
> - cs->dl_bw_cpu = cpu;
> - }
> + cs->dl_bw_cpu = cpu;
>
> out_success:
> /*
> diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
> index edca7849b165..7db4c87df83b 100644
> --- a/kernel/sched/deadline.c
> +++ b/kernel/sched/deadline.c
> @@ -3107,20 +3107,18 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
> static void set_cpus_allowed_dl(struct task_struct *p,
> struct affinity_context *ctx)
> {
> - struct root_domain *src_rd;
> struct rq *rq;
>
> WARN_ON_ONCE(!dl_task(p));
>
> rq = task_rq(p);
> - src_rd = rq->rd;
> /*
> * Migrating a SCHED_DEADLINE task between exclusive
> * cpusets (different root_domains) entails a bandwidth
> * update. We already made space for us in the destination
> * domain (see cpuset_can_attach()).
> */
> - if (!cpumask_intersects(src_rd->span, ctx->new_mask)) {
> + if (dl_task_needs_bw_move(p, ctx->new_mask)) {
> struct dl_bw *src_dl_b;
>
> src_dl_b = dl_bw_of(cpu_of(rq));
> @@ -3137,6 +3135,15 @@ static void set_cpus_allowed_dl(struct task_struct *p,
> set_cpus_allowed_common(p, ctx);
> }
>
> +bool dl_task_needs_bw_move(struct task_struct *p,
> + const struct cpumask *new_mask)
> +{
> + if (!dl_task(p))
> + return false;
> +
> + return !cpumask_intersects(task_rq(p)->rd->span, new_mask);
> +}
> +
> /* Assumes rq->lock is held */
> static void rq_online_dl(struct rq *rq)
> {
LGTM
Reviewed-by: Waiman Long <longman@redhat.com>
^ permalink raw reply
* Re: [PATCH 2/3] mm/zswap: Implement proactive writeback
From: Yosry Ahmed @ 2026-05-11 19:57 UTC (permalink / raw)
To: Nhat Pham
Cc: Hao Jia, akpm, tj, hannes, shakeel.butt, mhocko, mkoutny,
chengming.zhou, muchun.song, roman.gushchin, cgroups, linux-mm,
linux-kernel, linux-doc, Hao Jia
In-Reply-To: <CAKEwX=PLFRkfUvZyaYfwBv0QJ-8KAktvZvGA02Hod04H-RsS-Q@mail.gmail.com>
On Mon, May 11, 2026 at 12:49 PM Nhat Pham <nphamcs@gmail.com> wrote:
>
> On Mon, May 11, 2026 at 3:52 AM Hao Jia <jiahao.kernel@gmail.com> wrote:
> >
> > From: Hao Jia <jiahao1@lixiang.com>
> >
> > Zswap currently writes back pages to backing swap devices reactively,
> > triggered either by memory pressure via the shrinker or by the pool
> > reaching its size limit. This reactive approach offers no precise
> > control over when writeback happens, which can disturb latency-sensitive
> > workloads, and it cannot direct writeback at a specific memory cgroup.
> > However, there are scenarios where users might want to proactively
> > write back cold pages from zswap to the backing swap device, for
> > example, to free up memory for other applications or to prepare for
> > upcoming memory-intensive workloads.
> >
> > Therefore, implement a proactive writeback mechanism for zswap by
> > adding a new cgroup interface file memory.zswap.proactive_writeback
> > within the memory controller.
>
>
> We already have memory.reclaim, no? Would that not work to create
> headroom generally for your use case? Is there a reason why we are
> treating zswap memory as special here?
+1, why do we need to specifically proactively reclaim the compressed memory?
Also, if we do need to minimize the compressed memory and force higher
writeback rates, we can do so with memory.zswap.max, right?
^ permalink raw reply
* Re: [PATCH 2/3] mm/zswap: Implement proactive writeback
From: Nhat Pham @ 2026-05-11 19:54 UTC (permalink / raw)
To: Hao Jia
Cc: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny,
chengming.zhou, muchun.song, roman.gushchin, cgroups, linux-mm,
linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260511105149.75584-3-jiahao.kernel@gmail.com>
On Mon, May 11, 2026 at 3:52 AM Hao Jia <jiahao.kernel@gmail.com> wrote:
>
> From: Hao Jia <jiahao1@lixiang.com>
>
> Zswap currently writes back pages to backing swap devices reactively,
> triggered either by memory pressure via the shrinker or by the pool
> reaching its size limit. This reactive approach offers no precise
> control over when writeback happens, which can disturb latency-sensitive
> workloads, and it cannot direct writeback at a specific memory cgroup.
> However, there are scenarios where users might want to proactively
> write back cold pages from zswap to the backing swap device, for
> example, to free up memory for other applications or to prepare for
> upcoming memory-intensive workloads.
>
> Therefore, implement a proactive writeback mechanism for zswap by
> adding a new cgroup interface file memory.zswap.proactive_writeback
> within the memory controller.
>
> Users can trigger writeback by writing to this file with the following
> parameters:
> - max=<bytes>: The maximum amount of memory to write back (optional,
> default: unlimited).
> - <age>: The minimum age of the pages to write back. Only pages that
> have been in zswap for at least this duration will be written back.
>
> Example usage:
> # Write back pages older than 1 hour (3600 seconds), max 10MB
> echo "max=10M 3600" > memory.zswap.proactive_writeback
>
> The implementation consists of:
> 1. Add store_time to struct zswap_entry to record when each entry was
> inserted into zswap, used for proactive writeback age comparison.
> 2. Introduce struct zswap_shrink_walk_arg, passed as the cb_arg to
> list_lru_walk_one() in both the shrinker and proactive paths. It
> carries the per-invocation cutoff_time and proactive flag down to
> shrink_memcg_cb(), and propagates the encountered_page_in_swapcache
> out-signal from the callback back to the caller.
> 3. Modify the callback function shrink_memcg_cb() to proactively
> writeback zswap_entries that meet the time threshold.
> 4. Add zswap_proactive_writeback() as the proactive writeback driver:
> a per-node batched list_lru_walk_one() loop bounded by the
> writeback budget.
>
> Signed-off-by: Hao Jia <jiahao1@lixiang.com>
> ---
> Documentation/admin-guide/cgroup-v2.rst | 24 ++++
> include/linux/zswap.h | 8 ++
> mm/memcontrol.c | 76 ++++++++++
> mm/zswap.c | 176 ++++++++++++++++++++++--
> 4 files changed, 276 insertions(+), 8 deletions(-)
>
> diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
> index 6efd0095ed99..05b664b3b3e8 100644
> --- a/Documentation/admin-guide/cgroup-v2.rst
> +++ b/Documentation/admin-guide/cgroup-v2.rst
> @@ -1908,6 +1908,30 @@ The following nested keys are defined.
> This setting has no effect if zswap is disabled, and swapping
> is allowed unless memory.swap.max is set to 0.
>
> + memory.zswap.proactive_writeback
> + A write-only nested-keyed file which exists in non-root cgroups.
> +
> + This interface allows proactive writeback of pages from the zswap
> + pool to the backing swap device. This is useful to offload cold
> + pages from the zswap pool to the slower swap device. It is only
> + available if zswap writeback is enabled.
> +
> + Users can trigger writeback by writing to this file with the following
> + parameters:
> +
> + - "max=<bytes>" : Optional. The maximum amount of data to write back.
> + (default: unlimited). Please note that the kernel can over or under
> + writeback this value.
> +
> + - "<age>" : Required. The minimum age of the pages to write back
> + (in seconds). Only pages that have been in the zswap pool for at
> + least this amount of time will be written back.
> +
> + Example::
> +
> + # Write back pages older than 1 hour (3600 seconds), max 10MB
> + echo "max=10M 3600" > memory.zswap.proactive_writeback
> +
> memory.pressure
> A read-only nested-keyed file.
>
> diff --git a/include/linux/zswap.h b/include/linux/zswap.h
> index efa6b551217e..7a51b4f95017 100644
> --- a/include/linux/zswap.h
> +++ b/include/linux/zswap.h
> @@ -44,6 +44,8 @@ void zswap_lruvec_state_init(struct lruvec *lruvec);
> void zswap_folio_swapin(struct folio *folio);
> bool zswap_is_enabled(void);
> bool zswap_never_enabled(void);
> +int zswap_proactive_writeback(struct mem_cgroup *root, unsigned long nr_max_writeback,
> + ktime_t cutoff);
> #else
>
> struct zswap_lruvec_state {};
> @@ -78,6 +80,12 @@ static inline bool zswap_never_enabled(void)
> return true;
> }
>
> +static inline int zswap_proactive_writeback(struct mem_cgroup *root,
> + unsigned long nr_max_writeback, ktime_t cutoff)
> +{
> + return 0;
> +}
> +
> #endif
>
> #endif /* _LINUX_ZSWAP_H */
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 409c41359dc8..ba7f7b1954a8 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -70,6 +70,7 @@
> #include "memcontrol-v1.h"
>
> #include <linux/uaccess.h>
> +#include <linux/parser.h>
>
> #define CREATE_TRACE_POINTS
> #include <trace/events/memcg.h>
> @@ -5891,6 +5892,76 @@ static ssize_t zswap_writeback_write(struct kernfs_open_file *of,
> return nbytes;
> }
>
> +enum {
> + ZSWAP_WRITEBACK_MAX,
> + ZSWAP_WRITEBACK_AGE,
> + ZSWAP_WRITEBACK_ERR,
> +};
> +
> +static const match_table_t zswap_writeback_tokens = {
> + { ZSWAP_WRITEBACK_MAX, "max=%s" },
> + { ZSWAP_WRITEBACK_AGE, "%u" },
> + { ZSWAP_WRITEBACK_ERR, NULL },
> +};
> +
> +static ssize_t zswap_proactive_writeback_write(struct kernfs_open_file *of,
> + char *buf, size_t nbytes,
> + loff_t off)
> +{
> + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
> + unsigned long nr_max_writeback = ULONG_MAX;
> + substring_t args[MAX_OPT_ARGS];
> + unsigned int age_sec;
> + bool age_set = false;
> + ktime_t cutoff_time;
> + char *token, *end;
> + int err;
> +
> + if (!mem_cgroup_zswap_writeback_enabled(memcg))
> + return -EINVAL;
> +
> + buf = strstrip(buf);
> +
> + while ((token = strsep(&buf, " ")) != NULL) {
> + if (!strlen(token))
> + continue;
> +
> + switch (match_token(token, zswap_writeback_tokens, args)) {
> + case ZSWAP_WRITEBACK_MAX:
> + nr_max_writeback = memparse(args[0].from, &end);
> + if (*end != '\0')
> + return -EINVAL;
> + nr_max_writeback >>= PAGE_SHIFT;
> + break;
> + case ZSWAP_WRITEBACK_AGE:
> + if (age_set)
> + return -EINVAL;
> +
> + if (match_uint(&args[0], &age_sec))
> + return -EINVAL;
> + age_set = true;
> + break;
> + default:
> + return -EINVAL;
> + }
> + }
> +
> + if (!age_set || !age_sec || !nr_max_writeback)
> + return -EINVAL;
> +
> + cutoff_time = ktime_sub(ktime_get_boottime(),
> + ns_to_ktime((u64)age_sec * NSEC_PER_SEC));
> + /* age_sec >= uptime: no entry can be that old, skip the walk. */
> + if (ktime_to_ns(cutoff_time) <= 0)
> + return nbytes;
> +
> + err = zswap_proactive_writeback(memcg, nr_max_writeback, cutoff_time);
> + if (err)
> + return err;
> +
> + return nbytes;
> +}
> +
> static struct cftype zswap_files[] = {
> {
> .name = "zswap.current",
> @@ -5908,6 +5979,11 @@ static struct cftype zswap_files[] = {
> .seq_show = zswap_writeback_show,
> .write = zswap_writeback_write,
> },
> + {
> + .name = "zswap.proactive_writeback",
> + .flags = CFTYPE_NOT_ON_ROOT,
> + .write = zswap_proactive_writeback_write,
> + },
> { } /* terminate */
> };
> #endif /* CONFIG_ZSWAP */
> diff --git a/mm/zswap.c b/mm/zswap.c
> index 19538d6f169a..1173ac6836fa 100644
> --- a/mm/zswap.c
> +++ b/mm/zswap.c
> @@ -36,6 +36,7 @@
> #include <linux/workqueue.h>
> #include <linux/list_lru.h>
> #include <linux/zsmalloc.h>
> +#include <linux/timekeeping.h>
>
> #include "swap.h"
> #include "internal.h"
> @@ -160,6 +161,12 @@ struct zswap_pool {
> char tfm_name[CRYPTO_MAX_ALG_NAME];
> };
>
> +struct zswap_shrink_walk_arg {
> + ktime_t cutoff_time;
> + bool proactive;
> + bool encountered_page_in_swapcache;
> +};
> +
> /* Global LRU lists shared by all zswap pools. */
> static struct list_lru zswap_list_lru;
>
> @@ -183,6 +190,7 @@ static struct shrinker *zswap_shrinker;
> * handle - zsmalloc allocation handle that stores the compressed page data
> * objcg - the obj_cgroup that the compressed memory is charged to
> * lru - handle to the pool's lru used to evict pages.
> + * store_time - Time when the entry was stored, for proactive writeback.
> */
> struct zswap_entry {
> swp_entry_t swpentry;
> @@ -192,6 +200,7 @@ struct zswap_entry {
> unsigned long handle;
> struct obj_cgroup *objcg;
> struct list_head lru;
> + ktime_t store_time;
On the implementation side - will this blow up struct zswap_entry
memory footprint? If so, can you guard this behind a CONFIG option, if
we are to go this route?
^ permalink raw reply
* Re: [PATCH 0/3] mm/zswap: Implement per-cgroup proactive writeback
From: Nhat Pham @ 2026-05-11 19:53 UTC (permalink / raw)
To: Hao Jia
Cc: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny,
chengming.zhou, muchun.song, roman.gushchin, cgroups, linux-mm,
linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260511105149.75584-1-jiahao.kernel@gmail.com>
On Mon, May 11, 2026 at 3:52 AM Hao Jia <jiahao.kernel@gmail.com> wrote:
>
> From: Hao Jia <jiahao1@lixiang.com>
>
> Zswap currently writes back pages to backing swap devices reactively,
> triggered either by memory pressure via the shrinker or by the pool
> reaching its size limit. However, this reactive approach makes writeback
> timing indeterminate and can disrupt latency-sensitive workloads when
> eviction happens to coincide with a critical execution window.
You can make the same argument about ordinary memory reclaim :) That's
why we have kswapd (asynchronous reclaim ahead of time) and proactive
reclaim solutions (memory.reclaim), which would all target zswap as
well.
>
> Furthermore, in certain scenarios, it is desirable to trigger writeback
> in advance to free up memory. For example, users may want to prepare for
> an upcoming memory-intensive workload by flushing cold memory to the
> backing storage when the system is relatively idle.
Would memory.reclaim not work here? Why are we treating zswap memory
footprint as special here, and spare file and anon?
^ permalink raw reply
* Re: [PATCH 2/3] mm/zswap: Implement proactive writeback
From: Nhat Pham @ 2026-05-11 19:49 UTC (permalink / raw)
To: Hao Jia
Cc: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny,
chengming.zhou, muchun.song, roman.gushchin, cgroups, linux-mm,
linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260511105149.75584-3-jiahao.kernel@gmail.com>
On Mon, May 11, 2026 at 3:52 AM Hao Jia <jiahao.kernel@gmail.com> wrote:
>
> From: Hao Jia <jiahao1@lixiang.com>
>
> Zswap currently writes back pages to backing swap devices reactively,
> triggered either by memory pressure via the shrinker or by the pool
> reaching its size limit. This reactive approach offers no precise
> control over when writeback happens, which can disturb latency-sensitive
> workloads, and it cannot direct writeback at a specific memory cgroup.
> However, there are scenarios where users might want to proactively
> write back cold pages from zswap to the backing swap device, for
> example, to free up memory for other applications or to prepare for
> upcoming memory-intensive workloads.
>
> Therefore, implement a proactive writeback mechanism for zswap by
> adding a new cgroup interface file memory.zswap.proactive_writeback
> within the memory controller.
We already have memory.reclaim, no? Would that not work to create
headroom generally for your use case? Is there a reason why we are
treating zswap memory as special here?
^ permalink raw reply
* Re: [PATCH v2 00/10] sched: Flatten the pick
From: Tejun Heo @ 2026-05-11 19:23 UTC (permalink / raw)
To: Peter Zijlstra
Cc: mingo, longman, chenridong, juri.lelli, vincent.guittot,
dietmar.eggemann, rostedt, bsegall, mgorman, vschneid, hannes,
mkoutny, cgroups, linux-kernel, jstultz, kprateek.nayak, qyousef
In-Reply-To: <20260511113104.563854162@infradead.org>
Hello, Peter.
On Mon, May 11, 2026 at 01:31:04PM +0200, Peter Zijlstra wrote:
> So cgroup scheduling has always been a pain in the arse. The problems start
> with weight distribution and end with hierachical picks and it all sucks.
>
> The problems with weight distribution are related to that infernal global
> fraction:
>
> tg->w * grq_i->w
> ge_i->w = ----------------
> \Sum_j grq_j->w
>
> which we've approximated reasonably well by now. However, the immediate
> consequence of this fraction is that the total group weight (tg->w) gets
> fragmented across all your CPUs. And at 64 CPUs that means your per-cpu cgroup
> weight ends up being a nice 19 task worth. And more CPUs more tiny. Combine
> with the fact that 256 CPU systems are relatively common these days, this
> becomes painful.
>
> The common 'solution' is to inflate the group weight by 'nr_cpus'; the
> immediate problem with that is that when all load of a group gets concentrated
> on a single CPU, the per-cpu cgroup weight becomes insanely large, easily
> exceeding nice -20.
>
> Additionally there are numerical limits on the max weight you can have before
> the math starts suffering overflows. As such there is a definite limit on the
> total group weight. Which has annoyed people ;-)
>
> The first few patches add a knob /debug/sched/cgroup_mode and a few different
> options on how to deal with this. My favourite is 'concur', but obviously that
> is also the most expensive one :-/ It adds a tg->tasks counter which makes the
> update_tg_load_avg() thing more expensive.
Ignoring fixed math accuracy problems, isn't the root problem here that
every thread in the root cgroup competes as if each is its own cgroup? ie.
Isn't the canonical solution here to create an enveloping group, at least
for share calculation purposes, for root threads and then assign them some
weight so that they compete in the same way that other cgroups do? Then, the
different modes go away or rather whatever the user wants can be expressed
via root's weight if that's to be made configurable.
Thanks.
--
tejun
^ permalink raw reply
* Re: [RFC PATCH v5 20/29] sched/deadline: Allow deeper hierarchies of RT cgroups
From: Tejun Heo @ 2026-05-11 18:28 UTC (permalink / raw)
To: luca abeni
Cc: Peter Zijlstra, Yuri Andriaccio, Ingo Molnar, Juri Lelli,
Vincent Guittot, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, linux-kernel, Yuri Andriaccio,
hannes, mkoutny, cgroups
In-Reply-To: <20260507163058.2c435922@nowhere>
Hello,
On Thu, May 07, 2026 at 04:30:58PM +0200, luca abeni wrote:
...
> Just to better understand: would it make sense to allow non-{FIFO,RT}
> tasks to be in non-leaf cgroups (as allowed by the threaded CPU
> controller), while enforcing that FIFO/RR tasks can only be in leaf
> cgroups? Or would this be a hack that compromises the rt-CPU controller
> usefulness?
Code-wise, sure, but I don't think an interface like that would be a good
one. From user's pov, this amounts to adding restrictions on both whether a
controller can be enabled and whether tasks can be moved into some cgroups.
UNIX error reporting being what it is, this would come down to getting
-EINVAL or -EBUSY or whatever out of those operations. I don't think it's a
good idea to add subtle failure modes to these already pretty complex (but
currently w/ clearly-defined shared rules) operations. To users, this would
look like random arbitrary failures that are nearly impossible to decode
without tracing code.
If you want to enforce no-internal-threads, separating it out to its own
controller that doesn't support threaded mode would be the right direction.
Note that the only hard requirement here is that you don't want to get in
the way for people who are NOT interested in threaded rt control. If you
block enabling CPU control for e.g. cpu.max or block thread migration into a
cgroup, you'd be in the way; however, if all you say is "I don't support
sub-allocation in threaded mode" and e.g just fail writes to the knobs in
threaded cgroups, that does not get in the way. So, it's not like you *have*
to support full threaded mode. You just need to avoid hindering non-rt
operations.
> Yes, we have a bad default here.
> Would a default like "allow running FIFO/RR tasks without runtime
> enforcement" (this is what happens to FIFO/RR tasks running in the root
> control group) be acceptable?
Yes, if you can express that in a reasonable way in the config knobs, that'd
likely be an easier way. I don't know how to transition from
allowed-by-default to explicitly-allocated in such interface tho. Making
that reasonable and smooth would be the key factor in whether such approach
can be taken.
Thanks.
--
tejun
^ permalink raw reply
* Re: [RFC PATCH v5 20/29] sched/deadline: Allow deeper hierarchies of RT cgroups
From: Tejun Heo @ 2026-05-11 18:15 UTC (permalink / raw)
To: luca abeni
Cc: Peter Zijlstra, Yuri Andriaccio, Ingo Molnar, Juri Lelli,
Vincent Guittot, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, linux-kernel, Yuri Andriaccio,
hannes, mkoutny, cgroups
In-Reply-To: <20260511114004.0fd8d54c@luca64>
On Mon, May 11, 2026 at 11:40:04AM +0200, luca abeni wrote:
> We are discussing this issue with Yuri, and we have a doubt: if we
> disable the RT-CPU controller for a cgroup, would it be possible to
> enable it for its children?
> (In other words: if we want the RT-CPU controller to be enabled for
> some "leaf" cgroups, we need to enable it for their parents, right?)
Yeah, a cgroup has a controller available to it iff its parent enables that
controller, so all ancestors would have to enable it.
Thanks.
--
tejun
^ permalink raw reply
* Re: [PATCH 1/3] cgroup/cpuset: Fix deadline bandwidth leak in cpuset_can_attach()
From: Waiman Long @ 2026-05-11 17:54 UTC (permalink / raw)
To: Aaron Tomlin
Cc: tsbogend, paul, jmorris, serge, mingo, peterz, juri.lelli,
vincent.guittot, stephen.smalley.work, casey, tj, hannes, mkoutny,
chenridong, dietmar.eggemann, rostedt, bsegall, mgorman, vschneid,
kprateek.nayak, omosnace, kees, neelx, sean, chjohnst, steve,
mproche, nick.lange, cgroups, linux-mips, linux-fsdevel,
linux-security-module, selinux, linux-kernel
In-Reply-To: <aihz6zlfmcaxwb3ef4luisfpwqibwsajpphy5vzuksy3ftfkms@whhv2ax5plpb>
On 5/11/26 7:08 AM, Aaron Tomlin wrote:
> On Mon, May 11, 2026 at 01:10:02AM -0400, Waiman Long wrote:
>> On 5/9/26 12:48 PM, Aaron Tomlin wrote:
>>> During a cgroup migration, cpuset_can_attach() iterates over the
>>> provided taskset. If a task within the batch is a deadline (DL) task,
>>> the destination cpuset's DL metrics (i.e., nr_migrate_dl_tasks and
>>> sum_migrate_dl_bw) are appropriately incremented.
>>>
>>> However, if a subsequent task in the same migration batch fails the
>>> task_can_attach() check, the loop aborts and jumps directly to
>>> out_unlock. Consequently, any DL metrics accumulated from previously
>>> processed tasks in the batch remain permanently inflated in the
>>> destination cpuset. Because the migration is subsequently aborted by the
>>> cgroup core, cpuset_cancel_attach() is never invoked to unwind these
>>> specific increments.
>>>
>>> This behaviour results in a permanent leak of deadline bandwidth, which
>>> incorrectly restricts the admission control capacity of the destination
>>> cpuset.
>>>
>>> To resolve this, introduce an out_unlock_reset failure path that
>>> conditionally invokes reset_migrate_dl_data(). This guarantees that if a
>>> batch migration is aborted for any reason, the pending DL metrics are
>>> safely reset before returning the error.
>>>
>>> Fixes: 0a67b847e1f06 ("cpuset: Allow setscheduler regardless of manipulated task")
>> That is not the commit that introduced the bug. Anyway, there is already
>> another patch sent recently to fix this bug. See
>>
>> https://lore.kernel.org/lkml/20260509102031.97608-2-zhangguopeng@kylinos.cn/
>>
> Hi Waiman,
>
> Thank you for the follow up.
>
> Acknowledged. I will drop this patch in the next iteration due to [1].
>
> Please note, the sashiko AI Review bot reported: cpuset_can_attach()
> incorrectly assumes all migrating tasks originate from the same source
> cpuset. At first glance, this feedback is valid. I plan to submit a patch,
> if no solution was already proposed.
>
> [1]: https://lore.kernel.org/lkml/20260509102031.97608-2-zhangguopeng@kylinos.cn/
Yes, it does look like the AI feedback is valid. I will take a further
look into this.
Thanks,
Longman
^ permalink raw reply
* Re: [RFC PATCH v5 20/29] sched/deadline: Allow deeper hierarchies of RT cgroups
From: Tejun Heo @ 2026-05-11 17:52 UTC (permalink / raw)
To: Juri Lelli
Cc: luca abeni, Peter Zijlstra, Yuri Andriaccio, Ingo Molnar,
Vincent Guittot, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, linux-kernel, Yuri Andriaccio,
hannes, mkoutny, cgroups
In-Reply-To: <agGhi9_SG6vRnDVq@jlelli-thinkpadt14gen4.remote.csb>
Hello,
On Mon, May 11, 2026 at 11:29:47AM +0200, Juri Lelli wrote:
...
> While I like the automatic approach, I also fear that it might be more
> difficult to maintain/use from a systemd admin perspective, e.g. I
> cannot make a subgroup reservation bigger because there are threads
> running in the parent group which consume all the remaining (internal)
> bandwidth. If we make it explicit it seems easier to see where bandwidth
> is allocated at all levels.
>
> Peter? Tejun? What do we want to do with this interface?
blkcg on cgroup1 did soemthing similar for a while. It had a separate subdir
for knobs that apply to "internal threads". Effectivley, this becomes
creating a separate controller group for every cgroup as a sibling to its
children. It does work obviously but it is pretty ugly and unintuitive, both
in interface and implementation, and I'm skeptical this was actually useful
in any meaningful way. Nobody complained when we ripped it out.
If rt were to become its own cgroup controller, maybe one can just side-step
this by not supporting threaded mode at least at the beginning. If people
ask for it, hopefully we'll be able to develop better understanding of their
usecases and drive design that way. In practice, I don't think threaded mode
gets used all that much because usually only application processes
themselves know about their own threads, are not in the business of creating
their own cgroups (delegation to each application isn't common), and have
other ways of controlling their own threads. So, there's some chance that
this may not actually come up.
If rt stays as a part of cpu controller, my preference would be keeping the
config implicit for threaded mode at least at the beginning. ie. Don't get
in the way of people using threaded mode by blocking it but having some
reasonable and clear default (e.g. internal tasks have priority as suggested
or internal tasks get whatever is left over which may make more sense in the
allocation model) may be sufficient. If not, like in the other case, we can
make specific design decisions based on concrete use cases later.
Thanks.
--
tejun
^ permalink raw reply
* Re: [RFC PATCH v5 20/29] sched/deadline: Allow deeper hierarchies of RT cgroups
From: Tejun Heo @ 2026-05-11 17:37 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Yuri Andriaccio, Ingo Molnar, Juri Lelli, Vincent Guittot,
Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, linux-kernel, Luca Abeni, Yuri Andriaccio,
hannes, mkoutny, cgroups
In-Reply-To: <20260507105331.GQ1026330@noisy.programming.kicks-ass.net>
Hello, Peter.
On Thu, May 07, 2026 at 12:53:31PM +0200, Peter Zijlstra wrote:
...
> Looking at cpu_period_quota_parse() this thing takes two u64 values for:
> {runtime, period} but allows runtime to be the string "max".
>
> I think we'd want an optional extension to that and allow 3 values for:
> {runtime, period, deadline}, where if the deadline is not given, it will
> be the same as period.
Yeah, I don't know what's needed here but extending the interface as
necessary is completely fine.
> Right... this then means we need two controls, one to do hierarchical
> bandwidth distribution, and one to assign bandwidth to the internal
> group -- which is then subject to its own bandwidth distribution
> constraint.
>
> This might be a little confusing, but there is no way around that
> AFAICT.
Separating out the rt as a separate controller is one way and if the
configuration wants to stick to strict allocation model where nothing is
available by default unless explicitly allocated, this would be the only
way. Interface-wise, I think this is going to be fine but I suspect this
likely would complicated internal implementation quite a bit as now rt can't
piggyback on existing sched core cgroup infra - no task_group or
synchronization built around them - and has to build everything on its own.
It's not the end of the world but not ideal either.
> > - This has the same problem with cgroup1's rt cgroup sched support where
> > there is no way to have a permissive default configuration, which means
> > that users who don't really care about distributing rt shares
> > hierarchically would get blocked from running rt processes by default,
> > which basically forces distros to disable rt cgroup sched support. This is
> > not new but it'd be a shame to put in all the work and the end result is
> > that most people don't even have access to the feature.
>
> Right, but cgroup-v2 allows enabling/disabling specific controllers for
> a (sub)-hierarchy, right? So if the controller is not enabled (by
> default), it will fall back to putting the tasks in whatever parent does
> have it on, and by default the root group would have and would accept
> tasks.
>
> Additionally, I think we want a flag to allow non-priv tasks to use RT
> inside the controller -- after all, these tasks would be subject to
> strict bandwidth controls and cannot burn the system like unbounded/root
> FIFO tasks can.
>
> Does that all sound workable?
Yeah, if rt becomes its own controller, I don't see any fundamental
roadblocks. It'd involve a bunch of churn which may add to maintenance
overhead but it should work. An alternative would be coming up with some way
to express the default no-enforcement state through the config knobs. I'm
sure this would be doable too and if folks can figure out a reasonable
interface, it should be able to obtain basically the same functionality with
a lot less code.
Thanks.
--
tejun
^ permalink raw reply
* [PATCH v3 5/5] drm/amdgpu: Wire up dmem cgroup reclaim for VRAM manager
From: Thomas Hellström @ 2026-05-11 17:30 UTC (permalink / raw)
To: intel-xe
Cc: Thomas Hellström, Natalie Vock, Johannes Weiner, Tejun Heo,
Michal Koutný, cgroups, Huang Rui, Matthew Brost,
Matthew Auld, Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Simona Vetter, David Airlie, Christian König, Alex Deucher,
Rodrigo Vivi, dri-devel, amd-gfx, linux-kernel
In-Reply-To: <20260511173008.36526-1-thomas.hellstrom@linux.intel.com>
Register the VRAM manager with the dmem cgroup reclaim infrastructure
so that lowering dmem.max below current VRAM usage triggers TTM
eviction rather than failing with -EBUSY.
Guard place->flags in amdgpu_ttm_bo_eviction_valuable() against NULL,
as the TTM reclaim path passes a NULL place in cgroup drain mode.
v3:
- Rebased on fix for uninitialized list and buddy allocator on the
drmm_cgroup_register_region() error path.
Assisted-by: GitHub_Copilot:claude-sonnet-4.6
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 9 ++++++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 0dc68fb9d88e..334a177ae8d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1485,7 +1485,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
dma_resv_for_each_fence(&resv_cursor, bo->base.resv,
DMA_RESV_USAGE_BOOKKEEP, f) {
if (amdkfd_fence_check_mm(f, current->mm) &&
- !(place->flags & TTM_PL_FLAG_CONTIGUOUS))
+ !(place && (place->flags & TTM_PL_FLAG_CONTIGUOUS)))
return false;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index ac3f71d77140..a1f1ae264a40 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -916,6 +916,7 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
{
struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
struct ttm_resource_manager *man = &mgr->manager;
+ struct dmem_cgroup_region *cg;
int err;
ttm_resource_manager_init(man, &adev->mman.bdev,
@@ -932,9 +933,11 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
if (err)
return err;
- man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size);
- if (IS_ERR(man->cg))
- return PTR_ERR(man->cg);
+ cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram",
+ adev->gmc.real_vram_size);
+ if (IS_ERR(cg))
+ return PTR_ERR(cg);
+ ttm_resource_manager_set_dmem_region(man, cg);
ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
ttm_resource_manager_set_used(man, true);
--
2.54.0
^ permalink raw reply related
* [PATCH v3 4/5] drm/xe: Wire up dmem cgroup reclaim for VRAM manager
From: Thomas Hellström @ 2026-05-11 17:30 UTC (permalink / raw)
To: intel-xe
Cc: Thomas Hellström, Natalie Vock, Johannes Weiner, Tejun Heo,
Michal Koutný, cgroups, Huang Rui, Matthew Brost,
Matthew Auld, Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Simona Vetter, David Airlie, Christian König, Alex Deucher,
Rodrigo Vivi, dri-devel, amd-gfx, linux-kernel
In-Reply-To: <20260511173008.36526-1-thomas.hellstrom@linux.intel.com>
Register the VRAM manager with the dmem cgroup reclaim infrastructure
so that lowering dmem.max below current VRAM usage triggers TTM
eviction rather than failing with -EBUSY.
Assisted-by: GitHub_Copilot:claude-sonnet-4.6
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 5fd0d5506a7e..1bdcb3fee901 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -303,13 +303,6 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
struct ttm_resource_manager *man = &mgr->manager;
int err;
- if (mem_type != XE_PL_STOLEN) {
- const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1";
- man->cg = drmm_cgroup_register_region(&xe->drm, name, size);
- if (IS_ERR(man->cg))
- return PTR_ERR(man->cg);
- }
-
man->func = &xe_ttm_vram_mgr_func;
mgr->mem_type = mem_type;
mutex_init(&mgr->lock);
@@ -318,6 +311,18 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
mgr->visible_avail = io_size;
ttm_resource_manager_init(man, &xe->ttm, size);
+
+ if (mem_type != XE_PL_STOLEN) {
+ const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1";
+ struct dmem_cgroup_region *cg =
+ drmm_cgroup_register_region(&xe->drm, name, size);
+
+ if (IS_ERR(cg))
+ return PTR_ERR(cg);
+
+ ttm_resource_manager_set_dmem_region(man, cg);
+ }
+
err = gpu_buddy_init(&mgr->mm, man->size, default_page_size);
if (err)
return err;
--
2.54.0
^ permalink raw reply related
* [PATCH v3 3/5] drm/ttm: Hook up a cgroup-aware reclaim callback for the dmem controller
From: Thomas Hellström @ 2026-05-11 17:30 UTC (permalink / raw)
To: intel-xe
Cc: Thomas Hellström, Natalie Vock, Johannes Weiner, Tejun Heo,
Michal Koutný, cgroups, Huang Rui, Matthew Brost,
Matthew Auld, Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Simona Vetter, David Airlie, Christian König, Alex Deucher,
Rodrigo Vivi, dri-devel, amd-gfx, linux-kernel
In-Reply-To: <20260511173008.36526-1-thomas.hellstrom@linux.intel.com>
Add ttm_bo_evict_cgroup() to evict buffer objects charged to a specific
dmem cgroup pool from a resource manager's LRU until a byte target is
met. Add ttm_resource_manager_set_dmem_region() to register the TTM
eviction path as the reclaim callback for a dmem cgroup region.
The eviction context is interruptible; signals abort the operation and
propagate back through the write() syscall.
Introduce a new mode for the bo LRU walker so that sleeping locks
can be taken. This can be used when the caller doesn't hold any
previous dma_resv locks, and where it intends to hold at most
one lock at a time.
Like the rest of the TTM eviction this should sooner than later
be converted to full WW transactions.
v3:
- Fix ttm_resource_manager_set_dmem_region() storing an error pointer
in man->cg unconditionally. (Sashiko-bot)
- Fix kernel-doc function name format for ttm_bo_evict_cgroup() and
ttm_resource_manager_set_dmem_region().
Assisted-by: GitHub_Copilot:claude-sonnet-4.6
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/ttm_bo.c | 95 +++++++++++++++++++++++++++++-
drivers/gpu/drm/ttm/ttm_bo_util.c | 3 +-
drivers/gpu/drm/ttm/ttm_resource.c | 37 ++++++++++++
include/drm/ttm/ttm_bo.h | 10 ++++
include/drm/ttm/ttm_resource.h | 4 ++
5 files changed, 145 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index d85f0a37ac35..249d626dc061 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -515,12 +515,20 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *
{
struct ttm_bo_evict_walk *evict_walk =
container_of(walk, typeof(*evict_walk), walk);
+ /* Capture size before eviction in case res is cleared. */
+ s64 bo_size = bo->base.size;
s64 lret;
if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css,
evict_walk->try_low, &evict_walk->hit_low))
return 0;
+ /*
+ * evict_walk->place is NULL in cgroup drain mode. Drivers'
+ * eviction_valuable() callbacks must handle a NULL place, treating it
+ * as "any placement": the TTM base implementation already does so via
+ * ttm_resource_intersects().
+ */
if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place))
return 0;
@@ -536,11 +544,15 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *
goto out;
evict_walk->evicted++;
- if (evict_walk->res)
+ if (evict_walk->res) {
lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place,
evict_walk->res, NULL);
- if (lret == 0)
- return 1;
+ if (lret == 0)
+ return 1;
+ } else {
+ /* Cgroup drain: return bytes freed for byte-denominated progress. */
+ return bo_size;
+ }
out:
/* Errors that should terminate the walk. */
if (lret == -ENOSPC)
@@ -614,6 +626,83 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev,
return 0;
}
+/**
+ * ttm_bo_evict_cgroup() - Evict buffer objects charged to a specific cgroup.
+ * @bdev: The TTM device.
+ * @man: The resource manager whose LRU to walk.
+ * @limit_pool: The cgroup pool state whose members should be evicted.
+ * @target_bytes: Number of bytes to free.
+ * @ctx: The TTM operation context.
+ *
+ * Walk the LRU of @man and evict buffer objects that are charged to the
+ * cgroup identified by @limit_pool, until at least @target_bytes have been
+ * freed. Mirrors the two-pass (trylock -> sleeping-lock, low-watermark)
+ * strategy used by ttm_bo_evict_alloc().
+ *
+ * Return: >= @target_bytes on full success, 0..target_bytes-1 if partial,
+ * negative error code on fatal error.
+ */
+s64 ttm_bo_evict_cgroup(struct ttm_device *bdev,
+ struct ttm_resource_manager *man,
+ struct dmem_cgroup_pool_state *limit_pool,
+ s64 target_bytes,
+ struct ttm_operation_ctx *ctx)
+{
+ struct ttm_bo_evict_walk evict_walk = {
+ .walk = {
+ .ops = &ttm_evict_walk_ops,
+ .arg = { .ctx = ctx },
+ },
+ .limit_pool = limit_pool,
+ /* place, evictor, res left NULL: selects cgroup drain mode */
+ };
+ s64 lret, pass;
+
+ evict_walk.walk.arg.trylock_only = true;
+ lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, target_bytes);
+ if (lret < 0 || lret >= target_bytes)
+ return lret;
+
+ /* Second pass: also evict BOs at the low watermark. */
+ if (evict_walk.hit_low) {
+ evict_walk.try_low = true;
+ pass = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man,
+ target_bytes - lret);
+ if (pass < 0)
+ return pass;
+ lret += pass;
+ if (lret >= target_bytes)
+ return lret;
+ }
+
+ /* Full sleeping-lock pass for remaining target. */
+ evict_walk.try_low = evict_walk.hit_low = false;
+ evict_walk.walk.arg.trylock_only = false;
+
+retry:
+ evict_walk.walk.arg.sleeping_lock = true;
+ do {
+ evict_walk.evicted = 0;
+ pass = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man,
+ target_bytes - lret);
+ if (pass < 0) {
+ lret = pass;
+ goto out;
+ }
+ lret += pass;
+ } while (lret < target_bytes && evict_walk.evicted);
+
+ /* One more attempt if we hit the low limit during sleeping-lock pass. */
+ if (lret < target_bytes && evict_walk.hit_low && !evict_walk.try_low) {
+ evict_walk.try_low = true;
+ goto retry;
+ }
+
+out:
+ return lret;
+}
+EXPORT_SYMBOL(ttm_bo_evict_cgroup);
+
/**
* ttm_bo_pin - Pin the buffer object.
* @bo: The buffer object to pin
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index f83b7d5ec6c6..81c6a674c462 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -999,7 +999,8 @@ __ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs)
bo = res->bo;
if (ttm_lru_walk_trylock(curs, bo))
bo_locked = true;
- else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only)
+ else if ((!arg->ticket && !arg->sleeping_lock) || arg->ctx->no_wait_gpu ||
+ arg->trylock_only)
continue;
if (!ttm_bo_get_unless_zero(bo)) {
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 9f36631d48b6..6867ada16545 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -937,3 +937,40 @@ void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
#endif
}
EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
+
+static int ttm_resource_manager_dmem_reclaim(struct dmem_cgroup_pool_state *pool,
+ u64 target_bytes, void *priv)
+{
+ struct ttm_resource_manager *man = priv;
+ struct ttm_operation_ctx ctx = { .interruptible = true };
+ s64 freed;
+
+ freed = ttm_bo_evict_cgroup(man->bdev, man, pool, target_bytes, &ctx);
+ if (freed < 0)
+ return freed;
+
+ return freed >= (s64)target_bytes ? 0 : -ENOSPC;
+}
+
+/**
+ * ttm_resource_manager_set_dmem_region() - Associate a dmem cgroup region with a
+ * resource manager and register a reclaim
+ * callback.
+ * @man: The resource manager.
+ * @region: The dmem cgroup region to associate, may be NULL or IS_ERR().
+ *
+ * Sets @man->cg and registers ttm_resource_manager_dmem_reclaim() so that
+ * writing to dmem.max below current usage triggers TTM eviction rather than
+ * returning -EBUSY to userspace.
+ */
+void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man,
+ struct dmem_cgroup_region *region)
+{
+ if (!IS_ERR_OR_NULL(region)) {
+ man->cg = region;
+ dmem_cgroup_region_set_reclaim(region,
+ ttm_resource_manager_dmem_reclaim,
+ man);
+ }
+}
+EXPORT_SYMBOL(ttm_resource_manager_set_dmem_region);
diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h
index 8310bc3d55f9..32791c4db2a9 100644
--- a/include/drm/ttm/ttm_bo.h
+++ b/include/drm/ttm/ttm_bo.h
@@ -226,6 +226,11 @@ struct ttm_lru_walk_arg {
struct ww_acquire_ctx *ticket;
/** @trylock_only: Only use trylock for locking. */
bool trylock_only;
+ /**
+ * @sleeping_lock: Use sleeping locks even with %NULL @ticket.
+ * @trylock_only has precedence over this field.
+ */
+ bool sleeping_lock;
};
/**
@@ -431,6 +436,11 @@ void ttm_bo_unpin(struct ttm_buffer_object *bo);
int ttm_bo_evict_first(struct ttm_device *bdev,
struct ttm_resource_manager *man,
struct ttm_operation_ctx *ctx);
+s64 ttm_bo_evict_cgroup(struct ttm_device *bdev,
+ struct ttm_resource_manager *man,
+ struct dmem_cgroup_pool_state *limit_pool,
+ s64 target_bytes,
+ struct ttm_operation_ctx *ctx);
int ttm_bo_access(struct ttm_buffer_object *bo, unsigned long offset,
void *buf, int len, int write);
vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index 33e80f30b8b8..c187e6c8b871 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -39,6 +39,7 @@
struct dentry;
struct dmem_cgroup_device;
+struct dmem_cgroup_region;
struct drm_printer;
struct ttm_device;
struct ttm_resource_manager;
@@ -475,6 +476,9 @@ void ttm_resource_manager_init(struct ttm_resource_manager *man,
struct ttm_device *bdev,
uint64_t size);
+void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man,
+ struct dmem_cgroup_region *region);
+
int ttm_resource_manager_evict_all(struct ttm_device *bdev,
struct ttm_resource_manager *man);
--
2.54.0
^ permalink raw reply related
* [PATCH v3 2/5] cgroup/dmem: Add reclaim callback for lowering max below current usage
From: Thomas Hellström @ 2026-05-11 17:30 UTC (permalink / raw)
To: intel-xe
Cc: Thomas Hellström, Natalie Vock, Johannes Weiner, Tejun Heo,
Michal Koutný, cgroups, Huang Rui, Matthew Brost,
Matthew Auld, Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Simona Vetter, David Airlie, Christian König, Alex Deucher,
Rodrigo Vivi, dri-devel, amd-gfx, linux-kernel
In-Reply-To: <20260511173008.36526-1-thomas.hellstrom@linux.intel.com>
Add an optional reclaim callback to struct dmem_cgroup_region. When
dmem.max is set below the current usage of a cgroup pool, the new limit
is applied immediately (so that concurrent allocations are throttled
while reclaim is in progress) and then the driver is asked to evict
memory to bring usage back below the limit.
Reclaim is attempted up to a bounded number of times. No error is
returned to userspace if usage remains above the limit after reclaim,
and a pending signal will abort the reclaim loop early. This matches
the behavior of memory.max in the memory cgroup controller.
Also honor O_NONBLOCK so that if that flag is set during the
max value write, no reclaim is initiated. The idea is to avoid
charging the reclaim cost to the writer of the max value.
v2:
- Write max before reclaim is attempted (Maarten)
- Let signals abort the reclaim without error (Maarten)
- If a new max value is written with the O_NONBLOCK flag,
reclaim is not attempted (Maarten)
- Extract region from the pool parameter rather than
passing it explicitly to set_resource_xxx().
v3:
- Use an rwsem to protect reclaim callback registration and
region unregister against concurrent reclaim invocations,
ensuring reclaim_priv is visible when the callback is
invoked. (Sashiko-bot)
Assisted-by: GitHub_Copilot:claude-sonnet-4.6
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
include/linux/cgroup_dmem.h | 24 ++++++++
kernel/cgroup/dmem.c | 106 +++++++++++++++++++++++++++++++++---
2 files changed, 121 insertions(+), 9 deletions(-)
diff --git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h
index dd4869f1d736..c3bce21cbe80 100644
--- a/include/linux/cgroup_dmem.h
+++ b/include/linux/cgroup_dmem.h
@@ -14,6 +14,21 @@ struct dmem_cgroup_pool_state;
/* Opaque definition of a cgroup region, used internally */
struct dmem_cgroup_region;
+/**
+ * typedef dmem_cgroup_reclaim_fn_t - Reclaim callback for a dmem cgroup region.
+ * @pool: The cgroup pool that needs memory reclaimed.
+ * @target_bytes: Minimum number of bytes the driver should attempt to free.
+ * @priv: Private data registered with dmem_cgroup_region_set_reclaim().
+ *
+ * Called by the dmem cgroup controller when dmem.max is set below the current
+ * usage of @pool. The driver should evict at least @target_bytes of memory
+ * from @pool. May be called multiple times if usage remains above the limit.
+ *
+ * Return: 0 if progress was made, negative error code otherwise.
+ */
+typedef int (*dmem_cgroup_reclaim_fn_t)(struct dmem_cgroup_pool_state *pool,
+ u64 target_bytes, void *priv);
+
#if IS_ENABLED(CONFIG_CGROUP_DMEM)
struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) __printf(2,3);
void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region);
@@ -26,6 +41,9 @@ bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
bool ignore_low, bool *ret_hit_low);
void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool);
+void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
+ dmem_cgroup_reclaim_fn_t reclaim,
+ void *priv);
#else
static inline __printf(2,3) struct dmem_cgroup_region *
dmem_cgroup_register_region(u64 size, const char *name_fmt, ...)
@@ -62,5 +80,11 @@ bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
static inline void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
{ }
+static inline void
+dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
+ dmem_cgroup_reclaim_fn_t reclaim,
+ void *priv)
+{ }
+
#endif
#endif /* _CGROUP_DMEM_H */
diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
index 1ab1fb47f271..5fd5a1634d21 100644
--- a/kernel/cgroup/dmem.c
+++ b/kernel/cgroup/dmem.c
@@ -51,6 +51,20 @@ struct dmem_cgroup_region {
* No new pools should be added to the region afterwards.
*/
bool unregistered;
+
+ /**
+ * @reclaim: Optional callback invoked when dmem.max is set below the
+ * current usage of a pool. The driver should attempt to free at least
+ * @target_bytes from @pool. May be called multiple times if usage
+ * remains above the limit after returning.
+ */
+ dmem_cgroup_reclaim_fn_t reclaim;
+
+ /** @reclaim_priv: Private data passed to @reclaim. */
+ void *reclaim_priv;
+
+ /** @unregister_sem: Protect @reclaim while it is running. */
+ struct rw_semaphore unregister_sem;
};
struct dmemcg_state {
@@ -145,21 +159,58 @@ static void free_cg_pool(struct dmem_cgroup_pool_state *pool)
}
static void
-set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val)
+set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
{
page_counter_set_min(&pool->cnt, val);
}
static void
-set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val)
+set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
{
page_counter_set_low(&pool->cnt, val);
}
static void
-set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val)
+set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
{
- page_counter_set_max(&pool->cnt, val);
+ struct dmem_cgroup_region *region = pool->region;
+
+ /*
+ * Always update the limit, even if usage currently exceeds it.
+ * Concurrent allocations will be throttled against the new limit
+ * while reclaim is in progress.
+ */
+ xchg(&pool->cnt.max, (unsigned long)val);
+
+ if (nonblock || !READ_ONCE(region->reclaim))
+ return;
+
+ for (int retries = 5; retries > 0; retries--) {
+ u64 usage = page_counter_read(&pool->cnt);
+ int ret;
+
+ if (usage <= val)
+ break;
+
+ if (signal_pending(current))
+ break;
+
+ /* Block unregister until the reclaim callback completes. */
+ if (down_read_interruptible(®ion->unregister_sem))
+ break;
+
+ if (!region->reclaim) {
+ up_read(®ion->unregister_sem);
+ break;
+ }
+
+ ret = region->reclaim(pool, usage - val, region->reclaim_priv);
+ up_read(®ion->unregister_sem);
+ if (ret)
+ break;
+
+ cond_resched();
+ }
}
static u64 get_resource_low(struct dmem_cgroup_pool_state *pool)
@@ -184,9 +235,9 @@ static u64 get_resource_current(struct dmem_cgroup_pool_state *pool)
static void reset_all_resource_limits(struct dmem_cgroup_pool_state *rpool)
{
- set_resource_min(rpool, 0);
- set_resource_low(rpool, 0);
- set_resource_max(rpool, PAGE_COUNTER_MAX);
+ set_resource_min(rpool, 0, false);
+ set_resource_low(rpool, 0, false);
+ set_resource_max(rpool, PAGE_COUNTER_MAX, false);
}
static void dmemcs_offline(struct cgroup_subsys_state *css)
@@ -491,6 +542,12 @@ void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region)
region->unregistered = true;
spin_unlock(&dmemcg_lock);
+ /* Ensure all reclaim() callbacks have finished. */
+ down_write(®ion->unregister_sem);
+ /* Pairs with READ_ONCE() in set_resource_max() */
+ WRITE_ONCE(region->reclaim, NULL);
+ up_write(®ion->unregister_sem);
+
kref_put(®ion->ref, dmemcg_free_region);
}
EXPORT_SYMBOL_GPL(dmem_cgroup_unregister_region);
@@ -530,6 +587,7 @@ struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *fmt
INIT_LIST_HEAD(&ret->pools);
ret->name = region_name;
ret->size = size;
+ init_rwsem(&ret->unregister_sem);
kref_init(&ret->ref);
spin_lock(&dmemcg_lock);
@@ -568,6 +626,34 @@ void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
}
EXPORT_SYMBOL_GPL(dmem_cgroup_pool_state_put);
+/**
+ * dmem_cgroup_region_set_reclaim() - Register a reclaim callback on a region.
+ * @region: The region to register the callback for.
+ * @reclaim: Callback to invoke when dmem.max is set below current usage.
+ * Called with the pool that needs reclaiming and the number of
+ * bytes to free. Returns 0 on progress, negative on failure.
+ * @priv: Opaque pointer passed back to @reclaim.
+ *
+ * When dmem.max is lowered below the current usage of a cgroup pool, the
+ * dmem controller will call @reclaim with a target number of bytes to free.
+ * After @reclaim returns the controller retries setting the limit; if usage
+ * is still too high it calls @reclaim again, up to a bounded retry count.
+ */
+void dmem_cgroup_region_set_reclaim(struct dmem_cgroup_region *region,
+ dmem_cgroup_reclaim_fn_t reclaim,
+ void *priv)
+{
+ if (!region)
+ return;
+
+ down_write(®ion->unregister_sem);
+ region->reclaim_priv = priv;
+ /* Pairs with READ_ONCE() in set_resource_max() */
+ WRITE_ONCE(region->reclaim, reclaim);
+ up_write(®ion->unregister_sem);
+}
+EXPORT_SYMBOL_GPL(dmem_cgroup_region_set_reclaim);
+
static struct dmem_cgroup_pool_state *
get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region)
{
@@ -725,9 +811,10 @@ static int dmemcg_parse_limit(char *options, u64 *new_limit)
static ssize_t dmemcg_limit_write(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off,
- void (*apply)(struct dmem_cgroup_pool_state *, u64))
+ void (*apply)(struct dmem_cgroup_pool_state *, u64, bool))
{
struct dmemcg_state *dmemcs = css_to_dmemcs(of_css(of));
+ bool nonblock = of->file->f_flags & O_NONBLOCK;
int err = 0;
while (buf && !err) {
@@ -772,7 +859,8 @@ static ssize_t dmemcg_limit_write(struct kernfs_open_file *of,
}
/* And commit */
- apply(pool, new_limit);
+ apply(pool, new_limit, nonblock);
+
dmemcg_pool_put(pool);
out_put:
--
2.54.0
^ permalink raw reply related
* [PATCH v3 1/5] drm/amdgpu: Fix init ordering in amdgpu_vram_mgr_init()
From: Thomas Hellström @ 2026-05-11 17:30 UTC (permalink / raw)
To: intel-xe
Cc: Thomas Hellström, Sashiko-bot, Friedrich Vock,
Maarten Lankhorst, Tejun Heo, Maxime Ripard, Christian König,
Alex Deucher, amd-gfx, dri-devel, stable, Natalie Vock,
Johannes Weiner, Michal Koutný, cgroups, Huang Rui,
Matthew Brost, Matthew Auld, Maarten Lankhorst, Thomas Zimmermann,
Simona Vetter, David Airlie, Rodrigo Vivi, linux-kernel
In-Reply-To: <20260511173008.36526-1-thomas.hellstrom@linux.intel.com>
drmm_cgroup_register_region() is called before INIT_LIST_HEAD() and
gpu_buddy_init() in amdgpu_vram_mgr_init(). If it fails, the function
returns early and bypasses those initializations.
Since adev->mman.initialized is set to true before amdgpu_vram_mgr_init()
is called, a failure triggers amdgpu_ttm_fini(), which calls
amdgpu_vram_mgr_fini(), which then:
- Calls list_for_each_entry_safe() on reservations_pending and
reserved_pages, whose list_head::next pointers are zero-initialized
(NULL). The loop does not recognize them as empty and dereferences NULL.
- Calls gpu_buddy_fini(), which iterates free_trees[] unconditionally
via for_each_free_tree(). Since mm->free_trees is NULL
(never allocated), this dereferences NULL.
Both result in a kernel panic on the module load error path.
Fix by moving drmm_cgroup_register_region() to after the list and buddy
allocator are fully initialized, so the teardown path is safe to run.
Reported-by: Sashiko-bot <sashiko-bot@kernel.org>
Closes: https://sashiko.dev/#/patchset/20260428073116.15687-1-thomas.hellstrom@linux.intel.com?part=4
Fixes: 2b624a2c1865 ("drm/ttm: Handle cgroup based eviction in TTM")
Cc: Friedrich Vock <friedrich.vock@gmx.de>
Cc: Maarten Lankhorst <dev@lankhorst.se>
Cc: Tejun Heo <tj@kernel.org>
Cc: Maxime Ripard <mripard@kernel.org>
Cc: Christian König <christian.koenig@amd.com>
Cc: Alex Deucher <alexander.deucher@amd.com>
Cc: amd-gfx@lists.freedesktop.org
Cc: dri-devel@lists.freedesktop.org
Cc: <stable@vger.kernel.org> # v6.14+
Assisted-by: GitHub_Copilot:claude-sonnet-4.6
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 2a241a5b12c4..ac3f71d77140 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -918,9 +918,6 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
struct ttm_resource_manager *man = &mgr->manager;
int err;
- man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size);
- if (IS_ERR(man->cg))
- return PTR_ERR(man->cg);
ttm_resource_manager_init(man, &adev->mman.bdev,
adev->gmc.real_vram_size);
@@ -935,6 +932,10 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
if (err)
return err;
+ man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size);
+ if (IS_ERR(man->cg))
+ return PTR_ERR(man->cg);
+
ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
ttm_resource_manager_set_used(man, true);
return 0;
--
2.54.0
^ permalink raw reply related
* [PATCH v3 0/5] Add reclaim to the dmem cgroup controller
From: Thomas Hellström @ 2026-05-11 17:30 UTC (permalink / raw)
To: intel-xe
Cc: Thomas Hellström, Natalie Vock, Johannes Weiner, Tejun Heo,
Michal Koutný, cgroups, Huang Rui, Matthew Brost,
Matthew Auld, Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Simona Vetter, David Airlie, Christian König, Alex Deucher,
Rodrigo Vivi, dri-devel, amd-gfx, linux-kernel
When writing a "max" limit lower than the current usage, the
existing code silently failed. This series aims to improve
on that by returning -EBUSY on failure and also attempt
to synchronously reclaim device memory to push the usage
under the new max limit to avoid the error.
Patch 1 fixes a pre-existing amdgpu_vram_mgr_init() error path
Patch 2 implements and documents a reclaim callback interface
for the dmem controller.
Patch 3 implements a TTM reclaim callback.
Patch 4-5 hooks up the reclaim callback to the dmem cgroups-
aware drivers xe and amdgpu.
v2:
- Remove the error propagation that was in a previous series (Maarten)
- A number of updates in patch 1. See its commit message for
details (Maarten)
v3:
- Add patch 1 fixing a pre-existing amdgpu_vram_mgr_init() error path
bug where drmm_cgroup_register_region() was called before
INIT_LIST_HEAD() and gpu_buddy_init(), causing a kernel panic on
failure. (Sashiko-bot)
- Use an rwsem to protect reclaim callback registration and region
unregister against concurrent reclaim invocations. (Sashiko-bot)
- Fix ttm_resource_manager_set_dmem_region() storing an error pointer
in man->cg unconditionally. (Sashiko-bot)
- Fix kernel-doc function name format for ttm_bo_evict_cgroup() and
ttm_resource_manager_set_dmem_region().
User-space tests are at
https://patchwork.freedesktop.org/series/163935/
Test-with: 20260428065411.4222-1-thomas.hellstrom@linux.intel.com
Thomas Hellström (5):
drm/amdgpu: Fix init ordering in amdgpu_vram_mgr_init()
cgroup/dmem: Add reclaim callback for lowering max below current usage
drm/ttm: Hook up a cgroup-aware reclaim callback for the dmem
controller
drm/xe: Wire up dmem cgroup reclaim for VRAM manager
drm/amdgpu: Wire up dmem cgroup reclaim for VRAM manager
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 10 +-
drivers/gpu/drm/ttm/ttm_bo.c | 95 ++++++++++++++++-
drivers/gpu/drm/ttm/ttm_bo_util.c | 3 +-
drivers/gpu/drm/ttm/ttm_resource.c | 37 +++++++
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 19 ++--
include/drm/ttm/ttm_bo.h | 10 ++
include/drm/ttm/ttm_resource.h | 4 +
include/linux/cgroup_dmem.h | 24 +++++
kernel/cgroup/dmem.c | 106 +++++++++++++++++--
10 files changed, 286 insertions(+), 24 deletions(-)
--
2.54.0
^ permalink raw reply
* Re: [PATCH v3 05/12] mm, swap: unify large folio allocation
From: Kairui Song @ 2026-05-11 16:44 UTC (permalink / raw)
To: David Hildenbrand (Arm)
Cc: linux-mm, Andrew Morton, Zi Yan, Baolin Wang, Barry Song,
Hugh Dickins, Chris Li, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, Qi Zheng, linux-kernel, cgroups,
Yosry Ahmed, Lorenzo Stoakes, Dev Jain, Lance Yang, Michal Hocko,
Michal Hocko, Suren Baghdasaryan, Axel Rasmussen
In-Reply-To: <c72ead41-0bb3-4da0-856c-315dc552c722@kernel.org>
On Mon, May 11, 2026 at 11:15 PM David Hildenbrand (Arm)
<david@kernel.org> wrote:
>
> On 5/11/26 16:37, Kairui Song wrote:
> >
> > Yes, the current status is a bit odd, about two years ago I also
> > wanted to name it `swapin_direct()`.
> > https://lore.kernel.org/linux-mm/20240326185032.72159-3-ryncsn@gmail.com/
> >
> > But actually ZRAM or shmem would also benefit from supporting unified
> > readahead like this:
> > https://lore.kernel.org/linux-mm/20240102175338.62012-6-ryncsn@gmail.com/
> >
> > So calling it `swapin_entry` seems more future-proof. At some point in
> > the future we might remove `swapin_readahead`. All swapin operations
> > could have a unified or at least a per-device readahead policy like
> > the one in the link above, instead of the current policy where the
> > caller must decide whether to perform readahead.
> >
> > But any suggestion on naming is welcome :)
>
> The other proposal
>
> https://lore.kernel.org/all/tencent_CD11FE9B4A0B362E95E776C5F679598FAA07@qq.com/
>
> calls it
>
> swapin_synchronous_folio
>
> Maybe just swapin_sync_io()/swapin_sync() or sth like that?
Good idea, I can keep the swapin_sync name at this point. Sync io flag
still may remain for a longer time.
^ permalink raw reply
* Re: [PATCH v3 00/12] mm, swap: swap table phase IV: unify allocation and reduce static metadata
From: Chris Li @ 2026-05-11 16:34 UTC (permalink / raw)
To: Andrew Morton
Cc: kasong, linux-mm, David Hildenbrand, Zi Yan, Baolin Wang,
Barry Song, Hugh Dickins, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, Qi Zheng, linux-kernel, cgroups,
Yosry Ahmed, Lorenzo Stoakes, Dev Jain, Lance Yang, Michal Hocko,
Michal Hocko, Suren Baghdasaryan, Axel Rasmussen
In-Reply-To: <20260421-swap-table-p4-v3-0-2f23759a76bc@tencent.com>
On Mon, Apr 20, 2026 at 11:16 PM Kairui Song via B4 Relay
<devnull+kasong.tencent.com@kernel.org> wrote:
>
> This series unifies the allocation and charging of anon and shmem swap
> in folios, provides better synchronization, consolidates the metadata
> management, hence dropping the static array and map, and improves the
> performance. The static metadata overhead is now close to zero, and
> workload performance is slightly improved.
>
> For example, mounting a 1TB swap device saves about 512MB of memory:
>
> Before:
> free -m
> total used free shared buff/cache available
> Mem: 1464 805 346 1 382 658
> Swap: 1048575 0 1048575
>
> After:
> free -m
> total used free shared buff/cache available
> Mem: 1464 277 899 1 356 1187
> Swap: 1048575 0 1048575
>
> Memory usage is ~512M lower, and we now have a close to 0 static
> overhead. It was about 2 bytes per slot before, now roughly 0.09375
> bytes per slot (48 bytes ci info per cluster, which is 512 slots).
>
> Performance test is also looking good, testing Redis in a 1.5G VM using
> 5G ZRAM as swap:
>
> valkey-server --maxmemory 2560M
> redis-benchmark -r 3000000 -n 3000000 -d 1024 -c 12 -P 32 -t get
>
> Before: 3289011.918750 RPS
> After: 3312087.142241 RPS (0.99% better)
>
> Testing with build kernel under global pressure on a 48c96t system,
> limiting the total memory to 8G, using 12G ZRAM, 24 test runs,
> enabling THP:
>
> make -j96, using defconfig
>
> Before: user time 2904.59s system time 4773.99s
> After: user time 2909.38s system time 4641.55s (2.77% better)
>
> Testing with usemem on a 32c machine using 48G brd ramdisk and 16G
> RAM, 12 test run:
>
> usemem --init-time -O -y -x -n 48 1G
>
> Before: Throughput (Sum): 6482.58 MB/s Free Latency: 371371.67us
> After: Throughput (Sum): 6539.28 MB/s Free Latency: 363059.88us
>
> Seems similar, or slightly better.
>
> This series also reduces memory thrashing, I no longer see any:
> "Huh VM_FAULT_OOM leaked out to the #PF handler. Retrying PF", it was
> shown several times during stress testing before this series when under
> great pressure:
>
> Before: grep -Ri VM_FAULT_OOM <test logs> | wc -l => 18
> After: grep -Ri VM_FAULT_OOM <test logs> | wc -l => 0
>
> Signed-off-by: Kairui Song <kasong@tencent.com>
Hi Andrew,
I have given this swap table phase 4 series the first round of review.
Overall, it looks good with some minor nitpicks.
Can you add this to the mm-unstable for more exposures?
Thanks
Chris
> ---
> Changes in v3:
> - This is based on mm-unstable, also applies to mm-new, and has no
> conflict with YoungJun's tier series, and only trivial conflict with
> Baoquan's swapops due to filename change.
> - Fix zero map build issue on 32 bit archs [ YoungJun Park ]
> - Cleanup memcg table allocation helpers [ YoungJun Park ]
> - Fix WARN for non NUMA build:
> https://lore.kernel.org/linux-mm/CAMgjq7ANih7u7SJB8uWcQHS8XRJySNRc3ti9V-SVey0nGE3gLQ@mail.gmail.com/
> - Improve of commit messages.
> - Re-test several tests, the conclusion is the same as v2.
> - Link to v2: https://patch.msgid.link/20260417-swap-table-p4-v2-0-17f5d1015428@tencent.com
>
> Changes in v2:
> - Drop the RFC prefix and also the RFC part.
> - Now there is zero change to cgroup or refault tracking, RFC v1 changed
> some cgroup behavior. To archive that v2 use a standalone memcg_table
> for each cluster. It can be dropped or better optimized later if we
> have a better solution. The performance gain is partly cancelled
> compared to RFC v1 since we now need an extra allocation for free cluster
> isolation and peak memory usage is 2 bytes higher. But still looking
> good. That table size is accetable (1024 bytes), no RCU needed, and
> fits for kmalloc. Even if we keep it as it is in the future,
> it's still accetable.
> - Link to v1: https://lore.kernel.org/r/20260220-swap-table-p4-v1-0-104795d19815@tencent.com
>
> To: linux-mm@kvack.org
> Cc: Andrew Morton <akpm@linux-foundation.org>
> Cc: Chris Li <chrisl@kernel.org>
> Cc: Kairui Song <kasong@tencent.com>
> Cc: Kemeng Shi <shikemeng@huaweicloud.com>
> Cc: Nhat Pham <nphamcs@gmail.com>
> Cc: Baoquan He <bhe@redhat.com>
> Cc: Barry Song <baohua@kernel.org>
> Cc: Youngjun Park <youngjun.park@lge.com>
> Cc: Johannes Weiner <hannes@cmpxchg.org>
> Cc: Yosry Ahmed <yosry@kernel.org>
> Cc: Chengming Zhou <chengming.zhou@linux.dev>
> Cc: David Hildenbrand <david@kernel.org>
> Cc: Lorenzo Stoakes <ljs@kernel.org>
> Cc: Zi Yan <ziy@nvidia.com>
> Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
> Cc: Dev Jain <dev.jain@arm.com>
> Cc: Lance Yang <lance.yang@linux.dev>
> Cc: Hugh Dickins <hughd@google.com>
> Cc: Michal Hocko <mhocko@suse.com>
> Cc: Michal Hocko <mhocko@kernel.org>
> Cc: Roman Gushchin <roman.gushchin@linux.dev>
> Cc: Shakeel Butt <shakeel.butt@linux.dev>
> Cc: Muchun Song <muchun.song@linux.dev>
> Cc: Suren Baghdasaryan <surenb@google.com>
> Cc: Axel Rasmussen <axelrasmussen@google.com>
> Cc: Qi Zheng <zhengqi.arch@bytedance.com>
> Cc: linux-kernel@vger.kernel.org
> Cc: cgroups@vger.kernel.org
>
> ---
> Kairui Song (12):
> mm, swap: simplify swap cache allocation helper
> mm, swap: move common swap cache operations into standalone helpers
> mm/huge_memory: move THP gfp limit helper into header
> mm, swap: add support for stable large allocation in swap cache directly
> mm, swap: unify large folio allocation
> mm/memcg, swap: tidy up cgroup v1 memsw swap helpers
> mm, swap: support flexible batch freeing of slots in different memcgs
> mm, swap: delay and unify memcg lookup and charging for swapin
> mm, swap: consolidate cluster allocation helpers
> mm/memcg, swap: store cgroup id in cluster table directly
> mm/memcg: remove no longer used swap cgroup array
> mm, swap: merge zeromap into swap table
>
> MAINTAINERS | 1 -
> include/linux/huge_mm.h | 30 +++
> include/linux/memcontrol.h | 16 +-
> include/linux/swap.h | 19 +-
> include/linux/swap_cgroup.h | 47 ----
> mm/Makefile | 3 -
> mm/huge_memory.c | 2 +-
> mm/internal.h | 11 +-
> mm/memcontrol-v1.c | 66 +++---
> mm/memcontrol.c | 32 +--
> mm/memory.c | 88 ++------
> mm/page_io.c | 58 ++++-
> mm/shmem.c | 122 +++--------
> mm/swap.h | 91 +++-----
> mm/swap_cgroup.c | 172 ---------------
> mm/swap_state.c | 516 +++++++++++++++++++++++++-------------------
> mm/swap_table.h | 169 ++++++++++++---
> mm/swapfile.c | 212 +++++++++---------
> mm/vmscan.c | 2 +-
> mm/zswap.c | 25 +--
> 20 files changed, 783 insertions(+), 899 deletions(-)
> ---
> base-commit: f1541b40cd422d7e22273be9b7e9edfc9ea4f0d7
> change-id: 20260111-swap-table-p4-98ee92baa7c4
>
> Best regards,
> --
> Kairui Song <kasong@tencent.com>
>
>
^ permalink raw reply
* Re: [PATCH v3 12/12] mm, swap: merge zeromap into swap table
From: Chris Li @ 2026-05-11 16:30 UTC (permalink / raw)
To: kasong
Cc: linux-mm, Andrew Morton, David Hildenbrand, Zi Yan, Baolin Wang,
Barry Song, Hugh Dickins, Kemeng Shi, Nhat Pham, Baoquan He,
Johannes Weiner, Youngjun Park, Chengming Zhou, Roman Gushchin,
Shakeel Butt, Muchun Song, Qi Zheng, linux-kernel, cgroups,
Yosry Ahmed, Lorenzo Stoakes, Dev Jain, Lance Yang, Michal Hocko,
Michal Hocko, Suren Baghdasaryan, Axel Rasmussen
In-Reply-To: <20260421-swap-table-p4-v3-12-2f23759a76bc@tencent.com>
)
On Tue, Apr 21, 2026 at 8:16 AM Kairui Song via B4 Relay
<devnull+kasong.tencent.com@kernel.org> wrote:
>
> From: Kairui Song <kasong@tencent.com>
>
> By allocating one additional bit in the swap table entry's flags field
> alongside the count, we can store the zeromap inline
>
> For certain 32-bit archs, there might not be enough bits in the swap
> table to contain both PFN and flags. Therefore, conditionally let each
> cluster have a zeromap field at build time, and use that instead of the
> swap table for these archs. A few macros were moved to different headers
> for build time struct definition.
It might be worthwhile to mention the user-visible impact. For 64 bit
systems. The zeromap will store in the swap table, avoiding zeromap
allocation. It reduces the allocated memory. That is the happy path.
For certain 32-bit architectures, if the swapfile cluster is not fully
used, it will use less memory for zeromap. The empty cluster does not
allocate a zeromap. We still save memory. In the worst case, all
cluster are fully populated. We will use memory similar to the
previous zeromap implementation.
>
> Signed-off-by: Kairui Song <kasong@tencent.com>
Acked-by: Chris Li <chrisl@kernel.org>
> ---
> include/linux/swap.h | 1 -
> mm/memory.c | 11 +----
> mm/page_io.c | 58 ++++++++++++++++++++++----
> mm/swap.h | 51 +++++++++--------------
> mm/swap_state.c | 14 ++++---
> mm/swap_table.h | 115 +++++++++++++++++++++++++++++++++++++--------------
> mm/swapfile.c | 45 +++++++++-----------
> 7 files changed, 184 insertions(+), 111 deletions(-)
>
> diff --git a/include/linux/swap.h b/include/linux/swap.h
> index 57af4647d432..8f0f68e245ba 100644
> --- a/include/linux/swap.h
> +++ b/include/linux/swap.h
> @@ -253,7 +253,6 @@ struct swap_info_struct {
> struct plist_node list; /* entry in swap_active_head */
> signed char type; /* strange name for an index */
> unsigned int max; /* size of this swap device */
> - unsigned long *zeromap; /* kvmalloc'ed bitmap to track zero pages */
Nice.
> struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
> struct list_head free_clusters; /* free clusters list */
> struct list_head full_clusters; /* full clusters list */
> diff --git a/mm/memory.c b/mm/memory.c
> index 404734a5bcff..a45905f8728f 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -4595,13 +4595,11 @@ static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
>
> #ifdef CONFIG_TRANSPARENT_HUGEPAGE
> /*
> - * Check if the PTEs within a range are contiguous swap entries
> - * and have consistent swapcache, zeromap.
> + * Check if the PTEs within a range are contiguous swap entries.
> */
> static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
> {
> unsigned long addr;
> - softleaf_t entry;
> int idx;
> pte_t pte;
>
> @@ -4611,18 +4609,13 @@ static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
>
> if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
> return false;
> - entry = softleaf_from_pte(pte);
> - if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
> - return false;
> -
> /*
> * swap_read_folio() can't handle the case a large folio is hybridly
> * from different backends. And they are likely corner cases. Similar
> * things might be added once zswap support large folios.
> */
> - if (unlikely(swap_zeromap_batch(entry, nr_pages, NULL) != nr_pages))
> + if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
> return false;
> -
> return true;
> }
>
> diff --git a/mm/page_io.c b/mm/page_io.c
> index 70cea9e24d2f..c2557e72c381 100644
> --- a/mm/page_io.c
> +++ b/mm/page_io.c
> @@ -26,6 +26,7 @@
> #include <linux/delayacct.h>
> #include <linux/zswap.h>
> #include "swap.h"
> +#include "swap_table.h"
>
> static void __end_swap_bio_write(struct bio *bio)
> {
> @@ -204,15 +205,20 @@ static bool is_folio_zero_filled(struct folio *folio)
> static void swap_zeromap_folio_set(struct folio *folio)
> {
> struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
> - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
> int nr_pages = folio_nr_pages(folio);
> + struct swap_cluster_info *ci;
> swp_entry_t entry;
> unsigned int i;
>
> + VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
> + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
> +
> + ci = swap_cluster_get_and_lock(folio);
> for (i = 0; i < folio_nr_pages(folio); i++) {
> entry = page_swap_entry(folio_page(folio, i));
> - set_bit(swp_offset(entry), sis->zeromap);
> + __swap_table_set_zero(ci, swp_cluster_offset(entry));
> }
> + swap_cluster_unlock(ci);
>
> count_vm_events(SWPOUT_ZERO, nr_pages);
> if (objcg) {
> @@ -223,14 +229,19 @@ static void swap_zeromap_folio_set(struct folio *folio)
>
> static void swap_zeromap_folio_clear(struct folio *folio)
> {
> - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
> + struct swap_cluster_info *ci;
> swp_entry_t entry;
> unsigned int i;
>
> + VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
> + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
> +
> + ci = swap_cluster_get_and_lock(folio);
> for (i = 0; i < folio_nr_pages(folio); i++) {
> entry = page_swap_entry(folio_page(folio, i));
> - clear_bit(swp_offset(entry), sis->zeromap);
> + __swap_table_clear_zero(ci, swp_cluster_offset(entry));
> }
> + swap_cluster_unlock(ci);
> }
>
> /*
> @@ -255,10 +266,9 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
> }
>
> /*
> - * Use a bitmap (zeromap) to avoid doing IO for zero-filled pages.
> - * The bits in zeromap are protected by the locked swapcache folio
> - * and atomic updates are used to protect against read-modify-write
> - * corruption due to other zero swap entries seeing concurrent updates.
> + * Use the swap table zero mark to avoid doing IO for zero-filled
> + * pages. The zero mark is protected by the cluster lock, which is
> + * acquired internally by swap_zeromap_folio_set/clear.
> */
> if (is_folio_zero_filled(folio)) {
> swap_zeromap_folio_set(folio);
> @@ -509,16 +519,48 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
> mempool_free(sio, sio_pool);
> }
>
> +/*
> + * Return the count of contiguous swap entries that share the same
> + * zeromap status as the starting entry. If is_zerop is not NULL,
> + * it will return the zeromap status of the starting entry.
> + *
> + * Context: Caller must ensure the cluster containing the entries
> + * that will be checked won't be freed.
> + */
> +static int swap_zeromap_batch(swp_entry_t entry, int max_nr,
> + bool *is_zerop)
> +{
> + bool is_zero;
> + struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
> + unsigned int ci_start = swp_cluster_offset(entry), ci_off, ci_end;
> +
> + ci_off = ci_start;
> + ci_end = ci_off + max_nr;
Should we check ci_end less than the cluster's end and complain if not?
It seems using a for loop can be simpler. The loop index serves as a
counter as well.
Totally untested code:
int i;
rcu_read_lock();
is_zero = __swap_table_test_zero(ci, ci_start);
for (i =1; i < max_nr ; i++)
if (is_zero != __swap_table_test_zero(ci, ci_start + i))
break;
rcu_read_unlock();
if (is_zerop)
*is_zerop = is_zero;
return i;
Chris
> + break;
> + rcu_read_lock();
> + is_zero = __swap_table_test_zero(ci, ci_off);
> + if (is_zerop)
> + *is_zerop = is_zero;
> + while (++ci_off < ci_end) {
> + if (is_zero != __swap_table_test_zero(ci, ci_off))
> + break;
> + }
> + rcu_read_unlock();
> + return ci_off - ci_start;
> +}
> +
> static bool swap_read_folio_zeromap(struct folio *folio)
> {
> int nr_pages = folio_nr_pages(folio);
> struct obj_cgroup *objcg;
> bool is_zeromap;
>
> + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
> +
> /*
> * Swapping in a large folio that is partially in the zeromap is not
> * currently handled. Return true without marking the folio uptodate so
> * that an IO error is emitted (e.g. do_swap_page() will sigbus).
> + * Folio lock stabilizes the cluster and map, so the check is safe.
> */
> if (WARN_ON_ONCE(swap_zeromap_batch(folio->swap, nr_pages,
> &is_zeromap) != nr_pages))
> diff --git a/mm/swap.h b/mm/swap.h
> index e4ac7dbc1080..025ff4f0b021 100644
> --- a/mm/swap.h
> +++ b/mm/swap.h
> @@ -3,12 +3,29 @@
> #define _MM_SWAP_H
>
> #include <linux/atomic.h> /* for atomic_long_t */
> +#include <linux/mm.h> /* for PAGE_SHIFT */
> struct mempolicy;
> struct swap_iocb;
> struct swap_memcg_table;
>
> extern int page_cluster;
>
> +#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
> +#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
> +#elif defined(MAX_PHYSMEM_BITS)
> +#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
> +#else
> +#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
> +#endif
> +
> +/* Swap table marker, 0x1 means shadow, 0x2 means PFN (SWP_TB_PFN_MARK) */
> +#define SWAP_CACHE_PFN_MARK_BITS 2
> +/* At least 2 bits are needed to distinguish SWP_TB_COUNT_MAX, 1 and 0 */
> +#define SWAP_COUNT_MIN_BITS 2
> +/* If there are enough bits besides PFN and marker, store zero flag inline */
> +#define SWAP_TABLE_HAS_ZEROFLAG ((BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - \
> + SWAP_CACHE_PFN_BITS) > SWAP_COUNT_MIN_BITS)
> +
> #ifdef CONFIG_THP_SWAP
> #define SWAPFILE_CLUSTER HPAGE_PMD_NR
> #define swap_entry_order(order) (order)
> @@ -41,6 +58,9 @@ struct swap_cluster_info {
> unsigned int *extend_table; /* For large swap count, protected by ci->lock */
> #ifdef CONFIG_MEMCG
> struct swap_memcg_table *memcg_table; /* Swap table entries' cgroup record */
> +#endif
> +#if !SWAP_TABLE_HAS_ZEROFLAG
> + unsigned long *zero_bitmap;
> #endif
> struct list_head list;
> };
> @@ -314,31 +334,6 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
> return __swap_entry_to_info(folio->swap)->flags;
> }
>
> -/*
> - * Return the count of contiguous swap entries that share the same
> - * zeromap status as the starting entry. If is_zeromap is not NULL,
> - * it will return the zeromap status of the starting entry.
> - */
> -static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
> - bool *is_zeromap)
> -{
> - struct swap_info_struct *sis = __swap_entry_to_info(entry);
> - unsigned long start = swp_offset(entry);
> - unsigned long end = start + max_nr;
> - bool first_bit;
> -
> - first_bit = test_bit(start, sis->zeromap);
> - if (is_zeromap)
> - *is_zeromap = first_bit;
> -
> - if (max_nr <= 1)
> - return max_nr;
> - if (first_bit)
> - return find_next_zero_bit(sis->zeromap, end, start) - start;
> - else
> - return find_next_bit(sis->zeromap, end, start) - start;
> -}
> -
> #else /* CONFIG_SWAP */
> struct swap_iocb;
> static inline struct swap_cluster_info *swap_cluster_lock(
> @@ -476,11 +471,5 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
> {
> return 0;
> }
> -
> -static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
> - bool *has_zeromap)
> -{
> - return 0;
> -}
> #endif /* CONFIG_SWAP */
> #endif /* _MM_SWAP_H */
> diff --git a/mm/swap_state.c b/mm/swap_state.c
> index 71a3f128fcf0..fa4ef9f4a1d3 100644
> --- a/mm/swap_state.c
> +++ b/mm/swap_state.c
> @@ -159,6 +159,7 @@ static int __swap_cache_add_check(struct swap_cluster_info *ci,
> {
> unsigned int ci_off, ci_end;
> unsigned long old_tb;
> + bool is_zero;
>
> /*
> * If the target slot is not swapped out, return
> @@ -181,12 +182,14 @@ static int __swap_cache_add_check(struct swap_cluster_info *ci,
> if (nr == 1)
> return 0;
>
> + is_zero = __swap_table_test_zero(ci, ci_off);
> ci_off = round_down(ci_off, nr);
> ci_end = ci_off + nr;
> do {
> old_tb = __swap_table_get(ci, ci_off);
> if (unlikely(swp_tb_is_folio(old_tb) ||
> !__swp_tb_get_count(old_tb) ||
> + is_zero != __swap_table_test_zero(ci, ci_off) ||
> (memcg_id && *memcg_id != __swap_cgroup_get(ci, ci_off))))
> return -EBUSY;
> } while (++ci_off < ci_end);
> @@ -210,7 +213,7 @@ static void __swap_cache_do_add_folio(struct swap_cluster_info *ci,
> do {
> old_tb = __swap_table_get(ci, ci_off);
> VM_WARN_ON_ONCE(swp_tb_is_folio(old_tb));
> - __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_count(old_tb)));
> + __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb)));
> } while (++ci_off < ci_end);
>
> folio_ref_add(folio, nr_pages);
> @@ -246,7 +249,6 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci,
> struct folio *folio,
> swp_entry_t entry, void *shadow)
> {
> - int count;
> unsigned long old_tb;
> struct swap_info_struct *si;
> unsigned int ci_start, ci_off, ci_end;
> @@ -266,13 +268,13 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci,
> old_tb = __swap_table_get(ci, ci_off);
> WARN_ON_ONCE(!swp_tb_is_folio(old_tb) ||
> swp_tb_to_folio(old_tb) != folio);
> - count = __swp_tb_get_count(old_tb);
> - if (count)
> + if (__swp_tb_get_count(old_tb))
> folio_swapped = true;
> else
> need_free = true;
> /* If shadow is NULL, we set an empty shadow. */
> - __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow, count));
> + __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow,
> + __swp_tb_get_flags(old_tb)));
> } while (++ci_off < ci_end);
>
> folio->swap.val = 0;
> @@ -366,7 +368,7 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
> do {
> old_tb = __swap_table_get(ci, ci_off);
> WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || swp_tb_to_folio(old_tb) != old);
> - __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_count(old_tb)));
> + __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb)));
> } while (++ci_off < ci_end);
>
> /*
> diff --git a/mm/swap_table.h b/mm/swap_table.h
> index b2b02ee161b1..6cf1575eb26e 100644
> --- a/mm/swap_table.h
> +++ b/mm/swap_table.h
> @@ -26,12 +26,14 @@ struct swap_memcg_table {
> * Swap table entry type and bits layouts:
> *
> * NULL: |---------------- 0 ---------------| - Free slot
> - * Shadow: | SWAP_COUNT |---- SHADOW_VAL ---|1| - Swapped out slot
> - * PFN: | SWAP_COUNT |------ PFN -------|10| - Cached slot
> + * Shadow: |SWAP_COUNT|Z|---- SHADOW_VAL ---|1| - Swapped out slot
> + * PFN: |SWAP_COUNT|Z|------ PFN -------|10| - Cached slot
> * Pointer: |----------- Pointer ----------|100| - (Unused)
> * Bad: |------------- 1 -------------|1000| - Bad slot
> *
> - * SWAP_COUNT is `SWP_TB_COUNT_BITS` long, each entry is an atomic long.
> + * COUNT is `SWP_TB_COUNT_BITS` long, Z is the `SWP_TB_ZERO_FLAG` bit,
> + * and together they form the `SWP_TB_FLAGS_BITS` wide flags field.
> + * Each entry is an atomic long.
> *
> * Usages:
> *
> @@ -54,14 +56,6 @@ struct swap_memcg_table {
> * - Bad: Swap slot is reserved, protects swap header or holes on swap devices.
> */
>
> -#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
> -#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
> -#elif defined(MAX_PHYSMEM_BITS)
> -#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
> -#else
> -#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
> -#endif
> -
> /* NULL Entry, all 0 */
> #define SWP_TB_NULL 0UL
>
> @@ -69,22 +63,26 @@ struct swap_memcg_table {
> #define SWP_TB_SHADOW_MARK 0b1UL
>
> /* Cached: PFN */
> -#define SWP_TB_PFN_BITS (SWAP_CACHE_PFN_BITS + SWP_TB_PFN_MARK_BITS)
> +#define SWP_TB_PFN_BITS (SWAP_CACHE_PFN_BITS + SWAP_CACHE_PFN_MARK_BITS)
> #define SWP_TB_PFN_MARK 0b10UL
> -#define SWP_TB_PFN_MARK_BITS 2
> -#define SWP_TB_PFN_MARK_MASK (BIT(SWP_TB_PFN_MARK_BITS) - 1)
> +#define SWP_TB_PFN_MARK_MASK (BIT(SWAP_CACHE_PFN_MARK_BITS) - 1)
>
> -/* SWAP_COUNT part for PFN or shadow, the width can be shrunk or extended */
> -#define SWP_TB_COUNT_BITS min(4, BITS_PER_LONG - SWP_TB_PFN_BITS)
> +/* Flags: For PFN or shadow, contains SWAP_COUNT, width changes */
> +#define SWP_TB_FLAGS_BITS min(5, BITS_PER_LONG - SWP_TB_PFN_BITS)
> +#define SWP_TB_COUNT_BITS (SWP_TB_FLAGS_BITS - SWAP_TABLE_HAS_ZEROFLAG)
> +#define SWP_TB_FLAGS_MASK (~((~0UL) >> SWP_TB_FLAGS_BITS))
> #define SWP_TB_COUNT_MASK (~((~0UL) >> SWP_TB_COUNT_BITS))
> +#define SWP_TB_FLAGS_SHIFT (BITS_PER_LONG - SWP_TB_FLAGS_BITS)
> #define SWP_TB_COUNT_SHIFT (BITS_PER_LONG - SWP_TB_COUNT_BITS)
> #define SWP_TB_COUNT_MAX ((1 << SWP_TB_COUNT_BITS) - 1)
> +/* The first flag is zero bit (SWAP_TABLE_HAS_ZEROFLAG) */
> +#define SWP_TB_ZERO_FLAG BIT(BITS_PER_LONG - SWP_TB_FLAGS_BITS)
>
> /* Bad slot: ends with 0b1000 and rests of bits are all 1 */
> #define SWP_TB_BAD ((~0UL) << 3)
>
> /* Macro for shadow offset calculation */
> -#define SWAP_COUNT_SHIFT SWP_TB_COUNT_BITS
> +#define SWAP_COUNT_SHIFT SWP_TB_FLAGS_BITS
>
> /*
> * Helpers for casting one type of info into a swap table entry.
> @@ -102,40 +100,47 @@ static inline unsigned long __count_to_swp_tb(unsigned char count)
> * used (count > 0 && count < SWP_TB_COUNT_MAX), and
> * overflow (count == SWP_TB_COUNT_MAX).
> */
> - BUILD_BUG_ON(SWP_TB_COUNT_MAX < 2 || SWP_TB_COUNT_BITS < 2);
> + BUILD_BUG_ON(SWP_TB_COUNT_BITS < SWAP_COUNT_MIN_BITS);
> VM_WARN_ON(count > SWP_TB_COUNT_MAX);
> return ((unsigned long)count) << SWP_TB_COUNT_SHIFT;
> }
>
> -static inline unsigned long pfn_to_swp_tb(unsigned long pfn, unsigned int count)
> +static inline unsigned long __flags_to_swp_tb(unsigned char flags)
> +{
> + BUILD_BUG_ON(SWP_TB_FLAGS_BITS > BITS_PER_BYTE);
> + VM_WARN_ON(flags >> SWP_TB_FLAGS_BITS);
> + return ((unsigned long)flags) << SWP_TB_FLAGS_SHIFT;
> +}
> +
> +static inline unsigned long pfn_to_swp_tb(unsigned long pfn, unsigned char flags)
> {
> unsigned long swp_tb;
>
> BUILD_BUG_ON(sizeof(unsigned long) != sizeof(void *));
> BUILD_BUG_ON(SWAP_CACHE_PFN_BITS >
> - (BITS_PER_LONG - SWP_TB_PFN_MARK_BITS - SWP_TB_COUNT_BITS));
> + (BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - SWP_TB_FLAGS_BITS));
>
> - swp_tb = (pfn << SWP_TB_PFN_MARK_BITS) | SWP_TB_PFN_MARK;
> - VM_WARN_ON_ONCE(swp_tb & SWP_TB_COUNT_MASK);
> + swp_tb = (pfn << SWAP_CACHE_PFN_MARK_BITS) | SWP_TB_PFN_MARK;
> + VM_WARN_ON_ONCE(swp_tb & SWP_TB_FLAGS_MASK);
>
> - return swp_tb | __count_to_swp_tb(count);
> + return swp_tb | __flags_to_swp_tb(flags);
> }
>
> -static inline unsigned long folio_to_swp_tb(struct folio *folio, unsigned int count)
> +static inline unsigned long folio_to_swp_tb(struct folio *folio, unsigned char flags)
> {
> - return pfn_to_swp_tb(folio_pfn(folio), count);
> + return pfn_to_swp_tb(folio_pfn(folio), flags);
> }
>
> -static inline unsigned long shadow_to_swp_tb(void *shadow, unsigned int count)
> +static inline unsigned long shadow_to_swp_tb(void *shadow, unsigned char flags)
> {
> BUILD_BUG_ON((BITS_PER_XA_VALUE + 1) !=
> BITS_PER_BYTE * sizeof(unsigned long));
> BUILD_BUG_ON((unsigned long)xa_mk_value(0) != SWP_TB_SHADOW_MARK);
>
> VM_WARN_ON_ONCE(shadow && !xa_is_value(shadow));
> - VM_WARN_ON_ONCE(shadow && ((unsigned long)shadow & SWP_TB_COUNT_MASK));
> + VM_WARN_ON_ONCE(shadow && ((unsigned long)shadow & SWP_TB_FLAGS_MASK));
>
> - return (unsigned long)shadow | __count_to_swp_tb(count) | SWP_TB_SHADOW_MARK;
> + return (unsigned long)shadow | SWP_TB_SHADOW_MARK | __flags_to_swp_tb(flags);
> }
>
> /*
> @@ -173,14 +178,14 @@ static inline bool swp_tb_is_countable(unsigned long swp_tb)
> static inline struct folio *swp_tb_to_folio(unsigned long swp_tb)
> {
> VM_WARN_ON(!swp_tb_is_folio(swp_tb));
> - return pfn_folio((swp_tb & ~SWP_TB_COUNT_MASK) >> SWP_TB_PFN_MARK_BITS);
> + return pfn_folio((swp_tb & ~SWP_TB_FLAGS_MASK) >> SWAP_CACHE_PFN_MARK_BITS);
> }
>
> static inline void *swp_tb_to_shadow(unsigned long swp_tb)
> {
> VM_WARN_ON(!swp_tb_is_shadow(swp_tb));
> /* No shift needed, xa_value is stored as it is in the lower bits. */
> - return (void *)(swp_tb & ~SWP_TB_COUNT_MASK);
> + return (void *)(swp_tb & ~SWP_TB_FLAGS_MASK);
> }
>
> static inline unsigned char __swp_tb_get_count(unsigned long swp_tb)
> @@ -189,6 +194,12 @@ static inline unsigned char __swp_tb_get_count(unsigned long swp_tb)
> return ((swp_tb & SWP_TB_COUNT_MASK) >> SWP_TB_COUNT_SHIFT);
> }
>
> +static inline unsigned char __swp_tb_get_flags(unsigned long swp_tb)
> +{
> + VM_WARN_ON(!swp_tb_is_countable(swp_tb));
> + return ((swp_tb & SWP_TB_FLAGS_MASK) >> SWP_TB_FLAGS_SHIFT);
> +}
> +
> static inline int swp_tb_get_count(unsigned long swp_tb)
> {
> if (swp_tb_is_countable(swp_tb))
> @@ -253,6 +264,50 @@ static inline unsigned long swap_table_get(struct swap_cluster_info *ci,
> return swp_tb;
> }
>
> +static inline void __swap_table_set_zero(struct swap_cluster_info *ci,
> + unsigned int ci_off)
> +{
> +#if SWAP_TABLE_HAS_ZEROFLAG
> + unsigned long swp_tb = __swap_table_get(ci, ci_off);
> +
> + BUILD_BUG_ON(SWP_TB_ZERO_FLAG & ~SWP_TB_FLAGS_MASK);
> + VM_WARN_ON(!swp_tb_is_countable(swp_tb));
> + swp_tb |= SWP_TB_ZERO_FLAG;
> + __swap_table_set(ci, ci_off, swp_tb);
> +#else
> + __set_bit(ci_off, ci->zero_bitmap);
> +#endif
> +}
> +
> +static inline bool __swap_table_test_zero(struct swap_cluster_info *ci,
> + unsigned int ci_off)
> +{
> +#if SWAP_TABLE_HAS_ZEROFLAG
> + unsigned long swp_tb = __swap_table_get(ci, ci_off);
> +
> + VM_WARN_ON(!swp_tb_is_countable(swp_tb));
> + return !!(swp_tb & SWP_TB_ZERO_FLAG);
> +#else
> + return test_bit(ci_off, ci->zero_bitmap);
> +#endif
> +}
> +
> +static inline void __swap_table_clear_zero(struct swap_cluster_info *ci,
> + unsigned int ci_off)
> +{
> +
> +#if SWAP_TABLE_HAS_ZEROFLAG
> + unsigned long swp_tb = __swap_table_get(ci, ci_off);
> +
> + VM_WARN_ON(!swp_tb_is_countable(swp_tb));
> + swp_tb &= ~SWP_TB_ZERO_FLAG;
> + __swap_table_set(ci, ci_off, swp_tb);
> +#else
> + lockdep_assert_held(&ci->lock);
> + __clear_bit(ci_off, ci->zero_bitmap);
> +#endif
> +}
> +
> #ifdef CONFIG_MEMCG
> static inline void __swap_cgroup_set(struct swap_cluster_info *ci,
> unsigned int ci_off, unsigned long nr, unsigned short id)
> diff --git a/mm/swapfile.c b/mm/swapfile.c
> index 2172920e68d1..287d5807b8f7 100644
> --- a/mm/swapfile.c
> +++ b/mm/swapfile.c
> @@ -427,6 +427,11 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci)
> ci->memcg_table = NULL;
> #endif
>
> +#if !SWAP_TABLE_HAS_ZEROFLAG
> + kfree(ci->zero_bitmap);
> + ci->zero_bitmap = NULL;
> +#endif
> +
> table = (struct swap_table *)rcu_access_pointer(ci->table);
> if (!table)
> return;
> @@ -470,6 +475,13 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
> if (!ci->memcg_table)
> ret = -ENOMEM;
> #endif
> +
> +#if !SWAP_TABLE_HAS_ZEROFLAG
> + ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp);
> + if (!ci->zero_bitmap)
> + ret = -ENOMEM;
> +#endif
> +
> if (ret)
> swap_cluster_free_table(ci);
>
> @@ -926,8 +938,8 @@ static bool __swap_cluster_alloc_entries(struct swap_info_struct *si,
> order = 0;
> nr_pages = 1;
> swap_cluster_assert_empty(ci, ci_off, 1, false);
> - /* Sets a fake shadow as placeholder */
> - __swap_table_set(ci, ci_off, shadow_to_swp_tb(NULL, 1));
> + /* Fake shadow placeholder with no flag, hibernation does not use the zeromap */
> + __swap_table_set(ci, ci_off, __swp_tb_mk_count(shadow_to_swp_tb(NULL, 0), 1));
> } else {
> /* Allocation without folio is only possible with hibernation */
> WARN_ON_ONCE(1);
> @@ -1299,14 +1311,8 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
> void (*swap_slot_free_notify)(struct block_device *, unsigned long);
> unsigned int i;
>
> - /*
> - * Use atomic clear_bit operations only on zeromap instead of non-atomic
> - * bitmap_clear to prevent adjacent bits corruption due to simultaneous writes.
> - */
> - for (i = 0; i < nr_entries; i++) {
> - clear_bit(offset + i, si->zeromap);
> + for (i = 0; i < nr_entries; i++)
> zswap_invalidate(swp_entry(si->type, offset + i));
> - }
>
> if (si->flags & SWP_BLKDEV)
> swap_slot_free_notify =
> @@ -1891,7 +1897,11 @@ void __swap_cluster_free_entries(struct swap_info_struct *si,
> * ref, or after swap cache is dropped
> */
> VM_WARN_ON(!swp_tb_is_shadow(old_tb) || __swp_tb_get_count(old_tb) > 1);
> +
> + /* Resetting the slot to NULL also clears the inline flags. */
> __swap_table_set(ci, ci_off, null_to_swp_tb());
> + if (!SWAP_TABLE_HAS_ZEROFLAG)
> + __swap_table_clear_zero(ci, ci_off);
>
> /*
> * Uncharge swap slots by memcg in batches. Consecutive
> @@ -3024,7 +3034,6 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
> SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
> {
> struct swap_info_struct *p = NULL;
> - unsigned long *zeromap;
> struct swap_cluster_info *cluster_info;
> struct file *swap_file, *victim;
> struct address_space *mapping;
> @@ -3120,8 +3129,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
>
> swap_file = p->swap_file;
> p->swap_file = NULL;
> - zeromap = p->zeromap;
> - p->zeromap = NULL;
> maxpages = p->max;
> cluster_info = p->cluster_info;
> p->max = 0;
> @@ -3133,7 +3140,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
> mutex_unlock(&swapon_mutex);
> kfree(p->global_cluster);
> p->global_cluster = NULL;
> - kvfree(zeromap);
> free_swap_cluster_info(cluster_info, maxpages);
>
> inode = mapping->host;
> @@ -3665,17 +3671,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
> if (error)
> goto bad_swap_unlock_inode;
>
> - /*
> - * Use kvmalloc_array instead of bitmap_zalloc as the allocation order might
> - * be above MAX_PAGE_ORDER incase of a large swap file.
> - */
> - si->zeromap = kvmalloc_array(BITS_TO_LONGS(maxpages), sizeof(long),
> - GFP_KERNEL | __GFP_ZERO);
> - if (!si->zeromap) {
> - error = -ENOMEM;
> - goto bad_swap_unlock_inode;
> - }
> -
> if (si->bdev && bdev_stable_writes(si->bdev))
> si->flags |= SWP_STABLE_WRITES;
>
> @@ -3777,8 +3772,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
> destroy_swap_extents(si, swap_file);
> free_swap_cluster_info(si->cluster_info, si->max);
> si->cluster_info = NULL;
> - kvfree(si->zeromap);
> - si->zeromap = NULL;
> /*
> * Clear the SWP_USED flag after all resources are freed so
> * alloc_swap_info can reuse this si safely.
>
> --
> 2.53.0
>
>
^ permalink raw reply
* Re: [PATCH v2 10/10] sched/eevdf: Move to a single runqueue
From: K Prateek Nayak @ 2026-05-11 16:21 UTC (permalink / raw)
To: Peter Zijlstra, mingo
Cc: longman, chenridong, juri.lelli, vincent.guittot,
dietmar.eggemann, rostedt, bsegall, mgorman, vschneid, tj, hannes,
mkoutny, cgroups, linux-kernel, jstultz, qyousef
In-Reply-To: <20260511120628.206700041@infradead.org>
Hello Peter,
On 5/11/2026 5:01 PM, Peter Zijlstra wrote:
> @@ -9291,34 +9206,25 @@ static void wakeup_preempt_fair(struct r
> + se = pick_next_entity(rq, true);
> + if (!se)
> + goto again;
>
> p = task_of(se);
> - if (unlikely(throttled))
> + if (unlikely(check_cfs_rq_runtime(cfs_rq_of(se))))
> task_throttle_setup_work(p);
I think this bit should also be replicated in set_next_task() after
account_cfs_rq_runtime() since any part of the hierarchy may get
throttled as a result of failing to grab runtime.
Also check_cfs_rq_runtime() only sees if the cfs_rq is throttled
but the task can fail to run if it is on a throttled_hierarchy() too
so that should be the correct check here.
Something like below (only build tested on queue/sched/flat):
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e54da4c6c945..950c072244b2 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -9224,7 +9224,19 @@ struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf)
goto again;
p = task_of(se);
- if (unlikely(check_cfs_rq_runtime(cfs_rq_of(se))))
+ /*
+ * For cases where prev is picked again after
+ * being throttled, entity_tick() would have
+ * already marked its hierarchy as throttled.
+ *
+ * Add throttle work here since
+ * put_prev_set_next_task() is skipped on
+ * same task's selection.
+ *
+ * For other case, set_next_task_fair() will
+ * handle adding the throttle work.
+ */
+ if (throttled_hierarchy(cfs_rq_of(se)))
task_throttle_setup_work(p);
return p;
@@ -13819,6 +13831,12 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
if (on_rq)
weight = __calc_prop_weight(cfs_rq, se, weight);
}
+ /*
+ * Add throttle work if the bandwidth allocation above failed
+ * to grab any runtime and throttled the task's hierarchy.
+ */
+ if (throttled_hierarchy(task_cfs_rq(p)))
+ task_throttle_setup_work(p);
se = &p->se;
cfs_rq->curr = se;
---
> return p;
>
--
Thanks and Regards,
Prateek
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox