* [RFC PATCH 1/8] mm/memory-tiers: add node_to_tier_id and tier_id_to_nodemask
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 2:31 ` [RFC PATCH 2/8] mm/vmscan: add try_to_free_mem_cgroup_pages_nodemask liuqiqi
` (6 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Add two helpers to bridge NUMA nodes and memory tiers:
- node_to_tier_id(node):
Returns the tier ID for a given node, derived from its abstract
distance (adistance_start >> MEMTIER_CHUNK_BITS). The lookup is
lockless and low-cost, safe from any context including the memcg
charge fast path. Returns -1 if the node is not associated with
a memory tier.
- tier_id_to_nodemask(tier_id, nodes):
Resolves the nodemask belonging to a given tier. This operation
is sleepable (takes the tier mutex) and must not be called from
contexts that disallow blocking, such as the memcg charge fast
path. Suitable for reclaim and other sleepable contexts.
These helpers will be used by subsequent patches to implement per-tier
memcg accounting and reclaim.
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
include/linux/memory-tiers.h | 12 ++++++++
mm/memory-tiers.c | 58 ++++++++++++++++++++++++++++++++++++
2 files changed, 70 insertions(+)
diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h
index 7999c58629ee..7277a08b97c9 100644
--- a/include/linux/memory-tiers.h
+++ b/include/linux/memory-tiers.h
@@ -52,6 +52,8 @@ int mt_perf_to_adistance(struct access_coordinate *perf, int *adist);
struct memory_dev_type *mt_find_alloc_memory_type(int adist,
struct list_head *memory_types);
void mt_put_memory_types(struct list_head *memory_types);
+int node_to_tier_id(int node);
+int tier_id_to_nodemask(int tier_id, nodemask_t *nodes);
#ifdef CONFIG_NUMA_MIGRATION
int next_demotion_node(int node, const nodemask_t *allowed_mask);
void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets);
@@ -151,5 +153,15 @@ static inline struct memory_dev_type *mt_find_alloc_memory_type(int adist,
static inline void mt_put_memory_types(struct list_head *memory_types)
{
}
+
+static inline int node_to_tier_id(int node)
+{
+ return -1;
+}
+
+static inline int tier_id_to_nodemask(int tier_id, nodemask_t *nodes)
+{
+ return -ENOENT;
+}
#endif /* CONFIG_NUMA */
#endif /* _LINUX_MEMORY_TIERS_H */
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index 54851d8a195b..896d23ac7a01 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -156,6 +156,64 @@ static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memti
return nodes;
}
+/**
+ * node_to_tier_id() - Get the memory tier id of a node
+ * @node: The node to look up
+ *
+ * Cheap, lockless lookup safe for the memcg charge hot path. The tier id is
+ * the memory tier's device id, i.e. adistance_start >> MEMTIER_CHUNK_BITS.
+ *
+ * Return: the tier id, or -1 if the node has no memory tier.
+ */
+int node_to_tier_id(int node)
+{
+ pg_data_t *pgdat;
+ struct memory_tier *memtier;
+ int tier_id = -1;
+
+ pgdat = NODE_DATA(node);
+ if (!pgdat)
+ return -1;
+
+ rcu_read_lock();
+ memtier = rcu_dereference(pgdat->memtier);
+ if (memtier)
+ tier_id = memtier->dev.id;
+ rcu_read_unlock();
+ return tier_id;
+}
+EXPORT_SYMBOL_GPL(node_to_tier_id);
+
+/**
+ * tier_id_to_nodemask() - Get the set of nodes belonging to a memory tier
+ * @tier_id: The tier id to look up
+ * @nodes: Output nodemask, cleared and filled on success
+ *
+ * Takes the memory tier mutex, so callers must be in a sleepable context.
+ * The memcg charge path calls this only after its gfp-allow-blocking check
+ * (i.e., only when about to reclaim), never on the charge fast path.
+ *
+ * Return: 0 on success, -ENOENT if no tier with the given id exists.
+ */
+int tier_id_to_nodemask(int tier_id, nodemask_t *nodes)
+{
+ struct memory_tier *memtier;
+ int ret = -ENOENT;
+
+ nodes_clear(*nodes);
+ mutex_lock(&memory_tier_lock);
+ list_for_each_entry(memtier, &memory_tiers, list) {
+ if (memtier->dev.id == tier_id) {
+ *nodes = get_memtier_nodemask(memtier);
+ ret = 0;
+ break;
+ }
+ }
+ mutex_unlock(&memory_tier_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(tier_id_to_nodemask);
+
static void memory_tier_device_release(struct device *dev)
{
struct memory_tier *tier = to_memory_tier(dev);
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [RFC PATCH 2/8] mm/vmscan: add try_to_free_mem_cgroup_pages_nodemask
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
2026-08-18 2:31 ` [RFC PATCH 1/8] mm/memory-tiers: add node_to_tier_id and tier_id_to_nodemask liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 2:31 ` [RFC PATCH 3/8] mm/memcontrol: add per-tier page counter infrastructure and lifecycle liuqiqi
` (5 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
try_to_free_mem_cgroup_pages() reclaims from all nodes of a cgroup.
Refactor it to call a new helper,
try_to_free_mem_cgroup_pages_nodemask(), which constrains scanning to a
caller-supplied nodemask. The existing API is preserved as a wrapper
passing NULL, leaving all current callers unchanged.
The reclaim core already honours sc->nodemask for cgroup reclaim via
shrink_zones(), so this change integrates cleanly.
This is needed for per-tier memory control: reclaim must be scoped to
the nodes of the over-limit tier so that enforcing one tier's limit
does not evict pages from unrelated tiers. The new helper will be used
to enforce both tier.high and tier.max.
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
include/linux/swap.h | 6 ++++++
mm/vmscan.c | 26 ++++++++++++++++++++++++--
2 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 8f0f68e245ba..6a79d6995082 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -367,6 +367,12 @@ extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
gfp_t gfp_mask,
unsigned int reclaim_options,
int *swappiness);
+extern unsigned long try_to_free_mem_cgroup_pages_nodemask(struct mem_cgroup *memcg,
+ unsigned long nr_pages,
+ gfp_t gfp_mask,
+ unsigned int reclaim_options,
+ int *swappiness,
+ nodemask_t *nodemask);
extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem,
gfp_t gfp_mask, bool noswap,
pg_data_t *pgdat,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 35c3bb15ae96..e0602e681d96 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -6759,11 +6759,12 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
return sc.nr_reclaimed;
}
-unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
+unsigned long try_to_free_mem_cgroup_pages_nodemask(struct mem_cgroup *memcg,
unsigned long nr_pages,
gfp_t gfp_mask,
unsigned int reclaim_options,
- int *swappiness)
+ int *swappiness,
+ nodemask_t *nodemask)
{
unsigned long nr_reclaimed;
unsigned int noreclaim_flag;
@@ -6779,6 +6780,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
.may_unmap = 1,
.may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP),
.proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE),
+ .nodemask = nodemask,
};
/*
* Traverse the ZONELIST_FALLBACK zonelist of the current node to put
@@ -6799,6 +6801,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
return nr_reclaimed;
}
+
+unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
+ unsigned long nr_pages,
+ gfp_t gfp_mask,
+ unsigned int reclaim_options,
+ int *swappiness)
+{
+ return try_to_free_mem_cgroup_pages_nodemask(memcg, nr_pages, gfp_mask,
+ reclaim_options, swappiness, NULL);
+}
#else
unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
unsigned long nr_pages,
@@ -6808,6 +6820,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
{
return 0;
}
+
+unsigned long try_to_free_mem_cgroup_pages_nodemask(struct mem_cgroup *memcg,
+ unsigned long nr_pages,
+ gfp_t gfp_mask,
+ unsigned int reclaim_options,
+ int *swappiness,
+ nodemask_t *nodemask)
+{
+ return 0;
+}
#endif
static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [RFC PATCH 3/8] mm/memcontrol: add per-tier page counter infrastructure and lifecycle
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
2026-08-18 2:31 ` [RFC PATCH 1/8] mm/memory-tiers: add node_to_tier_id and tier_id_to_nodemask liuqiqi
2026-08-18 2:31 ` [RFC PATCH 2/8] mm/vmscan: add try_to_free_mem_cgroup_pages_nodemask liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 2:31 ` [RFC PATCH 4/8] mm/memcontrol: add per-tier charge and uncharge liuqiqi
` (4 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Introduce struct memcg_tier_counter, a hierarchical page_counter for
each (memcg, tier) pair. Each counter tracks memory usage on a specific
NUMA memory tier within a cgroup.
The counters are managed under the memcg lifecycle:
- Lookup: lockless and RCU-protected.
- Allocation: GFP_KERNEL, during css_alloc and memory tier hotplug.
- Parent hierarchy: established explicitly at creation time.
- Destruction: in css_free, alongside the memcg.
During css_alloc, counters are pre-allocated for the tiers of all
currently online nodes, ensuring the charge hot path never needs to
allocate memory.
This patch only lays the groundwork; the counters are not yet
integrated into charge/uncharge. The actual accounting logic is added
in a subsequent patch.
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
include/linux/memcontrol.h | 11 +++
mm/memcontrol.c | 141 +++++++++++++++++++++++++++++++++++++
2 files changed, 152 insertions(+)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index e1f46a0016fc..c0f5929a87bf 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -199,6 +199,14 @@ struct obj_cgroup {
* statistics based on the statistics developed by Rik Van Riel for clock-pro,
* to help the administrator determine what knobs to tune.
*/
+
+struct memcg_tier_counter {
+ struct page_counter counter;
+ int tier_id;
+ struct list_head list;
+ struct rcu_head rcu;
+};
+
struct mem_cgroup {
struct cgroup_subsys_state css;
@@ -320,6 +328,9 @@ struct mem_cgroup {
spinlock_t event_list_lock;
#endif /* CONFIG_MEMCG_V1 */
+ spinlock_t tier_lock;
+ struct list_head tier_counters;
+
struct mem_cgroup_per_node *nodeinfo[];
};
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6dc4888a90f3..70efe01bc36f 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -28,6 +28,8 @@
#include <linux/cgroup-defs.h>
#include <linux/page_counter.h>
#include <linux/memcontrol.h>
+#include <linux/memory-tiers.h>
+#include <linux/node.h>
#include <linux/cgroup.h>
#include <linux/cpuset.h>
#include <linux/sched/mm.h>
@@ -2360,6 +2362,89 @@ static void high_work_func(struct work_struct *work)
reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
}
+/*
+ * Find a tier counter for a given memcg and tier ID.
+ *
+ * Context: Caller must hold either:
+ * - The RCU read lock, for lockless lookup in the fast (charge) path.
+ * - memcg->tier_lock, for modifications in the slow (creation/free) path.
+ */
+static struct memcg_tier_counter *
+memcg_tier_counter_find(struct mem_cgroup *memcg, int tier_id)
+{
+ struct memcg_tier_counter *tc;
+
+ list_for_each_entry_rcu(tc, &memcg->tier_counters, list,
+ lockdep_is_held(&memcg->tier_lock))
+ if (tc->tier_id == tier_id)
+ return tc;
+ return NULL;
+}
+
+static struct page_counter *
+memcg_tier_parent_link(struct mem_cgroup *parent, int tier_id)
+{
+ struct memcg_tier_counter *ptc;
+ struct page_counter *pc;
+
+ if (!parent || mem_cgroup_is_root(parent))
+ return NULL;
+
+ rcu_read_lock();
+ ptc = memcg_tier_counter_find(parent, tier_id);
+ pc = ptc ? &ptc->counter : NULL;
+ rcu_read_unlock();
+ return pc;
+}
+
+static int memcg_tier_counter_create(struct mem_cgroup *memcg,
+ struct mem_cgroup *parent, int tier_id)
+{
+ struct page_counter *parent_cnt;
+ struct memcg_tier_counter *new;
+
+ /* Fast path (lockless RCU read): already exists -> nothing to do. */
+ rcu_read_lock();
+ if (memcg_tier_counter_find(memcg, tier_id)) {
+ rcu_read_unlock();
+ return 0;
+ }
+ rcu_read_unlock();
+
+ parent_cnt = memcg_tier_parent_link(parent, tier_id);
+
+ new = kzalloc_obj(*new);
+ if (!new)
+ return -ENOMEM;
+
+ new->tier_id = tier_id;
+ page_counter_init(&new->counter, parent_cnt, false);
+ page_counter_set_high(&new->counter, PAGE_COUNTER_MAX);
+ INIT_LIST_HEAD(&new->list);
+
+ spin_lock(&memcg->tier_lock);
+ if (memcg_tier_counter_find(memcg, tier_id)) {
+ spin_unlock(&memcg->tier_lock);
+ kfree(new);
+ return 0;
+ }
+ list_add_tail_rcu(&new->list, &memcg->tier_counters);
+ spin_unlock(&memcg->tier_lock);
+ return 0;
+}
+
+static void memcg_free_tier_counters(struct mem_cgroup *memcg)
+{
+ struct memcg_tier_counter *tc, *tmp;
+
+ spin_lock(&memcg->tier_lock);
+ list_for_each_entry_safe(tc, tmp, &memcg->tier_counters, list) {
+ list_del_rcu(&tc->list);
+ kfree_rcu(tc, rcu);
+ }
+ spin_unlock(&memcg->tier_lock);
+}
+
/*
* Clamp the maximum sleep time per allocation batch to 2 seconds. This is
* enough to still cause a significant slowdown in most cases, while still
@@ -4129,6 +4214,8 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
goto fail;
INIT_WORK(&memcg->high_work, high_work_func);
+ spin_lock_init(&memcg->tier_lock);
+ INIT_LIST_HEAD(&memcg->tier_counters);
vmpressure_init(&memcg->vmpressure);
INIT_LIST_HEAD(&memcg->memory_peaks);
INIT_LIST_HEAD(&memcg->swap_peaks);
@@ -4178,6 +4265,20 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
page_counter_init(&memcg->swap, &parent->swap, false);
+
+ {
+ int nid, tid;
+
+ for_each_online_node(nid) {
+ tid = node_to_tier_id(nid);
+ if (tid != -1 && memcg_tier_counter_create(memcg,
+ mem_cgroup_from_css(parent_css), tid)) {
+ memcg_free_tier_counters(memcg);
+ mem_cgroup_free(memcg);
+ return ERR_PTR(-ENOMEM);
+ }
+ }
+ }
#ifdef CONFIG_MEMCG_V1
memcg->memory.track_failcnt = !memcg_on_dfl;
WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
@@ -4338,6 +4439,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
vmpressure_cleanup(&memcg->vmpressure);
cancel_work_sync(&memcg->high_work);
+ memcg_free_tier_counters(memcg);
memcg1_remove_from_trees(memcg);
free_shrinker_info(memcg);
mem_cgroup_free(memcg);
@@ -5534,6 +5636,41 @@ __setup("cgroup.memory=", cgroup_memory);
* basically everything that doesn't depend on a specific mem_cgroup structure
* should be initialized from here.
*/
+#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_NUMA)
+/*
+ * Below MEMTIER_HOTPLUG_PRI (100): run after memory-tiers has set or
+ * cleared the node's tier association.
+ */
+#define MEMCG_TIER_NODE_PRI 90
+
+/* Memory hotplug callback: a node came online with a potentially new tier
+ * (e.g. CXL hotplug). Ensure every online memcg has a counter for this tier.
+ * Existing tiers hit in the RCU lookup, so this path does not allocate.
+ */
+static int __meminit memcg_tier_hotplug_cb(struct notifier_block *self,
+ unsigned long action, void *_arg)
+{
+ struct node_notify *nn = _arg;
+ struct mem_cgroup *memcg;
+ int tid;
+
+ if (action != NODE_ADDED_FIRST_MEMORY)
+ return notifier_from_errno(0);
+
+ tid = node_to_tier_id(nn->nid);
+ if (tid < 0)
+ return notifier_from_errno(0);
+
+ for_each_mem_cgroup(memcg) {
+ if (!mem_cgroup_is_root(memcg) &&
+ memcg_tier_counter_create(memcg, parent_mem_cgroup(memcg), tid))
+ pr_warn_ratelimited("memcg: tier %d counter alloc failed;"
+ " tier accounting degraded\n", tid);
+ }
+ return notifier_from_errno(0);
+}
+#endif
+
int __init mem_cgroup_init(void)
{
unsigned int memcg_size;
@@ -5553,6 +5690,10 @@ int __init mem_cgroup_init(void)
memcg_wq = alloc_workqueue("memcg", WQ_PERCPU, 0);
WARN_ON(!memcg_wq);
+#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_NUMA)
+ hotplug_node_notifier(memcg_tier_hotplug_cb, MEMCG_TIER_NODE_PRI);
+#endif
+
for_each_possible_cpu(cpu) {
INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
drain_local_memcg_stock);
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [RFC PATCH 4/8] mm/memcontrol: add per-tier charge and uncharge
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
` (2 preceding siblings ...)
2026-08-18 2:31 ` [RFC PATCH 3/8] mm/memcontrol: add per-tier page counter infrastructure and lifecycle liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 4:32 ` Tao Cui
2026-08-18 2:31 ` [RFC PATCH 5/8] mm/memcontrol: add per-cpu stock for tier charge/uncharge liuqiqi
` (3 subsequent siblings)
7 siblings, 1 reply; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Extend the memcg charging infrastructure to support per-tier accounting.
After a successful global charge (try_charge_memcg), charge_memcg()
attempts per-tier charging. If the tier charge fails, the global charge
is rolled back via refill_stock() to maintain a consistent state.
Per-tier limits are enforced via dedicated page_counter objects with
both a hard limit (max) and a soft limit (high):
- Hard limit: Charging beyond the tier's max triggers reclaim scoped
to that tier's NUMA nodes and retries the charge. If reclaim fails,
OOM is invoked, mirroring memory.max behavior. PF_MEMALLOC
allocations force-charge past the tier max without reclaim (recursion
guard); __GFP_NOFAIL/__GFP_HIGH allocations are force-charged past
the tier max only after reclaim and OOM both fail.
- Soft limit: The charge always succeeds, but if a tier exceeds its
high watermark, the charge path schedules tier_high_work to perform
asynchronous reclaim.
The folio lifecycle is updated accordingly:
- Charging: Attempt per-tier charging after global success; roll back
the global charge on tier failure.
- Uncharging: Directly uncharge the tier counter.
- Replacement: Force-charge the new folio to its tier; the old folio
is uncharged upon freeing.
- Migration: Keep the global usage unchanged; uncharge the old tier
and charge the new tier to reflect the folio's new location.
This patch implements per-tier accounting for LRU folios (anon and file).
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
include/linux/memcontrol.h | 1 +
mm/memcontrol.c | 198 ++++++++++++++++++++++++++++++++++++-
2 files changed, 197 insertions(+), 2 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index c0f5929a87bf..fa944e4bc5ad 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -330,6 +330,7 @@ struct mem_cgroup {
spinlock_t tier_lock;
struct list_head tier_counters;
+ struct work_struct tier_high_work;
struct mem_cgroup_per_node *nodeinfo[];
};
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 70efe01bc36f..30f24604010c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2433,6 +2433,46 @@ static int memcg_tier_counter_create(struct mem_cgroup *memcg,
return 0;
}
+static void memcg_charge_tier_id(struct mem_cgroup *memcg, int tier_id,
+ unsigned long nr_pages)
+{
+ struct memcg_tier_counter *tc;
+
+ if (tier_id < 0)
+ return;
+ rcu_read_lock();
+ tc = memcg_tier_counter_find(memcg, tier_id);
+ if (tc)
+ page_counter_charge(&tc->counter, nr_pages);
+ rcu_read_unlock();
+}
+
+static void memcg_uncharge_tier_id(struct mem_cgroup *memcg, int tier_id,
+ unsigned long nr_pages)
+{
+ struct memcg_tier_counter *tc;
+
+ if (tier_id < 0)
+ return;
+ rcu_read_lock();
+ tc = memcg_tier_counter_find(memcg, tier_id);
+ if (tc)
+ page_counter_uncharge(&tc->counter, nr_pages);
+ rcu_read_unlock();
+}
+
+static void memcg_charge_tier(struct mem_cgroup *memcg, struct folio *folio,
+ unsigned long nr_pages)
+{
+ memcg_charge_tier_id(memcg, node_to_tier_id(folio_nid(folio)), nr_pages);
+}
+
+static void memcg_uncharge_tier(struct mem_cgroup *memcg, struct folio *folio,
+ unsigned long nr_pages)
+{
+ memcg_uncharge_tier_id(memcg, node_to_tier_id(folio_nid(folio)), nr_pages);
+}
+
static void memcg_free_tier_counters(struct mem_cgroup *memcg)
{
struct memcg_tier_counter *tc, *tmp;
@@ -2445,6 +2485,49 @@ static void memcg_free_tier_counters(struct mem_cgroup *memcg)
spin_unlock(&memcg->tier_lock);
}
+static unsigned long
+reclaim_tier(struct mem_cgroup *memcg, unsigned int nr_pages, gfp_t gfp_mask,
+ nodemask_t *nmp)
+{
+ unsigned long nr_reclaimed, pflags;
+
+ psi_memstall_enter(&pflags);
+ nr_reclaimed = try_to_free_mem_cgroup_pages_nodemask(memcg, nr_pages, gfp_mask,
+ MEMCG_RECLAIM_MAY_SWAP, NULL, nmp);
+ psi_memstall_leave(&pflags);
+ return nr_reclaimed;
+}
+
+static void tier_high_work_func(struct work_struct *work)
+{
+ struct mem_cgroup *memcg;
+ struct memcg_tier_counter *tc;
+ /* Few tiers in practice (2-4); cap is generous. */
+ int over_ids[16];
+ int nr_over = 0;
+ int i;
+
+ memcg = container_of(work, struct mem_cgroup, tier_high_work);
+
+ spin_lock(&memcg->tier_lock);
+ list_for_each_entry(tc, &memcg->tier_counters, list) {
+ if (page_counter_read(&tc->counter) > READ_ONCE(tc->counter.high)) {
+ if (nr_over < ARRAY_SIZE(over_ids))
+ over_ids[nr_over++] = tc->tier_id;
+ }
+ }
+ spin_unlock(&memcg->tier_lock);
+
+ for (i = 0; i < nr_over; i++) {
+ nodemask_t nodes;
+
+ if (tier_id_to_nodemask(over_ids[i], &nodes) ||
+ nodes_empty(nodes))
+ continue;
+ reclaim_tier(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL, &nodes);
+ }
+}
+
/*
* Clamp the maximum sleep time per allocation batch to 2 seconds. This is
* enough to still cause a significant slowdown in most cases, while still
@@ -2873,6 +2956,90 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
return 0;
}
+static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
+ unsigned int nr_pages, int tier_id)
+{
+ struct memcg_tier_counter *tc;
+ struct page_counter *counter;
+ int nr_retries = MAX_RECLAIM_RETRIES;
+ unsigned long nr_reclaimed = 0;
+ bool passed_oom = false;
+ nodemask_t nodes, *nmp = NULL;
+
+ if (tier_id < 0)
+ return 0;
+
+ rcu_read_lock();
+ tc = memcg_tier_counter_find(memcg, tier_id);
+ rcu_read_unlock();
+ if (!tc)
+ return 0;
+
+retry:
+ if (page_counter_try_charge(&tc->counter, nr_pages, &counter))
+ goto success;
+
+ /* Over max -> reclaim. */
+ if (unlikely(current->flags & PF_MEMALLOC))
+ goto force;
+ if (unlikely(task_in_memcg_oom(current)))
+ goto nomem;
+ if (!gfpflags_allow_blocking(gfp_mask))
+ goto nomem;
+
+ if (!tier_id_to_nodemask(tier_id, &nodes) && !nodes_empty(nodes))
+ nmp = &nodes;
+
+ nr_reclaimed = reclaim_tier(memcg, nr_pages, gfp_mask, nmp);
+
+ if (page_counter_read(&tc->counter) + nr_pages <= READ_ONCE(tc->counter.max))
+ goto retry;
+ if (gfp_mask & __GFP_NORETRY)
+ goto nomem;
+ if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
+ goto retry;
+ if (nr_retries--)
+ goto retry;
+ if (gfp_mask & __GFP_RETRY_MAYFAIL)
+ goto nomem;
+ if (passed_oom && task_is_dying())
+ goto nomem;
+ if (mem_cgroup_oom(memcg, gfp_mask, get_order(nr_pages * PAGE_SIZE))) {
+ passed_oom = true; /* tier max is a hard limit: OOM, like memory.max */
+ nr_retries = MAX_RECLAIM_RETRIES;
+ goto retry;
+ }
+ goto nomem;
+success:
+ do {
+ struct memcg_tier_counter *tc_this;
+
+ rcu_read_lock();
+ tc_this = memcg_tier_counter_find(memcg, tier_id);
+ if (tc_this &&
+ page_counter_read(&tc_this->counter) > READ_ONCE(tc_this->counter.high) &&
+ !work_pending(&memcg->tier_high_work)) {
+ schedule_work(&memcg->tier_high_work);
+ rcu_read_unlock();
+ break;
+ }
+ rcu_read_unlock();
+ } while ((memcg = parent_mem_cgroup(memcg)));
+ return 0;
+nomem:
+ if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
+ return -ENOMEM;
+
+force:
+ /*
+ * Force-charge past the tier max for reclaim/privileged allocations
+ * (PF_MEMALLOC, or __GFP_NOFAIL/__GFP_HIGH that fell through from
+ * nomem) -- not skip -- so the page is in tier.current too.
+ */
+ page_counter_charge(&tc->counter, nr_pages);
+ return 0;
+}
+
static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
unsigned int nr_pages)
{
@@ -4216,6 +4383,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
INIT_WORK(&memcg->high_work, high_work_func);
spin_lock_init(&memcg->tier_lock);
INIT_LIST_HEAD(&memcg->tier_counters);
+ INIT_WORK(&memcg->tier_high_work, tier_high_work_func);
vmpressure_init(&memcg->vmpressure);
INIT_LIST_HEAD(&memcg->memory_peaks);
INIT_LIST_HEAD(&memcg->swap_peaks);
@@ -4439,6 +4607,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
vmpressure_cleanup(&memcg->vmpressure);
cancel_work_sync(&memcg->high_work);
+ cancel_work_sync(&memcg->tier_high_work);
memcg_free_tier_counters(memcg);
memcg1_remove_from_trees(memcg);
free_shrinker_info(memcg);
@@ -5224,8 +5393,17 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
objcg = get_obj_cgroup_from_memcg(memcg);
/* Do not account at the root objcg level. */
- if (!obj_cgroup_is_root(objcg))
+ if (!obj_cgroup_is_root(objcg)) {
ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio));
+ if (!ret) {
+ int tid = node_to_tier_id(folio_nid(folio));
+
+ ret = try_charge_memcg_tier(memcg, gfp, folio_nr_pages(folio), tid);
+ /* tier over max / OOM: undo the memory charge */
+ if (ret)
+ refill_stock(memcg, folio_nr_pages(folio));
+ }
+ }
if (ret) {
obj_cgroup_put(objcg);
return ret;
@@ -5385,8 +5563,11 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
ug->nr_kmem += nr_pages;
} else {
/* LRU pages aren't accounted at the root level */
- if (!obj_cgroup_is_root(objcg))
+ if (!obj_cgroup_is_root(objcg)) {
ug->nr_memory += nr_pages;
+ memcg_uncharge_tier(obj_cgroup_memcg(objcg), folio,
+ nr_pages);
+ }
ug->pgpgout++;
WARN_ON_ONCE(folio_unqueue_deferred_split(folio));
@@ -5461,6 +5642,7 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
page_counter_charge(&memcg->memory, nr_pages);
if (do_memsw_account())
page_counter_charge(&memcg->memsw, nr_pages);
+ memcg_charge_tier(memcg, new, nr_pages);
}
obj_cgroup_get(objcg);
@@ -5503,6 +5685,18 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)
if (!objcg)
return;
+ /* Re-account the per-tier breakdown if the folio moved across tiers. */
+ if (!obj_cgroup_is_root(objcg)) {
+ struct mem_cgroup *memcg = obj_cgroup_memcg(objcg);
+ int old_tier = node_to_tier_id(folio_nid(old));
+ int new_tier = node_to_tier_id(folio_nid(new));
+
+ if (old_tier != new_tier) {
+ memcg_uncharge_tier_id(memcg, old_tier, folio_nr_pages(old));
+ memcg_charge_tier_id(memcg, new_tier, folio_nr_pages(new));
+ }
+ }
+
/* Transfer the charge and the objcg ref */
commit_charge(new, objcg);
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [RFC PATCH 4/8] mm/memcontrol: add per-tier charge and uncharge
2026-08-18 2:31 ` [RFC PATCH 4/8] mm/memcontrol: add per-tier charge and uncharge liuqiqi
@ 2026-08-18 4:32 ` Tao Cui
0 siblings, 0 replies; 12+ messages in thread
From: Tao Cui @ 2026-08-18 4:32 UTC (permalink / raw)
To: liuqiqi, linux-mm
Cc: cui.tao, tj, mkoutny, hannes, mhocko, roman.gushchin,
shakeel.butt, muchun.song, akpm, cgroups, linux-kernel
Hi,Qiqi
在 2026/8/18 10:31, liuqiqi@kylinos.cn 写道:
> From: Qiqi Liu <liuqiqi@kylinos.cn>
>
> Extend the memcg charging infrastructure to support per-tier accounting.
> After a successful global charge (try_charge_memcg), charge_memcg()
> attempts per-tier charging. If the tier charge fails, the global charge
> is rolled back via refill_stock() to maintain a consistent state.
>
> Per-tier limits are enforced via dedicated page_counter objects with
> both a hard limit (max) and a soft limit (high):
>
> - Hard limit: Charging beyond the tier's max triggers reclaim scoped
> to that tier's NUMA nodes and retries the charge. If reclaim fails,
> OOM is invoked, mirroring memory.max behavior. PF_MEMALLOC
> allocations force-charge past the tier max without reclaim (recursion
> guard); __GFP_NOFAIL/__GFP_HIGH allocations are force-charged past
> the tier max only after reclaim and OOM both fail.
>
> - Soft limit: The charge always succeeds, but if a tier exceeds its
> high watermark, the charge path schedules tier_high_work to perform
> asynchronous reclaim.
>
> The folio lifecycle is updated accordingly:
>
> - Charging: Attempt per-tier charging after global success; roll back
> the global charge on tier failure.
> - Uncharging: Directly uncharge the tier counter.
> - Replacement: Force-charge the new folio to its tier; the old folio
> is uncharged upon freeing.
> - Migration: Keep the global usage unchanged; uncharge the old tier
> and charge the new tier to reflect the folio's new location.
>
> This patch implements per-tier accounting for LRU folios (anon and file).
>
> Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
> ---
> include/linux/memcontrol.h | 1 +
> mm/memcontrol.c | 198 ++++++++++++++++++++++++++++++++++++-
> 2 files changed, 197 insertions(+), 2 deletions(-)
>
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index c0f5929a87bf..fa944e4bc5ad 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -330,6 +330,7 @@ struct mem_cgroup {
>
> spinlock_t tier_lock;
> struct list_head tier_counters;
> + struct work_struct tier_high_work;
>
> struct mem_cgroup_per_node *nodeinfo[];
> };
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 70efe01bc36f..30f24604010c 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -2433,6 +2433,46 @@ static int memcg_tier_counter_create(struct mem_cgroup *memcg,
> return 0;
> }
>
> +static void memcg_charge_tier_id(struct mem_cgroup *memcg, int tier_id,
> + unsigned long nr_pages)
> +{
> + struct memcg_tier_counter *tc;
> +
> + if (tier_id < 0)
> + return;
> + rcu_read_lock();
> + tc = memcg_tier_counter_find(memcg, tier_id);
> + if (tc)
> + page_counter_charge(&tc->counter, nr_pages);
> + rcu_read_unlock();
> +}
> +
> +static void memcg_uncharge_tier_id(struct mem_cgroup *memcg, int tier_id,
> + unsigned long nr_pages)
> +{
> + struct memcg_tier_counter *tc;
> +
> + if (tier_id < 0)
> + return;
> + rcu_read_lock();
> + tc = memcg_tier_counter_find(memcg, tier_id);
> + if (tc)
> + page_counter_uncharge(&tc->counter, nr_pages);
> + rcu_read_unlock();
> +}
> +
> +static void memcg_charge_tier(struct mem_cgroup *memcg, struct folio *folio,
> + unsigned long nr_pages)
> +{
> + memcg_charge_tier_id(memcg, node_to_tier_id(folio_nid(folio)), nr_pages);
> +}
> +
> +static void memcg_uncharge_tier(struct mem_cgroup *memcg, struct folio *folio,
> + unsigned long nr_pages)
> +{
> + memcg_uncharge_tier_id(memcg, node_to_tier_id(folio_nid(folio)), nr_pages);
> +}
> +
> static void memcg_free_tier_counters(struct mem_cgroup *memcg)
> {
> struct memcg_tier_counter *tc, *tmp;
> @@ -2445,6 +2485,49 @@ static void memcg_free_tier_counters(struct mem_cgroup *memcg)
> spin_unlock(&memcg->tier_lock);
> }
>
> +static unsigned long
> +reclaim_tier(struct mem_cgroup *memcg, unsigned int nr_pages, gfp_t gfp_mask,
> + nodemask_t *nmp)
> +{
> + unsigned long nr_reclaimed, pflags;
> +
> + psi_memstall_enter(&pflags);
> + nr_reclaimed = try_to_free_mem_cgroup_pages_nodemask(memcg, nr_pages, gfp_mask,
> + MEMCG_RECLAIM_MAY_SWAP, NULL, nmp);
> + psi_memstall_leave(&pflags);
> + return nr_reclaimed;
> +}
> +
> +static void tier_high_work_func(struct work_struct *work)
> +{
> + struct mem_cgroup *memcg;
> + struct memcg_tier_counter *tc;
> + /* Few tiers in practice (2-4); cap is generous. */
> + int over_ids[16];
> + int nr_over = 0;
> + int i;
> +
> + memcg = container_of(work, struct mem_cgroup, tier_high_work);
> +
> + spin_lock(&memcg->tier_lock);
> + list_for_each_entry(tc, &memcg->tier_counters, list) {
> + if (page_counter_read(&tc->counter) > READ_ONCE(tc->counter.high)) {
> + if (nr_over < ARRAY_SIZE(over_ids))
> + over_ids[nr_over++] = tc->tier_id;
> + }
> + }
> + spin_unlock(&memcg->tier_lock);
> +
> + for (i = 0; i < nr_over; i++) {
> + nodemask_t nodes;
> +
> + if (tier_id_to_nodemask(over_ids[i], &nodes) ||
> + nodes_empty(nodes))
> + continue;
> + reclaim_tier(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL, &nodes);
> + }
> +}
> +
> /*
> * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
> * enough to still cause a significant slowdown in most cases, while still
> @@ -2873,6 +2956,90 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
> return 0;
> }
>
> +static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
> + unsigned int nr_pages, int tier_id)
> +{
> + struct memcg_tier_counter *tc;
> + struct page_counter *counter;
> + int nr_retries = MAX_RECLAIM_RETRIES;
> + unsigned long nr_reclaimed = 0;
> + bool passed_oom = false;
> + nodemask_t nodes, *nmp = NULL;
> +
> + if (tier_id < 0)
> + return 0;
> +
> + rcu_read_lock();
> + tc = memcg_tier_counter_find(memcg, tier_id);
> + rcu_read_unlock();
> + if (!tc)
> + return 0;
> +
> +retry:
> + if (page_counter_try_charge(&tc->counter, nr_pages, &counter))
> + goto success;
> +
> + /* Over max -> reclaim. */
> + if (unlikely(current->flags & PF_MEMALLOC))
> + goto force;
> + if (unlikely(task_in_memcg_oom(current)))
> + goto nomem;
> + if (!gfpflags_allow_blocking(gfp_mask))
> + goto nomem;
> +
> + if (!tier_id_to_nodemask(tier_id, &nodes) && !nodes_empty(nodes))
> + nmp = &nodes;
> +
> + nr_reclaimed = reclaim_tier(memcg, nr_pages, gfp_mask, nmp);
> +
If the nodemask lookup fails, nmp stays NULL and reclaim scans all
nodes - enforcing tier4.max evicts from tier22, and the freed pages
are on the wrong tier so it doesn't even help. I'd goto nomem here.
Also two nits: struct page_counter *counter is never read; and the
function relies on the caller's css reference to keep tc valid across
reclaim_tier()/mem_cgroup_oom() (the RCU lock only covers the list
walk) - worth a comment before someone adds a new caller.
Thanks,
Tao
> + if (page_counter_read(&tc->counter) + nr_pages <= READ_ONCE(tc->counter.max))
> + goto retry;
> + if (gfp_mask & __GFP_NORETRY)
> + goto nomem;
> + if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
> + goto retry;
> + if (nr_retries--)
> + goto retry;
> + if (gfp_mask & __GFP_RETRY_MAYFAIL)
> + goto nomem;
> + if (passed_oom && task_is_dying())
> + goto nomem;
> + if (mem_cgroup_oom(memcg, gfp_mask, get_order(nr_pages * PAGE_SIZE))) {
> + passed_oom = true; /* tier max is a hard limit: OOM, like memory.max */
> + nr_retries = MAX_RECLAIM_RETRIES;
> + goto retry;
> + }
> + goto nomem;
> +success:
> + do {
> + struct memcg_tier_counter *tc_this;
> +
> + rcu_read_lock();
> + tc_this = memcg_tier_counter_find(memcg, tier_id);
> + if (tc_this &&
> + page_counter_read(&tc_this->counter) > READ_ONCE(tc_this->counter.high) &&
> + !work_pending(&memcg->tier_high_work)) {
> + schedule_work(&memcg->tier_high_work);
> + rcu_read_unlock();
> + break;
> + }
> + rcu_read_unlock();
> + } while ((memcg = parent_mem_cgroup(memcg)));
> + return 0;
> +nomem:
> + if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
> + return -ENOMEM;
> +
> +force:
> + /*
> + * Force-charge past the tier max for reclaim/privileged allocations
> + * (PF_MEMALLOC, or __GFP_NOFAIL/__GFP_HIGH that fell through from
> + * nomem) -- not skip -- so the page is in tier.current too.
> + */
> + page_counter_charge(&tc->counter, nr_pages);
> + return 0;
> +}
> +
> static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
> unsigned int nr_pages)
> {
> @@ -4216,6 +4383,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
> INIT_WORK(&memcg->high_work, high_work_func);
> spin_lock_init(&memcg->tier_lock);
> INIT_LIST_HEAD(&memcg->tier_counters);
> + INIT_WORK(&memcg->tier_high_work, tier_high_work_func);
> vmpressure_init(&memcg->vmpressure);
> INIT_LIST_HEAD(&memcg->memory_peaks);
> INIT_LIST_HEAD(&memcg->swap_peaks);
> @@ -4439,6 +4607,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
>
> vmpressure_cleanup(&memcg->vmpressure);
> cancel_work_sync(&memcg->high_work);
> + cancel_work_sync(&memcg->tier_high_work);
> memcg_free_tier_counters(memcg);
> memcg1_remove_from_trees(memcg);
> free_shrinker_info(memcg);
> @@ -5224,8 +5393,17 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
>
> objcg = get_obj_cgroup_from_memcg(memcg);
> /* Do not account at the root objcg level. */
> - if (!obj_cgroup_is_root(objcg))
> + if (!obj_cgroup_is_root(objcg)) {
> ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio));
> + if (!ret) {
> + int tid = node_to_tier_id(folio_nid(folio));
> +
> + ret = try_charge_memcg_tier(memcg, gfp, folio_nr_pages(folio), tid);
> + /* tier over max / OOM: undo the memory charge */
> + if (ret)
> + refill_stock(memcg, folio_nr_pages(folio));
> + }
> + }
> if (ret) {
> obj_cgroup_put(objcg);
> return ret;
> @@ -5385,8 +5563,11 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
> ug->nr_kmem += nr_pages;
> } else {
> /* LRU pages aren't accounted at the root level */
> - if (!obj_cgroup_is_root(objcg))
> + if (!obj_cgroup_is_root(objcg)) {
> ug->nr_memory += nr_pages;
> + memcg_uncharge_tier(obj_cgroup_memcg(objcg), folio,
> + nr_pages);
> + }
> ug->pgpgout++;
>
> WARN_ON_ONCE(folio_unqueue_deferred_split(folio));
> @@ -5461,6 +5642,7 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
> page_counter_charge(&memcg->memory, nr_pages);
> if (do_memsw_account())
> page_counter_charge(&memcg->memsw, nr_pages);
> + memcg_charge_tier(memcg, new, nr_pages);
> }
>
> obj_cgroup_get(objcg);
> @@ -5503,6 +5685,18 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)
> if (!objcg)
> return;
>
> + /* Re-account the per-tier breakdown if the folio moved across tiers. */
> + if (!obj_cgroup_is_root(objcg)) {
> + struct mem_cgroup *memcg = obj_cgroup_memcg(objcg);
> + int old_tier = node_to_tier_id(folio_nid(old));
> + int new_tier = node_to_tier_id(folio_nid(new));
> +
> + if (old_tier != new_tier) {
> + memcg_uncharge_tier_id(memcg, old_tier, folio_nr_pages(old));
> + memcg_charge_tier_id(memcg, new_tier, folio_nr_pages(new));
> + }
> + }
> +
> /* Transfer the charge and the objcg ref */
> commit_charge(new, objcg);
>
^ permalink raw reply [flat|nested] 12+ messages in thread
* [RFC PATCH 5/8] mm/memcontrol: add per-cpu stock for tier charge/uncharge
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
` (3 preceding siblings ...)
2026-08-18 2:31 ` [RFC PATCH 4/8] mm/memcontrol: add per-tier charge and uncharge liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 2:31 ` [RFC PATCH 6/8] mm/memcontrol: add memory.tier control file liuqiqi
` (2 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Add tier_stock_pcp, a per-cpu cache with NR_TIER_STOCK slots, to cache
charge surplus for specific (memcg, tier_id) pairs.
The fast path, consume_tier_stock(), attempts to satisfy charges from
this cache. On a miss, try_charge_memcg_tier() charges in units of
max(MEMCG_CHARGE_BATCH, nr_pages), falling back to an exact nr_pages
charge when the batch does not fit, and refill_tier_stock() restocks
the surplus. Uncharging is handled by the same helper: refill_tier_stock()
drains a mismatched slot and caches the new pair.
Cached stock is drained via drain_all_tier_stock() during css_offline,
and from the charge path when a tier charge retries under limit pressure.
This is purely an optimization; the semantics remain identical to
per-page charging.
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
mm/memcontrol.c | 211 ++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 203 insertions(+), 8 deletions(-)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 30f24604010c..3aa55c287248 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2318,11 +2318,33 @@ void drain_all_stock(struct mem_cgroup *root_memcg)
mutex_unlock(&percpu_charge_mutex);
}
+/*
+ * Per-CPU tier charge/uncharge stock: caches up to NR_TIER_STOCK (memcg,
+ * tier_id) pairs so small charges/uncharges avoid atomics.
+ */
+#define NR_TIER_STOCK 7
+struct tier_stock_pcp {
+ local_lock_t lock;
+ uint8_t nr_pages[NR_TIER_STOCK];
+ struct mem_cgroup *cached[NR_TIER_STOCK];
+ int tier_id[NR_TIER_STOCK];
+ struct work_struct work;
+ unsigned long flags;
+ uint8_t drain_idx;
+};
+static DEFINE_PER_CPU(struct tier_stock_pcp, tier_stock);
+
+static void __drain_tier_stock(struct tier_stock_pcp *stock, int i);
+
static int memcg_hotplug_cpu_dead(unsigned int cpu)
{
/* no need for the local lock */
+ int i;
+
drain_obj_stock(&per_cpu(obj_stock, cpu));
drain_stock_fully(&per_cpu(memcg_stock, cpu));
+ for (i = 0; i < NR_TIER_STOCK; i++)
+ __drain_tier_stock(&per_cpu(tier_stock, cpu), i);
return 0;
}
@@ -2467,12 +2489,6 @@ static void memcg_charge_tier(struct mem_cgroup *memcg, struct folio *folio,
memcg_charge_tier_id(memcg, node_to_tier_id(folio_nid(folio)), nr_pages);
}
-static void memcg_uncharge_tier(struct mem_cgroup *memcg, struct folio *folio,
- unsigned long nr_pages)
-{
- memcg_uncharge_tier_id(memcg, node_to_tier_id(folio_nid(folio)), nr_pages);
-}
-
static void memcg_free_tier_counters(struct mem_cgroup *memcg)
{
struct memcg_tier_counter *tc, *tmp;
@@ -2956,14 +2972,173 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
return 0;
}
+/* Drain cached tier charge from a per-CPU stock slot.
+ * The drained count is returned to the tier's page_counter,
+ * and the memcg css reference is put.
+ */
+static void __drain_tier_stock(struct tier_stock_pcp *stock, int i)
+{
+ struct memcg_tier_counter *tc;
+ struct mem_cgroup *old = READ_ONCE(stock->cached[i]);
+ uint8_t stock_pages;
+
+ if (!old)
+ return;
+ stock_pages = READ_ONCE(stock->nr_pages[i]);
+ if (stock_pages) {
+ rcu_read_lock();
+ tc = memcg_tier_counter_find(old, READ_ONCE(stock->tier_id[i]));
+ if (tc)
+ page_counter_uncharge(&tc->counter, stock_pages);
+ rcu_read_unlock();
+ WRITE_ONCE(stock->nr_pages[i], 0);
+ }
+ css_put(&old->css);
+ WRITE_ONCE(stock->cached[i], NULL);
+ WRITE_ONCE(stock->tier_id[i], -1);
+}
+
+static void drain_local_tier_stock(struct work_struct *dummy)
+{
+ struct tier_stock_pcp *stock;
+ int i;
+
+ if (WARN_ON_ONCE(!in_task()))
+ return;
+ local_lock(&tier_stock.lock);
+ stock = this_cpu_ptr(&tier_stock);
+ for (i = 0; i < NR_TIER_STOCK; i++)
+ __drain_tier_stock(stock, i);
+ clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
+ local_unlock(&tier_stock.lock);
+}
+
+/* Drain per-CPU tier stock entries matching @memcg on all CPUs. */
+static void drain_all_tier_stock(struct mem_cgroup *memcg)
+{
+ int cpu, curcpu, i;
+
+ if (!mutex_trylock(&percpu_charge_mutex))
+ return;
+ migrate_disable();
+ curcpu = smp_processor_id();
+ for_each_online_cpu(cpu) {
+ struct tier_stock_pcp *stock = &per_cpu(tier_stock, cpu);
+
+ if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
+ continue;
+ for (i = 0; i < NR_TIER_STOCK; i++) {
+ if (READ_ONCE(stock->cached[i]) != memcg)
+ continue;
+ if (!test_and_set_bit(FLUSHING_CACHED_CHARGE,
+ &stock->flags)) {
+ if (cpu == curcpu)
+ drain_local_tier_stock(&stock->work);
+ else
+ schedule_drain_work(cpu, &stock->work);
+ }
+ break;
+ }
+ }
+ migrate_enable();
+ mutex_unlock(&percpu_charge_mutex);
+}
+
+/* Consume @nr_pages from the per-CPU tier stock if a matching slot has enough surplus */
+static bool consume_tier_stock(struct mem_cgroup *memcg, int tier_id,
+ unsigned int nr_pages)
+{
+ struct tier_stock_pcp *stock;
+ bool ret = false;
+ int i;
+ uint8_t pages;
+
+ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX);
+
+ if (nr_pages > MEMCG_CHARGE_BATCH)
+ return false;
+
+ local_lock(&tier_stock.lock);
+ stock = this_cpu_ptr(&tier_stock);
+
+ for (i = 0; i < NR_TIER_STOCK; i++) {
+ if (READ_ONCE(stock->cached[i]) == memcg &&
+ READ_ONCE(stock->tier_id[i]) == tier_id) {
+ pages = READ_ONCE(stock->nr_pages[i]);
+ if (pages >= nr_pages) {
+ WRITE_ONCE(stock->nr_pages[i], pages - nr_pages);
+ ret = true;
+ }
+ break;
+ }
+ }
+
+ local_unlock(&tier_stock.lock);
+ return ret;
+}
+
+/* Refund @nr_pages to the per-CPU tier stock. */
+static void refill_tier_stock(struct mem_cgroup *memcg, int tier_id,
+ unsigned int nr_pages)
+{
+ struct memcg_tier_counter *tc;
+ struct tier_stock_pcp *stock;
+ int empty_slot = -1;
+ uint8_t pages;
+ int i;
+
+ /* Too big to cache: direct uncharge, leave the stock untouched. */
+ if (nr_pages > MEMCG_CHARGE_BATCH) {
+ rcu_read_lock();
+ tc = memcg_tier_counter_find(memcg, tier_id);
+ if (tc)
+ page_counter_uncharge(&tc->counter, nr_pages);
+ rcu_read_unlock();
+ return;
+ }
+
+ local_lock(&tier_stock.lock);
+ stock = this_cpu_ptr(&tier_stock);
+
+ for (i = 0; i < NR_TIER_STOCK; i++) {
+ if (!READ_ONCE(stock->cached[i]) && empty_slot == -1)
+ empty_slot = i;
+ if (READ_ONCE(stock->cached[i]) == memcg &&
+ READ_ONCE(stock->tier_id[i]) == tier_id) {
+ pages = READ_ONCE(stock->nr_pages[i]) + nr_pages;
+ WRITE_ONCE(stock->nr_pages[i], pages);
+ if (pages > MEMCG_CHARGE_BATCH)
+ __drain_tier_stock(stock, i);
+ goto out;
+ }
+ }
+
+ /* Mismatch: pick a slot (empty or evict), drain, cache new. */
+ i = empty_slot;
+ if (i == -1) {
+ i = stock->drain_idx++;
+ if (stock->drain_idx == NR_TIER_STOCK)
+ stock->drain_idx = 0;
+ }
+ __drain_tier_stock(stock, i);
+ css_get(&memcg->css);
+ WRITE_ONCE(stock->cached[i], memcg);
+ WRITE_ONCE(stock->tier_id[i], tier_id);
+ WRITE_ONCE(stock->nr_pages[i], nr_pages);
+out:
+ local_unlock(&tier_stock.lock);
+}
+
static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
unsigned int nr_pages, int tier_id)
{
struct memcg_tier_counter *tc;
struct page_counter *counter;
+ unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
int nr_retries = MAX_RECLAIM_RETRIES;
unsigned long nr_reclaimed = 0;
bool passed_oom = false;
+ bool drained = false;
nodemask_t nodes, *nmp = NULL;
if (tier_id < 0)
@@ -2976,9 +3151,16 @@ static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
return 0;
retry:
- if (page_counter_try_charge(&tc->counter, nr_pages, &counter))
+ if (consume_tier_stock(memcg, tier_id, nr_pages))
+ return 0;
+ if (page_counter_try_charge(&tc->counter, batch, &counter))
goto success;
+ if (batch > nr_pages) {
+ batch = nr_pages;
+ goto retry;
+ }
+
/* Over max -> reclaim. */
if (unlikely(current->flags & PF_MEMALLOC))
goto force;
@@ -2994,6 +3176,13 @@ static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
if (page_counter_read(&tc->counter) + nr_pages <= READ_ONCE(tc->counter.max))
goto retry;
+
+ if (!drained) {
+ drain_all_tier_stock(memcg);
+ drained = true;
+ goto retry;
+ }
+
if (gfp_mask & __GFP_NORETRY)
goto nomem;
if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
@@ -3011,6 +3200,8 @@ static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
}
goto nomem;
success:
+ if (batch > nr_pages)
+ refill_tier_stock(memcg, tier_id, batch - nr_pages);
do {
struct memcg_tier_counter *tc_this;
@@ -4575,6 +4766,7 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
lru_gen_offline_memcg(memcg);
drain_all_stock(memcg);
+ drain_all_tier_stock(memcg);
mem_cgroup_private_id_put(memcg, 1);
}
@@ -5565,7 +5757,8 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
/* LRU pages aren't accounted at the root level */
if (!obj_cgroup_is_root(objcg)) {
ug->nr_memory += nr_pages;
- memcg_uncharge_tier(obj_cgroup_memcg(objcg), folio,
+ refill_tier_stock(obj_cgroup_memcg(objcg),
+ node_to_tier_id(folio_nid(folio)),
nr_pages);
}
ug->pgpgout++;
@@ -5893,6 +6086,8 @@ int __init mem_cgroup_init(void)
drain_local_memcg_stock);
INIT_WORK(&per_cpu_ptr(&obj_stock, cpu)->work,
drain_local_obj_stock);
+ INIT_WORK(&per_cpu(tier_stock, cpu).work,
+ drain_local_tier_stock);
}
memcg_size = struct_size_t(struct mem_cgroup, nodeinfo, nr_node_ids);
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [RFC PATCH 6/8] mm/memcontrol: add memory.tier control file
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
` (4 preceding siblings ...)
2026-08-18 2:31 ` [RFC PATCH 5/8] mm/memcontrol: add per-cpu stock for tier charge/uncharge liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 2:31 ` [RFC PATCH 7/8] mm/memcontrol: auto-derive tier high/max from memory.high/max liuqiqi
2026-08-18 2:31 ` [RFC PATCH 8/8] cgroup: add memory_tiered_limits cgroup mount option liuqiqi
7 siblings, 0 replies; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Introduce the memory.tier control file to expose per-tier usage and limits.
Counters are pre-allocated during css_alloc or memory hotplug:
tierN.current=<bytes> (read-only)
tierN.high=<bytes|max> (default: max)
tierN.max=<bytes|max> (default: max)
Writes are parsed via page_counter_memparse():
- high: Sets page_counter_set_high() and reclaims down to the new limit,
scoped to the tier's nodes, mirroring memory_high_write().
- max: Sets the limit via xchg() and reclaims or OOMs down to the new
limit, mirroring memory_max_write().
Since counters are pre-allocated for all online tiers, updates require
no memory allocation and return -ENOENT for unknown tiers.
Example:
# cat memory.tier
tier4.current=1048576
tier4.high=max
tier4.max=536870912
# echo "tier4.high=268435456" > memory.tier
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
mm/memcontrol.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 104 insertions(+)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 3aa55c287248..a6c6057f0e4f 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4822,6 +4822,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
{
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
+ struct memcg_tier_counter *tc;
page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
@@ -4835,6 +4836,13 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
memcg1_soft_limit_reset(memcg);
page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
memcg_wb_domain_size_changed(memcg);
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(tc, &memcg->tier_counters, list) {
+ page_counter_set_max(&tc->counter, PAGE_COUNTER_MAX);
+ page_counter_set_high(&tc->counter, PAGE_COUNTER_MAX);
+ }
+ rcu_read_unlock();
}
struct aggregate_control {
@@ -5331,6 +5339,96 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
return nbytes;
}
+static int memory_tier_show(struct seq_file *m, void *v)
+{
+ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+ struct memcg_tier_counter *tc;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(tc, &memcg->tier_counters, list) {
+ seq_printf(m, "tier%d.current=%llu\n", tc->tier_id,
+ (u64)page_counter_read(&tc->counter) * PAGE_SIZE);
+ seq_printf(m, "tier%d.high=", tc->tier_id);
+ seq_puts_memcg_tunable(m, READ_ONCE(tc->counter.high));
+ seq_printf(m, "tier%d.max=", tc->tier_id);
+ seq_puts_memcg_tunable(m, READ_ONCE(tc->counter.max));
+ }
+ rcu_read_unlock();
+ return 0;
+}
+
+static ssize_t memory_tier_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes, loff_t off)
+{
+ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
+ struct memcg_tier_counter *tc;
+ unsigned int nr_retries = MAX_RECLAIM_RETRIES;
+ bool drained = false;
+ nodemask_t nodes, *nmp = NULL;
+ unsigned long val;
+ char knob[8], *p;
+ int tier_id, err;
+
+ buf = strstrip(buf);
+ if (sscanf(buf, "tier%d.%7[^=]", &tier_id, knob) != 2)
+ return -EINVAL;
+ if (strcmp(knob, "high") && strcmp(knob, "max"))
+ return -EINVAL;
+ p = strchr(buf, '=');
+ if (!p)
+ return -EINVAL;
+ err = page_counter_memparse(p + 1, "max", &val);
+ if (err)
+ return err;
+
+ rcu_read_lock();
+ tc = memcg_tier_counter_find(memcg, tier_id);
+ rcu_read_unlock();
+ if (!tc)
+ return -ENOENT;
+
+ if (!tier_id_to_nodemask(tier_id, &nodes) && !nodes_empty(nodes))
+ nmp = &nodes;
+
+ if (!strcmp(knob, "high"))
+ page_counter_set_high(&tc->counter, val);
+ else
+ xchg(&tc->counter.max, val);
+
+ if (of->file->f_flags & O_NONBLOCK)
+ return nbytes;
+
+ for (;;) {
+ unsigned long nr_pages = page_counter_read(&tc->counter);
+
+ if (nr_pages <= val)
+ break;
+ if (signal_pending(current))
+ break;
+ if (!drained) {
+ drain_all_tier_stock(memcg);
+ drained = true;
+ continue;
+ }
+ if (reclaim_tier(memcg, nr_pages - val, GFP_KERNEL, nmp))
+ continue;
+ if (nr_retries) {
+ nr_retries--;
+ continue;
+ }
+ if (!strcmp(knob, "max")) {
+ memcg_memory_event(memcg, MEMCG_OOM);
+ if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
+ break;
+ cond_resched();
+ } else {
+ break;
+ }
+ }
+
+ return nbytes;
+}
+
/*
* Note: don't forget to update the 'samples/cgroup/memcg_event_listener'
* if any new events become available.
@@ -5500,6 +5598,12 @@ static struct cftype memory_files[] = {
.seq_show = memory_max_show,
.write = memory_max_write,
},
+ {
+ .name = "tier",
+ .flags = CFTYPE_NOT_ON_ROOT,
+ .seq_show = memory_tier_show,
+ .write = memory_tier_write,
+ },
{
.name = "events",
.flags = CFTYPE_NOT_ON_ROOT,
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [RFC PATCH 7/8] mm/memcontrol: auto-derive tier high/max from memory.high/max
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
` (5 preceding siblings ...)
2026-08-18 2:31 ` [RFC PATCH 6/8] mm/memcontrol: add memory.tier control file liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 4:46 ` Tao Cui
2026-08-18 2:31 ` [RFC PATCH 8/8] cgroup: add memory_tiered_limits cgroup mount option liuqiqi
7 siblings, 1 reply; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Per-tier limits are derived by default from the cgroup-wide memory.high
and memory.max knobs based on the tier's capacity ratio:
tierN.high = memory.high * tierN_capacity / total_capacity
tierN.max = memory.max * tierN_capacity / total_capacity
This provides zero-config tier partitioning. Setting memory.high or
memory.max automatically splits the budget across tiers proportionally
to their physical size. For instance, a 4G cgroup on a system with a
75%/25% DRAM/CXL split automatically receives ~3G DRAM and ~1G CXL.
The capacity cache and the derived limits are refreshed on node memory
hotplug, both when node memory is added and when it is removed.
Writing an explicit value pins the limit and disables auto-derivation
for that tier, decoupling it from subsequent memory.high/max updates.
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
include/linux/memcontrol.h | 2 +
mm/memcontrol.c | 162 +++++++++++++++++++++++++++++++++----
2 files changed, 150 insertions(+), 14 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index fa944e4bc5ad..8848bc5eeb24 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -205,6 +205,8 @@ struct memcg_tier_counter {
int tier_id;
struct list_head list;
struct rcu_head rcu;
+ bool max_derived; /* true: derive from memory.max by capacity ratio */
+ bool high_derived; /* true: derive from memory.high by capacity ratio */
};
struct mem_cgroup {
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a6c6057f0e4f..f39a702d2301 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2443,6 +2443,8 @@ static int memcg_tier_counter_create(struct mem_cgroup *memcg,
page_counter_init(&new->counter, parent_cnt, false);
page_counter_set_high(&new->counter, PAGE_COUNTER_MAX);
INIT_LIST_HEAD(&new->list);
+ new->max_derived = true;
+ new->high_derived = true;
spin_lock(&memcg->tier_lock);
if (memcg_tier_counter_find(memcg, tier_id)) {
@@ -2514,6 +2516,115 @@ reclaim_tier(struct mem_cgroup *memcg, unsigned int nr_pages, gfp_t gfp_mask,
return nr_reclaimed;
}
+/* Global per-tier capacity cache.
+ * Populated at boot and on node hotplug (walks online nodes only, no memcg touch).
+ * Stores (tier_id, capacity) pairs to support sparse tier IDs (e.g., 4, 22, 52)
+ * without requiring a dense index.
+ */
+#define NR_TIER_CAP_ENTRIES 16
+struct tier_cap_entry {
+ int tier_id;
+ unsigned long capacity;
+};
+static struct tier_cap_entry tier_cap_entries[NR_TIER_CAP_ENTRIES];
+static int nr_tier_entries;
+static unsigned long tier_total_capacity;
+/* Guards the per-tier capacity cache. */
+static DEFINE_SPINLOCK(tier_cap_lock);
+
+static void update_tier_capacity_cache(void)
+{
+ int nid;
+ int i;
+
+ spin_lock(&tier_cap_lock);
+ nr_tier_entries = 0;
+ tier_total_capacity = 0;
+ for (i = 0; i < NR_TIER_CAP_ENTRIES; i++) {
+ tier_cap_entries[i].tier_id = -1;
+ tier_cap_entries[i].capacity = 0;
+ }
+
+ for_each_online_node(nid) {
+ int tid = node_to_tier_id(nid);
+ unsigned long cap;
+
+ if (tid < 0)
+ continue;
+ cap = node_present_pages(nid);
+ for (i = 0; i < nr_tier_entries; i++)
+ if (tier_cap_entries[i].tier_id == tid) {
+ tier_cap_entries[i].capacity += cap;
+ tier_total_capacity += cap;
+ break;
+ }
+ if (i == nr_tier_entries && nr_tier_entries < NR_TIER_CAP_ENTRIES) {
+ tier_cap_entries[nr_tier_entries].tier_id = tid;
+ tier_cap_entries[nr_tier_entries].capacity = cap;
+ nr_tier_entries++;
+ tier_total_capacity += cap;
+ } else if (i == nr_tier_entries) {
+ pr_warn_ratelimited("memcg: tier capacity cache full (>%d tiers), tier %d not tracked\n",
+ NR_TIER_CAP_ENTRIES, tid);
+ }
+ }
+ spin_unlock(&tier_cap_lock);
+}
+
+/* Push derived high/max limits to tier counters of @memcg.
+ * Invoked on memory.high/max writes and capacity changes (hotplug).
+ */
+static void tier_update_derived_limits(struct mem_cgroup *memcg)
+{
+ struct memcg_tier_counter *tc;
+ struct tier_cap_entry local_entries[NR_TIER_CAP_ENTRIES];
+ unsigned long total;
+ int i, nr_entries;
+
+ spin_lock(&tier_cap_lock);
+ total = tier_total_capacity;
+ nr_entries = nr_tier_entries;
+ memcpy(local_entries, tier_cap_entries, sizeof(local_entries));
+ spin_unlock(&tier_cap_lock);
+
+ if (total == 0)
+ return;
+
+ /* Serialize flag check + limit store against memory.tier writes. */
+ spin_lock(&memcg->tier_lock);
+ list_for_each_entry_rcu(tc, &memcg->tier_counters, list,
+ lockdep_is_held(&memcg->tier_lock)) {
+ unsigned long cap = 0;
+
+ for (i = 0; i < nr_entries; i++)
+ if (local_entries[i].tier_id == tc->tier_id) {
+ cap = local_entries[i].capacity;
+ break;
+ }
+ if (cap == 0)
+ continue;
+ if (READ_ONCE(tc->max_derived)) {
+ unsigned long mem_max = READ_ONCE(memcg->memory.max);
+
+ if (mem_max == PAGE_COUNTER_MAX)
+ xchg(&tc->counter.max, PAGE_COUNTER_MAX);
+ else
+ xchg(&tc->counter.max,
+ mul_u64_u64_div_u64(mem_max, cap, total));
+ }
+ if (READ_ONCE(tc->high_derived)) {
+ unsigned long mem_high = READ_ONCE(memcg->memory.high);
+
+ if (mem_high == PAGE_COUNTER_MAX)
+ xchg(&tc->counter.high, PAGE_COUNTER_MAX);
+ else
+ xchg(&tc->counter.high,
+ mul_u64_u64_div_u64(mem_high, cap, total));
+ }
+ }
+ spin_unlock(&memcg->tier_lock);
+}
+
static void tier_high_work_func(struct work_struct *work)
{
struct mem_cgroup *memcg;
@@ -4841,6 +4952,8 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
list_for_each_entry_rcu(tc, &memcg->tier_counters, list) {
page_counter_set_max(&tc->counter, PAGE_COUNTER_MAX);
page_counter_set_high(&tc->counter, PAGE_COUNTER_MAX);
+ WRITE_ONCE(tc->max_derived, true);
+ WRITE_ONCE(tc->high_derived, true);
}
rcu_read_unlock();
}
@@ -5251,6 +5364,7 @@ static ssize_t memory_high_write(struct kernfs_open_file *of,
return err;
page_counter_set_high(&memcg->memory, high);
+ tier_update_derived_limits(memcg);
if (of->file->f_flags & O_NONBLOCK)
goto out;
@@ -5303,6 +5417,7 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
return err;
xchg(&memcg->memory.max, max);
+ tier_update_derived_limits(memcg);
if (of->file->f_flags & O_NONBLOCK)
goto out;
@@ -5390,10 +5505,17 @@ static ssize_t memory_tier_write(struct kernfs_open_file *of,
if (!tier_id_to_nodemask(tier_id, &nodes) && !nodes_empty(nodes))
nmp = &nodes;
- if (!strcmp(knob, "high"))
+ if (!strcmp(knob, "high")) {
+ spin_lock(&memcg->tier_lock);
+ WRITE_ONCE(tc->high_derived, false);
page_counter_set_high(&tc->counter, val);
- else
+ spin_unlock(&memcg->tier_lock);
+ } else {
+ spin_lock(&memcg->tier_lock);
+ WRITE_ONCE(tc->max_derived, false);
xchg(&tc->counter.max, val);
+ spin_unlock(&memcg->tier_lock);
+ }
if (of->file->f_flags & O_NONBLOCK)
return nbytes;
@@ -6145,18 +6267,28 @@ static int __meminit memcg_tier_hotplug_cb(struct notifier_block *self,
struct mem_cgroup *memcg;
int tid;
- if (action != NODE_ADDED_FIRST_MEMORY)
- return notifier_from_errno(0);
-
- tid = node_to_tier_id(nn->nid);
- if (tid < 0)
- return notifier_from_errno(0);
-
- for_each_mem_cgroup(memcg) {
- if (!mem_cgroup_is_root(memcg) &&
- memcg_tier_counter_create(memcg, parent_mem_cgroup(memcg), tid))
- pr_warn_ratelimited("memcg: tier %d counter alloc failed;"
- " tier accounting degraded\n", tid);
+ switch (action) {
+ case NODE_ADDED_FIRST_MEMORY:
+ tid = node_to_tier_id(nn->nid);
+ if (tid < 0)
+ return notifier_from_errno(0);
+
+ update_tier_capacity_cache();
+ for_each_mem_cgroup(memcg) {
+ if (!mem_cgroup_is_root(memcg) &&
+ memcg_tier_counter_create(memcg, parent_mem_cgroup(memcg), tid))
+ pr_warn_ratelimited("memcg: tier %d counter alloc failed;"
+ " tier accounting degraded\n", tid);
+ }
+ for_each_mem_cgroup(memcg)
+ tier_update_derived_limits(memcg);
+ break;
+ case NODE_REMOVED_LAST_MEMORY:
+ /* Node memory removed: refresh capacity and derived limits. */
+ update_tier_capacity_cache();
+ for_each_mem_cgroup(memcg)
+ tier_update_derived_limits(memcg);
+ break;
}
return notifier_from_errno(0);
}
@@ -6181,6 +6313,8 @@ int __init mem_cgroup_init(void)
memcg_wq = alloc_workqueue("memcg", WQ_PERCPU, 0);
WARN_ON(!memcg_wq);
+ update_tier_capacity_cache();
+
#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_NUMA)
hotplug_node_notifier(memcg_tier_hotplug_cb, MEMCG_TIER_NODE_PRI);
#endif
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [RFC PATCH 7/8] mm/memcontrol: auto-derive tier high/max from memory.high/max
2026-08-18 2:31 ` [RFC PATCH 7/8] mm/memcontrol: auto-derive tier high/max from memory.high/max liuqiqi
@ 2026-08-18 4:46 ` Tao Cui
0 siblings, 0 replies; 12+ messages in thread
From: Tao Cui @ 2026-08-18 4:46 UTC (permalink / raw)
To: liuqiqi, linux-mm
Cc: cui.tao, tj, mkoutny, hannes, mhocko, roman.gushchin,
shakeel.butt, muchun.song, akpm, cgroups, linux-kernel
在 2026/8/18 10:31, liuqiqi@kylinos.cn 写道:
> From: Qiqi Liu <liuqiqi@kylinos.cn>
>
> Per-tier limits are derived by default from the cgroup-wide memory.high
> and memory.max knobs based on the tier's capacity ratio:
>
> tierN.high = memory.high * tierN_capacity / total_capacity
> tierN.max = memory.max * tierN_capacity / total_capacity
>
> This provides zero-config tier partitioning. Setting memory.high or
> memory.max automatically splits the budget across tiers proportionally
> to their physical size. For instance, a 4G cgroup on a system with a
> 75%/25% DRAM/CXL split automatically receives ~3G DRAM and ~1G CXL.
>
> The capacity cache and the derived limits are refreshed on node memory
> hotplug, both when node memory is added and when it is removed.
>
> Writing an explicit value pins the limit and disables auto-derivation
> for that tier, decoupling it from subsequent memory.high/max updates.
>
> Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
> ---
> include/linux/memcontrol.h | 2 +
> mm/memcontrol.c | 162 +++++++++++++++++++++++++++++++++----
> 2 files changed, 150 insertions(+), 14 deletions(-)
>
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index fa944e4bc5ad..8848bc5eeb24 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -205,6 +205,8 @@ struct memcg_tier_counter {
> int tier_id;
> struct list_head list;
> struct rcu_head rcu;
> + bool max_derived; /* true: derive from memory.max by capacity ratio */
> + bool high_derived; /* true: derive from memory.high by capacity ratio */
> };
>
> struct mem_cgroup {
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index a6c6057f0e4f..f39a702d2301 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -2443,6 +2443,8 @@ static int memcg_tier_counter_create(struct mem_cgroup *memcg,
> page_counter_init(&new->counter, parent_cnt, false);
> page_counter_set_high(&new->counter, PAGE_COUNTER_MAX);
> INIT_LIST_HEAD(&new->list);
> + new->max_derived = true;
> + new->high_derived = true;
>
> spin_lock(&memcg->tier_lock);
> if (memcg_tier_counter_find(memcg, tier_id)) {
> @@ -2514,6 +2516,115 @@ reclaim_tier(struct mem_cgroup *memcg, unsigned int nr_pages, gfp_t gfp_mask,
> return nr_reclaimed;
> }
>
> +/* Global per-tier capacity cache.
> + * Populated at boot and on node hotplug (walks online nodes only, no memcg touch).
> + * Stores (tier_id, capacity) pairs to support sparse tier IDs (e.g., 4, 22, 52)
> + * without requiring a dense index.
> + */
> +#define NR_TIER_CAP_ENTRIES 16
> +struct tier_cap_entry {
> + int tier_id;
> + unsigned long capacity;
> +};
> +static struct tier_cap_entry tier_cap_entries[NR_TIER_CAP_ENTRIES];
> +static int nr_tier_entries;
> +static unsigned long tier_total_capacity;
> +/* Guards the per-tier capacity cache. */
> +static DEFINE_SPINLOCK(tier_cap_lock);
> +
"Populated at boot" doesn't hold: the boot-time call sits in
mem_cgroup_init(), which runs from start_kernel() (init/main.c:1177),
before any initcall. Nodes are assigned tiers in
memory_tier_late_init() (late_initcall), which fires no node notifier.
So tier_total_capacity stays 0 and every memory.high/max write hits
the early return in tier_update_derived_limits(). The hotplug
callback is the only other refresh point - your QEMU setup likely
works because the CXL tier comes up through it.
Either hook a refresh into the tier-assignment path in memory-tiers,
or compute lazily when the cache is empty.
Thanks,
Tao
> +static void update_tier_capacity_cache(void)
> +{
> + int nid;
> + int i;
> +
> + spin_lock(&tier_cap_lock);
> + nr_tier_entries = 0;
> + tier_total_capacity = 0;
> + for (i = 0; i < NR_TIER_CAP_ENTRIES; i++) {
> + tier_cap_entries[i].tier_id = -1;
> + tier_cap_entries[i].capacity = 0;
> + }
> +
> + for_each_online_node(nid) {
> + int tid = node_to_tier_id(nid);
> + unsigned long cap;
> +
> + if (tid < 0)
> + continue;
> + cap = node_present_pages(nid);
> + for (i = 0; i < nr_tier_entries; i++)
> + if (tier_cap_entries[i].tier_id == tid) {
> + tier_cap_entries[i].capacity += cap;
> + tier_total_capacity += cap;
> + break;
> + }
> + if (i == nr_tier_entries && nr_tier_entries < NR_TIER_CAP_ENTRIES) {
> + tier_cap_entries[nr_tier_entries].tier_id = tid;
> + tier_cap_entries[nr_tier_entries].capacity = cap;
> + nr_tier_entries++;
> + tier_total_capacity += cap;
> + } else if (i == nr_tier_entries) {
> + pr_warn_ratelimited("memcg: tier capacity cache full (>%d tiers), tier %d not tracked\n",
> + NR_TIER_CAP_ENTRIES, tid);
> + }
> + }
> + spin_unlock(&tier_cap_lock);
> +}
> +
> +/* Push derived high/max limits to tier counters of @memcg.
> + * Invoked on memory.high/max writes and capacity changes (hotplug).
> + */
> +static void tier_update_derived_limits(struct mem_cgroup *memcg)
> +{
> + struct memcg_tier_counter *tc;
> + struct tier_cap_entry local_entries[NR_TIER_CAP_ENTRIES];
> + unsigned long total;
> + int i, nr_entries;
> +
> + spin_lock(&tier_cap_lock);
> + total = tier_total_capacity;
> + nr_entries = nr_tier_entries;
> + memcpy(local_entries, tier_cap_entries, sizeof(local_entries));
> + spin_unlock(&tier_cap_lock);
> +
> + if (total == 0)
> + return;
> +
> + /* Serialize flag check + limit store against memory.tier writes. */
> + spin_lock(&memcg->tier_lock);
> + list_for_each_entry_rcu(tc, &memcg->tier_counters, list,
> + lockdep_is_held(&memcg->tier_lock)) {
> + unsigned long cap = 0;
> +
> + for (i = 0; i < nr_entries; i++)
> + if (local_entries[i].tier_id == tc->tier_id) {
> + cap = local_entries[i].capacity;
> + break;
> + }
> + if (cap == 0)
> + continue;
> + if (READ_ONCE(tc->max_derived)) {
> + unsigned long mem_max = READ_ONCE(memcg->memory.max);
> +
> + if (mem_max == PAGE_COUNTER_MAX)
> + xchg(&tc->counter.max, PAGE_COUNTER_MAX);
> + else
> + xchg(&tc->counter.max,
> + mul_u64_u64_div_u64(mem_max, cap, total));
> + }
> + if (READ_ONCE(tc->high_derived)) {
> + unsigned long mem_high = READ_ONCE(memcg->memory.high);
> +
> + if (mem_high == PAGE_COUNTER_MAX)
> + xchg(&tc->counter.high, PAGE_COUNTER_MAX);
> + else
> + xchg(&tc->counter.high,
> + mul_u64_u64_div_u64(mem_high, cap, total));
> + }
> + }
> + spin_unlock(&memcg->tier_lock);
> +}
> +
> static void tier_high_work_func(struct work_struct *work)
> {
> struct mem_cgroup *memcg;
> @@ -4841,6 +4952,8 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
> list_for_each_entry_rcu(tc, &memcg->tier_counters, list) {
> page_counter_set_max(&tc->counter, PAGE_COUNTER_MAX);
> page_counter_set_high(&tc->counter, PAGE_COUNTER_MAX);
> + WRITE_ONCE(tc->max_derived, true);
> + WRITE_ONCE(tc->high_derived, true);
> }
> rcu_read_unlock();
> }
> @@ -5251,6 +5364,7 @@ static ssize_t memory_high_write(struct kernfs_open_file *of,
> return err;
>
> page_counter_set_high(&memcg->memory, high);
> + tier_update_derived_limits(memcg);
>
> if (of->file->f_flags & O_NONBLOCK)
> goto out;
> @@ -5303,6 +5417,7 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
> return err;
>
> xchg(&memcg->memory.max, max);
> + tier_update_derived_limits(memcg);
>
> if (of->file->f_flags & O_NONBLOCK)
> goto out;
> @@ -5390,10 +5505,17 @@ static ssize_t memory_tier_write(struct kernfs_open_file *of,
> if (!tier_id_to_nodemask(tier_id, &nodes) && !nodes_empty(nodes))
> nmp = &nodes;
>
> - if (!strcmp(knob, "high"))
> + if (!strcmp(knob, "high")) {
> + spin_lock(&memcg->tier_lock);
> + WRITE_ONCE(tc->high_derived, false);
> page_counter_set_high(&tc->counter, val);
> - else
> + spin_unlock(&memcg->tier_lock);
> + } else {
> + spin_lock(&memcg->tier_lock);
> + WRITE_ONCE(tc->max_derived, false);
> xchg(&tc->counter.max, val);
> + spin_unlock(&memcg->tier_lock);
> + }
>
> if (of->file->f_flags & O_NONBLOCK)
> return nbytes;
> @@ -6145,18 +6267,28 @@ static int __meminit memcg_tier_hotplug_cb(struct notifier_block *self,
> struct mem_cgroup *memcg;
> int tid;
>
> - if (action != NODE_ADDED_FIRST_MEMORY)
> - return notifier_from_errno(0);
> -
> - tid = node_to_tier_id(nn->nid);
> - if (tid < 0)
> - return notifier_from_errno(0);
> -
> - for_each_mem_cgroup(memcg) {
> - if (!mem_cgroup_is_root(memcg) &&
> - memcg_tier_counter_create(memcg, parent_mem_cgroup(memcg), tid))
> - pr_warn_ratelimited("memcg: tier %d counter alloc failed;"
> - " tier accounting degraded\n", tid);
> + switch (action) {
> + case NODE_ADDED_FIRST_MEMORY:
> + tid = node_to_tier_id(nn->nid);
> + if (tid < 0)
> + return notifier_from_errno(0);
> +
> + update_tier_capacity_cache();
> + for_each_mem_cgroup(memcg) {
> + if (!mem_cgroup_is_root(memcg) &&
> + memcg_tier_counter_create(memcg, parent_mem_cgroup(memcg), tid))
> + pr_warn_ratelimited("memcg: tier %d counter alloc failed;"
> + " tier accounting degraded\n", tid);
> + }
> + for_each_mem_cgroup(memcg)
> + tier_update_derived_limits(memcg);
> + break;
> + case NODE_REMOVED_LAST_MEMORY:
> + /* Node memory removed: refresh capacity and derived limits. */
> + update_tier_capacity_cache();
> + for_each_mem_cgroup(memcg)
> + tier_update_derived_limits(memcg);
> + break;
> }
> return notifier_from_errno(0);
> }
> @@ -6181,6 +6313,8 @@ int __init mem_cgroup_init(void)
> memcg_wq = alloc_workqueue("memcg", WQ_PERCPU, 0);
> WARN_ON(!memcg_wq);
>
> + update_tier_capacity_cache();
> +
> #if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_NUMA)
> hotplug_node_notifier(memcg_tier_hotplug_cb, MEMCG_TIER_NODE_PRI);
> #endif
^ permalink raw reply [flat|nested] 12+ messages in thread
* [RFC PATCH 8/8] cgroup: add memory_tiered_limits cgroup mount option
2026-08-18 2:31 [RFC PATCH 0/8] mm/memcontrol: introduce per-tier memory accounting and control liuqiqi
` (6 preceding siblings ...)
2026-08-18 2:31 ` [RFC PATCH 7/8] mm/memcontrol: auto-derive tier high/max from memory.high/max liuqiqi
@ 2026-08-18 2:31 ` liuqiqi
2026-08-18 4:56 ` Tao Cui
7 siblings, 1 reply; 12+ messages in thread
From: liuqiqi @ 2026-08-18 2:31 UTC (permalink / raw)
To: linux-mm
Cc: tj, mkoutny, hannes, mhocko, roman.gushchin, shakeel.butt,
muchun.song, akpm, cgroups, linux-kernel, Qiqi Liu
From: Qiqi Liu <liuqiqi@kylinos.cn>
Introduce the memory_tiered_limits cgroup v2 mount option to enable
tier-aware memory control. The option follows the same pattern as
memory_recursiveprot and memory_hugetlb_accounting.
When enabled, per-tier accounting, charging, stock batching, auto-
derivation, and the memory.tier control file are active. When disabled
(default), all tier-specific code is gated at entry points: no counters
are created, no charge/uncharge occurs, and memory.tier remains empty.
This ensures no measurable overhead for systems that do not opt in.
Usage:
mount -t cgroup2 none /sys/fs/cgroup -o memory_tiered_limits
Because the cgroup2 mount is owned by the init system, early-boot
cgroups are created before userspace can specify mount options. To
cover these boot-time cgroups, mirror the cgroup_favordynmods approach
and add a kernel command-line parameter,
cgroup_memory_tiered_limits=<bool>.
This sets CGRP_ROOT_MEMORY_TIERED_LIMITS on all cgroup2 mounts by
default. The mount option remains available for runtime, per-mount
control.
Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
---
include/linux/cgroup-defs.h | 5 +++++
include/linux/memcontrol.h | 12 ++++++++++++
kernel/cgroup/cgroup.c | 21 +++++++++++++++++++++
mm/memcontrol.c | 22 ++++++++++++++++++----
4 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index de2cd6238c2a..4324d68dbce9 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -129,6 +129,11 @@ enum {
* Enable legacy local pids.events.
*/
CGRP_ROOT_PIDS_LOCAL_EVENTS = (1 << 20),
+
+ /*
+ * Enable tier-aware limits for the memory controller.
+ */
+ CGRP_ROOT_MEMORY_TIERED_LIMITS = (1 << 21),
};
/* cftype->flags */
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 8848bc5eeb24..094b9a839977 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -544,6 +544,18 @@ static inline bool mem_cgroup_disabled(void)
return !cgroup_subsys_enabled(memory_cgrp_subsys);
}
+#ifdef CONFIG_NUMA
+static inline bool mem_cgroup_tiered_limits(void)
+{
+ return cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_TIERED_LIMITS;
+}
+#else
+static inline bool mem_cgroup_tiered_limits(void)
+{
+ return false;
+}
+#endif
+
static inline void mem_cgroup_protection(struct mem_cgroup *root,
struct mem_cgroup *memcg,
unsigned long *min,
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 38f8d9df8fbc..da94bcda0859 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -231,6 +231,7 @@ static u32 have_release_callback __read_mostly;
static u32 have_canfork_callback __read_mostly;
static bool have_favordynmods __ro_after_init = IS_ENABLED(CONFIG_CGROUP_FAVOR_DYNMODS);
+static bool have_memory_tiered_limits __ro_after_init;
/*
* Write protected by cgroup_mutex and write-lock of cgroup_threadgroup_rwsem,
@@ -1985,6 +1986,7 @@ enum cgroup2_param {
Opt_memory_recursiveprot,
Opt_memory_hugetlb_accounting,
Opt_pids_localevents,
+ Opt_memory_tiered_limits,
nr__cgroup2_params
};
@@ -1995,6 +1997,7 @@ static const struct fs_parameter_spec cgroup2_fs_parameters[] = {
fsparam_flag("memory_recursiveprot", Opt_memory_recursiveprot),
fsparam_flag("memory_hugetlb_accounting", Opt_memory_hugetlb_accounting),
fsparam_flag("pids_localevents", Opt_pids_localevents),
+ fsparam_flag("memory_tiered_limits", Opt_memory_tiered_limits),
{}
};
@@ -2027,6 +2030,9 @@ static int cgroup2_parse_param(struct fs_context *fc, struct fs_parameter *param
case Opt_pids_localevents:
ctx->flags |= CGRP_ROOT_PIDS_LOCAL_EVENTS;
return 0;
+ case Opt_memory_tiered_limits:
+ ctx->flags |= CGRP_ROOT_MEMORY_TIERED_LIMITS;
+ return 0;
}
return -EINVAL;
}
@@ -2068,6 +2074,11 @@ static void apply_cgroup_root_flags(unsigned int root_flags)
cgrp_dfl_root.flags |= CGRP_ROOT_PIDS_LOCAL_EVENTS;
else
cgrp_dfl_root.flags &= ~CGRP_ROOT_PIDS_LOCAL_EVENTS;
+
+ if (root_flags & CGRP_ROOT_MEMORY_TIERED_LIMITS)
+ cgrp_dfl_root.flags |= CGRP_ROOT_MEMORY_TIERED_LIMITS;
+ else
+ cgrp_dfl_root.flags &= ~CGRP_ROOT_MEMORY_TIERED_LIMITS;
}
}
@@ -2085,6 +2096,8 @@ static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root
seq_puts(seq, ",memory_hugetlb_accounting");
if (cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS)
seq_puts(seq, ",pids_localevents");
+ if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_TIERED_LIMITS)
+ seq_puts(seq, ",memory_tiered_limits");
return 0;
}
@@ -2363,6 +2376,8 @@ static int cgroup_init_fs_context(struct fs_context *fc)
if (have_favordynmods)
ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS;
+ if (have_memory_tiered_limits)
+ ctx->flags |= CGRP_ROOT_MEMORY_TIERED_LIMITS;
return 0;
}
@@ -7214,6 +7229,12 @@ static int __init cgroup_favordynmods_setup(char *str)
}
__setup("cgroup_favordynmods=", cgroup_favordynmods_setup);
+static int __init cgroup_memory_tiered_limits_setup(char *str)
+{
+ return (kstrtobool(str, &have_memory_tiered_limits) == 0);
+}
+__setup("cgroup_memory_tiered_limits=", cgroup_memory_tiered_limits_setup);
+
/**
* css_tryget_online_from_dir - get corresponding css from a cgroup dentry
* @dentry: directory dentry of interest
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index f39a702d2301..891051f164ff 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2462,7 +2462,7 @@ static void memcg_charge_tier_id(struct mem_cgroup *memcg, int tier_id,
{
struct memcg_tier_counter *tc;
- if (tier_id < 0)
+ if (!mem_cgroup_tiered_limits() || tier_id < 0)
return;
rcu_read_lock();
tc = memcg_tier_counter_find(memcg, tier_id);
@@ -2476,7 +2476,7 @@ static void memcg_uncharge_tier_id(struct mem_cgroup *memcg, int tier_id,
{
struct memcg_tier_counter *tc;
- if (tier_id < 0)
+ if (!mem_cgroup_tiered_limits() || tier_id < 0)
return;
rcu_read_lock();
tc = memcg_tier_counter_find(memcg, tier_id);
@@ -2581,6 +2581,9 @@ static void tier_update_derived_limits(struct mem_cgroup *memcg)
unsigned long total;
int i, nr_entries;
+ if (!mem_cgroup_tiered_limits())
+ return;
+
spin_lock(&tier_cap_lock);
total = tier_total_capacity;
nr_entries = nr_tier_entries;
@@ -3198,6 +3201,9 @@ static void refill_tier_stock(struct mem_cgroup *memcg, int tier_id,
uint8_t pages;
int i;
+ if (!mem_cgroup_tiered_limits())
+ return;
+
/* Too big to cache: direct uncharge, leave the stock untouched. */
if (nr_pages > MEMCG_CHARGE_BATCH) {
rcu_read_lock();
@@ -3252,7 +3258,7 @@ static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
bool drained = false;
nodemask_t nodes, *nmp = NULL;
- if (tier_id < 0)
+ if (!mem_cgroup_tiered_limits() || tier_id < 0)
return 0;
rcu_read_lock();
@@ -4736,7 +4742,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
page_counter_init(&memcg->swap, &parent->swap, false);
- {
+ if (mem_cgroup_tiered_limits()) {
int nid, tid;
for_each_online_node(nid) {
@@ -5459,6 +5465,8 @@ static int memory_tier_show(struct seq_file *m, void *v)
struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
struct memcg_tier_counter *tc;
+ if (!mem_cgroup_tiered_limits())
+ return 0;
rcu_read_lock();
list_for_each_entry_rcu(tc, &memcg->tier_counters, list) {
seq_printf(m, "tier%d.current=%llu\n", tc->tier_id,
@@ -5484,6 +5492,9 @@ static ssize_t memory_tier_write(struct kernfs_open_file *of,
char knob[8], *p;
int tier_id, err;
+ if (!mem_cgroup_tiered_limits())
+ return -EOPNOTSUPP;
+
buf = strstrip(buf);
if (sscanf(buf, "tier%d.%7[^=]", &tier_id, knob) != 2)
return -EINVAL;
@@ -6267,6 +6278,9 @@ static int __meminit memcg_tier_hotplug_cb(struct notifier_block *self,
struct mem_cgroup *memcg;
int tid;
+ if (!mem_cgroup_tiered_limits())
+ return notifier_from_errno(0);
+
switch (action) {
case NODE_ADDED_FIRST_MEMORY:
tid = node_to_tier_id(nn->nid);
--
2.43.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [RFC PATCH 8/8] cgroup: add memory_tiered_limits cgroup mount option
2026-08-18 2:31 ` [RFC PATCH 8/8] cgroup: add memory_tiered_limits cgroup mount option liuqiqi
@ 2026-08-18 4:56 ` Tao Cui
0 siblings, 0 replies; 12+ messages in thread
From: Tao Cui @ 2026-08-18 4:56 UTC (permalink / raw)
To: liuqiqi, linux-mm
Cc: cui.tao, tj, mkoutny, hannes, mhocko, roman.gushchin,
shakeel.butt, muchun.song, akpm, cgroups, linux-kernel
在 2026/8/18 10:31, liuqiqi@kylinos.cn 写道:
> From: Qiqi Liu <liuqiqi@kylinos.cn>
>
> Introduce the memory_tiered_limits cgroup v2 mount option to enable
> tier-aware memory control. The option follows the same pattern as
> memory_recursiveprot and memory_hugetlb_accounting.
>
> When enabled, per-tier accounting, charging, stock batching, auto-
> derivation, and the memory.tier control file are active. When disabled
> (default), all tier-specific code is gated at entry points: no counters
> are created, no charge/uncharge occurs, and memory.tier remains empty.
> This ensures no measurable overhead for systems that do not opt in.
>
> Usage:
> mount -t cgroup2 none /sys/fs/cgroup -o memory_tiered_limits
>
> Because the cgroup2 mount is owned by the init system, early-boot
> cgroups are created before userspace can specify mount options. To
> cover these boot-time cgroups, mirror the cgroup_favordynmods approach
> and add a kernel command-line parameter,
> cgroup_memory_tiered_limits=<bool>.
> This sets CGRP_ROOT_MEMORY_TIERED_LIMITS on all cgroup2 mounts by
> default. The mount option remains available for runtime, per-mount
> control.
>
> Signed-off-by: Qiqi Liu <liuqiqi@kylinos.cn>
> ---
> include/linux/cgroup-defs.h | 5 +++++
> include/linux/memcontrol.h | 12 ++++++++++++
> kernel/cgroup/cgroup.c | 21 +++++++++++++++++++++
> mm/memcontrol.c | 22 ++++++++++++++++++----
> 4 files changed, 56 insertions(+), 4 deletions(-)
>
> diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
> index de2cd6238c2a..4324d68dbce9 100644
> --- a/include/linux/cgroup-defs.h
> +++ b/include/linux/cgroup-defs.h
> @@ -129,6 +129,11 @@ enum {
> * Enable legacy local pids.events.
> */
> CGRP_ROOT_PIDS_LOCAL_EVENTS = (1 << 20),
> +
> + /*
> + * Enable tier-aware limits for the memory controller.
> + */
> + CGRP_ROOT_MEMORY_TIERED_LIMITS = (1 << 21),
> };
>
> /* cftype->flags */
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index 8848bc5eeb24..094b9a839977 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -544,6 +544,18 @@ static inline bool mem_cgroup_disabled(void)
> return !cgroup_subsys_enabled(memory_cgrp_subsys);
> }
>
> +#ifdef CONFIG_NUMA
> +static inline bool mem_cgroup_tiered_limits(void)
> +{
> + return cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_TIERED_LIMITS;
> +}
> +#else
> +static inline bool mem_cgroup_tiered_limits(void)
> +{
> + return false;
> +}
> +#endif
> +
> static inline void mem_cgroup_protection(struct mem_cgroup *root,
> struct mem_cgroup *memcg,
> unsigned long *min,
> diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
> index 38f8d9df8fbc..da94bcda0859 100644
> --- a/kernel/cgroup/cgroup.c
> +++ b/kernel/cgroup/cgroup.c
> @@ -231,6 +231,7 @@ static u32 have_release_callback __read_mostly;
> static u32 have_canfork_callback __read_mostly;
>
> static bool have_favordynmods __ro_after_init = IS_ENABLED(CONFIG_CGROUP_FAVOR_DYNMODS);
> +static bool have_memory_tiered_limits __ro_after_init;
>
> /*
> * Write protected by cgroup_mutex and write-lock of cgroup_threadgroup_rwsem,
> @@ -1985,6 +1986,7 @@ enum cgroup2_param {
> Opt_memory_recursiveprot,
> Opt_memory_hugetlb_accounting,
> Opt_pids_localevents,
> + Opt_memory_tiered_limits,
> nr__cgroup2_params
> };
>
> @@ -1995,6 +1997,7 @@ static const struct fs_parameter_spec cgroup2_fs_parameters[] = {
> fsparam_flag("memory_recursiveprot", Opt_memory_recursiveprot),
> fsparam_flag("memory_hugetlb_accounting", Opt_memory_hugetlb_accounting),
> fsparam_flag("pids_localevents", Opt_pids_localevents),
> + fsparam_flag("memory_tiered_limits", Opt_memory_tiered_limits),
> {}
> };
>
> @@ -2027,6 +2030,9 @@ static int cgroup2_parse_param(struct fs_context *fc, struct fs_parameter *param
> case Opt_pids_localevents:
> ctx->flags |= CGRP_ROOT_PIDS_LOCAL_EVENTS;
> return 0;
> + case Opt_memory_tiered_limits:
> + ctx->flags |= CGRP_ROOT_MEMORY_TIERED_LIMITS;
> + return 0;
> }
> return -EINVAL;
> }
> @@ -2068,6 +2074,11 @@ static void apply_cgroup_root_flags(unsigned int root_flags)
> cgrp_dfl_root.flags |= CGRP_ROOT_PIDS_LOCAL_EVENTS;
> else
> cgrp_dfl_root.flags &= ~CGRP_ROOT_PIDS_LOCAL_EVENTS;
> +
> + if (root_flags & CGRP_ROOT_MEMORY_TIERED_LIMITS)
> + cgrp_dfl_root.flags |= CGRP_ROOT_MEMORY_TIERED_LIMITS;
> + else
> + cgrp_dfl_root.flags &= ~CGRP_ROOT_MEMORY_TIERED_LIMITS;
> }
> }
>
> @@ -2085,6 +2096,8 @@ static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root
> seq_puts(seq, ",memory_hugetlb_accounting");
> if (cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS)
> seq_puts(seq, ",pids_localevents");
> + if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_TIERED_LIMITS)
> + seq_puts(seq, ",memory_tiered_limits");
> return 0;
> }
>
> @@ -2363,6 +2376,8 @@ static int cgroup_init_fs_context(struct fs_context *fc)
>
> if (have_favordynmods)
> ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS;
> + if (have_memory_tiered_limits)
> + ctx->flags |= CGRP_ROOT_MEMORY_TIERED_LIMITS;
>
> return 0;
> }
> @@ -7214,6 +7229,12 @@ static int __init cgroup_favordynmods_setup(char *str)
> }
> __setup("cgroup_favordynmods=", cgroup_favordynmods_setup);
>
> +static int __init cgroup_memory_tiered_limits_setup(char *str)
> +{
> + return (kstrtobool(str, &have_memory_tiered_limits) == 0);
> +}
> +__setup("cgroup_memory_tiered_limits=", cgroup_memory_tiered_limits_setup);
> +
> /**
> * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
> * @dentry: directory dentry of interest
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index f39a702d2301..891051f164ff 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -2462,7 +2462,7 @@ static void memcg_charge_tier_id(struct mem_cgroup *memcg, int tier_id,
> {
> struct memcg_tier_counter *tc;
>
> - if (tier_id < 0)
> + if (!mem_cgroup_tiered_limits() || tier_id < 0)
The cover says remount "affects newly created cgroups only", but the
gate is the live global flag on both charge and uncharge.
(same in memcg_uncharge_tier_id() and refill_tier_stock()).
> return;
> rcu_read_lock();
> tc = memcg_tier_counter_find(memcg, tier_id);
> @@ -2476,7 +2476,7 @@ static void memcg_uncharge_tier_id(struct mem_cgroup *memcg, int tier_id,
> {
> struct memcg_tier_counter *tc;
>
> - if (tier_id < 0)
> + if (!mem_cgroup_tiered_limits() || tier_id < 0)
> return;
> rcu_read_lock();
> tc = memcg_tier_counter_find(memcg, tier_id);
> @@ -2581,6 +2581,9 @@ static void tier_update_derived_limits(struct mem_cgroup *memcg)
> unsigned long total;
> int i, nr_entries;
>
> + if (!mem_cgroup_tiered_limits())
> + return;
> +
> spin_lock(&tier_cap_lock);
> total = tier_total_capacity;
> nr_entries = nr_tier_entries;
> @@ -3198,6 +3201,9 @@ static void refill_tier_stock(struct mem_cgroup *memcg, int tier_id,
> uint8_t pages;
> int i;
>
> + if (!mem_cgroup_tiered_limits())
> + return;
> +
> /* Too big to cache: direct uncharge, leave the stock untouched. */
> if (nr_pages > MEMCG_CHARGE_BATCH) {
> rcu_read_lock();
> @@ -3252,7 +3258,7 @@ static int try_charge_memcg_tier(struct mem_cgroup *memcg, gfp_t gfp_mask,
> bool drained = false;
> nodemask_t nodes, *nmp = NULL;
>
> - if (tier_id < 0)
> + if (!mem_cgroup_tiered_limits() || tier_id < 0)
> return 0;
>
> rcu_read_lock();
> @@ -4736,7 +4742,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
> page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
> page_counter_init(&memcg->swap, &parent->swap, false);
>
> - {
> + if (mem_cgroup_tiered_limits()) {
> int nid, tid;
>
> for_each_online_node(nid) {
> @@ -5459,6 +5465,8 @@ static int memory_tier_show(struct seq_file *m, void *v)
> struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
> struct memcg_tier_counter *tc;
>
> + if (!mem_cgroup_tiered_limits())
> + return 0;
> rcu_read_lock();
> list_for_each_entry_rcu(tc, &memcg->tier_counters, list) {
> seq_printf(m, "tier%d.current=%llu\n", tc->tier_id,
> @@ -5484,6 +5492,9 @@ static ssize_t memory_tier_write(struct kernfs_open_file *of,
> char knob[8], *p;
> int tier_id, err;
>
> + if (!mem_cgroup_tiered_limits())
> + return -EOPNOTSUPP;
> +
> buf = strstrip(buf);
> if (sscanf(buf, "tier%d.%7[^=]", &tier_id, knob) != 2)
> return -EINVAL;
> @@ -6267,6 +6278,9 @@ static int __meminit memcg_tier_hotplug_cb(struct notifier_block *self,
> struct mem_cgroup *memcg;
> int tid;
>
> + if (!mem_cgroup_tiered_limits())
> + return notifier_from_errno(0);
> +
> switch (action) {
> case NODE_ADDED_FIRST_MEMORY:
> tid = node_to_tier_id(nn->nid);
^ permalink raw reply [flat|nested] 12+ messages in thread