[RFC PATCH v3 14/14] mm/page_alloc: steer allocations away from exhausted memory tiers

[Joshua Hahn <joshua.hahnjy@gmail.com>]

A memcg only finds out that it is over a tier's limit at
try_charge_memcg time, after the page has already been allocated on the
full tier's node. This triggers reclaim to push memory to lower tiers or
to swap, much like how zone_reclaim_mode triggers reclaim when a node is
full.

This causes a lot of unnecessary churn. Instead of allocating a page on
a full tier only to immediately reclaim, make the page allocator
aware of tier limits at allocation time and steer the first allocation
attempt to nodes belonging to tiers with tiered limit headroom.

This only affects the ac->nodemask for the fastpath, meaning if no
node can satisfy this allocation, it falls back to the caller's nodemask
and places a page on a full tier, triggering reclaim. Moreover, if it
turns out that the intersection of the allocation context nodemask and
the under-limit nodemask is empty, skip the steering.

This steering only affects MIGRATE_MOVABLE allocations. We try not to
steer kernel allocations using NUMA_NO_NODE which would prefer to remain
on higher tiers, even if they are full.

This steering also does not affect alloc_pages_bulk_noprof, since
its callers do not charge their memory allocations to a tier.

No-op unless the system has tiered memcg limits enabled.

Signed-off-by: Joshua Hahn <joshua.hahnjy@gmail.com>
---
 include/linux/memcontrol.h |  6 +++++
 mm/memcontrol.c            | 46 ++++++++++++++++++++++++++++++++++++++
 mm/page_alloc.c            | 19 +++++++++++++---
 3 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index a7c366b431a0e..b700f3224cbaa 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -655,6 +655,7 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
 }
 
 bool mem_cgroup_tier_over_limit(struct folio *folio, int dst_nid);
+bool mem_cgroup_tier_allowed_nodemask(nodemask_t *mask);
 
 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
 
@@ -1179,6 +1180,11 @@ static inline bool mem_cgroup_tier_over_limit(struct folio *folio, int dst_nid)
 	return false;
 }
 
+static inline bool mem_cgroup_tier_allowed_nodemask(nodemask_t *mask)
+{
+	return false;
+}
+
 static inline int mem_cgroup_charge(struct folio *folio,
 		struct mm_struct *mm, gfp_t gfp)
 {
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 05611a01aa082..e499e58baa287 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2625,6 +2625,52 @@ bool mem_cgroup_tier_over_limit(struct folio *folio, int dst_nid)
 	return false;
 }
 
+/*
+ * Returns whether a mem_cgroup is above a tier's limit.
+ * The nodemask becomes populated with nodes that are under their limits.
+ */
+bool mem_cgroup_tier_allowed_nodemask(nodemask_t *mask)
+{
+	struct mem_cgroup *memcg;
+	int nr_tier_slots;
+	bool restricted = false;
+
+	if (!mem_cgroup_tiered_limits())
+		return false;
+
+	nr_tier_slots = mt_nr_tier_slots();
+	*mask = node_states[N_MEMORY];
+
+	rcu_read_lock();
+	memcg = active_memcg();
+	if (!memcg && in_task() && current->mm)
+		memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
+
+	for (; memcg && !mem_cgroup_is_root(memcg);
+	     memcg = parent_mem_cgroup(memcg)) {
+		if (READ_ONCE(memcg->memory.max) == PAGE_COUNTER_MAX &&
+		    READ_ONCE(memcg->memory.high) == PAGE_COUNTER_MAX)
+			continue;
+
+		for (int slot = 0; slot < nr_tier_slots; slot++) {
+			struct page_counter *tier = &memcg->tier[slot];
+			unsigned long limit;
+
+			limit = min(READ_ONCE(tier->high),
+				    READ_ONCE(tier->max));
+
+			if (page_counter_read(tier) <= limit)
+				continue;
+
+			restricted = true;
+			nodes_andnot(*mask, *mask, *mt_tier_nodes(slot));
+		}
+	}
+	rcu_read_unlock();
+
+	return restricted;
+}
+
 /*
  * Get the number of jiffies that we should penalise a mischievous cgroup which
  * is exceeding its memory.high by checking both it and its ancestors.
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 0aeca106a4fde..7ca0acc37d2af 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5153,7 +5153,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
 		int preferred_nid, nodemask_t *nodemask,
 		struct alloc_context *ac, gfp_t *alloc_gfp,
-		unsigned int *alloc_flags)
+		unsigned int *alloc_flags, nodemask_t *tier_nodes)
 {
 	ac->highest_zoneidx = gfp_zone(gfp_mask);
 	ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
@@ -5187,6 +5187,17 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
 	/* Dirty zone balancing only done in the fast path */
 	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
 
+	if (mem_cgroup_tiered_limits() && tier_nodes &&
+	    !(gfp_mask & __GFP_THISNODE) &&
+	    ac->migratetype == MIGRATE_MOVABLE &&
+	    mem_cgroup_tier_allowed_nodemask(tier_nodes)) {
+		if (ac->nodemask)
+			nodes_and(*tier_nodes, *tier_nodes, *ac->nodemask);
+		/* tier_nodes must outlive the function call since ac uses it */
+		if (!nodes_empty(*tier_nodes))
+			ac->nodemask = tier_nodes;
+	}
+
 	/*
 	 * The preferred zone is used for statistics but crucially it is
 	 * also used as the starting point for the zonelist iterator. It
@@ -5269,7 +5280,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 
 	/* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
 	gfp &= gfp_allowed_mask;
-	if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &gfp, &alloc_flags))
+	if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &gfp,
+				 &alloc_flags, NULL))
 		goto out;
 
 	/* Find an allowed local zone that meets the low watermark. */
@@ -5451,6 +5463,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 		.alloc_flags = alloc_flags,
 	};
 	unsigned int fastpath_alloc_flags = alloc_flags;
+	nodemask_t tier_nodes;
 
 	/* Other flags could be supported later if needed. */
 	if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG)))
@@ -5481,7 +5494,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 	gfp = current_gfp_context(gfp);
 	alloc_gfp = gfp;
 	if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac,
-			&alloc_gfp, &fastpath_alloc_flags))
+			&alloc_gfp, &fastpath_alloc_flags, &tier_nodes))
 		return NULL;
 
 	if (!(alloc_flags & ALLOC_NOLOCK)) {
-- 
2.53.0-Meta