From: "Harry Yoo (Meta)" <harry@kernel.org>
To: Johannes Weiner <hannes@cmpxchg.org>,
Michal Hocko <mhocko@kernel.org>,
Roman Gushchin <roman.gushchin@linux.dev>,
Shakeel Butt <shakeel.butt@linux.dev>,
Muchun Song <muchun.song@linux.dev>,
Andrew Morton <akpm@linux-foundation.org>,
Harry Yoo <harry@kernel.org>,
Alexei Starovoitov <ast@kernel.org>,
Vlastimil Babka <vbabka@kernel.org>, Hao Li <hao.li@linux.dev>,
Christoph Lameter <cl@gentwo.org>,
David Rientjes <rientjes@google.com>,
Sebastian Andrzej Siewior <bigeasy@linutronix.de>,
Clark Williams <clrkwllms@kernel.org>,
Steven Rostedt <rostedt@goodmis.org>
Cc: kernel-team@meta.com, cgroups@vger.kernel.org,
linux-mm@kvack.org, linux-kernel@vger.kernel.org,
linux-rt-devel@lists.linux.dev
Subject: [PATCH for-next v3 6/7] mm/slab: refactor free_to_pcs()
Date: Sun, 15 Nov 2026 15:09:40 +0000 [thread overview]
Message-ID: <20261115-kmalloc-nolock-fixes-v3-6-9034dd4ecaab@kernel.org> (raw)
In-Reply-To: <20261115-kmalloc-nolock-fixes-v3-0-9034dd4ecaab@kernel.org>
Every caller of free_to_pcs() has to call can_free_to_pcs() before it.
Refactor this by adding a slab parameter and calling can_free_to_pcs()
from free_to_pcs(). To do this, move can_free_to_pcs() above
free_to_pcs().
Note that can_free_to_pcs() cannot be folded into free_to_pcs() as
free_to_pcs_bulk() still needs it.
While at it, let free_to_pcs() take free_flags instead of allow_spin.
No functional change intended.
Signed-off-by: Harry Yoo (Meta) <harry@kernel.org>
---
mm/slub.c | 119 ++++++++++++++++++++++++++++++++------------------------------
1 file changed, 61 insertions(+), 58 deletions(-)
diff --git a/mm/slub.c b/mm/slub.c
index aec9fb6e2235..fd3de3bee1de 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6148,15 +6148,71 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return pcs;
}
+static __always_inline bool can_free_to_pcs(struct slab *slab)
+{
+ int slab_node;
+ int numa_node;
+
+ if (!IS_ENABLED(CONFIG_NUMA))
+ goto check_pfmemalloc;
+
+ slab_node = slab_nid(slab);
+
+#ifdef CONFIG_HAVE_MEMORYLESS_NODES
+ /*
+ * numa_mem_id() points to the closest node with memory so only allow
+ * objects from that node to the percpu sheaves
+ */
+ numa_node = numa_mem_id();
+
+ if (likely(slab_node == numa_node))
+ goto check_pfmemalloc;
+#else
+
+ /*
+ * numa_mem_id() is only a wrapper to numa_node_id() which is where this
+ * cpu belongs to, but it might be a memoryless node anyway. We don't
+ * know what the closest node is.
+ */
+ numa_node = numa_node_id();
+
+ /* freed object is from this cpu's node, proceed */
+ if (likely(slab_node == numa_node))
+ goto check_pfmemalloc;
+
+ /*
+ * Freed object isn't from this cpu's node, but that node is memoryless
+ * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
+ * Proceed as it's better to cache remote objects than falling back to
+ * the slowpath for everything. The allocation side can never obtain
+ * a local object anyway, if none exist. We don't have numa_mem_id() to
+ * point to the closest node as we would on a proper memoryless node
+ * setup.
+ */
+ if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
+ goto check_pfmemalloc;
+#endif
+
+ return false;
+
+check_pfmemalloc:
+ return likely(!slab_test_pfmemalloc(slab));
+}
+
/*
* Free an object to the percpu sheaves.
* The object is expected to have passed slab_free_hook() already.
*/
static __fastpath_inline
-bool free_to_pcs(struct kmem_cache *s, void *object, bool allow_spin)
+bool free_to_pcs(struct kmem_cache *s, struct slab *slab, void *object,
+ unsigned int free_flags)
{
+ bool allow_spin = free_flags_allow_spinning(free_flags);
struct slub_percpu_sheaves *pcs;
+ if (unlikely(!can_free_to_pcs(slab)))
+ return false;
+
if (!local_trylock(&s->cpu_sheaves->lock))
return false;
@@ -6355,57 +6411,6 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
return false;
}
-static __always_inline bool can_free_to_pcs(struct slab *slab)
-{
- int slab_node;
- int numa_node;
-
- if (!IS_ENABLED(CONFIG_NUMA))
- goto check_pfmemalloc;
-
- slab_node = slab_nid(slab);
-
-#ifdef CONFIG_HAVE_MEMORYLESS_NODES
- /*
- * numa_mem_id() points to the closest node with memory so only allow
- * objects from that node to the percpu sheaves
- */
- numa_node = numa_mem_id();
-
- if (likely(slab_node == numa_node))
- goto check_pfmemalloc;
-#else
-
- /*
- * numa_mem_id() is only a wrapper to numa_node_id() which is where this
- * cpu belongs to, but it might be a memoryless node anyway. We don't
- * know what the closest node is.
- */
- numa_node = numa_node_id();
-
- /* freed object is from this cpu's node, proceed */
- if (likely(slab_node == numa_node))
- goto check_pfmemalloc;
-
- /*
- * Freed object isn't from this cpu's node, but that node is memoryless
- * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
- * Proceed as it's better to cache remote objects than falling back to
- * the slowpath for everything. The allocation side can never obtain
- * a local object anyway, if none exist. We don't have numa_mem_id() to
- * point to the closest node as we would on a proper memoryless node
- * setup.
- */
- if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
- goto check_pfmemalloc;
-#endif
-
- return false;
-
-check_pfmemalloc:
- return likely(!slab_test_pfmemalloc(slab));
-}
-
/*
* Try to free as many objects (already processed by free hooks) as possible to
* a single per-cpu sheaf.
@@ -6642,7 +6647,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
if (unlikely(!slab_free_hook(s, object, free_flags)))
return;
- if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true)))
+ if (likely(free_to_pcs(s, slab, object, free_flags)))
return;
__slab_free(s, slab, object, object, 1, addr);
@@ -6656,7 +6661,6 @@ void memcg_alloc_abort_single(struct kmem_cache *s, void *object,
unsigned int alloc_flags)
{
struct slab *slab = virt_to_slab(object);
- bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
unsigned int free_flags = to_free_flags(alloc_flags);
if (slab_want_init_on_free(s))
@@ -6667,11 +6671,10 @@ void memcg_alloc_abort_single(struct kmem_cache *s, void *object,
if (unlikely(!slab_free_hook(s, object, free_flags)))
return;
- if (likely(can_free_to_pcs(slab)) &&
- likely(free_to_pcs(s, object, allow_spin)))
+ if (likely(free_to_pcs(s, slab, object, free_flags)))
return;
- if (likely(allow_spin))
+ if (likely(free_flags_allow_spinning(free_flags)))
__slab_free(s, slab, object, object, 1, _RET_IP_);
else
defer_free(s, object);
@@ -6991,7 +6994,7 @@ void kfree_nolock(const void *object)
if (unlikely(!slab_free_hook(s, x, free_flags)))
return;
- if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false)))
+ if (likely(free_to_pcs(s, slab, x, free_flags)))
return;
/*
--
2.54.0
next prev parent reply other threads:[~2026-11-15 15:10 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-15 15:09 [PATCH for-next v3 0/7] memcg,slab: kmalloc_nolock() fixes Harry Yoo (Meta)
2026-11-15 15:09 ` [PATCH for-next v3 1/7] mm/memcontrol: defer obj_cgroup_release() to irq_work Harry Yoo (Meta)
2026-11-15 15:09 ` [PATCH for-next v3 2/7] mm/slab: pass free_flags to slab_free_hook() Harry Yoo (Meta)
2026-11-15 15:09 ` [PATCH for-next v3 3/7] mm/slab: handle SLAB_FREE_NOLOCK in slab_free_hook() instead of open coding Harry Yoo (Meta)
2026-11-16 8:26 ` sashiko-bot
2026-11-15 15:09 ` [PATCH for-next v3 4/7] mm/slab: fix a deadlock in memcg_alloc_abort_single() Harry Yoo (Meta)
2026-11-15 15:09 ` [PATCH for-next v3 5/7] mm/slab: serialize deferred_work_barrier() Harry Yoo (Meta)
2026-11-15 15:09 ` Harry Yoo (Meta) [this message]
2026-11-15 15:09 ` [PATCH for-next v3 7/7] mm/slab: call debug_locks and KCSAN checks in kfree_nolock() Harry Yoo (Meta)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261115-kmalloc-nolock-fixes-v3-6-9034dd4ecaab@kernel.org \
--to=harry@kernel.org \
--cc=akpm@linux-foundation.org \
--cc=ast@kernel.org \
--cc=bigeasy@linutronix.de \
--cc=cgroups@vger.kernel.org \
--cc=cl@gentwo.org \
--cc=clrkwllms@kernel.org \
--cc=hannes@cmpxchg.org \
--cc=hao.li@linux.dev \
--cc=kernel-team@meta.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-rt-devel@lists.linux.dev \
--cc=mhocko@kernel.org \
--cc=muchun.song@linux.dev \
--cc=rientjes@google.com \
--cc=roman.gushchin@linux.dev \
--cc=rostedt@goodmis.org \
--cc=shakeel.butt@linux.dev \
--cc=vbabka@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.