* [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths
@ 2026-06-18 10:08 Hao Li
2026-06-18 15:21 ` Hao Li
0 siblings, 1 reply; 2+ messages in thread
From: Hao Li @ 2026-06-18 10:08 UTC (permalink / raw)
To: vbabka, harry
Cc: akpm, cl, rientjes, roman.gushchin, linux-mm, linux-kernel,
Hao Li
Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
NUMA policy independently. Since they are called consecutively in paths
like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
redundant computations.
Introduce a helper function to resolve the NUMA policy once, eliminating
the duplicated code and reducing execution overhead.
Signed-off-by: Hao Li <hao.li@linux.dev>
---
mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
1 file changed, 29 insertions(+), 43 deletions(-)
diff --git a/mm/slub.c b/mm/slub.c
index 62e9cd46916f..45e9f379b7da 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
return object;
}
+static __always_inline int apply_numa_policy(int node)
+{
+#ifdef CONFIG_NUMA
+ if (static_branch_unlikely(&strict_numa) &&
+ node == NUMA_NO_NODE) {
+
+ struct mempolicy *mpol = current->mempolicy;
+
+ if (mpol) {
+ /*
+ * Special BIND rule support. If the local node
+ * is in permitted set then do not redirect
+ * to a particular node.
+ * Otherwise we apply the memory policy to get
+ * the node we need to allocate on.
+ */
+ if (mpol->mode != MPOL_BIND ||
+ !node_isset(numa_mem_id(), mpol->nodes))
+ node = mempolicy_slab_node();
+ }
+ }
+#endif
+ return node;
+}
+
static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
const struct slab_alloc_context *ac)
{
void *object;
-#ifdef CONFIG_NUMA
- if (static_branch_unlikely(&strict_numa) &&
- node == NUMA_NO_NODE) {
-
- struct mempolicy *mpol = current->mempolicy;
-
- if (mpol) {
- /*
- * Special BIND rule support. If the local node
- * is in permitted set then do not redirect
- * to a particular node.
- * Otherwise we apply the memory policy to get
- * the node we need to allocate on.
- */
- if (mpol->mode != MPOL_BIND ||
- !node_isset(numa_mem_id(), mpol->nodes))
- node = mempolicy_slab_node();
- }
- }
-#endif
-
object = ___slab_alloc(s, gfpflags, node, ac);
return object;
@@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
bool node_requested;
void *object;
-#ifdef CONFIG_NUMA
- if (static_branch_unlikely(&strict_numa) &&
- node == NUMA_NO_NODE) {
-
- struct mempolicy *mpol = current->mempolicy;
-
- if (mpol) {
- /*
- * Special BIND rule support. If the local node
- * is in permitted set then do not redirect
- * to a particular node.
- * Otherwise we apply the memory policy to get
- * the node we need to allocate on.
- */
- if (mpol->mode != MPOL_BIND ||
- !node_isset(numa_mem_id(), mpol->nodes))
-
- node = mempolicy_slab_node();
- }
- }
-#endif
-
node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
/*
@@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
if (unlikely(object))
goto out;
+ node = apply_numa_policy(node);
+
object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
if (unlikely(!object))
@@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
*/
return NULL;
+ node = apply_numa_policy(node);
+
ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
if (ret)
goto success;
--
2.54.0
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths
2026-06-18 10:08 [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths Hao Li
@ 2026-06-18 15:21 ` Hao Li
0 siblings, 0 replies; 2+ messages in thread
From: Hao Li @ 2026-06-18 15:21 UTC (permalink / raw)
To: vbabka, harry; +Cc: akpm, cl, rientjes, roman.gushchin, linux-mm, linux-kernel
On Thu, Jun 18, 2026 at 06:08:52PM +0800, Hao Li wrote:
> Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
> NUMA policy independently. Since they are called consecutively in paths
> like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
> redundant computations.
>
> Introduce a helper function to resolve the NUMA policy once, eliminating
> the duplicated code and reducing execution overhead.
>
> Signed-off-by: Hao Li <hao.li@linux.dev>
> ---
> mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
> 1 file changed, 29 insertions(+), 43 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index 62e9cd46916f..45e9f379b7da 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
> return object;
> }
>
> +static __always_inline int apply_numa_policy(int node)
> +{
> +#ifdef CONFIG_NUMA
> + if (static_branch_unlikely(&strict_numa) &&
> + node == NUMA_NO_NODE) {
> +
> + struct mempolicy *mpol = current->mempolicy;
> +
> + if (mpol) {
> + /*
> + * Special BIND rule support. If the local node
> + * is in permitted set then do not redirect
> + * to a particular node.
> + * Otherwise we apply the memory policy to get
> + * the node we need to allocate on.
> + */
> + if (mpol->mode != MPOL_BIND ||
> + !node_isset(numa_mem_id(), mpol->nodes))
> + node = mempolicy_slab_node();
> + }
> + }
> +#endif
> + return node;
> +}
> +
> static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
> const struct slab_alloc_context *ac)
> {
> void *object;
>
> -#ifdef CONFIG_NUMA
> - if (static_branch_unlikely(&strict_numa) &&
> - node == NUMA_NO_NODE) {
> -
> - struct mempolicy *mpol = current->mempolicy;
> -
> - if (mpol) {
> - /*
> - * Special BIND rule support. If the local node
> - * is in permitted set then do not redirect
> - * to a particular node.
> - * Otherwise we apply the memory policy to get
> - * the node we need to allocate on.
> - */
> - if (mpol->mode != MPOL_BIND ||
> - !node_isset(numa_mem_id(), mpol->nodes))
> - node = mempolicy_slab_node();
> - }
> - }
> -#endif
> -
> object = ___slab_alloc(s, gfpflags, node, ac);
>
> return object;
oh!, __slab_alloc_node is almost an empty wrapper
Maybe I need to eliminate it completely...
> @@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
> bool node_requested;
> void *object;
>
> -#ifdef CONFIG_NUMA
> - if (static_branch_unlikely(&strict_numa) &&
> - node == NUMA_NO_NODE) {
> -
> - struct mempolicy *mpol = current->mempolicy;
> -
> - if (mpol) {
> - /*
> - * Special BIND rule support. If the local node
> - * is in permitted set then do not redirect
> - * to a particular node.
> - * Otherwise we apply the memory policy to get
> - * the node we need to allocate on.
> - */
> - if (mpol->mode != MPOL_BIND ||
> - !node_isset(numa_mem_id(), mpol->nodes))
> -
> - node = mempolicy_slab_node();
> - }
> - }
> -#endif
> -
> node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
>
> /*
> @@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
> if (unlikely(object))
> goto out;
>
> + node = apply_numa_policy(node);
> +
> object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
>
> if (unlikely(!object))
> @@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
> */
> return NULL;
>
> + node = apply_numa_policy(node);
> +
> ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
> if (ret)
> goto success;
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2026-06-18 15:22 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-06-18 10:08 [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths Hao Li
2026-06-18 15:21 ` Hao Li
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.