[PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths

[PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths

[Hao Li]

Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
NUMA policy independently. Since they are called consecutively in paths
like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
redundant computations.

Introduce a helper function to resolve the NUMA policy once, eliminating
the duplicated code and reducing execution overhead.

Signed-off-by: Hao Li <hao.li@linux.dev>
---
 mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
 1 file changed, 29 insertions(+), 43 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 62e9cd46916f..45e9f379b7da 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return object;
 }
 
+static __always_inline int apply_numa_policy(int node)
+{
+#ifdef CONFIG_NUMA
+	if (static_branch_unlikely(&strict_numa) &&
+			node == NUMA_NO_NODE) {
+
+		struct mempolicy *mpol = current->mempolicy;
+
+		if (mpol) {
+			/*
+			 * Special BIND rule support. If the local node
+			 * is in permitted set then do not redirect
+			 * to a particular node.
+			 * Otherwise we apply the memory policy to get
+			 * the node we need to allocate on.
+			 */
+			if (mpol->mode != MPOL_BIND ||
+					!node_isset(numa_mem_id(), mpol->nodes))
+				node = mempolicy_slab_node();
+		}
+	}
+#endif
+	return node;
+}
+
 static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
 			       const struct slab_alloc_context *ac)
 {
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	object = ___slab_alloc(s, gfpflags, node, ac);
 
 	return object;
@@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
 	bool node_requested;
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			 node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
 
 	/*
@@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
 	if (unlikely(object))
 		goto out;
 
+	node = apply_numa_policy(node);
+
 	object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
 
 	if (unlikely(!object))
@@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
 		 */
 		return NULL;
 
+	node = apply_numa_policy(node);
+
 	ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
 	if (ret)
 		goto success;
-- 
2.54.0

Re: [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths

[Hao Li]

On Thu, Jun 18, 2026 at 06:08:52PM +0800, Hao Li wrote:
> Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
> NUMA policy independently. Since they are called consecutively in paths
> like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
> redundant computations.
> 
> Introduce a helper function to resolve the NUMA policy once, eliminating
> the duplicated code and reducing execution overhead.
> 
> Signed-off-by: Hao Li <hao.li@linux.dev>
> ---
>  mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
>  1 file changed, 29 insertions(+), 43 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 62e9cd46916f..45e9f379b7da 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
>  	return object;
>  }
>  
> +static __always_inline int apply_numa_policy(int node)
> +{
> +#ifdef CONFIG_NUMA
> +	if (static_branch_unlikely(&strict_numa) &&
> +			node == NUMA_NO_NODE) {
> +
> +		struct mempolicy *mpol = current->mempolicy;
> +
> +		if (mpol) {
> +			/*
> +			 * Special BIND rule support. If the local node
> +			 * is in permitted set then do not redirect
> +			 * to a particular node.
> +			 * Otherwise we apply the memory policy to get
> +			 * the node we need to allocate on.
> +			 */
> +			if (mpol->mode != MPOL_BIND ||
> +					!node_isset(numa_mem_id(), mpol->nodes))
> +				node = mempolicy_slab_node();
> +		}
> +	}
> +#endif
> +	return node;
> +}
> +
>  static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
>  			       const struct slab_alloc_context *ac)
>  {
>  	void *object;
>  
> -#ifdef CONFIG_NUMA
> -	if (static_branch_unlikely(&strict_numa) &&
> -			node == NUMA_NO_NODE) {
> -
> -		struct mempolicy *mpol = current->mempolicy;
> -
> -		if (mpol) {
> -			/*
> -			 * Special BIND rule support. If the local node
> -			 * is in permitted set then do not redirect
> -			 * to a particular node.
> -			 * Otherwise we apply the memory policy to get
> -			 * the node we need to allocate on.
> -			 */
> -			if (mpol->mode != MPOL_BIND ||
> -					!node_isset(numa_mem_id(), mpol->nodes))
> -				node = mempolicy_slab_node();
> -		}
> -	}
> -#endif
> -
>  	object = ___slab_alloc(s, gfpflags, node, ac);
>  
>  	return object;

oh!, __slab_alloc_node is almost an empty wrapper
Maybe I need to eliminate it completely...

> @@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
>  	bool node_requested;
>  	void *object;
>  
> -#ifdef CONFIG_NUMA
> -	if (static_branch_unlikely(&strict_numa) &&
> -			 node == NUMA_NO_NODE) {
> -
> -		struct mempolicy *mpol = current->mempolicy;
> -
> -		if (mpol) {
> -			/*
> -			 * Special BIND rule support. If the local node
> -			 * is in permitted set then do not redirect
> -			 * to a particular node.
> -			 * Otherwise we apply the memory policy to get
> -			 * the node we need to allocate on.
> -			 */
> -			if (mpol->mode != MPOL_BIND ||
> -					!node_isset(numa_mem_id(), mpol->nodes))
> -
> -				node = mempolicy_slab_node();
> -		}
> -	}
> -#endif
> -
>  	node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
>  
>  	/*
> @@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
>  	if (unlikely(object))
>  		goto out;
>  
> +	node = apply_numa_policy(node);
> +
>  	object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
>  
>  	if (unlikely(!object))
> @@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
>  		 */
>  		return NULL;
>  
> +	node = apply_numa_policy(node);
> +
>  	ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
>  	if (ret)
>  		goto success;
> -- 
> 2.54.0
> 

Re: [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths

[Harry Yoo]

[-- Attachment #1.1: Type: text/plain, Size: 1509 bytes --]



On 6/18/26 7:08 PM, Hao Li wrote:
> Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
> NUMA policy independently. Since they are called consecutively in paths
> like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
> redundant computations.

It uses a static key, so probably just slightly larger code when disabled.

By inlining both __slab_alloc_node and alloc_from_pcs(), I assume the
compiler would have deduplicated it, but there is a patch in
slab/for-next that makes it not inlined anymore :)

> Introduce a helper function to resolve the NUMA policy once, eliminating
> the duplicated code and reducing execution overhead.

Nice! I think there's no reason why we shouldn't do this.

> Signed-off-by: Hao Li <hao.li@linux.dev>
> ---
>  mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
>  1 file changed, 29 insertions(+), 43 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 62e9cd46916f..45e9f379b7da 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
>  	return object;
>  }
>  
> +static __always_inline int apply_numa_policy(int node)

apply_numa_policy() is bit confusing because we usually don't apply
mempolicy for each object (unless strict_numa is set), but rather when
grabbing new slabs.

perhaps apply_strict_numa[_policy]() will be a better name?


-- 
Cheers,
Harry / Hyeonggon


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths

[Hao Li]

On Fri, Jun 19, 2026 at 02:50:04PM +0900, Harry Yoo wrote:
> 
> 
> On 6/18/26 7:08 PM, Hao Li wrote:
> > Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
> > NUMA policy independently. Since they are called consecutively in paths
> > like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
> > redundant computations.
> 
> It uses a static key, so probably just slightly larger code when disabled.

Yeah, make sense. The performance impact in this case should be negligible.

> 
> By inlining both __slab_alloc_node and alloc_from_pcs(), I assume the
> compiler would have deduplicated it, but there is a patch in
> slab/for-next that makes it not inlined anymore :)

Thanks, I hadn't thought about it from that angle before, and that's a
interesting point.

Building on that, I thought about it a bit more. While in theory the compiler
might be able to help us here, some of the information inside
mempolicy_slab_node() seems only could be determined at runtime. So I suspect
the compiler might not be able to infer that the two code blocks are equivalent
and deduplicate it.

> 
> > Introduce a helper function to resolve the NUMA policy once, eliminating
> > the duplicated code and reducing execution overhead.
> 
> Nice! I think there's no reason why we shouldn't do this.

Thanks!

> 
> > Signed-off-by: Hao Li <hao.li@linux.dev>
> > ---
> >  mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
> >  1 file changed, 29 insertions(+), 43 deletions(-)
> > 
> > diff --git a/mm/slub.c b/mm/slub.c
> > index 62e9cd46916f..45e9f379b7da 100644
> > --- a/mm/slub.c
> > +++ b/mm/slub.c
> > @@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
> >  	return object;
> >  }
> >  
> > +static __always_inline int apply_numa_policy(int node)
> 
> apply_numa_policy() is bit confusing because we usually don't apply
> mempolicy for each object (unless strict_numa is set), but rather when
> grabbing new slabs.

Yes, this make sense. it's only for strict numa mode.

> 
> perhaps apply_strict_numa[_policy]() will be a better name?

Agree, apply_strict_numa_policy is indeed a better name!

-- 
Thanks,
Hao