[PATCH v2] mempool: optimize mempool resizing

[PATCH v2] mempool: optimize mempool resizing

[Vitaly Wool]

Resizing mempool to a bigger size currently requires a new allocation and
a data copy to a new larger elements array which doesn't go well with the
idea of having a fast and deadlock free memory allocations during exreme
load.

This situation can't be completely avoided but its occurrence can be
vastly minimized if we track the actual number of slots in the elements
array. This patch introduces a new member of struct mempool which is set
when the array is allocated in mempool_init_node() and only changes when
mempool_resize() replaces the array with a bigger one.  Growing a pool to a
size that still fits the existing array now just raises min_nr and refills
the pool.

A racing resize may enlarge the array while the lock is dropped for the
allocation, so __elements_size is checked again once the lock is
retaken, and the unneeded array is freed.  The array being replaced is
now freed after dropping the lock.

Signed-off-by: Vitaly Vul <vitaly.vul@partner.samsung.com>
---
 include/linux/mempool.h |  1 +
 mm/mempool.c            | 43 ++++++++++++++++++++++++++---------------
 2 files changed, 28 insertions(+), 16 deletions(-)

Changelog:
* v2: all the API / behavioral changes are dropped, only the
  optimization remains

diff --git a/include/linux/mempool.h b/include/linux/mempool.h
index 6da502aef2f7..a8a411290bf0 100644
--- a/include/linux/mempool.h
+++ b/include/linux/mempool.h
@@ -20,6 +20,7 @@ typedef struct mempool {
 	int min_nr;		/* nr of elements at *elements */
 	int curr_nr;		/* Current nr of elements at *elements */
 	void **elements;
+	int __elements_size;	/* nr of slots in *elements, never shrinks */
 
 	void *pool_data;
 	mempool_alloc_t *alloc;
diff --git a/mm/mempool.c b/mm/mempool.c
index cb74e718b2c6..1911a696737b 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -261,6 +261,7 @@ int mempool_init_node(struct mempool *pool, int min_nr,
 					    gfp_mask, node_id);
 	if (!pool->elements)
 		return -ENOMEM;
+	pool->__elements_size = max(1, min_nr);
 
 	/*
 	 * First pre-allocate the guaranteed number of buffers,
@@ -353,6 +354,8 @@ EXPORT_SYMBOL(mempool_create_node_noprof);
  * This function shrinks/grows the pool. In the case of growing,
  * it cannot be guaranteed that the pool will be grown to the new
  * size immediately, but new mempool_free() calls will refill it.
+ * Shrinking keeps the elements array, so growing the pool again up to
+ * the largest size it ever had does not reallocate it.
  * This function may sleep.
  *
  * Note, the caller must guarantee that no mempool_destroy is called
@@ -364,7 +367,7 @@ EXPORT_SYMBOL(mempool_create_node_noprof);
 int mempool_resize(struct mempool *pool, int new_min_nr)
 {
 	void *element;
-	void **new_elements;
+	void **new_elements = NULL;
 	unsigned long flags;
 
 	BUG_ON(new_min_nr <= 0);
@@ -381,24 +384,31 @@ int mempool_resize(struct mempool *pool, int new_min_nr)
 		pool->min_nr = new_min_nr;
 		goto out_unlock;
 	}
-	spin_unlock_irqrestore(&pool->lock, flags);
-
-	/* Grow the pool */
-	new_elements = kmalloc_objs(*new_elements, new_min_nr);
-	if (!new_elements)
-		return -ENOMEM;
 
-	spin_lock_irqsave(&pool->lock, flags);
-	if (unlikely(new_min_nr <= pool->min_nr)) {
-		/* Raced, other resize will do our work */
+	/*
+	 * Grow the pool.  The elements array is only reallocated if it is too
+	 * small, it may still be large from before an earlier shrink.
+	 */
+	if (new_min_nr > pool->__elements_size) {
 		spin_unlock_irqrestore(&pool->lock, flags);
-		kfree(new_elements);
-		goto out;
+		new_elements = kmalloc_objs(*new_elements, new_min_nr);
+		if (!new_elements)
+			return -ENOMEM;
+
+		spin_lock_irqsave(&pool->lock, flags);
+		if (unlikely(new_min_nr <= pool->min_nr)) {
+			/* Raced, other resize will do our work */
+			goto out_unlock;
+		}
+		/* A racing resize may have enlarged the array meanwhile */
+		if (new_min_nr > pool->__elements_size) {
+			memcpy(new_elements, pool->elements,
+			       pool->curr_nr * sizeof(*new_elements));
+			/* The old array is freed below, outside the lock */
+			swap(pool->elements, new_elements);
+			pool->__elements_size = new_min_nr;
+		}
 	}
-	memcpy(new_elements, pool->elements,
-			pool->curr_nr * sizeof(*new_elements));
-	kfree(pool->elements);
-	pool->elements = new_elements;
 	pool->min_nr = new_min_nr;
 
 	while (pool->curr_nr < pool->min_nr) {
@@ -418,6 +428,7 @@ int mempool_resize(struct mempool *pool, int new_min_nr)
 out_unlock:
 	spin_unlock_irqrestore(&pool->lock, flags);
 out:
+	kfree(new_elements);
 	return 0;
 }
 EXPORT_SYMBOL(mempool_resize);
-- 
2.39.5

Re: [PATCH v2] mempool: optimize mempool resizing

[Harry Yoo]

I think this has unresolved feedback from Christoph [1] and Damien [2],
so adding them to Cc list.

[1] https://lore.kernel.org/linux-mm/20260626042252.GA7997@lst.de
[2] https://lore.kernel.org/linux-mm/9104ac5d-ed2b-409d-8220-512dd4581966@kernel.org

On Tue, Oct 13, 2026 at 11:14:08AM +0200, Vitaly Wool wrote:
> Resizing mempool to a bigger size currently requires a new allocation and
> a data copy to a new larger elements array which doesn't go well with the
> idea of having a fast and deadlock free memory allocations during exreme
> load.

Please elaborate the problematic scenario more in detail.
Have you obsesrved the problem in practice?

> This situation can't be completely avoided but its occurrence can be
> vastly minimized if we track the actual number of slots in the elements
> array. This patch introduces a new member of struct mempool which is set
> when the array is allocated in mempool_init_node() and only changes when
> mempool_resize() replaces the array with a bigger one.  Growing a pool to a
> size that still fits the existing array now just raises min_nr and refills
> the pool.
> 
> A racing resize may enlarge the array while the lock is dropped for the
> allocation, so __elements_size is checked again once the lock is
> retaken, and the unneeded array is freed.  The array being replaced is
> now freed after dropping the lock.
> 
> Signed-off-by: Vitaly Vul <vitaly.vul@partner.samsung.com>
> ---
>  include/linux/mempool.h |  1 +
>  mm/mempool.c            | 43 ++++++++++++++++++++++++++---------------
>  2 files changed, 28 insertions(+), 16 deletions(-)
> 
> Changelog:
> * v2: all the API / behavioral changes are dropped, only the
>   optimization remains
> 
> diff --git a/include/linux/mempool.h b/include/linux/mempool.h
> index 6da502aef2f7..a8a411290bf0 100644
> --- a/include/linux/mempool.h
> +++ b/include/linux/mempool.h
> @@ -20,6 +20,7 @@ typedef struct mempool {
>  	int min_nr;		/* nr of elements at *elements */
>  	int curr_nr;		/* Current nr of elements at *elements */
>  	void **elements;
> +	int __elements_size;	/* nr of slots in *elements, never shrinks */
>  
>  	void *pool_data;
>  	mempool_alloc_t *alloc;
> diff --git a/mm/mempool.c b/mm/mempool.c
> index cb74e718b2c6..1911a696737b 100644
> --- a/mm/mempool.c
> +++ b/mm/mempool.c
> @@ -261,6 +261,7 @@ int mempool_init_node(struct mempool *pool, int min_nr,
>  					    gfp_mask, node_id);
>  	if (!pool->elements)
>  		return -ENOMEM;
> +	pool->__elements_size = max(1, min_nr);
>  
>  	/*
>  	 * First pre-allocate the guaranteed number of buffers,
> @@ -353,6 +354,8 @@ EXPORT_SYMBOL(mempool_create_node_noprof);
>   * This function shrinks/grows the pool. In the case of growing,
>   * it cannot be guaranteed that the pool will be grown to the new
>   * size immediately, but new mempool_free() calls will refill it.
> + * Shrinking keeps the elements array, so growing the pool again up to
> + * the largest size it ever had does not reallocate it.
>   * This function may sleep.
>   *
>   * Note, the caller must guarantee that no mempool_destroy is called
> @@ -364,7 +367,7 @@ EXPORT_SYMBOL(mempool_create_node_noprof);
>  int mempool_resize(struct mempool *pool, int new_min_nr)
>  {
>  	void *element;
> -	void **new_elements;
> +	void **new_elements = NULL;
>  	unsigned long flags;
>  
>  	BUG_ON(new_min_nr <= 0);
> @@ -381,24 +384,31 @@ int mempool_resize(struct mempool *pool, int new_min_nr)
>  		pool->min_nr = new_min_nr;
>  		goto out_unlock;
>  	}
> -	spin_unlock_irqrestore(&pool->lock, flags);
> -
> -	/* Grow the pool */
> -	new_elements = kmalloc_objs(*new_elements, new_min_nr);
> -	if (!new_elements)
> -		return -ENOMEM;
>  
> -	spin_lock_irqsave(&pool->lock, flags);
> -	if (unlikely(new_min_nr <= pool->min_nr)) {
> -		/* Raced, other resize will do our work */
> +	/*
> +	 * Grow the pool.  The elements array is only reallocated if it is too
> +	 * small, it may still be large from before an earlier shrink.
> +	 */
> +	if (new_min_nr > pool->__elements_size) {
>  		spin_unlock_irqrestore(&pool->lock, flags);
> -		kfree(new_elements);
> -		goto out;
> +		new_elements = kmalloc_objs(*new_elements, new_min_nr);
> +		if (!new_elements)
> +			return -ENOMEM;
> +
> +		spin_lock_irqsave(&pool->lock, flags);
> +		if (unlikely(new_min_nr <= pool->min_nr)) {
> +			/* Raced, other resize will do our work */
> +			goto out_unlock;
> +		}
> +		/* A racing resize may have enlarged the array meanwhile */
> +		if (new_min_nr > pool->__elements_size) {
> +			memcpy(new_elements, pool->elements,
> +			       pool->curr_nr * sizeof(*new_elements));
> +			/* The old array is freed below, outside the lock */
> +			swap(pool->elements, new_elements);
> +			pool->__elements_size = new_min_nr;
> +		}
>  	}
> -	memcpy(new_elements, pool->elements,
> -			pool->curr_nr * sizeof(*new_elements));
> -	kfree(pool->elements);
> -	pool->elements = new_elements;
>  	pool->min_nr = new_min_nr;
>  
>  	while (pool->curr_nr < pool->min_nr) {
> @@ -418,6 +428,7 @@ int mempool_resize(struct mempool *pool, int new_min_nr)
>  out_unlock:
>  	spin_unlock_irqrestore(&pool->lock, flags);
>  out:
> +	kfree(new_elements);
>  	return 0;
>  }
>  EXPORT_SYMBOL(mempool_resize);
> -- 
> 2.39.5
> 

-- 
Cheers,
Harry / Hyeonggon