RE: [PATCH] stack: introduce pile

[Konstantin Ananyev <konstantin.ananyev@huawei.com>]

> diff --git a/lib/stack/rte_stack.h b/lib/stack/rte_stack.h
> index fd17ac791d..ca11f1d296 100644
> --- a/lib/stack/rte_stack.h
> +++ b/lib/stack/rte_stack.h
> @@ -1,5 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   * Copyright(c) 2019 Intel Corporation
> + * Copyright(c) 2026 SmartShare Systems
>   */
> 
>  /**
> @@ -28,11 +29,47 @@
>  #define RTE_STACK_NAMESIZE (RTE_MEMZONE_NAMESIZE - \
>  			   sizeof(RTE_STACK_MZ_PREFIX) + 1)
> 
> +static_assert(((sizeof(void *) * RTE_STACK_PILE_BULK_SIZE) &
> RTE_CACHE_LINE_MASK) == 0,
> +		"Pile bulk size must be divisible by CPU cache line size");
> +static_assert(RTE_IS_POWER_OF_2(RTE_STACK_PILE_BULK_SIZE),
> +		"Pile bulk size must be power of 2");
> +
> +/* Note: Also used as solo (single-object) pile element. */
>  struct rte_stack_lf_elem {
>  	void *data;			/**< Data pointer */
>  	struct rte_stack_lf_elem *next;	/**< Next pointer */
>  };
> 
> +/*
> + * Bulk (multi-object) pile element.
> + * Inherited from the rte_stack_lf_elem (single-object) class,
> + * and extended with an array for holding a bulk of object pointers.
> + */
> +struct rte_stack_pile_bulk_elem {
> +	/* The first part must be ABI compatible with the rte_stack_lf_elem
> parent class. */
> +	void *data;                             /**< Unused, for rte_stack_lf_elem
> compatibility */

Why simply not: 
struct rte_stack_pile_bulk_elem {
	struct rte_stack_lf_elem elem;
              ....
};
?
That way you just need a static_assert that elem always first (offset == 0).
Wouldn't need that clumsy asserts below.

> +	struct rte_stack_pile_bulk_elem *next;  /**< Next pointer */
> +	/* The second part differs. */
> +	alignas(RTE_CACHE_LINE_SIZE)
> +	void *objs[RTE_STACK_PILE_BULK_SIZE];   /**< Bulk (multi-object)
> pointers */
> +};
> +
> +static_assert(sizeof(struct rte_stack_lf_elem) ==
> +		sizeof(struct rte_stack_lf_elem *) + sizeof(void *),
> +		"Parent type has changed");
> +static_assert(RTE_SIZEOF_FIELD(struct rte_stack_lf_elem, next) ==
> +		RTE_SIZEOF_FIELD(struct rte_stack_pile_bulk_elem, next),
> +		"Inherited type mismatch");
> +static_assert(offsetof(struct rte_stack_lf_elem, next) ==
> +		offsetof(struct rte_stack_pile_bulk_elem, next),
> +		"Inherited type mismatch");
> +static_assert(RTE_SIZEOF_FIELD(struct rte_stack_lf_elem, data) ==
> +		RTE_SIZEOF_FIELD(struct rte_stack_pile_bulk_elem, data),
> +		"Inherited type mismatch");
> +static_assert(offsetof(struct rte_stack_lf_elem, data) ==
> +		offsetof(struct rte_stack_pile_bulk_elem, data),
> +		"Inherited type mismatch");
> +
>  struct __rte_aligned(16) rte_stack_lf_head {
>  	struct rte_stack_lf_elem *top; /**< Stack top */
>  	uint64_t cnt; /**< Modification counter for avoiding ABA problem */
> @@ -51,12 +88,36 @@ struct rte_stack_lf_list {
>  struct rte_stack_lf {
>  	/** LIFO list of elements */
>  	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_list used;
> +	RTE_CACHE_GUARD;
>  	/** LIFO list of free elements */
>  	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_list free;
> +	RTE_CACHE_GUARD;
>  	/** LIFO elements */
>  	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_elem elems[];
>  };
> 
> +/* Pile structure containing three lock-free LIFO-like lists:
> + *  - A list of elements, each element holding a bulk of pointers to objects.
> + *  - A list of elements, each element holding one pointer to an object.
> + *  - A list of free linked-list elements.
> + */
> +struct rte_stack_pile {
> +	/** LIFO list of bulk (multi-object) elements */
> +	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_list bulk;
> +	RTE_CACHE_GUARD;
> +	/** LIFO list of solo (single-object) elements */
> +	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_list solo;
> +	RTE_CACHE_GUARD;
> +	/** LIFO list of free bulk elements */
> +	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_list free_bulk;
> +	RTE_CACHE_GUARD;
> +	/** LIFO list of free solo elements */
> +	alignas(RTE_CACHE_LINE_SIZE) struct rte_stack_lf_list free_solo;
> +	RTE_CACHE_GUARD;
> +	/** LIFO elements follow, first bulk, then solo */
> +	alignas(RTE_CACHE_LINE_SIZE) void *elems[];
> +};
> +
>  /* Structure containing the LIFO, its current length, and a lock for mutual
>   * exclusion.
>   */
> @@ -78,6 +139,7 @@ struct __rte_cache_aligned rte_stack {
>  	uint32_t flags; /**< Flags supplied at creation. */
>  	union {
>  		struct rte_stack_lf stack_lf; /**< Lock-free LIFO structure. */
> +		struct rte_stack_pile stack_pile; /**< Lock-free pile (LIFO-like)
> structure. */
>  		struct rte_stack_std stack_std;	/**< LIFO structure. */
>  	};
>  };
> @@ -88,8 +150,19 @@ struct __rte_cache_aligned rte_stack {
>   */
>  #define RTE_STACK_F_LF 0x0001
> 
> +/**
> + * The stack-like pile uses lock-free push and pop functions.
> + * It is optimized for bulks of objects, and is not strictly LIFO.
> + * This flag is only supported on x86_64 or arm64 platforms, currently.
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior
> notice.
> + */
> +#define RTE_STACK_F_PILE 0x0002
> +
>  #include "rte_stack_std.h"
>  #include "rte_stack_lf.h"
> +#include "rte_stack_pile.h"
> 
>  #ifdef __cplusplus
>  extern "C" {
> @@ -115,6 +188,8 @@ rte_stack_push(struct rte_stack *s, void * const
> *obj_table, unsigned int n)
> 
>  	if (s->flags & RTE_STACK_F_LF)
>  		return __rte_stack_lf_push(s, obj_table, n);
> +	else if (s->flags & RTE_STACK_F_PILE)
> +		return __rte_stack_pile_push(s, obj_table, n);

I understand that you followed the common pattern, but might be
consider putting rte_scak_pile-* funcions into .c?
pile push/pop functions are quite big and heavily branched, do
we really want all that code to be always inlined for any stack pushs/pops?
Again, we can still keep inlned version in internal header for mempool driver usage.  

>  	else
>  		return __rte_stack_std_push(s, obj_table, n);
>  }
> @@ -139,6 +214,8 @@ rte_stack_pop(struct rte_stack *s, void **obj_table,
> unsigned int n)
> 
>  	if (s->flags & RTE_STACK_F_LF)
>  		return __rte_stack_lf_pop(s, obj_table, n);
> +	else if (s->flags & RTE_STACK_F_PILE)
> +		return __rte_stack_pile_pop(s, obj_table, n);
>  	else
>  		return __rte_stack_std_pop(s, obj_table, n);
>  }
> @@ -158,6 +235,8 @@ rte_stack_count(struct rte_stack *s)
> 
>  	if (s->flags & RTE_STACK_F_LF)
>  		return __rte_stack_lf_count(s);
> +	else if (s->flags & RTE_STACK_F_PILE)
> +		return __rte_stack_pile_count(s);
>  	else
>  		return __rte_stack_std_count(s);
>  }
> diff --git a/lib/stack/rte_stack_lf.h b/lib/stack/rte_stack_lf.h
> index f2b012cd0e..1ee9330c57 100644
> --- a/lib/stack/rte_stack_lf.h
> +++ b/lib/stack/rte_stack_lf.h
> @@ -79,6 +79,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table,
> unsigned int n)
>  		return 0;
> 
>  	/* Pop n used elements */
> +	__rte_assume(obj_table != NULL);
>  	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
>  					 n, obj_table, &last);
>  	if (unlikely(first == NULL))
> diff --git a/lib/stack/rte_stack_pile.c b/lib/stack/rte_stack_pile.c
> new file mode 100644
> index 0000000000..5884163313
> --- /dev/null
> +++ b/lib/stack/rte_stack_pile.c
> @@ -0,0 +1,35 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2026 SmartShare Systems
> + */
> +
> +#include "rte_stack.h"
> +
> +void
> +rte_stack_pile_init(struct rte_stack *s, unsigned int count)
> +{
> +	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) /
> RTE_STACK_PILE_BULK_SIZE;
> +	struct rte_stack_pile_bulk_elem *bulk_elems =
> +			(struct rte_stack_pile_bulk_elem *)(s->stack_pile.elems);
> +	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem
> *)&bulk_elems[bulk];
> +	unsigned int i;
> +
> +	for (i = 0; i < bulk; i++)
> +		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
> +					  &bulk_elems[i], &bulk_elems[i], 1);
> +	for (i = 0; i < count; i++)
> +		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
> +					  &solo_elems[i], &solo_elems[i], 1);
> +}
> +
> +ssize_t
> +rte_stack_pile_get_memsize(unsigned int count)
> +{
> +	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) /
> RTE_STACK_PILE_BULK_SIZE;
> +	ssize_t sz = offsetof(struct rte_stack, stack_pile.elems);
> +	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem);
> +	sz += count * sizeof(struct rte_stack_lf_elem);
> +	sz = RTE_CACHE_LINE_ROUNDUP(sz);
> +	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
> +
> +	return sz;
> +}
> diff --git a/lib/stack/rte_stack_pile.h b/lib/stack/rte_stack_pile.h
> new file mode 100644
> index 0000000000..b747434f3b
> --- /dev/null
> +++ b/lib/stack/rte_stack_pile.h
> @@ -0,0 +1,334 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2026 SmartShare Systems
> + */
> +
> +#ifndef _RTE_STACK_PILE_H_
> +#define _RTE_STACK_PILE_H_
> +
> +#include <rte_memcpy.h>
> +
> +#include "rte_stack_lf.h"
> +#ifdef RTE_STACK_LF_SUPPORTED
> +/**
> + * Indicates that RTE_STACK_F_PILE is supported.
> + */
> +#define RTE_STACK_PILE_SUPPORTED
> +#endif
> +
> +static __rte_always_inline unsigned int
> +__rte_stack_pile_count(struct rte_stack *s)
> +{
> +	/* stack_lf_push() and stack_lf_pop() do not update the list's contents
> +	 * and stack_lf->len atomically, which can cause the list to appear
> +	 * shorter than it actually is if this function is called while other
> +	 * threads are modifying the list.
> +	 *
> +	 * However, given the inherently approximate nature of the get_count
> +	 * callback -- even if the list and its size were updated atomically,
> +	 * the size could change between when get_count executes and when
> the
> +	 * value is returned to the caller -- this is acceptable.
> +	 *
> +	 * The stack_lf->len updates are placed such that the list may appear to
> +	 * have fewer elements than it does, but will never appear to have more
> +	 * elements. If the mempool is near-empty to the point that this is a
> +	 * concern, the user should consider increasing the mempool size.
> +	 */
> +#ifdef RTE_USE_C11_MEM_MODEL
> +	return RTE_MIN((unsigned int)s->capacity,
> +			(unsigned int)rte_atomic_load_explicit(&s-
> >stack_pile.bulk.len,
> +			rte_memory_order_relaxed) *
> RTE_STACK_PILE_BULK_SIZE +
> +			(unsigned int)rte_atomic_load_explicit(&s-
> >stack_pile.solo.len,
> +			rte_memory_order_relaxed));
> +#else /* FIXME: Remove if removed from lock-free stack. */
> +	/* NOTE: review for potential ordering optimization */
> +	return RTE_MIN((unsigned int)s->capacity,
> +			(unsigned int)rte_atomic_load_explicit(&s-
> >stack_pile.bulk.len,
> +			rte_memory_order_seq_cst) *
> RTE_STACK_PILE_BULK_SIZE +
> +			(unsigned int)rte_atomic_load_explicit(&s-
> >stack_pile.solo.len,
> +			rte_memory_order_seq_cst));
> +#endif
> +}
> +
> +static __rte_always_inline void
> +__rte_stack_pile_bulk_push_elems(struct rte_stack_lf_list *list,
> +		struct rte_stack_pile_bulk_elem *first,
> +		struct rte_stack_pile_bulk_elem *last,
> +		unsigned int num)
> +{
> +	__rte_stack_lf_push_elems(list,
> +		(struct rte_stack_lf_elem *)first,
> +		(struct rte_stack_lf_elem *)last,
> +		num);
> +}
> +
> +static __rte_always_inline struct rte_stack_pile_bulk_elem *
> +__rte_stack_pile_bulk_pop_elems(struct rte_stack_lf_list *list,
> +		unsigned int num,
> +		void **obj_table,
> +		struct rte_stack_pile_bulk_elem **last)
> +{
> +	struct rte_stack_pile_bulk_elem *first = (struct rte_stack_pile_bulk_elem
> *)
> +			__rte_stack_lf_pop_elems(list, num, NULL,
> +			(struct rte_stack_lf_elem **)last);
> +	if (first == NULL)
> +		return NULL;
> +
> +	if (obj_table != NULL) {
> +		/*
> +		 * Traverse the list to copy the bulks.
> +		 * Note:
> +		 * Done here to minimize the time spent in the retry loop in
> +		 * __rte_stack_lf_pop_elems(),
> +		 * and to avoid modifying __rte_stack_lf_pop_elems().
> +		 */
> +		struct rte_stack_pile_bulk_elem *tmp = first;
> +		for (unsigned int i = 0; i < num; i++, tmp = tmp->next)
> +			rte_memcpy(&obj_table[i * RTE_STACK_PILE_BULK_SIZE],
> tmp->objs,
> +					sizeof(void *) *
> RTE_STACK_PILE_BULK_SIZE);
> +	}
> +
> +	return first;
> +}
> +
> +/**
> + * Push several objects on the pile (lock-free, MT-safe).
> + *
> + * @param s
> + *   A pointer to the pile structure.
> + * @param obj_table
> + *   A pointer to a table of void * pointers (objects).
> + * @param n
> + *   The number of objects to push on the pile from the obj_table.
> + * @return
> + *   Actual number of objects pushed (either 0 or *n*).
> + */
> +static __rte_always_inline unsigned int
> +__rte_stack_pile_push(struct rte_stack *s,
> +		void * const *obj_table,
> +		unsigned int n)
> +{
> +	RTE_ASSERT(s != NULL);
> +	RTE_ASSERT(obj_table != NULL);
> +
> +	struct rte_stack_pile *pile = &s->stack_pile;
> +	struct rte_stack_pile_bulk_elem *bulk_first = NULL, *bulk_last = NULL,
> *tmp_bulk;
> +	struct rte_stack_lf_elem *solo_first = NULL, *solo_last = NULL,
> *tmp_solo;
> +	unsigned int n_bulk = n / RTE_STACK_PILE_BULK_SIZE;
> +	unsigned int n_solo = n & (RTE_STACK_PILE_BULK_SIZE - 1);
> +	unsigned int i;
> +
> +	if (unlikely(n_bulk == 0)) {
> +		if (unlikely(n_solo == 0))
> +			return 0;
> +		goto solo;
> +	}
> +
> +	/* Allocate n_bulk elements from the free list. */
> +	bulk_first = __rte_stack_pile_bulk_pop_elems(&pile->free_bulk, n_bulk,
> NULL, &bulk_last);
> +	if (unlikely(bulk_first == NULL))
> +		return 0; /* Failed. */
> +
> +	if (likely(n_solo == 0))
> +		goto bulk;
> +
> +solo:
> +	/* Allocate n_solo elements from the free list. */
> +	solo_first = __rte_stack_lf_pop_elems(&pile->free_solo, n_solo, NULL,
> &solo_last);
> +	if (unlikely(solo_first == NULL)) {
> +		/* Failed. Roll back. */
> +		if (n_bulk > 0)
> +			__rte_stack_pile_bulk_push_elems(&pile->free_bulk,
> +					bulk_first, bulk_last, n_bulk);
> +		return 0;
> +	}
> +
> +	/*
> +	 * Construct the solo elements.
> +	 * Copy objects in reverse order.
> +	 */
> +	tmp_solo = solo_first;
> +	__rte_assume(n_solo > 0);
> +	__rte_assume(n_solo < RTE_STACK_PILE_BULK_SIZE);
> +	for (i = 0; i < n_solo; i++, tmp_solo = tmp_solo->next)
> +		tmp_solo->data = obj_table[n_bulk * RTE_STACK_PILE_BULK_SIZE
> + n_solo - i - 1];
> +
> +	/* Push them to the solo list. */
> +	__rte_stack_lf_push_elems(&pile->solo, solo_first, solo_last, n_solo);
> +
> +	if (unlikely(n_bulk == 0))
> +		return n; /* Done. */
> +
> +bulk:
> +	/*
> +	 * Construct the bulk elements.
> +	 * Copy bulks in reverse order, but ignore the object order within each
> bulk.
> +	 */
> +	tmp_bulk = bulk_first;
> +	__rte_assume(n_bulk > 0);
> +	for (i = 0; i < n_bulk; i++, tmp_bulk = tmp_bulk->next)
> +		rte_memcpy(tmp_bulk->objs, &obj_table[(n_bulk - i - 1) *
> RTE_STACK_PILE_BULK_SIZE],
> +				sizeof(void *) * RTE_STACK_PILE_BULK_SIZE);
> +
> +	/* Push them to the bulk list. */
> +	__rte_stack_pile_bulk_push_elems(&pile->bulk, bulk_first, bulk_last,
> n_bulk);
> +
> +	return n;
> +}
> +
> +/**
> + * Pop several objects from the pile (lock-free, MT-safe).
> + *
> + * @param s
> + *   A pointer to the pile structure.
> + * @param obj_table
> + *   A pointer to a table of void * pointers (objects).
> + * @param n
> + *   The number of objects to pull from the pile.
> + * @return
> + *   Actual number of objects popped (either 0 or *n*).
> + */
> +static __rte_always_inline unsigned int
> +__rte_stack_pile_pop(struct rte_stack *s,
> +		void **obj_table,
> +		unsigned int n)
> +{
> +	RTE_ASSERT(s != NULL);
> +	RTE_ASSERT(obj_table != NULL);
> +
> +	struct rte_stack_pile *pile = &s->stack_pile;
> +	struct rte_stack_pile_bulk_elem *bulk_first = NULL, *bulk_last = NULL;
> +	struct rte_stack_lf_elem *solo_first = NULL, *solo_last = NULL,
> *tmp_solo;
> +	alignas(RTE_CACHE_LINE_SIZE) void
> *obj_frag[RTE_STACK_PILE_BULK_SIZE];
> +	struct rte_stack_pile_bulk_elem *frag = NULL;
> +	unsigned int n_bulk = n / RTE_STACK_PILE_BULK_SIZE;
> +	unsigned int n_solo = n & (RTE_STACK_PILE_BULK_SIZE - 1);
> +	unsigned int i;
> +
> +	if (unlikely(n_bulk == 0)) {
> +		if (unlikely(n_solo == 0))
> +			return 0;
> +		goto solo;
> +	}
> +
> +bulk:
> +	/* Fetch n_bulk * RTE_STACK_PILE_BULK_SIZE objects as bulk elements.
> */
> +	bulk_first = __rte_stack_pile_bulk_pop_elems(&pile->bulk, n_bulk,
> obj_table, &bulk_last);
> +	if (unlikely(bulk_first == NULL)) {

That's quite heavy construction...
If it fail, might be go straight to solo is better?
As an alternative: introduction _burst_ version of _lf_pop_elems - it would really help here:
bot in terms of performance and code compaction.
 
> +		/*
> +		 * Not available.
> +		 * Retry with fewer bulk elements; objects to be fetched as solo
> elements instead.
> +		 */
> +		n_solo += RTE_STACK_PILE_BULK_SIZE;
> +		n_bulk--;
> +		if (n_bulk > 0)
> +			goto bulk;
> +		else
> +			goto solo;
> +	}
> +
> +	if (likely(n_solo == 0))
> +		goto done;
> +
> +solo:
> +	/* Fetch n_solo objects as solo elements. */
> +	solo_first = __rte_stack_lf_pop_elems(&pile->solo, n_solo,
> +			&obj_table[n_bulk * RTE_STACK_PILE_BULK_SIZE],
> &solo_last);
> +	if (solo_first != NULL)
> +		goto done;
> +
> +	/* Solo elements not available. Try fragmentation. */
> +	if (unlikely(n_solo >= RTE_STACK_PILE_BULK_SIZE))
> +		goto fail; /* Ran out of bulk elements above. Don't try to fetch
> one more. */
> +

Fragmentation procedure is quite big piece of code, my suggestion would be
to put it into separate inline helper function (for readability and ease to maintain).
As a generic comment - there are a lot fo gotos, probably some of them unavoidable,
but makes code really hard to follow, please consider splitting big functions into
smaller chunks whenever possible. 

> +	/* Fetch a fragmentation element as a bulk element. */
> +	frag = __rte_stack_pile_bulk_pop_elems(&pile->bulk, 1, obj_frag, NULL);
> +	if (unlikely(frag == NULL))
> +		goto fail;
> +
> +	/* Get n_solo objects from the fragmentation element. */
> +	__rte_assume(n_solo > 0);
> +	__rte_assume(n_solo < RTE_STACK_PILE_BULK_SIZE);
> +	for (i = 0; i < n_solo; i++)
> +		obj_table[n_bulk * RTE_STACK_PILE_BULK_SIZE + i] = obj_frag[i];
> +
> +	/* Fetch free elements for the excess objects. */
> +	__rte_assume(RTE_STACK_PILE_BULK_SIZE - n_solo > 0);
> +	__rte_assume(RTE_STACK_PILE_BULK_SIZE - n_solo <
> RTE_STACK_PILE_BULK_SIZE);
> +	solo_first = __rte_stack_lf_pop_elems(&pile->free_solo,
> +			RTE_STACK_PILE_BULK_SIZE - n_solo, NULL, &solo_last);
> +	if (unlikely(solo_first == NULL))
> +		goto fail;
> +
> +	/* Construct the solo elements from the excess objects. */
> +	tmp_solo = solo_first;
> +	__rte_assume(n_solo > 0);
> +	__rte_assume(n_solo < RTE_STACK_PILE_BULK_SIZE);
> +	for (i = n_solo; i < RTE_STACK_PILE_BULK_SIZE; i++, tmp_solo = tmp_solo-
> >next)
> +		tmp_solo->data = obj_frag[i];
> +
> +	/* Push the excess objects as solo elements. */
> +	__rte_stack_lf_push_elems(&pile->solo, solo_first, solo_last,
> +			RTE_STACK_PILE_BULK_SIZE - n_solo);
> +	n_solo = 0;
> +
> +	/* Add the fragmentation element to the bulk elements, so it can be
> freed with them. */
> +	if (n_bulk > 0)
> +		bulk_last->next = frag;
> +	else
> +		bulk_first = frag;
> +	bulk_last = frag;
> +	n_bulk++;
> +
> +done:
> +	/* Success. Free the elements. */
> +	if (n_bulk > 0)
> +		__rte_stack_pile_bulk_push_elems(&pile->free_bulk, bulk_first,
> bulk_last, n_bulk);
> +	if (n_solo > 0)
> +		__rte_stack_lf_push_elems(&pile->free_solo, solo_first,
> solo_last, n_solo);
> +
> +	return n;
> +
> +fail:
> +	/* Failed. Roll back. */
> +	if (frag != NULL) {
> +		/*
> +		 * No further action than this is required to roll the
> fragmentation
> +		 * element back into the pile of bulk elements, as the objects in
> +		 * the fragmentation element are intact.
> +		 */
> +		if (n_bulk > 0)
> +			bulk_last->next = frag;
> +		else
> +			bulk_first = frag;
> +		bulk_last = frag;
> +		n_bulk += 1;
> +	}
> +	if (n_bulk > 0)
> +		__rte_stack_pile_bulk_push_elems(&pile->bulk, bulk_first,
> bulk_last, n_bulk);
> +
> +	return 0;
> +}
> +
> +/**
> + * @internal Initialize a pile stack.
> + *
> + * @param s
> + *   A pointer to the stack structure.
> + * @param count
> + *   The size of the stack.
> + */
> +void
> +rte_stack_pile_init(struct rte_stack *s, unsigned int count);
> +
> +/**
> + * @internal Return the memory required for a pile stack.
> + *
> + * @param count
> + *   The size of the stack.
> + * @return
> + *   The bytes to allocate for a pile stack.
> + */
> +ssize_t
> +rte_stack_pile_get_memsize(unsigned int count);
> +
> +#endif /* _RTE_STACK_PILE_H_ */
> --
> 2.43.0