[PATCH] mbuf: avoid temporary array for bulk free

[Nam Tran <hoangnamtran18122005@gmail.com>]

rte_pktmbuf_free_bulk() currently stages freeable mbufs in a
temporary array before returning them to their mempool. For flat
packet arrays, this requires copying pointers even though the
original array already contains contiguous freeable mbufs.

Track contiguous same-pool runs in the input array and pass them
directly to rte_mbuf_raw_free_bulk(). Flush a run when encountering
a NULL mbuf, an mbuf retained by reference counting, or a pool
change. Preserve the existing array-based implementation as the
fallback for chained packets.

On an ARM64 Linux test environment, same-binary A/B measurements
using rte_rdtsc showed lower median timer ticks per call for flat
bulk frees:

  burst  32:  2.05 ->  1.50
  burst  64:  5.16 ->  4.52
  burst 128: 11.16 ->  6.90
  burst 256: 26.46 -> 19.73

This corresponds to reductions of approximately 12% to 38% across
the tested burst sizes.

Add coverage for NULL entries, mixed mempools, and shared mbufs.

Signed-off-by: Nam Tran <hoangnamtran18122005@gmail.com>
---
 app/test/test_mbuf.c | 53 +++++++++++++++++++++++++++++++++
 lib/mbuf/rte_mbuf.c  | 71 ++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 121 insertions(+), 3 deletions(-)

diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c
index db23259745..2aed37b225 100644
--- a/app/test/test_mbuf.c
+++ b/app/test/test_mbuf.c
@@ -833,6 +833,59 @@ test_pktmbuf_pool_bulk(void)
 		goto err;
 	}
 
+	printf("Test bulk free with NULL entries.\n");
+
+	ret = rte_pktmbuf_alloc_bulk(pool, mbufs, 4);
+	if (ret != 0)
+		goto err;
+
+	m = mbufs[1];
+	mbufs[1] = NULL;
+	rte_pktmbuf_free_bulk(mbufs, 4);
+	rte_pktmbuf_free(m);
+
+	if (!rte_mempool_full(pool)) {
+		printf("mempool not full after NULL-entry bulk free\n");
+		goto err;
+	}
+
+	printf("Test bulk free with multiple pools.\n");
+
+	for (i = 0; i < 4; i++) {
+		mbufs[i] = rte_pktmbuf_alloc((i & 1) ? pool2 : pool);
+		if (mbufs[i] == NULL)
+			goto err;
+	}
+
+	rte_pktmbuf_free_bulk(mbufs, 4);
+
+	if (!(rte_mempool_full(pool) && rte_mempool_full(pool2))) {
+		printf("mempools not full after mixed-pool bulk free\n");
+		goto err;
+	}
+
+	printf("Test bulk free with shared mbuf.\n");
+
+	ret = rte_pktmbuf_alloc_bulk(pool, mbufs, 4);
+	if (ret != 0)
+		goto err;
+
+	m = mbufs[1];
+	rte_mbuf_refcnt_update(m, 1);
+	rte_pktmbuf_free_bulk(mbufs, 4);
+
+	if (rte_mbuf_refcnt_read(m) != 1) {
+		printf("shared mbuf reference count incorrect\n");
+		goto err;
+	}
+
+	rte_pktmbuf_free(m);
+
+	if (!rte_mempool_full(pool)) {
+		printf("mempool not full after shared mbuf free\n");
+		goto err;
+	}
+
 	printf("Test bulk free of single long chain.\n");
 
 	/* Bulk allocate all mbufs in the pool, in one go. */
diff --git a/lib/mbuf/rte_mbuf.c b/lib/mbuf/rte_mbuf.c
index 005bfaa573..796631a033 100644
--- a/lib/mbuf/rte_mbuf.c
+++ b/lib/mbuf/rte_mbuf.c
@@ -555,15 +555,15 @@ __rte_pktmbuf_free_seg_via_array(struct rte_mbuf *m,
  */
 #define RTE_PKTMBUF_FREE_PENDING_SZ 64
 
-/* Free a bulk of packet mbufs back into their original mempools. */
-RTE_EXPORT_SYMBOL(rte_pktmbuf_free_bulk)
-void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
+static void
+__rte_pktmbuf_free_bulk_fallback(struct rte_mbuf **mbufs, unsigned int count)
 {
 	struct rte_mbuf *m, *m_next, *pending[RTE_PKTMBUF_FREE_PENDING_SZ];
 	unsigned int idx, nb_pending = 0;
 
 	for (idx = 0; idx < count; idx++) {
 		m = mbufs[idx];
+
 		if (unlikely(m == NULL))
 			continue;
 
@@ -582,6 +582,71 @@ void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
 		rte_mbuf_raw_free_bulk(pending[0]->pool, pending, nb_pending);
 }
 
+/* Free a bulk of packet mbufs back into their original mempools. */
+RTE_EXPORT_SYMBOL(rte_pktmbuf_free_bulk)
+void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
+{
+	struct rte_mempool *run_pool = NULL;
+	unsigned int run_start = 0;
+	unsigned int run_count = 0;
+	unsigned int idx;
+
+	for (idx = 0; idx < count; idx++) {
+		struct rte_mbuf *m = mbufs[idx];
+
+		if (unlikely(m == NULL)) {
+			if (run_count != 0) {
+				rte_mbuf_raw_free_bulk(run_pool,
+						&mbufs[run_start], run_count);
+				run_count = 0;
+			}
+			continue;
+		}
+
+		__rte_mbuf_sanity_check(m, 1);
+
+		/*
+		 * Preserve the generic path for chained packets. No mbuf in
+		 * this suffix has been modified yet.
+		 */
+		if (unlikely(m->next != NULL)) {
+			if (run_count != 0)
+				rte_mbuf_raw_free_bulk(run_pool,
+						&mbufs[run_start], run_count);
+
+			__rte_pktmbuf_free_bulk_fallback(&mbufs[idx],
+					count - idx);
+			return;
+		}
+
+		m = rte_pktmbuf_prefree_seg(m);
+		if (unlikely(m == NULL)) {
+			if (run_count != 0) {
+				rte_mbuf_raw_free_bulk(run_pool,
+						&mbufs[run_start], run_count);
+				run_count = 0;
+			}
+			continue;
+		}
+
+		if (run_count != 0 && m->pool != run_pool) {
+			rte_mbuf_raw_free_bulk(run_pool,
+					&mbufs[run_start], run_count);
+			run_count = 0;
+		}
+
+		if (run_count == 0) {
+			run_pool = m->pool;
+			run_start = idx;
+		}
+
+		run_count++;
+	}
+
+	if (run_count != 0)
+		rte_mbuf_raw_free_bulk(run_pool, &mbufs[run_start], run_count);
+}
+
 /* Creates a shallow copy of mbuf */
 RTE_EXPORT_SYMBOL(rte_pktmbuf_clone)
 struct rte_mbuf *
-- 
2.51.0