[PATCH v4-S3 65/65] net/dpaa2: add Tx fast path for no-confirmation case

[Prashant Gupta <prashant.gupta_3@nxp.com>]

From: Jun Yang <jun.yang@nxp.com>

Split a dedicated Tx fast path out of dpaa2_dev_tx() for the common
no-confirmation configuration, keeping the hot enqueue loop free of the
Tx-conf and timestamp bookkeeping needed only by the slow path.

- Add dpaa2_dev_tx_fast() for the DPAA2_TX_NO_CONF case. dpaa2_dev_tx()
  builds the enqueue descriptor and dispatches straight to it, so the
  common path no longer runs the per-burst Tx-conf drain or the
  per-packet timestamp accounting.
- Factor the retry enqueue loop into dpaa2_dev_tx_enqueue_multiple() and
  the ordered-queue tail into dpaa2_dev_tx_multi_ordered(), shared by
  both the fast and slow paths to remove duplicated code.
- Zero-initialise fd_arr in the slow path and mark the rarely taken
  branches (congestion retry, VLAN insert, ordered, timestamp) unlikely.

This reduces per-packet work on the common Tx path.

Signed-off-by: Jun Yang <jun.yang@nxp.com>
---
 drivers/net/dpaa2/dpaa2_rxtx.c | 158 +++++++++++++++++++++++++--------
 1 file changed, 122 insertions(+), 36 deletions(-)

diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c
index 2af5a4745d..312c9bd7d5 100644
--- a/drivers/net/dpaa2/dpaa2_rxtx.c
+++ b/drivers/net/dpaa2/dpaa2_rxtx.c
@@ -1511,6 +1511,114 @@ dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs,
 	return dpaa2_dev_tx_multi_txq_ordered(mq, bufs, nb_pkts);
 }
 
+static __rte_always_inline uint16_t __rte_hot
+dpaa2_dev_tx_enqueue_multiple(struct qbman_swp *swp,
+	const struct qbman_fd *fd, uint16_t to_send,
+	const struct qbman_eq_desc *eqdesc)
+{
+	uint16_t loop = 0, retry_count = 0;
+	int eq_ret;
+
+	while (loop < to_send) {
+		eq_ret = qbman_swp_enqueue_multiple(swp, eqdesc,
+			&fd[loop], NULL, to_send - loop);
+		if (unlikely(eq_ret < 0)) {
+			retry_count++;
+			if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
+				break;
+		} else {
+			loop += eq_ret;
+			retry_count = 0;
+		}
+	}
+
+	return loop;
+}
+
+static __rte_noinline uint16_t
+dpaa2_dev_tx_multi_ordered(struct dpaa2_queue *dpaa2_q,
+	struct rte_mbuf **bufs, uint16_t nb_pkts)
+{
+	uint16_t to_send, order_tx = 0, total = 0;
+
+order_tx_again:
+	to_send = (nb_pkts > dpaa2_eqcr_size) ? dpaa2_eqcr_size : nb_pkts;
+	order_tx = dpaa2_dev_tx_ordered(dpaa2_q, bufs, to_send);
+	bufs += order_tx;
+	nb_pkts -= order_tx;
+	total += order_tx;
+	if (nb_pkts && order_tx >= to_send)
+		goto order_tx_again;
+
+	return total;
+}
+
+static __rte_always_inline uint16_t __rte_hot
+dpaa2_dev_tx_fast(struct dpaa2_queue *dpaa2_q, struct rte_mbuf **bufs,
+	uint16_t nb_pkts, struct qbman_swp *swp, struct qbman_fd *fd_arr,
+	const struct qbman_eq_desc *eqdesc, struct rte_mempool *hw_mp)
+{
+	/* Function to transmit the frames to given device and VQ*/
+	uint32_t loop, retry_count, frames_to_send;
+	int32_t ret, order = false;
+	uint16_t num_tx = 0, eq_ret = 0;
+
+tx_again:
+	/*Check if the queue is congested*/
+	retry_count = 0;
+	while (qbman_result_SCN_state(dpaa2_q->cscn)) {
+		retry_count++;
+		/* Retry for some time before giving up */
+		if (retry_count > CONG_RETRY_COUNT)
+			goto skip_tx;
+	}
+
+	frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? dpaa2_eqcr_size : nb_pkts;
+
+	for (loop = 0; loop < frames_to_send; loop++) {
+		if (unlikely(dpaa2_mbuf_seq_is_flag_set((*bufs),
+			(DPAA2_ENQUEUE_FLAG_ORP | QBMAN_ENQUEUE_FLAG_DCA)))) {
+			order = true;
+			break;
+		}
+
+		if (unlikely((*bufs)->ol_flags & RTE_MBUF_F_TX_VLAN)) {
+			ret = rte_vlan_insert(bufs);
+			if (ret)
+				break;
+		}
+		if (likely(RTE_MBUF_DIRECT(*bufs) &&
+			(*bufs)->pool->ops_index == hw_mp->ops_index &&
+			(*bufs)->nb_segs == 1 && rte_mbuf_refcnt_read(*bufs) == 1)) {
+			DPAA2_MBUF_TO_CONTIG_FD((*bufs), &fd_arr[loop],
+				mempool_to_bpid((*bufs)->pool));
+			ret = 0;
+		} else if (unlikely((*bufs)->nb_segs > 1)) {
+			ret = dpaa2_dev_tx_mbuf_to_sg_fd(hw_mp, *bufs, &fd_arr[loop],
+				dpaa2_q, DPAA2_TX_NO_CONF, false);
+		} else {
+			ret = dpaa2_dev_tx_mbuf_to_simple_fd(hw_mp, *bufs, &fd_arr[loop],
+				dpaa2_q, DPAA2_TX_NO_CONF, false);
+		}
+		if (ret)
+			break;
+		bufs++;
+	}
+
+	eq_ret = dpaa2_dev_tx_enqueue_multiple(swp, fd_arr, loop, eqdesc);
+	num_tx += eq_ret;
+	nb_pkts -= eq_ret;
+	if (unlikely(nb_pkts > 0 && !ret && !order))
+		goto tx_again;
+
+skip_tx:
+	dpaa2_q->tx_pkts += num_tx;
+	if (unlikely(order && eq_ret >= loop))
+		num_tx += dpaa2_dev_tx_multi_ordered(dpaa2_q, bufs, nb_pkts);
+
+	return num_tx;
+}
+
 /*
  * Callback to handle sending packets through WRIOP based interface
  */
@@ -1519,12 +1627,12 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	/* Function to transmit the frames to given device and VQ*/
 	uint32_t loop, retry_count, i, ptp_n, frames_to_send;
-	int32_t ret, tstamp[MAX_TX_RING_SLOTS], ptp_set, order = false, eq_ret = 0;
-	struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
+	int32_t ret, tstamp[MAX_TX_RING_SLOTS], ptp_set, order = false;
+	struct qbman_fd fd_arr[MAX_TX_RING_SLOTS] = {0};
 	struct qbman_eq_desc eqdesc;
 	struct dpaa2_queue *dpaa2_q = queue;
 	struct qbman_swp *swp;
-	uint16_t num_tx = 0, order_tx;
+	uint16_t num_tx = 0, eq_ret = 0;
 	struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 	struct dpaa2_dev_priv *priv = eth_data->dev_private;
 	struct rte_mempool *hw_mp;
@@ -1549,15 +1657,15 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	}
 	hw_mp = priv->bp_list->mp;
 
-	if (dpaa2_q->tx_conf_queue)
-		dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue, false);
-
 	/*Prepare enqueue descriptor*/
 	qbman_eq_desc_clear(&eqdesc);
 	qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
 	qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid);
+	if (likely(priv->tx_conf_type == DPAA2_TX_NO_CONF))
+		return dpaa2_dev_tx_fast(dpaa2_q, bufs, nb_pkts, swp, fd_arr, &eqdesc, hw_mp);
 
-	/*Clear the unused FD fields before sending*/
+	if (dpaa2_q->tx_conf_queue)
+		dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue, false);
 
 tx_again:
 	/*Check if the queue is congested*/
@@ -1617,27 +1725,13 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		bufs++;
 	}
 
-	frames_to_send = loop;
-	loop = 0;
-	retry_count = 0;
-	while (loop < frames_to_send) {
-		eq_ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
-			&fd_arr[loop], NULL, frames_to_send - loop);
-		if (unlikely(eq_ret < 0)) {
-			retry_count++;
-			if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
-				break;
-		} else {
-			loop += eq_ret;
-			retry_count = 0;
-		}
-	}
+	eq_ret = dpaa2_dev_tx_enqueue_multiple(swp, fd_arr, loop, &eqdesc);
+	num_tx += eq_ret;
+	nb_pkts -= eq_ret;
 
-	num_tx += loop;
-	nb_pkts -= loop;
 	if (priv->tx_conf_type != DPAA2_TX_NO_CONF) {
-		for (i = 0; i < loop; i++) {
-			if (tstamp[i]) {
+		for (i = 0; i < eq_ret; i++) {
+			if (unlikely(tstamp[i])) {
 				rte_atomic_fetch_add_explicit(&dpaa2_q->tx_conf_queue->ts_to_cnfd,
 					1, rte_memory_order_relaxed);
 			}
@@ -1648,16 +1742,8 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 
 skip_tx:
 	dpaa2_q->tx_pkts += num_tx;
-	if (order && eq_ret >= 0) {
-order_tx_again:
-		frames_to_send = (nb_pkts > dpaa2_eqcr_size) ? dpaa2_eqcr_size : nb_pkts;
-		nb_pkts -= frames_to_send;
-		order_tx = dpaa2_dev_tx_ordered(queue, bufs, frames_to_send);
-		num_tx += order_tx;
-		bufs += frames_to_send;
-		if (nb_pkts && order_tx >= frames_to_send)
-			goto order_tx_again;
-	}
+	if (unlikely(order && eq_ret >= loop))
+		num_tx += dpaa2_dev_tx_multi_ordered(dpaa2_q, bufs, nb_pkts);
 
 	return num_tx;
 }
-- 
2.43.0