From: Bruce Richardson <bruce.richardson@intel.com>
To: dev@dpdk.org
Cc: Bruce Richardson <bruce.richardson@intel.com>,
Vladimir Medvedkin <vladimir.medvedkin@intel.com>
Subject: [PATCH 06/13] net/intel: move iavf descriptor writing functions to common
Date: Thu, 3 Sep 2026 18:01:35 +0100 [thread overview]
Message-ID: <20260903170140.360477-7-bruce.richardson@intel.com> (raw)
In-Reply-To: <20260903170140.360477-1-bruce.richardson@intel.com>
In order to promote reuse of code, move the Tx vector functions for
writing descriptors from the iavf driver to the common folder in a new
tx_vec_x86.h file. This will allow reuse in later commits by other
drivers such as ice.
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
drivers/net/intel/common/tx_vec_x86.h | 513 ++++++++++++++++++
drivers/net/intel/iavf/iavf_rxtx_vec_avx2.c | 285 +---------
drivers/net/intel/iavf/iavf_rxtx_vec_avx512.c | 272 +---------
drivers/net/intel/iavf/iavf_rxtx_vec_common.h | 77 +--
4 files changed, 532 insertions(+), 615 deletions(-)
create mode 100644 drivers/net/intel/common/tx_vec_x86.h
diff --git a/drivers/net/intel/common/tx_vec_x86.h b/drivers/net/intel/common/tx_vec_x86.h
new file mode 100644
index 0000000000..b65bc9fc78
--- /dev/null
+++ b/drivers/net/intel/common/tx_vec_x86.h
@@ -0,0 +1,513 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 Intel Corporation
+ */
+
+#ifndef _COMMON_INTEL_TX_VEC_X86_H_
+#define _COMMON_INTEL_TX_VEC_X86_H_
+
+#include <stdint.h>
+
+#include <rte_mbuf.h>
+
+#include "tx.h"
+
+static __rte_always_inline void
+ci_fill_ctx_desc_tunneling(uint64_t *low_ctx_qw, struct rte_mbuf *pkt)
+{
+ if (pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
+ uint64_t eip_typ = CI_TX_CTX_EIPT_NONE;
+ uint64_t eip_len = 0;
+ uint64_t eip_noinc = 0;
+ /* Default - IP_ID is increment in each segment of LSO */
+
+ switch (pkt->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 |
+ RTE_MBUF_F_TX_OUTER_IPV6 |
+ RTE_MBUF_F_TX_OUTER_IP_CKSUM)) {
+ case RTE_MBUF_F_TX_OUTER_IPV4:
+ eip_typ = CI_TX_CTX_EIPT_IPV4_NO_CSUM;
+ eip_len = pkt->outer_l3_len >> 2;
+ break;
+ case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM:
+ eip_typ = CI_TX_CTX_EIPT_IPV4;
+ eip_len = pkt->outer_l3_len >> 2;
+ break;
+ case RTE_MBUF_F_TX_OUTER_IPV6:
+ eip_typ = CI_TX_CTX_EIPT_IPV6;
+ eip_len = pkt->outer_l3_len >> 2;
+ break;
+ }
+
+ /* L4TUNT: L4 Tunneling Type */
+ switch (pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
+ case RTE_MBUF_F_TX_TUNNEL_IPIP:
+ /* for non UDP / GRE tunneling, set to 00b */
+ break;
+ case RTE_MBUF_F_TX_TUNNEL_VXLAN:
+ case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
+ case RTE_MBUF_F_TX_TUNNEL_GTP:
+ case RTE_MBUF_F_TX_TUNNEL_GENEVE:
+ eip_typ |= CI_TXD_CTX_UDP_TUNNELING;
+ break;
+ case RTE_MBUF_F_TX_TUNNEL_GRE:
+ eip_typ |= CI_TXD_CTX_GRE_TUNNELING;
+ break;
+ default:
+ PMD_TX_LOG(ERR, "Tunnel type not supported");
+ return;
+ }
+
+ /* L4TUNLEN: L4 Tunneling Length, in Words
+ *
+ * We depend on app to set rte_mbuf.l2_len correctly.
+ * For IP in GRE it should be set to the length of the GRE
+ * header;
+ * For MAC in GRE or MAC in UDP it should be set to the length
+ * of the GRE or UDP headers plus the inner MAC up to including
+ * its last Ethertype.
+ * If MPLS labels exists, it should include them as well.
+ */
+ eip_typ |= (pkt->l2_len >> 1) << CI_TXD_CTX_QW0_NATLEN_S;
+
+ /**
+ * Calculate the tunneling UDP checksum.
+ * Shall be set only if L4TUNT = 01b and EIPT is not zero
+ */
+ if ((eip_typ & (CI_TX_CTX_EIPT_IPV4 |
+ CI_TX_CTX_EIPT_IPV6 |
+ CI_TX_CTX_EIPT_IPV4_NO_CSUM)) &&
+ (eip_typ & CI_TXD_CTX_UDP_TUNNELING) &&
+ (pkt->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM))
+ eip_typ |= CI_TXD_CTX_QW0_L4T_CS_M;
+
+ *low_ctx_qw = eip_typ << CI_TXD_CTX_QW0_EIPT_S |
+ eip_len << CI_TXD_CTX_QW0_EIPLEN_S |
+ eip_noinc << CI_TXD_CTX_QW0_EIP_NOINC_S;
+
+ } else {
+ *low_ctx_qw = 0;
+ }
+}
+
+static __rte_always_inline void
+ci_tx_vec_offload(struct rte_mbuf *tx_pkt, uint64_t *txd_hi,
+ enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
+{
+ uint64_t ol_flags = tx_pkt->ol_flags;
+ uint32_t td_cmd = 0;
+ uint32_t td_offset = 0;
+
+ /* Set MACLEN */
+ if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
+ td_offset |= (tx_pkt->outer_l2_len >> 1) << CI_TX_DESC_LEN_MACLEN_S;
+ else
+ td_offset |= (tx_pkt->l2_len >> 1) << CI_TX_DESC_LEN_MACLEN_S;
+
+ /* Enable L3 checksum offloads */
+ if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
+ if (ol_flags & RTE_MBUF_F_TX_IPV4) {
+ td_cmd |= CI_TX_DESC_CMD_IIPT_IPV4_CSUM;
+ td_offset |= (tx_pkt->l3_len >> 2) << CI_TX_DESC_LEN_IPLEN_S;
+ }
+ } else if (ol_flags & RTE_MBUF_F_TX_IPV4) {
+ td_cmd |= CI_TX_DESC_CMD_IIPT_IPV4;
+ td_offset |= (tx_pkt->l3_len >> 2) << CI_TX_DESC_LEN_IPLEN_S;
+ } else if (ol_flags & RTE_MBUF_F_TX_IPV6) {
+ td_cmd |= CI_TX_DESC_CMD_IIPT_IPV6;
+ td_offset |= (tx_pkt->l3_len >> 2) << CI_TX_DESC_LEN_IPLEN_S;
+ }
+
+ /* Enable L4 checksum offloads */
+ switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
+ case RTE_MBUF_F_TX_TCP_CKSUM:
+ td_cmd |= CI_TX_DESC_CMD_L4T_EOFT_TCP;
+ td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << CI_TX_DESC_LEN_L4_LEN_S;
+ break;
+ case RTE_MBUF_F_TX_SCTP_CKSUM:
+ td_cmd |= CI_TX_DESC_CMD_L4T_EOFT_SCTP;
+ td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << CI_TX_DESC_LEN_L4_LEN_S;
+ break;
+ case RTE_MBUF_F_TX_UDP_CKSUM:
+ td_cmd |= CI_TX_DESC_CMD_L4T_EOFT_UDP;
+ td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << CI_TX_DESC_LEN_L4_LEN_S;
+ break;
+ default:
+ break;
+ }
+
+ *txd_hi |= ((uint64_t)td_offset) << CI_TXD_QW1_OFFSET_S;
+
+ if (ol_flags & RTE_MBUF_F_TX_QINQ) {
+ td_cmd |= CI_TX_DESC_CMD_IL2TAG1;
+ /* L2Tag1 always carries a tag for QinQ: the outer tag if that's
+ * where it is placed, otherwise the inner.
+ */
+ if (qinq_outer_pos == CI_TAG_IN_DATA_DESC)
+ *txd_hi |= ((uint64_t)tx_pkt->vlan_tci_outer << CI_TXD_QW1_L2TAG1_S);
+ else
+ *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << CI_TXD_QW1_L2TAG1_S);
+ } else if (ol_flags & RTE_MBUF_F_TX_VLAN && single_vlan_pos == CI_TAG_IN_DATA_DESC) {
+ td_cmd |= CI_TX_DESC_CMD_IL2TAG1;
+ *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << CI_TXD_QW1_L2TAG1_S);
+ }
+
+ *txd_hi |= ((uint64_t)td_cmd) << CI_TXD_QW1_CMD_S;
+}
+
+static __rte_always_inline void
+ci_vtx1(volatile struct ci_tx_desc *txdp,
+ struct rte_mbuf *pkt, uint64_t flags, bool offload,
+ enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
+{
+ uint64_t high_qw = (CI_TX_DESC_DTYPE_DATA |
+ ((uint64_t)flags << CI_TXD_QW1_CMD_S) |
+ ((uint64_t)pkt->data_len << CI_TXD_QW1_TX_BUF_SZ_S));
+ if (offload)
+ ci_tx_vec_offload(pkt, &high_qw, single_vlan_pos, qinq_outer_pos);
+
+ __m128i descriptor = _mm_set_epi64x(high_qw, pkt->buf_iova + pkt->data_off);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, txdp), descriptor);
+}
+
+#ifdef __AVX2__
+
+static __rte_always_inline void
+ci_vtx_avx2(volatile struct ci_tx_desc *txdp,
+ struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags, bool offload,
+ enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
+{
+ const uint64_t hi_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
+
+ /* if unaligned on 32-bit boundary, do one to align */
+ if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) {
+ ci_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
+ nb_pkts--; txdp++; pkt++;
+ }
+
+ /* do two at a time while possible, in bursts */
+ for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) {
+ uint64_t hi_qw3 = hi_qw_tmpl |
+ ((uint64_t)pkt[3]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ if (offload)
+ ci_tx_vec_offload(pkt[3], &hi_qw3, single_vlan_pos, qinq_outer_pos);
+ uint64_t hi_qw2 = hi_qw_tmpl |
+ ((uint64_t)pkt[2]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ if (offload)
+ ci_tx_vec_offload(pkt[2], &hi_qw2, single_vlan_pos, qinq_outer_pos);
+ uint64_t hi_qw1 = hi_qw_tmpl |
+ ((uint64_t)pkt[1]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ if (offload)
+ ci_tx_vec_offload(pkt[1], &hi_qw1, single_vlan_pos, qinq_outer_pos);
+ uint64_t hi_qw0 = hi_qw_tmpl |
+ ((uint64_t)pkt[0]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ if (offload)
+ ci_tx_vec_offload(pkt[0], &hi_qw0, single_vlan_pos, qinq_outer_pos);
+
+ __m256i desc2_3 =
+ _mm256_set_epi64x
+ (hi_qw3,
+ pkt[3]->buf_iova + pkt[3]->data_off,
+ hi_qw2,
+ pkt[2]->buf_iova + pkt[2]->data_off);
+ __m256i desc0_1 =
+ _mm256_set_epi64x
+ (hi_qw1,
+ pkt[1]->buf_iova + pkt[1]->data_off,
+ hi_qw0,
+ pkt[0]->buf_iova + pkt[0]->data_off);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp + 2), desc2_3);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), desc0_1);
+ }
+
+ /* do any last ones */
+ while (nb_pkts) {
+ ci_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
+ txdp++; pkt++; nb_pkts--;
+ }
+}
+
+static __rte_always_inline void
+ci_vtx1_ctx_avx2(volatile struct ci_tx_desc *txdp, struct rte_mbuf *pkt,
+ uint64_t flags, bool offload, enum ci_l2tag_pos single_vlan_pos,
+ enum ci_l2tag_pos qinq_outer_pos, bool ptype_lldp_enabled)
+{
+ uint64_t high_ctx_qw = CI_TX_DESC_DTYPE_CTX;
+ uint64_t low_ctx_qw = 0;
+
+ if (offload) {
+ ci_fill_ctx_desc_tunneling(&low_ctx_qw, pkt);
+ if (pkt->ol_flags & RTE_MBUF_F_TX_QINQ) {
+ uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
+ (uint64_t)pkt->vlan_tci_outer :
+ (uint64_t)pkt->vlan_tci;
+ high_ctx_qw |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw |= qinq_tag << CI_TXD_CTX_QW0_L2TAG2_S;
+ } else if ((pkt->ol_flags & RTE_MBUF_F_TX_VLAN) &&
+ single_vlan_pos == CI_TAG_IN_CTX_DESC) {
+ high_ctx_qw |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw |= (uint64_t)pkt->vlan_tci << CI_TXD_CTX_QW0_L2TAG2_S;
+ }
+ }
+ if (IAVF_CHECK_TX_LLDP(pkt, ptype_lldp_enabled))
+ high_ctx_qw |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
+ uint64_t high_data_qw = (CI_TX_DESC_DTYPE_DATA |
+ ((uint64_t)flags << CI_TXD_QW1_CMD_S) |
+ ((uint64_t)pkt->data_len << CI_TXD_QW1_TX_BUF_SZ_S));
+ if (offload)
+ ci_tx_vec_offload(pkt, &high_data_qw, single_vlan_pos, qinq_outer_pos);
+
+ __m256i ctx_data_desc = _mm256_set_epi64x(high_data_qw, pkt->buf_iova + pkt->data_off,
+ high_ctx_qw, low_ctx_qw);
+
+ /* tx_id is always even in ctx mode, so txdp is always 32-byte aligned */
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), ctx_data_desc);
+}
+
+static __rte_always_inline void
+ci_vtx_ctx_avx2(volatile struct ci_tx_desc *txdp,
+ struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags,
+ bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos,
+ bool ptype_lldp_enabled)
+{
+ uint64_t hi_data_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
+
+ for (; nb_pkts > 1; txdp += 4, pkt += 2, nb_pkts -= 2) {
+ uint64_t hi_ctx_qw1 = CI_TX_DESC_DTYPE_CTX;
+ uint64_t hi_ctx_qw0 = CI_TX_DESC_DTYPE_CTX;
+ uint64_t low_ctx_qw1 = 0;
+ uint64_t low_ctx_qw0 = 0;
+ uint64_t hi_data_qw1 = 0;
+ uint64_t hi_data_qw0 = 0;
+
+ hi_data_qw1 = hi_data_qw_tmpl |
+ ((uint64_t)pkt[1]->data_len <<
+ CI_TXD_QW1_TX_BUF_SZ_S);
+ hi_data_qw0 = hi_data_qw_tmpl |
+ ((uint64_t)pkt[0]->data_len <<
+ CI_TXD_QW1_TX_BUF_SZ_S);
+
+ if (offload) {
+ /* tunnel fill assigns low_ctx_qw1; must run before QinQ/VLAN OR below */
+ ci_fill_ctx_desc_tunneling(&low_ctx_qw1, pkt[1]);
+ if (pkt[1]->ol_flags & RTE_MBUF_F_TX_QINQ) {
+ uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
+ (uint64_t)pkt[1]->vlan_tci_outer :
+ (uint64_t)pkt[1]->vlan_tci;
+ hi_ctx_qw1 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw1 |= qinq_tag << CI_TXD_CTX_QW0_L2TAG2_S;
+ } else if (pkt[1]->ol_flags & RTE_MBUF_F_TX_VLAN &&
+ single_vlan_pos == CI_TAG_IN_CTX_DESC) {
+ hi_ctx_qw1 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw1 |= (uint64_t)pkt[1]->vlan_tci << CI_TXD_CTX_QW0_L2TAG2_S;
+ }
+ }
+ if (IAVF_CHECK_TX_LLDP(pkt[1], ptype_lldp_enabled))
+ hi_ctx_qw1 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
+
+ if (offload) {
+ /* tunnel fill assigns low_ctx_qw0; must run before QinQ/VLAN OR below */
+ ci_fill_ctx_desc_tunneling(&low_ctx_qw0, pkt[0]);
+ if (pkt[0]->ol_flags & RTE_MBUF_F_TX_QINQ) {
+ uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
+ (uint64_t)pkt[0]->vlan_tci_outer :
+ (uint64_t)pkt[0]->vlan_tci;
+ hi_ctx_qw0 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw0 |= qinq_tag << CI_TXD_CTX_QW0_L2TAG2_S;
+ } else if (pkt[0]->ol_flags & RTE_MBUF_F_TX_VLAN &&
+ single_vlan_pos == CI_TAG_IN_CTX_DESC) {
+ hi_ctx_qw0 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw0 |= (uint64_t)pkt[0]->vlan_tci << CI_TXD_CTX_QW0_L2TAG2_S;
+ }
+ }
+ if (IAVF_CHECK_TX_LLDP(pkt[0], ptype_lldp_enabled))
+ hi_ctx_qw0 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
+
+ if (offload) {
+ ci_tx_vec_offload(pkt[1], &hi_data_qw1, single_vlan_pos, qinq_outer_pos);
+ ci_tx_vec_offload(pkt[0], &hi_data_qw0, single_vlan_pos, qinq_outer_pos);
+ }
+
+ __m256i desc2_3 = _mm256_set_epi64x
+ (hi_data_qw1, pkt[1]->buf_iova + pkt[1]->data_off,
+ hi_ctx_qw1, low_ctx_qw1);
+ __m256i desc0_1 = _mm256_set_epi64x
+ (hi_data_qw0, pkt[0]->buf_iova + pkt[0]->data_off,
+ hi_ctx_qw0, low_ctx_qw0);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp + 2), desc2_3);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), desc0_1);
+ }
+
+ if (nb_pkts)
+ ci_vtx1_ctx_avx2(txdp, *pkt, flags, offload,
+ single_vlan_pos, qinq_outer_pos, ptype_lldp_enabled);
+}
+
+#endif /* __AVX2__ */
+
+#ifdef __AVX512VL__
+
+static __rte_always_inline void
+ci_vtx_avx512(volatile struct ci_tx_desc *txdp,
+ struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags,
+ bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
+{
+ const uint64_t hi_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
+
+ /* if unaligned on 32-bit boundary, do one to align */
+ if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) {
+ ci_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
+ nb_pkts--; txdp++; pkt++;
+ }
+
+ /* do 4 at a time while possible, in bursts */
+ for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) {
+ uint64_t hi_qw3 = hi_qw_tmpl |
+ ((uint64_t)pkt[3]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ uint64_t hi_qw2 = hi_qw_tmpl |
+ ((uint64_t)pkt[2]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ uint64_t hi_qw1 = hi_qw_tmpl |
+ ((uint64_t)pkt[1]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ uint64_t hi_qw0 = hi_qw_tmpl |
+ ((uint64_t)pkt[0]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ if (offload) {
+ ci_tx_vec_offload(pkt[3], &hi_qw3, single_vlan_pos, qinq_outer_pos);
+ ci_tx_vec_offload(pkt[2], &hi_qw2, single_vlan_pos, qinq_outer_pos);
+ ci_tx_vec_offload(pkt[1], &hi_qw1, single_vlan_pos, qinq_outer_pos);
+ ci_tx_vec_offload(pkt[0], &hi_qw0, single_vlan_pos, qinq_outer_pos);
+ }
+
+ __m512i desc0_3 =
+ _mm512_set_epi64
+ (hi_qw3,
+ pkt[3]->buf_iova + pkt[3]->data_off,
+ hi_qw2,
+ pkt[2]->buf_iova + pkt[2]->data_off,
+ hi_qw1,
+ pkt[1]->buf_iova + pkt[1]->data_off,
+ hi_qw0,
+ pkt[0]->buf_iova + pkt[0]->data_off);
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, txdp), desc0_3);
+ }
+
+ /* do any last ones */
+ while (nb_pkts) {
+ ci_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
+ txdp++; pkt++; nb_pkts--;
+ }
+}
+
+static __rte_always_inline void
+ci_vtx1_ctx_avx512(volatile struct ci_tx_desc *txdp, struct rte_mbuf *pkt,
+ uint64_t flags, bool offload, enum ci_l2tag_pos single_vlan_pos,
+ enum ci_l2tag_pos qinq_outer_pos, bool lldp_enabled)
+{
+ uint64_t high_ctx_qw = CI_TX_DESC_DTYPE_CTX;
+ uint64_t low_ctx_qw = 0;
+
+ if (offload) {
+ ci_fill_ctx_desc_tunneling(&low_ctx_qw, pkt);
+ if (pkt->ol_flags & RTE_MBUF_F_TX_QINQ) {
+ uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
+ (uint64_t)pkt->vlan_tci_outer :
+ (uint64_t)pkt->vlan_tci;
+ high_ctx_qw |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw |= qinq_tag << CI_TXD_CTX_QW0_L2TAG2_S;
+ } else if ((pkt->ol_flags & RTE_MBUF_F_TX_VLAN) &&
+ single_vlan_pos == CI_TAG_IN_CTX_DESC) {
+ high_ctx_qw |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw |= (uint64_t)pkt->vlan_tci << CI_TXD_CTX_QW0_L2TAG2_S;
+ }
+ }
+ if (IAVF_CHECK_TX_LLDP(pkt, lldp_enabled))
+ high_ctx_qw |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
+ uint64_t high_data_qw = (CI_TX_DESC_DTYPE_DATA |
+ ((uint64_t)flags << CI_TXD_QW1_CMD_S) |
+ ((uint64_t)pkt->data_len << CI_TXD_QW1_TX_BUF_SZ_S));
+ if (offload)
+ ci_tx_vec_offload(pkt, &high_data_qw, single_vlan_pos, qinq_outer_pos);
+
+ __m256i ctx_data_desc = _mm256_set_epi64x
+ (high_data_qw, pkt->buf_iova + pkt->data_off,
+ high_ctx_qw, low_ctx_qw);
+
+ /* tx_id is always even in ctx mode, so txdp is always 32-byte aligned */
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), ctx_data_desc);
+}
+
+static __rte_always_inline void
+ci_vtx_ctx_avx512(volatile struct ci_tx_desc *txdp,
+ struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags,
+ bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos,
+ bool lldp_enabled)
+{
+ uint64_t hi_data_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
+
+ for (; nb_pkts > 1; txdp += 4, pkt += 2, nb_pkts -= 2) {
+ uint64_t hi_ctx_qw1 = CI_TX_DESC_DTYPE_CTX;
+ uint64_t hi_ctx_qw0 = CI_TX_DESC_DTYPE_CTX;
+ uint64_t low_ctx_qw1 = 0;
+ uint64_t low_ctx_qw0 = 0;
+ uint64_t hi_data_qw1 = 0;
+ uint64_t hi_data_qw0 = 0;
+
+ hi_data_qw1 = hi_data_qw_tmpl |
+ ((uint64_t)pkt[1]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+ hi_data_qw0 = hi_data_qw_tmpl |
+ ((uint64_t)pkt[0]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
+
+ if (offload) {
+ /* tunnel fill assigns low_ctx_qw1; must run before QinQ/VLAN OR below */
+ ci_fill_ctx_desc_tunneling(&low_ctx_qw1, pkt[1]);
+ if (pkt[1]->ol_flags & RTE_MBUF_F_TX_QINQ) {
+ uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
+ (uint64_t)pkt[1]->vlan_tci_outer :
+ (uint64_t)pkt[1]->vlan_tci;
+ hi_ctx_qw1 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw1 |= qinq_tag << CI_TXD_CTX_QW0_L2TAG2_S;
+ } else if (pkt[1]->ol_flags & RTE_MBUF_F_TX_VLAN &&
+ single_vlan_pos == CI_TAG_IN_CTX_DESC) {
+ hi_ctx_qw1 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw1 |= (uint64_t)pkt[1]->vlan_tci << CI_TXD_CTX_QW0_L2TAG2_S;
+ }
+ }
+ if (IAVF_CHECK_TX_LLDP(pkt[1], lldp_enabled))
+ hi_ctx_qw1 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
+
+ if (offload) {
+ /* tunnel fill assigns low_ctx_qw0; must run before QinQ/VLAN OR below */
+ ci_fill_ctx_desc_tunneling(&low_ctx_qw0, pkt[0]);
+ if (pkt[0]->ol_flags & RTE_MBUF_F_TX_QINQ) {
+ uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
+ (uint64_t)pkt[0]->vlan_tci_outer :
+ (uint64_t)pkt[0]->vlan_tci;
+ hi_ctx_qw0 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw0 |= qinq_tag << CI_TXD_CTX_QW0_L2TAG2_S;
+ } else if (pkt[0]->ol_flags & RTE_MBUF_F_TX_VLAN &&
+ single_vlan_pos == CI_TAG_IN_CTX_DESC) {
+ hi_ctx_qw0 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
+ low_ctx_qw0 |= (uint64_t)pkt[0]->vlan_tci << CI_TXD_CTX_QW0_L2TAG2_S;
+ }
+ }
+ if (IAVF_CHECK_TX_LLDP(pkt[0], lldp_enabled))
+ hi_ctx_qw0 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
+
+ if (offload) {
+ ci_tx_vec_offload(pkt[1], &hi_data_qw1, single_vlan_pos, qinq_outer_pos);
+ ci_tx_vec_offload(pkt[0], &hi_data_qw0, single_vlan_pos, qinq_outer_pos);
+ }
+
+ __m512i desc0_3 = _mm512_set_epi64
+ (hi_data_qw1, pkt[1]->buf_iova + pkt[1]->data_off,
+ hi_ctx_qw1, low_ctx_qw1,
+ hi_data_qw0, pkt[0]->buf_iova + pkt[0]->data_off,
+ hi_ctx_qw0, low_ctx_qw0);
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, txdp), desc0_3);
+ }
+
+ if (nb_pkts)
+ ci_vtx1_ctx_avx512(txdp, *pkt, flags, offload,
+ single_vlan_pos, qinq_outer_pos, lldp_enabled);
+}
+
+#endif /* __AVX512VL__ */
+
+#endif /* _COMMON_INTEL_TX_VEC_X86_H_ */
diff --git a/drivers/net/intel/iavf/iavf_rxtx_vec_avx2.c b/drivers/net/intel/iavf/iavf_rxtx_vec_avx2.c
index f95aabe577..1d0492edd9 100644
--- a/drivers/net/intel/iavf/iavf_rxtx_vec_avx2.c
+++ b/drivers/net/intel/iavf/iavf_rxtx_vec_avx2.c
@@ -1617,77 +1617,6 @@ iavf_recv_scattered_pkts_vec_avx2_flex_rxd_offload(void *rx_queue,
}
-static __rte_always_inline void
-iavf_vtx1(volatile struct ci_tx_desc *txdp,
- struct rte_mbuf *pkt, uint64_t flags, bool offload,
- enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
-{
- uint64_t high_qw = (CI_TX_DESC_DTYPE_DATA |
- ((uint64_t)flags << CI_TXD_QW1_CMD_S) |
- ((uint64_t)pkt->data_len << CI_TXD_QW1_TX_BUF_SZ_S));
- if (offload)
- iavf_txd_enable_offload(pkt, &high_qw, single_vlan_pos, qinq_outer_pos);
-
- __m128i descriptor = _mm_set_epi64x(high_qw,
- pkt->buf_iova + pkt->data_off);
- _mm_store_si128(RTE_CAST_PTR(__m128i *, txdp), descriptor);
-}
-
-static __rte_always_inline void
-iavf_vtx(volatile struct ci_tx_desc *txdp,
- struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags, bool offload,
- enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
-{
- const uint64_t hi_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
-
- /* if unaligned on 32-bit boundary, do one to align */
- if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) {
- iavf_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
- nb_pkts--; txdp++; pkt++;
- }
-
- /* do two at a time while possible, in bursts */
- for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) {
- uint64_t hi_qw3 = hi_qw_tmpl |
- ((uint64_t)pkt[3]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- if (offload)
- iavf_txd_enable_offload(pkt[3], &hi_qw3, single_vlan_pos, qinq_outer_pos);
- uint64_t hi_qw2 = hi_qw_tmpl |
- ((uint64_t)pkt[2]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- if (offload)
- iavf_txd_enable_offload(pkt[2], &hi_qw2, single_vlan_pos, qinq_outer_pos);
- uint64_t hi_qw1 = hi_qw_tmpl |
- ((uint64_t)pkt[1]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- if (offload)
- iavf_txd_enable_offload(pkt[1], &hi_qw1, single_vlan_pos, qinq_outer_pos);
- uint64_t hi_qw0 = hi_qw_tmpl |
- ((uint64_t)pkt[0]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- if (offload)
- iavf_txd_enable_offload(pkt[0], &hi_qw0, single_vlan_pos, qinq_outer_pos);
-
- __m256i desc2_3 =
- _mm256_set_epi64x
- (hi_qw3,
- pkt[3]->buf_iova + pkt[3]->data_off,
- hi_qw2,
- pkt[2]->buf_iova + pkt[2]->data_off);
- __m256i desc0_1 =
- _mm256_set_epi64x
- (hi_qw1,
- pkt[1]->buf_iova + pkt[1]->data_off,
- hi_qw0,
- pkt[0]->buf_iova + pkt[0]->data_off);
- _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp + 2), desc2_3);
- _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), desc0_1);
- }
-
- /* do any last ones */
- while (nb_pkts) {
- iavf_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
- txdp++; pkt++; nb_pkts--;
- }
-}
-
static __rte_always_inline uint16_t
iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts, bool offload)
@@ -1721,11 +1650,11 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
if (nb_commit >= n) {
ci_tx_backlog_entry_vec(txep, tx_pkts, n);
- iavf_vtx(txdp, tx_pkts, n - 1, flags, offload, vlan_pos, vlan_pos);
+ ci_vtx_avx2(txdp, tx_pkts, n - 1, flags, offload, vlan_pos, vlan_pos);
tx_pkts += (n - 1);
txdp += (n - 1);
- iavf_vtx1(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos);
+ ci_vtx1(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos);
nb_commit = (uint16_t)(nb_commit - n);
@@ -1739,7 +1668,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
ci_tx_backlog_entry_vec(txep, tx_pkts, nb_commit);
- iavf_vtx(txdp, tx_pkts, nb_commit, flags, offload, vlan_pos, vlan_pos);
+ ci_vtx_avx2(txdp, tx_pkts, nb_commit, flags, offload, vlan_pos, vlan_pos);
tx_id = (uint16_t)(tx_id + nb_commit);
if (tx_id > txq->tx_next_rs) {
@@ -1756,207 +1685,6 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
return nb_pkts;
}
-static inline void
-iavf_fill_ctx_desc_tunneling_avx2(uint64_t *low_ctx_qw, struct rte_mbuf *pkt)
-{
- if (pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
- uint64_t eip_typ = CI_TX_CTX_EIPT_NONE;
- uint64_t eip_len = 0;
- uint64_t eip_noinc = 0;
- /* Default - IP_ID is increment in each segment of LSO */
-
- switch (pkt->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 |
- RTE_MBUF_F_TX_OUTER_IPV6 |
- RTE_MBUF_F_TX_OUTER_IP_CKSUM)) {
- case RTE_MBUF_F_TX_OUTER_IPV4:
- eip_typ = CI_TX_CTX_EIPT_IPV4_NO_CSUM;
- eip_len = pkt->outer_l3_len >> 2;
- break;
- case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM:
- eip_typ = CI_TX_CTX_EIPT_IPV4;
- eip_len = pkt->outer_l3_len >> 2;
- break;
- case RTE_MBUF_F_TX_OUTER_IPV6:
- eip_typ = CI_TX_CTX_EIPT_IPV6;
- eip_len = pkt->outer_l3_len >> 2;
- break;
- }
-
- /* L4TUNT: L4 Tunneling Type */
- switch (pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
- case RTE_MBUF_F_TX_TUNNEL_IPIP:
- /* for non UDP / GRE tunneling, set to 00b */
- break;
- case RTE_MBUF_F_TX_TUNNEL_VXLAN:
- case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
- case RTE_MBUF_F_TX_TUNNEL_GTP:
- case RTE_MBUF_F_TX_TUNNEL_GENEVE:
- eip_typ |= CI_TXD_CTX_UDP_TUNNELING;
- break;
- case RTE_MBUF_F_TX_TUNNEL_GRE:
- eip_typ |= CI_TXD_CTX_GRE_TUNNELING;
- break;
- default:
- PMD_TX_LOG(ERR, "Tunnel type not supported");
- return;
- }
-
- /* L4TUNLEN: L4 Tunneling Length, in Words
- *
- * We depend on app to set rte_mbuf.l2_len correctly.
- * For IP in GRE it should be set to the length of the GRE
- * header;
- * For MAC in GRE or MAC in UDP it should be set to the length
- * of the GRE or UDP headers plus the inner MAC up to including
- * its last Ethertype.
- * If MPLS labels exists, it should include them as well.
- */
- eip_typ |= (pkt->l2_len >> 1) << CI_TXD_CTX_QW0_NATLEN_S;
-
- /**
- * Calculate the tunneling UDP checksum.
- * Shall be set only if L4TUNT = 01b and EIPT is not zero
- */
- if ((eip_typ & (CI_TX_CTX_EIPT_IPV4 |
- CI_TX_CTX_EIPT_IPV6 |
- CI_TX_CTX_EIPT_IPV4_NO_CSUM)) &&
- (eip_typ & CI_TXD_CTX_UDP_TUNNELING) &&
- (pkt->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM))
- eip_typ |= CI_TXD_CTX_QW0_L4T_CS_M;
-
- *low_ctx_qw = eip_typ << CI_TXD_CTX_QW0_EIPT_S |
- eip_len << CI_TXD_CTX_QW0_EIPLEN_S |
- eip_noinc << CI_TXD_CTX_QW0_EIP_NOINC_S;
-
- } else {
- *low_ctx_qw = 0;
- }
-}
-
-static __rte_always_inline void
-ctx_vtx1(volatile struct ci_tx_desc *txdp, struct rte_mbuf *pkt,
- uint64_t flags, bool offload, enum ci_l2tag_pos single_vlan_pos,
- enum ci_l2tag_pos qinq_outer_pos, bool ptype_lldp_enabled)
-{
- uint64_t high_ctx_qw = IAVF_TX_DESC_DTYPE_CONTEXT;
- uint64_t low_ctx_qw = 0;
-
- if (offload) {
- iavf_fill_ctx_desc_tunneling_avx2(&low_ctx_qw, pkt);
- if (pkt->ol_flags & RTE_MBUF_F_TX_QINQ) {
- uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
- (uint64_t)pkt->vlan_tci_outer :
- (uint64_t)pkt->vlan_tci;
- high_ctx_qw |= IAVF_TX_CTX_DESC_IL2TAG2 << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw |= qinq_tag << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- } else if ((pkt->ol_flags & RTE_MBUF_F_TX_VLAN) &&
- single_vlan_pos == CI_TAG_IN_CTX_DESC) {
- high_ctx_qw |= IAVF_TX_CTX_DESC_IL2TAG2 << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw |= (uint64_t)pkt->vlan_tci << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- }
- }
- if (IAVF_CHECK_TX_LLDP(pkt, ptype_lldp_enabled))
- high_ctx_qw |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- uint64_t high_data_qw = (IAVF_TX_DESC_DTYPE_DATA |
- ((uint64_t)flags << IAVF_TXD_QW1_CMD_SHIFT) |
- ((uint64_t)pkt->data_len << IAVF_TXD_QW1_TX_BUF_SZ_SHIFT));
- if (offload)
- iavf_txd_enable_offload(pkt, &high_data_qw, single_vlan_pos, qinq_outer_pos);
-
- __m256i ctx_data_desc = _mm256_set_epi64x(high_data_qw, pkt->buf_iova + pkt->data_off,
- high_ctx_qw, low_ctx_qw);
-
- /* tx_id is always even in ctx mode, so txdp is always 32-byte aligned */
- _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), ctx_data_desc);
-}
-
-static __rte_always_inline void
-ctx_vtx(volatile struct ci_tx_desc *txdp,
- struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags,
- bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos,
- bool ptype_lldp_enabled)
-{
- uint64_t hi_data_qw_tmpl = (IAVF_TX_DESC_DTYPE_DATA |
- ((uint64_t)flags << IAVF_TXD_QW1_CMD_SHIFT));
-
- for (; nb_pkts > 1; txdp += 4, pkt += 2, nb_pkts -= 2) {
- uint64_t hi_ctx_qw1 = IAVF_TX_DESC_DTYPE_CONTEXT;
- uint64_t hi_ctx_qw0 = IAVF_TX_DESC_DTYPE_CONTEXT;
- uint64_t low_ctx_qw1 = 0;
- uint64_t low_ctx_qw0 = 0;
- uint64_t hi_data_qw1 = 0;
- uint64_t hi_data_qw0 = 0;
-
- hi_data_qw1 = hi_data_qw_tmpl |
- ((uint64_t)pkt[1]->data_len <<
- IAVF_TXD_QW1_TX_BUF_SZ_SHIFT);
- hi_data_qw0 = hi_data_qw_tmpl |
- ((uint64_t)pkt[0]->data_len <<
- IAVF_TXD_QW1_TX_BUF_SZ_SHIFT);
-
- if (offload) {
- /* tunnel fill assigns low_ctx_qw1; must run before QinQ/VLAN OR below */
- iavf_fill_ctx_desc_tunneling_avx2(&low_ctx_qw1, pkt[1]);
- if (pkt[1]->ol_flags & RTE_MBUF_F_TX_QINQ) {
- uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
- (uint64_t)pkt[1]->vlan_tci_outer :
- (uint64_t)pkt[1]->vlan_tci;
- hi_ctx_qw1 |= IAVF_TX_CTX_DESC_IL2TAG2 <<
- IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw1 |= qinq_tag << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- } else if (pkt[1]->ol_flags & RTE_MBUF_F_TX_VLAN &&
- single_vlan_pos == CI_TAG_IN_CTX_DESC) {
- hi_ctx_qw1 |=
- IAVF_TX_CTX_DESC_IL2TAG2 << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw1 |=
- (uint64_t)pkt[1]->vlan_tci << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- }
- }
- if (IAVF_CHECK_TX_LLDP(pkt[1], ptype_lldp_enabled))
- hi_ctx_qw1 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << IAVF_TXD_CTX_QW1_CMD_SHIFT;
-
- if (offload) {
- /* tunnel fill assigns low_ctx_qw0; must run before QinQ/VLAN OR below */
- iavf_fill_ctx_desc_tunneling_avx2(&low_ctx_qw0, pkt[0]);
- if (pkt[0]->ol_flags & RTE_MBUF_F_TX_QINQ) {
- uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
- (uint64_t)pkt[0]->vlan_tci_outer :
- (uint64_t)pkt[0]->vlan_tci;
- hi_ctx_qw0 |= IAVF_TX_CTX_DESC_IL2TAG2 <<
- IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw0 |= qinq_tag << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- } else if (pkt[0]->ol_flags & RTE_MBUF_F_TX_VLAN &&
- single_vlan_pos == CI_TAG_IN_CTX_DESC) {
- hi_ctx_qw0 |=
- IAVF_TX_CTX_DESC_IL2TAG2 << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw0 |=
- (uint64_t)pkt[0]->vlan_tci << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- }
- }
- if (IAVF_CHECK_TX_LLDP(pkt[0], ptype_lldp_enabled))
- hi_ctx_qw0 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << IAVF_TXD_CTX_QW1_CMD_SHIFT;
-
- if (offload) {
- iavf_txd_enable_offload(pkt[1], &hi_data_qw1, single_vlan_pos, qinq_outer_pos);
- iavf_txd_enable_offload(pkt[0], &hi_data_qw0, single_vlan_pos, qinq_outer_pos);
- }
-
- __m256i desc2_3 =
- _mm256_set_epi64x
- (hi_data_qw1, pkt[1]->buf_iova + pkt[1]->data_off,
- hi_ctx_qw1, low_ctx_qw1);
- __m256i desc0_1 =
- _mm256_set_epi64x
- (hi_data_qw0, pkt[0]->buf_iova + pkt[0]->data_off,
- hi_ctx_qw0, low_ctx_qw0);
- _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp + 2), desc2_3);
- _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), desc0_1);
- }
-
- if (nb_pkts)
- ctx_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos, ptype_lldp_enabled);
-}
-
static __rte_always_inline uint16_t
iavf_xmit_fixed_burst_vec_avx2_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts, bool offload)
@@ -1994,10 +1722,11 @@ iavf_xmit_fixed_burst_vec_avx2_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
nb_mbuf = n >> 1;
ci_tx_backlog_entry_vec(txep, tx_pkts, nb_mbuf);
- ctx_vtx(txdp, tx_pkts, nb_mbuf - 1, flags, offload, vlan_pos, vlan_pos, lldp_enabled);
+ ci_vtx_ctx_avx2(txdp, tx_pkts, nb_mbuf - 1, flags, offload,
+ vlan_pos, vlan_pos, lldp_enabled);
tx_pkts += (nb_mbuf - 1);
txdp += (n - 2);
- ctx_vtx1(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos, lldp_enabled);
+ ci_vtx1_ctx_avx2(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos, lldp_enabled);
nb_commit = (uint16_t)(nb_commit - n);
@@ -2011,7 +1740,7 @@ iavf_xmit_fixed_burst_vec_avx2_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
nb_mbuf = nb_commit >> 1;
ci_tx_backlog_entry_vec(txep, tx_pkts, nb_mbuf);
- ctx_vtx(txdp, tx_pkts, nb_mbuf, flags, offload, vlan_pos, vlan_pos, lldp_enabled);
+ ci_vtx_ctx_avx2(txdp, tx_pkts, nb_mbuf, flags, offload, vlan_pos, vlan_pos, lldp_enabled);
tx_id = (uint16_t)(tx_id + nb_commit);
if (tx_id > txq->tx_next_rs) {
diff --git a/drivers/net/intel/iavf/iavf_rxtx_vec_avx512.c b/drivers/net/intel/iavf/iavf_rxtx_vec_avx512.c
index 74d91fff31..0c738e4882 100644
--- a/drivers/net/intel/iavf/iavf_rxtx_vec_avx512.c
+++ b/drivers/net/intel/iavf/iavf_rxtx_vec_avx512.c
@@ -1827,266 +1827,8 @@ tx_backlog_entry_avx512(struct ci_tx_entry_vec *txep,
txep[i].mbuf = tx_pkts[i];
}
-static __rte_always_inline void
-iavf_vtx1(volatile struct ci_tx_desc *txdp,
- struct rte_mbuf *pkt, uint64_t flags,
- bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
-{
- uint64_t high_qw = (CI_TX_DESC_DTYPE_DATA |
- ((uint64_t)flags << CI_TXD_QW1_CMD_S) |
- ((uint64_t)pkt->data_len << CI_TXD_QW1_TX_BUF_SZ_S));
- if (offload)
- iavf_txd_enable_offload(pkt, &high_qw, single_vlan_pos, qinq_outer_pos);
-
- __m128i descriptor = _mm_set_epi64x(high_qw,
- pkt->buf_iova + pkt->data_off);
- _mm_storeu_si128(RTE_CAST_PTR(__m128i *, txdp), descriptor);
-}
-
#define IAVF_TX_LEN_MASK 0xAA
#define IAVF_TX_OFF_MASK 0x55
-static __rte_always_inline void
-iavf_vtx(volatile struct ci_tx_desc *txdp,
- struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags,
- bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos)
-{
- const uint64_t hi_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
-
- /* if unaligned on 32-bit boundary, do one to align */
- if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) {
- iavf_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
- nb_pkts--; txdp++; pkt++;
- }
-
- /* do 4 at a time while possible, in bursts */
- for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) {
- uint64_t hi_qw3 = hi_qw_tmpl |
- ((uint64_t)pkt[3]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- uint64_t hi_qw2 = hi_qw_tmpl |
- ((uint64_t)pkt[2]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- uint64_t hi_qw1 = hi_qw_tmpl |
- ((uint64_t)pkt[1]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- uint64_t hi_qw0 = hi_qw_tmpl |
- ((uint64_t)pkt[0]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- if (offload) {
- iavf_txd_enable_offload(pkt[3], &hi_qw3, single_vlan_pos, qinq_outer_pos);
- iavf_txd_enable_offload(pkt[2], &hi_qw2, single_vlan_pos, qinq_outer_pos);
- iavf_txd_enable_offload(pkt[1], &hi_qw1, single_vlan_pos, qinq_outer_pos);
- iavf_txd_enable_offload(pkt[0], &hi_qw0, single_vlan_pos, qinq_outer_pos);
- }
-
- __m512i desc0_3 =
- _mm512_set_epi64
- (hi_qw3,
- pkt[3]->buf_iova + pkt[3]->data_off,
- hi_qw2,
- pkt[2]->buf_iova + pkt[2]->data_off,
- hi_qw1,
- pkt[1]->buf_iova + pkt[1]->data_off,
- hi_qw0,
- pkt[0]->buf_iova + pkt[0]->data_off);
- _mm512_storeu_si512(RTE_CAST_PTR(void *, txdp), desc0_3);
- }
-
- /* do any last ones */
- while (nb_pkts) {
- iavf_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos);
- txdp++; pkt++; nb_pkts--;
- }
-}
-
-static __rte_always_inline void
-iavf_fill_ctx_desc_tunneling_avx512(uint64_t *low_ctx_qw, struct rte_mbuf *pkt)
-{
- if (pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
- uint64_t eip_typ = CI_TX_CTX_EIPT_NONE;
- uint64_t eip_len = 0;
- uint64_t eip_noinc = 0;
- /* Default - IP_ID is increment in each segment of LSO */
-
- switch (pkt->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 |
- RTE_MBUF_F_TX_OUTER_IPV6 |
- RTE_MBUF_F_TX_OUTER_IP_CKSUM)) {
- case RTE_MBUF_F_TX_OUTER_IPV4:
- eip_typ = CI_TX_CTX_EIPT_IPV4_NO_CSUM;
- eip_len = pkt->outer_l3_len >> 2;
- break;
- case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM:
- eip_typ = CI_TX_CTX_EIPT_IPV4;
- eip_len = pkt->outer_l3_len >> 2;
- break;
- case RTE_MBUF_F_TX_OUTER_IPV6:
- eip_typ = CI_TX_CTX_EIPT_IPV6;
- eip_len = pkt->outer_l3_len >> 2;
- break;
- }
-
- /* L4TUNT: L4 Tunneling Type */
- switch (pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
- case RTE_MBUF_F_TX_TUNNEL_IPIP:
- /* for non UDP / GRE tunneling, set to 00b */
- break;
- case RTE_MBUF_F_TX_TUNNEL_VXLAN:
- case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
- case RTE_MBUF_F_TX_TUNNEL_GTP:
- case RTE_MBUF_F_TX_TUNNEL_GENEVE:
- eip_typ |= CI_TXD_CTX_UDP_TUNNELING;
- break;
- case RTE_MBUF_F_TX_TUNNEL_GRE:
- eip_typ |= CI_TXD_CTX_GRE_TUNNELING;
- break;
- default:
- PMD_TX_LOG(ERR, "Tunnel type not supported");
- return;
- }
-
- /* L4TUNLEN: L4 Tunneling Length, in Words
- *
- * We depend on app to set rte_mbuf.l2_len correctly.
- * For IP in GRE it should be set to the length of the GRE
- * header;
- * For MAC in GRE or MAC in UDP it should be set to the length
- * of the GRE or UDP headers plus the inner MAC up to including
- * its last Ethertype.
- * If MPLS labels exists, it should include them as well.
- */
- eip_typ |= (pkt->l2_len >> 1) << CI_TXD_CTX_QW0_NATLEN_S;
-
- /**
- * Calculate the tunneling UDP checksum.
- * Shall be set only if L4TUNT = 01b and EIPT is not zero
- */
- if ((eip_typ & (CI_TX_CTX_EIPT_IPV4 |
- CI_TX_CTX_EIPT_IPV6 |
- CI_TX_CTX_EIPT_IPV4_NO_CSUM)) &&
- (eip_typ & CI_TXD_CTX_UDP_TUNNELING) &&
- (pkt->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM))
- eip_typ |= CI_TXD_CTX_QW0_L4T_CS_M;
-
- *low_ctx_qw = eip_typ << CI_TXD_CTX_QW0_EIPT_S |
- eip_len << CI_TXD_CTX_QW0_EIPLEN_S |
- eip_noinc << CI_TXD_CTX_QW0_EIP_NOINC_S;
-
- } else {
- *low_ctx_qw = 0;
- }
-}
-
-static __rte_always_inline void
-ctx_vtx1(volatile struct ci_tx_desc *txdp, struct rte_mbuf *pkt,
- uint64_t flags, bool offload, enum ci_l2tag_pos single_vlan_pos,
- enum ci_l2tag_pos qinq_outer_pos, bool lldp_enabled)
-{
- uint64_t high_ctx_qw = IAVF_TX_DESC_DTYPE_CONTEXT;
- uint64_t low_ctx_qw = 0;
-
- if (offload) {
- iavf_fill_ctx_desc_tunneling_avx512(&low_ctx_qw, pkt);
- if (pkt->ol_flags & RTE_MBUF_F_TX_QINQ) {
- uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
- (uint64_t)pkt->vlan_tci_outer :
- (uint64_t)pkt->vlan_tci;
- high_ctx_qw |= IAVF_TX_CTX_DESC_IL2TAG2 << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw |= qinq_tag << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- } else if ((pkt->ol_flags & RTE_MBUF_F_TX_VLAN) &&
- single_vlan_pos == CI_TAG_IN_CTX_DESC) {
- high_ctx_qw |= IAVF_TX_CTX_DESC_IL2TAG2 << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- low_ctx_qw |= (uint64_t)pkt->vlan_tci << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- }
- }
- if (IAVF_CHECK_TX_LLDP(pkt, lldp_enabled))
- high_ctx_qw |= IAVF_TX_CTX_DESC_SWTCH_UPLINK
- << IAVF_TXD_CTX_QW1_CMD_SHIFT;
- uint64_t high_data_qw = (CI_TX_DESC_DTYPE_DATA |
- ((uint64_t)flags << CI_TXD_QW1_CMD_S) |
- ((uint64_t)pkt->data_len << CI_TXD_QW1_TX_BUF_SZ_S));
- if (offload)
- iavf_txd_enable_offload(pkt, &high_data_qw, single_vlan_pos, qinq_outer_pos);
-
- __m256i ctx_data_desc = _mm256_set_epi64x(high_data_qw, pkt->buf_iova + pkt->data_off,
- high_ctx_qw, low_ctx_qw);
-
- /* tx_id is always even in ctx mode, so txdp is always 32-byte aligned */
- _mm256_store_si256(RTE_CAST_PTR(__m256i *, txdp), ctx_data_desc);
-}
-
-static __rte_always_inline void
-ctx_vtx(volatile struct ci_tx_desc *txdp,
- struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags,
- bool offload, enum ci_l2tag_pos single_vlan_pos, enum ci_l2tag_pos qinq_outer_pos,
- bool lldp_enabled)
-{
- uint64_t hi_data_qw_tmpl = (CI_TX_DESC_DTYPE_DATA | (flags << CI_TXD_QW1_CMD_S));
-
- for (; nb_pkts > 1; txdp += 4, pkt += 2, nb_pkts -= 2) {
- uint64_t hi_ctx_qw1 = IAVF_TX_DESC_DTYPE_CONTEXT;
- uint64_t hi_ctx_qw0 = IAVF_TX_DESC_DTYPE_CONTEXT;
- uint64_t low_ctx_qw1 = 0;
- uint64_t low_ctx_qw0 = 0;
- uint64_t hi_data_qw1 = 0;
- uint64_t hi_data_qw0 = 0;
-
- hi_data_qw1 = hi_data_qw_tmpl |
- ((uint64_t)pkt[1]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
- hi_data_qw0 = hi_data_qw_tmpl |
- ((uint64_t)pkt[0]->data_len << CI_TXD_QW1_TX_BUF_SZ_S);
-
- if (offload) {
- /* tunnel fill assigns low_ctx_qw1; must run before QinQ/VLAN OR below */
- iavf_fill_ctx_desc_tunneling_avx512(&low_ctx_qw1, pkt[1]);
- if (pkt[1]->ol_flags & RTE_MBUF_F_TX_QINQ) {
- uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
- (uint64_t)pkt[1]->vlan_tci_outer :
- (uint64_t)pkt[1]->vlan_tci;
- hi_ctx_qw1 |= CI_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
- low_ctx_qw1 |= qinq_tag << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- } else if (pkt[1]->ol_flags & RTE_MBUF_F_TX_VLAN &&
- single_vlan_pos == CI_TAG_IN_CTX_DESC) {
- hi_ctx_qw1 |= IAVF_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
- low_ctx_qw1 |=
- (uint64_t)pkt[1]->vlan_tci << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- }
- }
- if (IAVF_CHECK_TX_LLDP(pkt[1], lldp_enabled))
- hi_ctx_qw1 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK
- << CI_TXD_QW1_CMD_S;
-
- if (offload) {
- /* tunnel fill assigns low_ctx_qw0; must run before QinQ/VLAN OR below */
- iavf_fill_ctx_desc_tunneling_avx512(&low_ctx_qw0, pkt[0]);
- if (pkt[0]->ol_flags & RTE_MBUF_F_TX_QINQ) {
- uint64_t qinq_tag = qinq_outer_pos == CI_TAG_IN_CTX_DESC ?
- (uint64_t)pkt[0]->vlan_tci_outer :
- (uint64_t)pkt[0]->vlan_tci;
- hi_ctx_qw0 |= IAVF_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
- low_ctx_qw0 |= qinq_tag << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- } else if (pkt[0]->ol_flags & RTE_MBUF_F_TX_VLAN &&
- single_vlan_pos == CI_TAG_IN_CTX_DESC) {
- hi_ctx_qw0 |= IAVF_TX_CTX_DESC_IL2TAG2 << CI_TXD_QW1_CMD_S;
- low_ctx_qw0 |=
- (uint64_t)pkt[0]->vlan_tci << IAVF_TXD_CTX_QW0_L2TAG2_PARAM;
- }
- }
- if (IAVF_CHECK_TX_LLDP(pkt[0], lldp_enabled))
- hi_ctx_qw0 |= IAVF_TX_CTX_DESC_SWTCH_UPLINK << CI_TXD_QW1_CMD_S;
-
- if (offload) {
- iavf_txd_enable_offload(pkt[1], &hi_data_qw1, single_vlan_pos, qinq_outer_pos);
- iavf_txd_enable_offload(pkt[0], &hi_data_qw0, single_vlan_pos, qinq_outer_pos);
- }
-
- __m512i desc0_3 =
- _mm512_set_epi64
- (hi_data_qw1, pkt[1]->buf_iova + pkt[1]->data_off,
- hi_ctx_qw1, low_ctx_qw1,
- hi_data_qw0, pkt[0]->buf_iova + pkt[0]->data_off,
- hi_ctx_qw0, low_ctx_qw0);
- _mm512_storeu_si512(RTE_CAST_PTR(void *, txdp), desc0_3);
- }
-
- if (nb_pkts)
- ctx_vtx1(txdp, *pkt, flags, offload, single_vlan_pos, qinq_outer_pos, lldp_enabled);
-}
static __rte_always_inline uint16_t
iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
@@ -2122,11 +1864,11 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
if (nb_commit >= n) {
tx_backlog_entry_avx512(txep, tx_pkts, n);
- iavf_vtx(txdp, tx_pkts, n - 1, flags, offload, vlan_pos, vlan_pos);
+ ci_vtx_avx512(txdp, tx_pkts, n - 1, flags, offload, vlan_pos, vlan_pos);
tx_pkts += (n - 1);
txdp += (n - 1);
- iavf_vtx1(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos);
+ ci_vtx1(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos);
nb_commit = (uint16_t)(nb_commit - n);
@@ -2141,7 +1883,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
tx_backlog_entry_avx512(txep, tx_pkts, nb_commit);
- iavf_vtx(txdp, tx_pkts, nb_commit, flags, offload, vlan_pos, vlan_pos);
+ ci_vtx_avx512(txdp, tx_pkts, nb_commit, flags, offload, vlan_pos, vlan_pos);
tx_id = (uint16_t)(tx_id + nb_commit);
if (tx_id > txq->tx_next_rs) {
@@ -2195,10 +1937,12 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
nb_mbuf = n >> 1;
tx_backlog_entry_avx512(txep, tx_pkts, nb_mbuf);
- ctx_vtx(txdp, tx_pkts, nb_mbuf - 1, flags, offload, vlan_pos, vlan_pos, lldp_enabled);
+ ci_vtx_ctx_avx512(txdp, tx_pkts, nb_mbuf - 1, flags, offload,
+ vlan_pos, vlan_pos, lldp_enabled);
tx_pkts += (nb_mbuf - 1);
txdp += (n - 2);
- ctx_vtx1(txdp, *tx_pkts++, rs, offload, vlan_pos, vlan_pos, lldp_enabled);
+ ci_vtx1_ctx_avx512(txdp, *tx_pkts++, rs, offload,
+ vlan_pos, vlan_pos, lldp_enabled);
nb_commit = (uint16_t)(nb_commit - n);
@@ -2212,7 +1956,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
nb_mbuf = nb_commit >> 1;
tx_backlog_entry_avx512(txep, tx_pkts, nb_mbuf);
- ctx_vtx(txdp, tx_pkts, nb_mbuf, flags, offload, vlan_pos, vlan_pos, lldp_enabled);
+ ci_vtx_ctx_avx512(txdp, tx_pkts, nb_mbuf, flags, offload, vlan_pos, vlan_pos, lldp_enabled);
tx_id = (uint16_t)(tx_id + nb_commit);
if (tx_id > txq->tx_next_rs) {
diff --git a/drivers/net/intel/iavf/iavf_rxtx_vec_common.h b/drivers/net/intel/iavf/iavf_rxtx_vec_common.h
index 74446fcf4a..61478036ec 100644
--- a/drivers/net/intel/iavf/iavf_rxtx_vec_common.h
+++ b/drivers/net/intel/iavf/iavf_rxtx_vec_common.h
@@ -11,6 +11,10 @@
#include "iavf.h"
#include "iavf_rxtx.h"
+#ifdef RTE_ARCH_X86
+#include "../common/tx_vec_x86.h"
+#endif
+
static inline int
iavf_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)
{
@@ -119,77 +123,4 @@ iavf_tx_vec_dev_check_default(struct rte_eth_dev *dev)
return ret;
}
-static __rte_always_inline void
-iavf_txd_enable_offload(__rte_unused struct rte_mbuf *tx_pkt,
- uint64_t *txd_hi, enum ci_l2tag_pos single_vlan_pos,
- enum ci_l2tag_pos qinq_outer_pos)
-{
- uint64_t ol_flags = tx_pkt->ol_flags;
- uint32_t td_cmd = 0;
- uint32_t td_offset = 0;
-
- /* Set MACLEN */
- if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
- td_offset |= (tx_pkt->outer_l2_len >> 1)
- << CI_TX_DESC_LEN_MACLEN_S;
- else
- td_offset |= (tx_pkt->l2_len >> 1)
- << CI_TX_DESC_LEN_MACLEN_S;
-
- /* Enable L3 checksum offloads */
- if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
- if (ol_flags & RTE_MBUF_F_TX_IPV4) {
- td_cmd |= CI_TX_DESC_CMD_IIPT_IPV4_CSUM;
- td_offset |= (tx_pkt->l3_len >> 2) <<
- CI_TX_DESC_LEN_IPLEN_S;
- }
- } else if (ol_flags & RTE_MBUF_F_TX_IPV4) {
- td_cmd |= CI_TX_DESC_CMD_IIPT_IPV4;
- td_offset |= (tx_pkt->l3_len >> 2) <<
- CI_TX_DESC_LEN_IPLEN_S;
- } else if (ol_flags & RTE_MBUF_F_TX_IPV6) {
- td_cmd |= CI_TX_DESC_CMD_IIPT_IPV6;
- td_offset |= (tx_pkt->l3_len >> 2) <<
- CI_TX_DESC_LEN_IPLEN_S;
- }
-
- /* Enable L4 checksum offloads */
- switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
- case RTE_MBUF_F_TX_TCP_CKSUM:
- td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
- td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
- IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
- break;
- case RTE_MBUF_F_TX_SCTP_CKSUM:
- td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP;
- td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
- IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
- break;
- case RTE_MBUF_F_TX_UDP_CKSUM:
- td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP;
- td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
- IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
- break;
- default:
- break;
- }
-
- *txd_hi |= ((uint64_t)td_offset) << CI_TXD_QW1_OFFSET_S;
-
- if (ol_flags & RTE_MBUF_F_TX_QINQ) {
- td_cmd |= IAVF_TX_DESC_CMD_IL2TAG1;
- /* L2Tag1 always carries a tag for QinQ: the outer tag if that's
- * where it is placed, otherwise the inner.
- */
- if (qinq_outer_pos == CI_TAG_IN_DATA_DESC)
- *txd_hi |= ((uint64_t)tx_pkt->vlan_tci_outer << CI_TXD_QW1_L2TAG1_S);
- else
- *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << CI_TXD_QW1_L2TAG1_S);
- } else if (ol_flags & RTE_MBUF_F_TX_VLAN && single_vlan_pos == CI_TAG_IN_DATA_DESC) {
- td_cmd |= CI_TX_DESC_CMD_IL2TAG1;
- *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << CI_TXD_QW1_L2TAG1_S);
- }
-
- *txd_hi |= ((uint64_t)td_cmd) << CI_TXD_QW1_CMD_S;
-}
#endif
--
2.53.0
next prev parent reply other threads:[~2026-09-03 17:02 UTC|newest]
Thread overview: 69+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-03 17:01 [PATCH 00/13] Consolidate ice and iavf vector Tx paths Bruce Richardson
2026-09-03 17:01 ` [PATCH 01/13] net/iavf: remove unnecessary alignment calls Bruce Richardson
2026-09-14 14:20 ` Loftus, Ciara
2026-09-03 17:01 ` [PATCH 02/13] net/intel: make Tx context flag common Bruce Richardson
2026-09-14 14:21 ` Loftus, Ciara
2026-09-03 17:01 ` [PATCH 03/13] net/iavf: use separate params for VLAN and QinQ position Bruce Richardson
2026-09-14 14:24 ` Loftus, Ciara
2026-09-03 17:01 ` [PATCH 04/13] net/iavf: deduplicate tunnel field fill functions Bruce Richardson
2026-09-04 7:01 ` David Marchand
2026-09-03 17:01 ` [PATCH 05/13] net/intel: define common macros for tunneling bit-shifts Bruce Richardson
2026-09-14 14:25 ` Loftus, Ciara
2026-09-03 17:01 ` Bruce Richardson [this message]
2026-09-14 14:34 ` [PATCH 06/13] net/intel: move iavf descriptor writing functions to common Loftus, Ciara
2026-09-03 17:01 ` [PATCH 07/13] net/intel: use function callback for lldp Bruce Richardson
2026-09-03 17:01 ` [PATCH 08/13] net/ice: use common descriptor creation functions Bruce Richardson
2026-09-14 14:15 ` Loftus, Ciara
2026-09-14 17:02 ` Bruce Richardson
2026-09-03 17:01 ` [PATCH 09/13] net/intel: move vector Tx paths to common Bruce Richardson
2026-09-14 14:38 ` Loftus, Ciara
2026-09-03 17:03 ` [PATCH 10/13] net/ice: use common AVX Tx functions Bruce Richardson
2026-09-14 14:41 ` Loftus, Ciara
2026-09-03 17:04 ` [PATCH 11/13] net/intel: add common vector Tx fns with context handling Bruce Richardson
2026-09-14 14:57 ` Loftus, Ciara
2026-09-03 17:04 ` [PATCH 12/13] net/intel: improve Tx path selection logic Bruce Richardson
2026-09-14 14:59 ` Loftus, Ciara
2026-09-03 17:04 ` [PATCH 13/13] net/ice: enable context desc offloads for vector Tx Bruce Richardson
2026-09-14 14:18 ` Loftus, Ciara
2026-09-14 16:35 ` Bruce Richardson
2026-09-04 7:10 ` [PATCH 00/13] Consolidate ice and iavf vector Tx paths David Marchand
2026-09-15 13:36 ` [PATCH v2 00/15] consolidate ice and iavf Tx vector paths Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 01/15] net/iavf: remove unnecessary alignment calls Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 02/15] net/intel: make Tx context flag common Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 03/15] net/iavf: use separate params for VLAN and QinQ position Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 04/15] net/iavf: deduplicate tunnel field fill functions Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 05/15] net/intel: define common macros for tunneling bit-shifts Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 06/15] net/intel: move iavf descriptor writing functions to common Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 07/15] net/intel: use function callback for lldp Bruce Richardson
2026-09-16 8:56 ` Loftus, Ciara
2026-09-15 13:36 ` [PATCH v2 08/15] net/intel: allow building without IOVA in mbuf Bruce Richardson
2026-09-16 8:57 ` Loftus, Ciara
2026-09-15 13:36 ` [PATCH v2 09/15] net/ice: use common descriptor creation functions Bruce Richardson
2026-09-16 8:58 ` Loftus, Ciara
2026-09-15 13:36 ` [PATCH v2 10/15] net/intel: move vector Tx paths to common Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 11/15] net/ice: use common AVX Tx functions Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 12/15] net/intel: add common vector Tx fns with context handling Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 13/15] net/intel: improve Tx path selection logic Bruce Richardson
2026-09-15 13:36 ` [PATCH v2 14/15] net/ice: enable context desc offloads for vector Tx Bruce Richardson
2026-09-16 9:03 ` Loftus, Ciara
2026-09-15 13:36 ` [PATCH v2 15/15] net/intel: consolidate Tx vector offload path checks Bruce Richardson
2026-09-16 9:02 ` Loftus, Ciara
2026-09-16 10:36 ` [PATCH v3 00/15] consolidate ice and iavf Tx vector paths Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 01/15] net/iavf: remove unnecessary alignment calls Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 02/15] net/intel: make Tx context flag common Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 03/15] net/iavf: use separate params for VLAN and QinQ position Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 04/15] net/iavf: deduplicate tunnel field fill functions Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 05/15] net/intel: define common macros for tunneling bit-shifts Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 06/15] net/intel: move iavf descriptor writing functions to common Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 07/15] net/intel: use function callback for lldp Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 08/15] net/intel: allow building without IOVA in mbuf Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 09/15] net/ice: use common descriptor creation functions Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 10/15] net/intel: move vector Tx paths to common Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 11/15] net/ice: use common AVX Tx functions Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 12/15] net/intel: add common vector Tx fns with context handling Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 13/15] net/intel: improve Tx path selection logic Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 14/15] net/ice: enable context desc offloads for vector Tx Bruce Richardson
2026-09-16 13:00 ` Loftus, Ciara
2026-09-16 13:06 ` Bruce Richardson
2026-09-16 10:36 ` [PATCH v3 15/15] net/intel: consolidate Tx vector offload path checks Bruce Richardson
2026-09-16 13:09 ` [PATCH v3 00/15] consolidate ice and iavf Tx vector paths Bruce Richardson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260903170140.360477-7-bruce.richardson@intel.com \
--to=bruce.richardson@intel.com \
--cc=dev@dpdk.org \
--cc=vladimir.medvedkin@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.