* [PATCH v10 01/20] net/sxe2: support AVX512 vectorized path for Rx and Tx
From: liujie5 @ 2026-06-06 1:07 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260606010726.2256170-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add AVX512 vector data path for Rx and Tx burst functions.
The decision to use AVX512 is based on:
1. CPU hardware flags (AVX512F, AVX512BW).
2. Compiler support (CC_AVX512_SUPPORT).
3. Max SIMD bitwidth configuration.
Performance shows approximately X% improvement in small packet
forwarding scenarios.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 24 +
drivers/net/sxe2/sxe2_drv_cmd.h | 80 +--
drivers/net/sxe2/sxe2_ethdev.c | 2 +-
drivers/net/sxe2/sxe2_txrx.c | 92 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 18 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 ++++++++++++++++++++++++
7 files changed, 1099 insertions(+), 60 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 6b2eb75b0e..7bd0d8120c 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -15,6 +15,30 @@ includes += include_directories('../../common/sxe2')
if arch_subdir == 'x86'
sources += files('sxe2_txrx_vec_sse.c')
+
+ sxe2_avx512_cpu_support =(
+ cc.get_define('__AVX512F__', args: machine_args) != '' and
+ cc.get_define('__AVX512BW__', args: machine_args) != '')
+
+ sxe2_avx512_cc_support = (
+ not machine_args.contains('-mno-avx512f') and
+ cc.has_argument('-mavx512f') and
+ cc.has_argument('-mavx512bw'))
+
+ if sxe2_avx512_cpu_support == true or sxe2_avx512_cc_support == true
+ cflags += ['-DCC_AVX512_SUPPORT']
+ avx512_args = [cflags, '-mavx512f', '-mavx512bw']
+ if cc.has_argument('-march=skylake-avx512')
+ avx512_args += '-march=skylake-avx512'
+ endif
+ sxe2_avx512_lib = static_library('sxe2_avx512_lib', 'sxe2_txrx_vec_avx512.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev, static_rte_bus_pci],
+ include_directories: includes,
+ c_args: avx512_args)
+ objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
+ endif
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index bba6476c2e..ccc9c20ef4 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -67,20 +67,20 @@ enum sxe2_dev_type {
SXE2_DEV_T_MAX,
};
-struct sxe2_drv_queue_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_caps {
uint16_t queues_cnt;
uint16_t base_idx_in_pf;
-};
+} __rte_packed_end;
-struct sxe2_drv_msix_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_msix_caps {
uint16_t msix_vectors_cnt;
uint16_t base_idx_in_func;
-};
+} __rte_packed_end;
-struct sxe2_drv_rss_hash_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rss_hash_caps {
uint16_t hash_key_size;
uint16_t lut_key_size;
-};
+} __rte_packed_end;
enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_BOTH = 0,
@@ -89,18 +89,18 @@ enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_MAX,
};
-struct sxe2_drv_vsi_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_caps {
uint16_t func_id;
uint16_t dpdk_vsi_id;
uint16_t kernel_vsi_id;
uint16_t vsi_type;
-};
+} __rte_packed_end;
-struct sxe2_drv_representor_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_representor_caps {
uint16_t cnt_repr_vf;
uint8_t rsv[2];
struct sxe2_drv_vsi_caps repr_vf_id[256];
-};
+} __rte_packed_end;
enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_NOTSET = 0,
@@ -111,25 +111,25 @@ enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_UNKNOWN,
};
-struct sxe2_switchdev_mode_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_mode_info {
uint8_t pf_id;
uint8_t is_switchdev;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_switchdev_cpvsi_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_cpvsi_info {
uint16_t cp_vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_txsch_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_txsch_caps {
uint8_t layer_cap;
uint8_t tm_mid_node_num;
uint8_t prio_num;
uint8_t rev;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_caps_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_caps_resp {
struct sxe2_drv_queue_caps queue_caps;
struct sxe2_drv_msix_caps msix_caps;
struct sxe2_drv_rss_hash_caps rss_hash_caps;
@@ -141,24 +141,24 @@ struct sxe2_drv_dev_caps_resp {
uint8_t dev_type;
uint8_t rev;
uint32_t cap_flags;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_info_resp {
uint64_t dsn;
uint16_t vsi_id;
uint8_t rsv[2];
uint8_t mac_addr[SXE2_ETH_ALEN];
uint8_t rsv2[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_fw_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_fw_info_resp {
uint8_t main_version_id;
uint8_t sub_version_id;
uint8_t fix_version_id;
uint8_t build_id;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_ctxt {
uint64_t dma_addr;
uint32_t max_lro_size;
uint32_t split_type_mask;
@@ -170,62 +170,62 @@ struct sxe2_drv_rxq_ctxt {
uint8_t keep_crc_en;
uint8_t split_en;
uint8_t desc_size;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
uint16_t max_frame_size;
uint8_t rsv[2];
struct sxe2_drv_rxq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_ctxt {
uint64_t dma_addr;
uint32_t sched_mode;
uint16_t queue_id;
uint16_t depth;
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
struct sxe2_drv_txq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_q_switch_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_q_switch_req {
uint16_t q_idx;
uint16_t vsi_id;
uint8_t is_enable;
uint8_t sched_mode;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_create_req_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_create_req_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_free_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_free_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 8d66e5d8c5..e0f7002138 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -891,7 +891,7 @@ static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
static int32_t sxe2_parse_eth_devargs(struct rte_device *dev,
struct rte_eth_devargs *eth_da)
{
- int ret = 0;
+ int32_t ret = 0;
if (dev->devargs == NULL)
return 0;
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 8d17535301..aa1c474088 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -157,6 +157,19 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
tx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
+#endif
+ }
+ if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
if (ret != 0)
@@ -172,14 +185,25 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
tx_mode_flags = adapter->q_ctxt.tx_mode_flags;
}
-#ifdef RTE_ARCH_X86
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
- if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
- dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ dev->tx_pkt_prepare = NULL;
+#ifdef RTE_ARCH_X86
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
+ }
+#endif
} else {
- dev->tx_pkt_prepare = NULL;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ }
}
} else {
#endif
@@ -201,8 +225,16 @@ static const struct {
} sxe2_tx_burst_infos[] = {
{ sxe2_tx_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
- { sxe2_tx_pkts_vec_sse, "Vector SSE" },
- { sxe2_tx_pkts_vec_sse_simple, "Vector SSE Simple" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_tx_pkts_vec_avx512,
+ "Vector AVX512" },
+ { sxe2_tx_pkts_vec_avx512_simple,
+ "Vector AVX512 Simple" },
+#endif
+ { sxe2_tx_pkts_vec_sse,
+ "Vector SSE" },
+ { sxe2_tx_pkts_vec_sse_simple,
+ "Vector SSE Simple" },
#endif
};
@@ -288,6 +320,20 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
rx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
+#endif
+ }
+ if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
+ rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
if (ret != 0)
@@ -301,7 +347,16 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
#ifdef RTE_ARCH_X86
if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
- dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
+#endif
+ } else {
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ }
return;
}
#endif
@@ -315,19 +370,30 @@ static const struct {
eth_rx_burst_t rx_burst;
const char *info;
} sxe2_rx_burst_infos[] = {
- { sxe2_rx_pkts_scattered, "Scalar Scattered" },
- { sxe2_rx_pkts_scattered_split, "Scalar Scattered split" },
+ { sxe2_rx_pkts_scattered,
+ "Scalar Scattered" },
+ { sxe2_rx_pkts_scattered_split,
+ "Scalar Scattered split" },
#ifdef RTE_ARCH_X86
- { sxe2_rx_pkts_scattered_vec_sse_offload, "Vector SSE Scattered" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_rx_pkts_scattered_vec_avx512,
+ "Vector AVX512 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx512_offload,
+ "Offload Vector AVX512 Scattered" },
+#endif
+ { sxe2_rx_pkts_scattered_vec_sse_offload,
+ "Vector SSE Scattered" },
#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
- __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode)
+ __rte_unused uint16_t queue_id,
+ struct rte_eth_burst_mode *mode)
{
eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
int32_t ret = -EINVAL;
uint32_t i, size;
+
size = RTE_DIM(sxe2_rx_burst_infos);
for (i = 0; i < size; ++i) {
if (pkt_burst == sxe2_rx_burst_infos[i].rx_burst) {
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index 8df4954d86..cf004f5eb2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -165,16 +165,54 @@ static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
return;
}
i = txq->next_dd - (txq->rs_thresh - 1);
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
+#ifdef CC_AVX512_SUPPORT
+ struct rte_eth_dev *dev;
+ struct sxe2_tx_buffer_vec *buffer_vec;
+
+ dev = &rte_eth_devices[txq->port_id];
+
+ if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
+ dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
+ buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for ( ; i < txq->next_use; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ } else {
+#endif
+ buffer = txq->buffer_ring;
+ buffer = txq->buffer_ring;
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer[i].mbuf);
+ buffer[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
buffer[i].mbuf = NULL;
}
}
- i = 0;
+#ifdef CC_AVX512_SUPPORT
}
+#endif
+
for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 04ff4d96a5..af7c8d12b2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,15 +11,19 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
- SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE)
+ SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
+ SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
- SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE)
+ SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
+ SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -54,6 +58,16 @@ uint16_t sxe2_tx_pkts_vec_sse(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_
uint16_t sxe2_tx_pkts_vec_sse_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_sse_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_ctx_offload(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
new file mode 100644
index 0000000000..2aec8037dd
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -0,0 +1,897 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_TEST
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_queue *txq)
+{
+ struct sxe2_tx_buffer_vec *buffer;
+ struct rte_mbuf *mbuf;
+ struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
+ struct rte_mempool *mp;
+ struct rte_mempool_cache *cache;
+ void **cache_objs;
+ uint32_t copied;
+ uint32_t i;
+ int32_t ret;
+ uint16_t rs_thresh;
+ uint16_t free_num;
+
+ if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
+ (txq->desc_ring[txq->next_dd].wb.dd &
+ rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
+ ret = 0;
+ goto l_end;
+ }
+
+ rs_thresh = txq->rs_thresh;
+
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += txq->next_dd - (rs_thresh - 1);
+
+ if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
+ (rs_thresh & 31) == 0) {
+ mp = buffer[0].mbuf->pool;
+ cache = rte_mempool_default_cache(mp, rte_lcore_id());
+
+ if (cache == NULL || cache->len)
+ goto normal;
+
+ if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer, rs_thresh);
+ goto done;
+ }
+ cache_objs = &cache->objs[cache->len];
+
+ copied = 0;
+ while (copied < rs_thresh) {
+ const __m512i objs0 = _mm512_loadu_si512(&buffer[copied]);
+ const __m512i objs1 = _mm512_loadu_si512(&buffer[copied + 8]);
+ const __m512i objs2 = _mm512_loadu_si512(&buffer[copied + 16]);
+ const __m512i objs3 = _mm512_loadu_si512(&buffer[copied + 24]);
+
+ _mm512_storeu_si512(&cache_objs[copied], objs0);
+ _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
+ _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
+ _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
+ copied += 32;
+ }
+ cache->len += rs_thresh;
+
+ if (cache->len >= cache->flushthresh) {
+ (void)rte_mempool_ops_enqueue_bulk(mp,
+ &cache->objs[cache->size], cache->len - cache->size);
+ cache->len = cache->size;
+ }
+ goto done;
+ }
+
+normal:
+ mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
+
+ if (likely(mbuf)) {
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+
+ if (likely(mbuf)) {
+ if (likely(mbuf->pool == mbuf_free_arr[0]->pool)) {
+ mbuf_free_arr[free_num] = mbuf;
+ free_num++;
+ } else {
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+ }
+ }
+ }
+
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+ } else {
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+ if (mbuf != NULL)
+ rte_mempool_put(mbuf->pool, mbuf);
+ }
+ }
+
+done:
+ txq->desc_free_num += txq->rs_thresh;
+ txq->next_dd += txq->rs_thresh;
+ if (txq->next_dd >= txq->ring_depth)
+ txq->next_dd = txq->rs_thresh - 1;
+ ret = rs_thresh;
+
+l_end:
+ return ret;
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline
+void sxe2_tx_desc_fill_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m512i desc_group;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ while (pkts_num > 3) {
+ desc3_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group =
+ _mm512_set_epi64(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]),
+ desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, desc), desc_group);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx512(desc, *pkts, desc_cmd, with_offloads);
+
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline void
+sxe2_tx_pkts_mbuf_fill_avx512(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ uint16_t i;
+
+ for (i = 0; i < nb_pkts; ++i)
+ buffer[i].mbuf = tx_pkts[i];
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer_vec *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec_avx512(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx512 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += next_use;
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx512(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = txq->desc_ring;
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ }
+
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num =
+ sxe2_tx_pkts_vec_avx512_batch(txq, tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx512(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read),
+ dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ? (rxq->ring_depth - 1) :
+ (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m512i mbufs4_7;
+ __m512i mbufs0_3;
+ __m256i mbufs6_7;
+ __m256i mbufs4_5;
+ __m256i mbufs2_3;
+ __m256i mbufs0_1;
+ uint32_t bit_num = 0;
+ uint16_t done_num = 0;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx512(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) & SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m512i crc_adjust =
+ _mm512_set4_epi32(0, -rxq->crc_len, -rxq->crc_len, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+
+ const __m512i rvp_shuf_mask =
+ _mm512_set4_epi32((7 << 24) | (6 << 16) | (5 << 8) | 4,
+ (3 << 24) | (2 << 16) | (13 << 8) | 12,
+ (0xFFU << 24) | (0xFF << 16) | (13 << 8) | 12,
+ 0xFFFFFFFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 8, 0, 10, 2, 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ const __m512i descs4_7 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs4_5), descs6_7, 1);
+ const __m512i descs0_3 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs0_1), descs2_3, 1);
+
+ if (split_rxe_flags != NULL) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs4_7 = _mm512_shuffle_epi8(descs4_7, rvp_shuf_mask);
+ mbufs0_3 = _mm512_shuffle_epi8(descs0_3, rvp_shuf_mask);
+
+ mbufs4_7 = _mm512_add_epi32(mbufs4_7, crc_adjust);
+ mbufs0_3 = _mm512_add_epi32(mbufs0_3, crc_adjust);
+
+ const __m512i ptype_mask = _mm512_set1_epi64(SXE2_RX_FLEX_DESC_PTYPE_M <<
+ SXE2_RX_FLEX_DESC_PTYPE_S);
+
+ __m512i ptypes4_7 = _mm512_and_si512(descs4_7, ptype_mask);
+ __m512i ptypes0_3 = _mm512_and_si512(descs0_3, ptype_mask);
+
+ const __m256i ptypes6_7 = _mm512_extracti64x4_epi64(ptypes4_7, 1);
+ const __m256i ptypes4_5 = _mm512_extracti64x4_epi64(ptypes4_7, 0);
+ const __m256i ptypes2_3 = _mm512_extracti64x4_epi64(ptypes0_3, 1);
+ const __m256i ptypes0_1 = _mm512_extracti64x4_epi64(ptypes0_3, 0);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 13);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 5);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 13);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 5);
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 13);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 5);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 13);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 5);
+
+ const __m512i ptype_mask4_7 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype7],
+ 0, 0, 0, ptype_tbl[ptype6],
+ 0, 0, 0, ptype_tbl[ptype5],
+ 0, 0, 0, ptype_tbl[ptype4]);
+ const __m512i ptype_mask0_3 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype3],
+ 0, 0, 0, ptype_tbl[ptype2],
+ 0, 0, 0, ptype_tbl[ptype1],
+ 0, 0, 0, ptype_tbl[ptype0]);
+
+ mbufs4_7 = _mm512_or_si512(mbufs4_7, ptype_mask4_7);
+ mbufs0_3 = _mm512_or_si512(mbufs0_3, ptype_mask0_3);
+
+ mbufs6_7 = _mm512_extracti64x4_epi64(mbufs4_7, 1);
+ mbufs4_5 = _mm512_extracti64x4_epi64(mbufs4_7, 0);
+ mbufs2_3 = _mm512_extracti64x4_epi64(mbufs0_3, 1);
+ mbufs0_1 = _mm512_extracti64x4_epi64(mbufs0_3, 0);
+
+ const __m512i staterr_per_mask =
+ _mm512_set_epi32(0x17, 0x1F, 0x07, 0x0F,
+ 0x13, 0x1B, 0x03, 0x0B,
+ 0x16, 0x1E, 0x06, 0x0E,
+ 0x12, 0x1A, 0x02, 0x0A);
+ __m512i qw1_0_7 = _mm512_permutex2var_epi32(descs4_7,
+ staterr_per_mask,
+ descs0_3);
+
+ __m256i staterrs0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 0);
+
+ __m256i stu_len0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 1);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0xC0001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0,
+ 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+
+ const __m256i vlan_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 8), 0x04);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 4), 0x04);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(mbuf_flags, 4), 0x04);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] = _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 =
+ _mm256_and_si256(staterrs0_7, umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+ __m128i umbcast_bits_128 =
+ _mm_packs_epi32(_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256(umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags) {
+ const __m256i eop_rxe_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+ const __m128i eop_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits =
+ _mm_or_si128(not_eop_bits,
+ _mm_srli_epi16(_mm_and_si128(eop_rxe_bits,
+ rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((rxq->processing_idx & 1) == 1 && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ !split_rxe_flags64[0] && !split_rxe_flags64[1] &&
+ !split_rxe_flags64[2] && !split_rxe_flags64[3]) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num = 0;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue, rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM, offload);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue,
+ rx_pkts + done_num, nb_pkts, offload);
+
+end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
+
+#endif
--
2.52.0
^ permalink raw reply related
* [PATCH v10 00/20] net/sxe2: added Linkdata sxe2 ethernet driver
From: liujie5 @ 2026-06-06 1:07 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260604015404.1552953-21-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch set implements core functionality for the SXE2 PMD,
including basic driver framework, data path setup, and advanced
offload features (VLAN, RSS,TM, PTP etc.).
V10:
- Addressed AI comments
Jie Liu (20):
net/sxe2: support AVX512 vectorized path for Rx and Tx
net/sxe2: add AVX2 vector data path for Rx and Tx
drivers: add supported packet types get callback
net/sxe2: support L2 filtering and MAC config
drivers: support RSS feature
net/sxe2: support TM hierarchy and shaping
net/sxe2: support IPsec inline protocol offload
net/sxe2: support statistics and multi-process
drivers: interrupt handling
net/sxe2: add NEON vec Rx/Tx burst functions
drivers: add support for VF representors
net/sxe2: add support for custom UDP tunnel ports
net/sxe2: support firmware version reading
net/sxe2: implement get monitor address
common/sxe2: add shared SFP module definitions
net/sxe2: support SFP module info and EEPROM access
net/sxe2: implement private dump info
net/sxe2: add mbuf validation in Tx debug mode
drivers: add testpmd commands for private features
net/sxe2: update sxe2 feature matrix docs
doc/guides/nics/features/sxe2.ini | 56 +
drivers/common/sxe2/sxe2_common.c | 156 ++
drivers/common/sxe2/sxe2_common.h | 4 +
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++
drivers/common/sxe2/sxe2_ioctl_chnl.c | 178 +-
drivers/common/sxe2/sxe2_ioctl_chnl_func.h | 18 +
drivers/common/sxe2/sxe2_msg.h | 118 ++
drivers/common/sxe2/sxe2_ptype.h | 1793 ++++++++++++++++++
drivers/net/sxe2/meson.build | 56 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 1587 +++++++++++++++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 139 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 521 +++++-
drivers/net/sxe2/sxe2_dump.c | 304 +++
drivers/net/sxe2/sxe2_dump.h | 12 +
drivers/net/sxe2/sxe2_ethdev.c | 1531 +++++++++++++++-
drivers/net/sxe2/sxe2_ethdev.h | 115 +-
drivers/net/sxe2/sxe2_ethdev_repr.c | 610 ++++++
drivers/net/sxe2/sxe2_ethdev_repr.h | 32 +
drivers/net/sxe2/sxe2_filter.c | 895 +++++++++
drivers/net/sxe2/sxe2_filter.h | 100 +
drivers/net/sxe2/sxe2_flow.c | 1391 ++++++++++++++
drivers/net/sxe2/sxe2_flow.h | 30 +
drivers/net/sxe2/sxe2_flow_define.h | 144 ++
drivers/net/sxe2/sxe2_flow_parse_action.c | 1182 ++++++++++++
drivers/net/sxe2/sxe2_flow_parse_action.h | 23 +
drivers/net/sxe2/sxe2_flow_parse_engine.c | 106 ++
drivers/net/sxe2/sxe2_flow_parse_engine.h | 13 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 1935 ++++++++++++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 46 +
drivers/net/sxe2/sxe2_ipsec.c | 1565 ++++++++++++++++
drivers/net/sxe2/sxe2_ipsec.h | 254 +++
drivers/net/sxe2/sxe2_irq.c | 1025 +++++++++++
drivers/net/sxe2/sxe2_irq.h | 25 +
drivers/net/sxe2/sxe2_mac.c | 535 ++++++
drivers/net/sxe2/sxe2_mac.h | 84 +
drivers/net/sxe2/sxe2_mp.c | 414 +++++
drivers/net/sxe2/sxe2_mp.h | 67 +
drivers/net/sxe2/sxe2_queue.c | 17 +-
drivers/net/sxe2/sxe2_rss.c | 584 ++++++
drivers/net/sxe2/sxe2_rss.h | 81 +
drivers/net/sxe2/sxe2_rx.c | 38 +
drivers/net/sxe2/sxe2_rx.h | 2 +
drivers/net/sxe2/sxe2_security.c | 335 ++++
drivers/net/sxe2/sxe2_security.h | 77 +
drivers/net/sxe2/sxe2_stats.c | 591 ++++++
drivers/net/sxe2/sxe2_stats.h | 39 +
drivers/net/sxe2/sxe2_switchdev.c | 332 ++++
drivers/net/sxe2/sxe2_switchdev.h | 33 +
drivers/net/sxe2/sxe2_testpmd.c | 733 ++++++++
drivers/net/sxe2/sxe2_testpmd_lib.c | 969 ++++++++++
drivers/net/sxe2/sxe2_testpmd_lib.h | 142 ++
drivers/net/sxe2/sxe2_tm.c | 1169 ++++++++++++
drivers/net/sxe2/sxe2_tm.h | 78 +
drivers/net/sxe2/sxe2_tx.c | 7 +
drivers/net/sxe2/sxe2_txrx.c | 176 +-
drivers/net/sxe2/sxe2_txrx.h | 4 +
drivers/net/sxe2/sxe2_txrx_check_mbuf.c | 595 ++++++
drivers/net/sxe2/sxe2_txrx_check_mbuf.h | 38 +
drivers/net/sxe2/sxe2_txrx_poll.c | 243 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 38 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 776 ++++++++
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 +++++++++
drivers/net/sxe2/sxe2_txrx_vec_common.h | 1 +
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 721 ++++++++
drivers/net/sxe2/sxe2_vsi.c | 146 ++
drivers/net/sxe2/sxe2_vsi.h | 12 +-
drivers/net/sxe2/sxe2vf_regs.h | 85 +
68 files changed, 26575 insertions(+), 127 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/common/sxe2/sxe2_msg.h
create mode 100644 drivers/common/sxe2/sxe2_ptype.h
create mode 100644 drivers/net/sxe2/sxe2_dump.c
create mode 100644 drivers/net/sxe2/sxe2_dump.h
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.c
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.h
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
create mode 100644 drivers/net/sxe2/sxe2_flow.c
create mode 100644 drivers/net/sxe2/sxe2_flow.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.h
create mode 100644 drivers/net/sxe2/sxe2_ipsec.c
create mode 100644 drivers/net/sxe2/sxe2_ipsec.h
create mode 100644 drivers/net/sxe2/sxe2_irq.c
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
create mode 100644 drivers/net/sxe2/sxe2_mp.c
create mode 100644 drivers/net/sxe2/sxe2_mp.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
create mode 100644 drivers/net/sxe2/sxe2_security.c
create mode 100644 drivers/net/sxe2/sxe2_security.h
create mode 100644 drivers/net/sxe2/sxe2_stats.c
create mode 100644 drivers/net/sxe2/sxe2_stats.h
create mode 100644 drivers/net/sxe2/sxe2_switchdev.c
create mode 100644 drivers/net/sxe2/sxe2_switchdev.h
create mode 100644 drivers/net/sxe2/sxe2_testpmd.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.h
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
create mode 100644 drivers/net/sxe2/sxe2vf_regs.h
--
2.52.0
^ permalink raw reply
* [PATCH v9 10/10] dts: add selective Rx tests
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Luca Vizzarro, Patrick Robb
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
Add TestSuite_rx_split with 7 test cases:
- 3 positive: headers only, payload only, two non-contiguous segments
- 4 negative: missing offload flag, out-of-range, overlap, all-discard
Add selective Rx capability detection via testpmd "show port info".
The test suite could be completed later for the basic buffer split
configuration based on offsets or protocols.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/api/capabilities.py | 2 +
dts/api/testpmd/__init__.py | 17 ++
dts/api/testpmd/types.py | 6 +
dts/framework/testbed_model/capability.py | 2 +
dts/tests/TestSuite_rx_split.py | 277 ++++++++++++++++++++++
5 files changed, 304 insertions(+)
create mode 100644 dts/tests/TestSuite_rx_split.py
diff --git a/dts/api/capabilities.py b/dts/api/capabilities.py
index 09bc538523..b0c1d81d36 100644
--- a/dts/api/capabilities.py
+++ b/dts/api/capabilities.py
@@ -136,6 +136,8 @@ class NicCapability(IntEnum):
#: Device supports all VLAN capabilities.
PORT_RX_OFFLOAD_VLAN = auto()
QUEUE_RX_OFFLOAD_VLAN = auto()
+ #: Device supports selective Rx.
+ SELECTIVE_RX = auto()
#: Device supports Rx queue setup after device started.
RUNTIME_RX_QUEUE_SETUP = auto()
#: Device supports Tx queue setup after device started.
diff --git a/dts/api/testpmd/__init__.py b/dts/api/testpmd/__init__.py
index e9187440bb..6973a64573 100644
--- a/dts/api/testpmd/__init__.py
+++ b/dts/api/testpmd/__init__.py
@@ -1409,6 +1409,23 @@ def get_capabilities_show_port_info(
self.ports[0].device_capabilities,
)
+ def get_capabilities_selective_rx(
+ self,
+ supported_capabilities: MutableSet["NicCapability"],
+ unsupported_capabilities: MutableSet["NicCapability"],
+ ) -> None:
+ """Get selective Rx capability from show port info.
+
+ Args:
+ supported_capabilities: Supported capabilities will be added to this set.
+ unsupported_capabilities: Unsupported capabilities will be added to this set.
+ """
+ port_info = self.show_port_info(self.ports[0].id)
+ if port_info.selective_rx:
+ supported_capabilities.add(NicCapability.SELECTIVE_RX)
+ else:
+ unsupported_capabilities.add(NicCapability.SELECTIVE_RX)
+
def get_capabilities_mcast_filtering(
self,
supported_capabilities: MutableSet["NicCapability"],
diff --git a/dts/api/testpmd/types.py b/dts/api/testpmd/types.py
index 0d322aece2..6f1eaf47cc 100644
--- a/dts/api/testpmd/types.py
+++ b/dts/api/testpmd/types.py
@@ -614,6 +614,12 @@ def _validate(info: str) -> str | None:
metadata=VLANOffloadFlag.make_parser(),
)
+ #: Selective Rx support
+ selective_rx: bool = field(
+ default=False,
+ metadata=TextParser.find(r"Selective Rx: supported"),
+ )
+
#: Maximum size of RX buffer
max_rx_bufsize: int | None = field(
default=None, metadata=TextParser.find_int(r"Maximum size of RX buffer: (\d+)")
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 96e1cd449f..b10799ea4b 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -324,6 +324,8 @@ def mapping(cap: NicCapability) -> TestPmdNicCapability:
| NicCapability.FLOW_SHARED_OBJECT_KEEP
):
return (TestPmd.get_capabilities_show_port_info, None)
+ case NicCapability.SELECTIVE_RX:
+ return (TestPmd.get_capabilities_selective_rx, None)
case NicCapability.MCAST_FILTERING:
return (TestPmd.get_capabilities_mcast_filtering, None)
case NicCapability.FLOW_CTRL:
diff --git a/dts/tests/TestSuite_rx_split.py b/dts/tests/TestSuite_rx_split.py
new file mode 100644
index 0000000000..0c7913bbd8
--- /dev/null
+++ b/dts/tests/TestSuite_rx_split.py
@@ -0,0 +1,277 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 NVIDIA Corporation & Affiliates
+
+"""Rx split test suite.
+
+Test configuring a packet split on Rx,
+and discarding some segments (selective Rx) at NIC level.
+"""
+
+from collections.abc import Callable
+from typing import Any
+
+from scapy.layers.inet import IP
+from scapy.layers.l2 import Ether
+from scapy.packet import Packet, Raw
+
+from api.capabilities import (
+ NicCapability,
+ requires_nic_capability,
+)
+from api.packet import adjust_addresses, send_packet_and_capture
+from api.test import fail, verify
+from api.testpmd import TestPmd
+from api.testpmd.config import SimpleForwardingModes
+from api.testpmd.types import RxOffloadCapability, TxOffloadCapability
+from framework.exception import InteractiveCommandExecutionError
+from framework.test_suite import TestSuite, func_test
+
+PAYLOAD = bytes(range(256))
+ETHER_HDR_LEN = len(Ether())
+IP_HDR_LEN = len(IP())
+ETHER_IP_HDR_LEN = ETHER_HDR_LEN + IP_HDR_LEN
+ETHER_MIN_FRAME_LEN = 60
+
+
+@requires_nic_capability(NicCapability.PORT_RX_OFFLOAD_BUFFER_SPLIT)
+@requires_nic_capability(NicCapability.SELECTIVE_RX)
+class TestRxSplit(TestSuite):
+ """Rx split test suite.
+
+ Configure testpmd with various Rx segment offset/length combinations
+ and verify that only the requested portions of the packet are received
+ and forwarded.
+ """
+
+ def _create_testpmd(self, **kwargs: Any) -> TestPmd:
+ """Create a TestPmd instance with defaults overridden by kwargs."""
+ defaults: dict[str, Any] = {
+ "forward_mode": SimpleForwardingModes.io,
+ "rx_offloads": RxOffloadCapability.BUFFER_SPLIT | RxOffloadCapability.SCATTER,
+ }
+ return TestPmd(**{**defaults, **kwargs})
+
+ def _build_packet(self) -> Packet:
+ """Build a test packet with an incrementing byte pattern payload."""
+ packet = Ether() / IP() / Raw(load=PAYLOAD)
+ return adjust_addresses([packet])[0]
+
+ def _start_and_verify(self, testpmd: TestPmd, expected: Callable[[bytes], bytes]) -> None:
+ """Start testpmd, send the default packet, and verify received bytes."""
+ testpmd.start()
+ packet = self._build_packet()
+ self._send_and_verify(testpmd, packet, expected(bytes(packet)))
+
+ def _send_and_verify(self, testpmd: TestPmd, tg_packet: Packet, expected: bytes) -> None:
+ """Clear stats, send a packet, and verify received content and stats.
+
+ Args:
+ testpmd: The running testpmd instance.
+ tg_packet: The packet to send by Scapy on the TG.
+ expected: Expected raw bytes received by testpmd on the SUT.
+ """
+ testpmd.clear_port_stats_all(verify=False)
+
+ # TG send, SUT receive and forward back, then TG capture
+ sut_len = len(expected)
+ capture_len = max(sut_len, ETHER_MIN_FRAME_LEN)
+ received = send_packet_and_capture(tg_packet)
+ verify(
+ len(received) > 0,
+ "Did not receive any packets.",
+ )
+
+ recv_bytes = bytes(received[0])
+ verify(
+ len(recv_bytes) == capture_len,
+ f"Expected packet length {capture_len}, got {len(recv_bytes)}.",
+ )
+ verify(
+ recv_bytes[:sut_len] == expected,
+ "Received packet content does not match expected bytes.",
+ )
+
+ all_stats, _ = testpmd.show_port_stats_all()
+ total_rx_packets = sum(s.rx_packets for s in all_stats)
+ total_rx_bytes = sum(s.rx_bytes for s in all_stats)
+ verify(
+ total_rx_packets == 1,
+ f"Expected 1 Rx packet, got {total_rx_packets}.",
+ )
+ verify(
+ total_rx_bytes == sut_len,
+ f"Expected {sut_len} Rx bytes, got {total_rx_bytes}.",
+ )
+
+ def _verify_port_start_fails(self, testpmd: TestPmd, message: str) -> None:
+ """Verify that starting ports fails, then drain testpmd output."""
+ try:
+ testpmd.start_all_ports()
+ fail(message)
+ except InteractiveCommandExecutionError:
+ testpmd.stop(verify=False)
+
+ @func_test
+ def selective_rx_headers(self) -> None:
+ """Keep only the Ethernet + IP headers, discard the rest.
+
+ Steps:
+ Start testpmd with rxpkts, mbuf-size and buffer split enabled.
+ Configure the payload discard segment with length 0 (rest).
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet has Ether + IP headers only.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_length=[ETHER_IP_HDR_LEN, 0],
+ mbuf_size=[256, 0],
+ ) as testpmd:
+
+ def expected(packet: bytes) -> bytes:
+ return packet[:ETHER_IP_HDR_LEN]
+
+ self._start_and_verify(testpmd, expected)
+
+ @func_test
+ def selective_rx_headers_discard_length(self) -> None:
+ """Keep only the Ethernet + IP headers, discard the remaining length.
+
+ Steps:
+ Start testpmd with rxpkts, mbuf-size and buffer split enabled.
+ Configure the payload discard segment with an explicit length.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet has Ether + IP headers only.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_length=[ETHER_IP_HDR_LEN, len(PAYLOAD)],
+ mbuf_size=[256, 0],
+ ) as testpmd:
+
+ def expected(packet: bytes) -> bytes:
+ return packet[:ETHER_IP_HDR_LEN]
+
+ self._start_and_verify(testpmd, expected)
+
+ @func_test
+ def selective_rx_payload_only(self) -> None:
+ """Skip the Ethernet + IP headers, keep only the payload.
+
+ Steps:
+ Start testpmd with rxpkts, mbuf-size and buffer split enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet is matching the original payload.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_length=[ETHER_IP_HDR_LEN, len(PAYLOAD)],
+ mbuf_size=[0, 512],
+ ) as testpmd:
+
+ def expected(_: bytes) -> bytes:
+ return PAYLOAD
+
+ self._start_and_verify(testpmd, expected)
+
+ @func_test
+ def selective_rx_two_segments(self) -> None:
+ """Keep the IP header and the middle of the payload, skip the rest.
+
+ Steps:
+ Start testpmd with rxpkts, mbuf-size, buffer split
+ and multi-segment Tx enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet is matching the IP header and middle of payload.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ payload_offset = 100
+ payload_length = 100
+ with self._create_testpmd(
+ tx_offloads=TxOffloadCapability.MULTI_SEGS,
+ rx_segments_length=[ETHER_HDR_LEN, IP_HDR_LEN, payload_offset, payload_length, 0],
+ mbuf_size=[0, 256, 0, 256, 0],
+ ) as testpmd:
+
+ def expected(packet: bytes) -> bytes:
+ payload_start = ETHER_IP_HDR_LEN + payload_offset
+ return (
+ packet[ETHER_HDR_LEN:ETHER_IP_HDR_LEN]
+ + packet[payload_start : payload_start + payload_length]
+ )
+
+ self._start_and_verify(testpmd, expected)
+
+ @func_test
+ def selective_rx_no_offload(self) -> None:
+ """Configure selective Rx with buffer split disabled.
+
+ Steps:
+ Start testpmd with rxpkts, mbuf-size including a discard segment,
+ buffer split disabled, and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_offloads=None,
+ rx_segments_length=[ETHER_IP_HDR_LEN, 0],
+ mbuf_size=[256, 0],
+ disable_device_start=True,
+ ) as testpmd:
+ self._verify_port_start_fails(
+ testpmd,
+ "Expected configuration to fail with buffer split disabled.",
+ )
+
+ @func_test
+ def selective_rx_segment_exceeds_mbuf(self) -> None:
+ """Configure selective Rx with segment length exceeding mbuf capacity.
+
+ Steps:
+ Start testpmd with rxpkts larger than mbuf-size,
+ buffer split enabled, and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_length=[4096, 0],
+ mbuf_size=[128, 0],
+ disable_device_start=True,
+ ) as testpmd:
+ self._verify_port_start_fails(
+ testpmd,
+ "Expected configuration to fail with segment > mbuf size.",
+ )
+
+ @func_test
+ def selective_rx_all_discard(self) -> None:
+ """Configure selective Rx with only discard segment.
+
+ Steps:
+ Start testpmd with only discard segment,
+ buffer split enabled, and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_length=[0],
+ mbuf_size=[0],
+ disable_device_start=True,
+ ) as testpmd:
+ self._verify_port_start_fails(
+ testpmd,
+ "Expected configuration to fail with only discard segment.",
+ )
--
2.54.0
^ permalink raw reply related
* [PATCH v9 09/10] dts: use specific types for Rx/Tx offloads
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Luca Vizzarro, Patrick Robb
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
Testpmd Rx and Tx offload parameters only accepted integer masks.
This forced tests to pass enum values through .value
when using RxOffloadCapability or TxOffloadCapability.
Allow these parameters to take either typed offload flags
or raw integer masks,
and convert both forms to the hexadecimal mask
expected by the testpmd command line.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/api/testpmd/config.py | 11 +++++++++--
dts/framework/params/__init__.py | 14 ++++++++++++++
dts/framework/params/types.py | 5 +++--
3 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/dts/api/testpmd/config.py b/dts/api/testpmd/config.py
index 1e59bccd08..f0581843ca 100644
--- a/dts/api/testpmd/config.py
+++ b/dts/api/testpmd/config.py
@@ -19,6 +19,7 @@
YesNoSwitch,
bracketed,
comma_separated,
+ hex_from_flag_or_int,
hex_from_flag_value,
modify_str,
str_from_flag_value,
@@ -26,6 +27,8 @@
from framework.params.eal import EalParams
from framework.utils import StrEnum
+from .types import RxOffloadCapability, TxOffloadCapability
+
class PortTopology(StrEnum):
"""Enum representing the port topology."""
@@ -577,12 +580,16 @@ class TestPmdParams(EalParams):
)
multi_rx_mempool: Switch = None
rx_shared_queue: Switch | int = field(default=None, metadata=Params.long("rxq-share"))
- rx_offloads: int | None = field(default=None, metadata=Params.convert_value(hex))
+ rx_offloads: RxOffloadCapability | int | None = field(
+ default=None, metadata=Params.convert_value(hex_from_flag_or_int)
+ )
rx_mq_mode: RXMultiQueueMode | None = None
tx_queues: int | None = field(default=None, metadata=Params.long("txq"))
tx_ring: TXRingParams | None = None
- tx_offloads: int | None = field(default=None, metadata=Params.convert_value(hex))
+ tx_offloads: TxOffloadCapability | int | None = field(
+ default=None, metadata=Params.convert_value(hex_from_flag_or_int)
+ )
eth_link_speed: int | None = None
disable_link_check: Switch = None
diff --git a/dts/framework/params/__init__.py b/dts/framework/params/__init__.py
index e6a2d3c903..b5bae9dad9 100644
--- a/dts/framework/params/__init__.py
+++ b/dts/framework/params/__init__.py
@@ -130,6 +130,20 @@ def hex_from_flag_value(flag: Flag) -> str:
return hex(flag.value)
+def hex_from_flag_or_int(value: Flag | int) -> str:
+ """Returns a :class:`enum.Flag` or integer value converted to hexadecimal.
+
+ Args:
+ value: An instance of :class:`Flag` or an integer.
+
+ Returns:
+ The value in hexadecimal representation.
+ """
+ if isinstance(value, Flag):
+ return hex_from_flag_value(value)
+ return hex(value)
+
+
class ParamsModifier(TypedDict, total=False):
"""Params modifiers dict compatible with the :func:`dataclasses.field` metadata parameter."""
diff --git a/dts/framework/params/types.py b/dts/framework/params/types.py
index 3c7650474c..a9f3749083 100644
--- a/dts/framework/params/types.py
+++ b/dts/framework/params/types.py
@@ -53,6 +53,7 @@ def create_testpmd(**kwargs: Unpack[TestPmdParamsDict]):
TXRingParams,
TxUDPPortPair,
)
+from api.testpmd.types import RxOffloadCapability, TxOffloadCapability
from framework.params import Switch, YesNoSwitch
from framework.testbed_model.cpu import LogicalCoreList
from framework.testbed_model.port import Port
@@ -175,11 +176,11 @@ class TestPmdParamsDict(EalParamsDict, total=False):
rx_segments_length: list[int] | None
multi_rx_mempool: Switch
rx_shared_queue: Switch | int
- rx_offloads: int | None
+ rx_offloads: RxOffloadCapability | int | None
rx_mq_mode: RXMultiQueueMode | None
tx_queues: int | None
tx_ring: TXRingParams | None
- tx_offloads: int | None
+ tx_offloads: TxOffloadCapability | int | None
eth_link_speed: int | None
disable_link_check: Switch
disable_device_start: Switch
--
2.54.0
^ permalink raw reply related
* [PATCH v9 08/10] dts: fix topology capability comparison
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Luca Vizzarro, Patrick Robb,
Juraj Linkeš, Dean Marx, Jeremy Spewock
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
TopologyCapability.__gt__() was delegating to __lt__(),
which caused infinite recursion when "other" is not a TopologyCapability:
other.__lt__(self) returns NotImplemented,
Python retries with self.__gt__(other),
and the cycle repeats.
dts/framework/testbed_model/capability.py", line 579, in __gt__
return other < self
^^^^^^^^^^^^
RecursionError: maximum recursion depth exceeded
Similarly, __le__() was delegating to "not __gt__()",
which returns True for non-comparable types instead of False.
Fix both by checking is_comparable_with() first
and comparing topology_type directly, consistent with __lt__().
Fixes: 039256daa8bf ("dts: add topology capability")
Cc: stable@dpdk.org
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/framework/testbed_model/capability.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 960370fc72..96e1cd449f 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -574,7 +574,9 @@ def __gt__(self, other: Any) -> bool:
Returns:
:data:`True` if the instance's topology type is more complex than the compared object's.
"""
- return other < self
+ if not self.is_comparable_with(other):
+ return False
+ return self.topology_type > other.topology_type
def __le__(self, other: Any) -> bool:
"""Compare the :attr:`~TopologyCapability.topology_type`s.
@@ -586,7 +588,9 @@ def __le__(self, other: Any) -> bool:
:data:`True` if the instance's topology type is less complex or equal than
the compared object's.
"""
- return not self > other
+ if not self.is_comparable_with(other):
+ return False
+ return self.topology_type <= other.topology_type
def __hash__(self):
"""Each instance is identified by :attr:`topology_type`."""
--
2.54.0
^ permalink raw reply related
* [PATCH v9 07/10] common/mlx5: remove callbacks for MR registration
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Dariusz Sosnowski, Viacheslav Ovsiienko,
Bing Zhao, Ori Kam, Suanming Mou, Matan Azrad, Fan Zhang,
Ashish Gupta
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
The functions register/unregister for a Memory Region (MR)
were not called directly.
There are only 2 implementations for Linux and Windows,
no need of handling this difference with function pointers.
The callback pointers are replaced with direct calls
and link time decision based on the Operating System.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/common/mlx5/linux/mlx5_common_verbs.c | 26 +++----------
drivers/common/mlx5/mlx5_common.c | 6 +--
drivers/common/mlx5/mlx5_common_mr.c | 37 ++++++++-----------
drivers/common/mlx5/mlx5_common_mr.h | 26 +++----------
drivers/common/mlx5/windows/mlx5_common_os.c | 23 ++----------
drivers/compress/mlx5/mlx5_compress.c | 4 +-
drivers/crypto/mlx5/mlx5_crypto.h | 2 -
drivers/crypto/mlx5/mlx5_crypto_gcm.c | 6 +--
drivers/net/mlx5/mlx5.h | 3 +-
drivers/net/mlx5/mlx5_flow_aso.c | 21 +++++------
drivers/net/mlx5/mlx5_flow_hw.c | 11 ++----
drivers/net/mlx5/mlx5_flow_quota.c | 6 +--
drivers/net/mlx5/mlx5_hws_cnt.c | 19 ++++------
13 files changed, 61 insertions(+), 129 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index 6d44e1f566..5e23c5844d 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -106,10 +106,10 @@ mlx5_set_context_attr(struct rte_device *dev, struct ibv_context *ctx)
* @return
* 0 on successful registration, -1 otherwise
*/
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_common_verbs_reg_mr)
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_reg_mr)
int
-mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr)
+mlx5_os_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr)
{
struct ibv_mr *ibv_mr;
@@ -136,9 +136,9 @@ mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
* pmd_mr struct set with lkey, address, length and pointer to mr object
*
*/
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_common_verbs_dereg_mr)
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_dereg_mr)
void
-mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
+mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
{
if (pmd_mr && pmd_mr->obj != NULL) {
claim_zero(mlx5_glue->dereg_mr(pmd_mr->obj));
@@ -146,22 +146,6 @@ mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
}
}
-/**
- * Set the reg_mr and dereg_mr callbacks.
- *
- * @param[out] reg_mr_cb
- * Pointer to reg_mr func
- * @param[out] dereg_mr_cb
- * Pointer to dereg_mr func
- */
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_set_reg_mr_cb)
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
-{
- *reg_mr_cb = mlx5_common_verbs_reg_mr;
- *dereg_mr_cb = mlx5_common_verbs_dereg_mr;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
struct mlx5_pmd_mr *
mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index f87dc9d773..f4b8904366 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -1135,7 +1135,7 @@ mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
return -1;
}
mr = mlx5_create_mr_ext(dev->pd, (uintptr_t)addr, len,
- SOCKET_ID_ANY, dev->mr_scache.reg_mr_cb);
+ SOCKET_ID_ANY);
if (!mr) {
DRV_LOG(WARNING, "Device %s unable to DMA map", rte_dev->name);
rte_errno = EINVAL;
@@ -1165,7 +1165,7 @@ mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
ret = mlx5_mr_expand_cache(&dev->mr_scache, size,
rte_dev->numa_node);
if (ret < 0) {
- mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+ mlx5_mr_free(mr);
rte_errno = ret;
return -1;
}
@@ -1221,7 +1221,7 @@ mlx5_common_dev_dma_unmap(struct rte_device *rte_dev, void *addr,
}
LIST_REMOVE(mr, mr);
DRV_LOG(DEBUG, "MR(%p) is removed from list.", (void *)mr);
- mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+ mlx5_mr_free(mr);
mlx5_mr_rebuild_cache(&dev->mr_scache);
/*
* No explicit wmb is needed after updating dev_gen due to
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index 64ffc7f4ea..aa2d5e88a4 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -492,12 +492,12 @@ mlx5_mr_lookup_cache(struct mlx5_mr_share_cache *share_cache,
* Pointer to MR to free.
*/
void
-mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb)
+mlx5_mr_free(struct mlx5_mr *mr)
{
if (mr == NULL)
return;
DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
- dereg_mr_cb(&mr->pmd_mr);
+ mlx5_os_dereg_mr(&mr->pmd_mr);
rte_bitmap_free(mr->ms_bmp);
mlx5_free(mr);
}
@@ -545,7 +545,7 @@ mlx5_mr_garbage_collect(struct mlx5_mr_share_cache *share_cache)
struct mlx5_mr *mr = mr_next;
mr_next = LIST_NEXT(mr, mr);
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
}
}
@@ -821,7 +821,7 @@ mlx5_mr_create_primary(void *pd,
data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
data.end = data.start + msl->page_sz;
rte_mcfg_mem_read_unlock();
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
goto alloc_resources;
}
MLX5_ASSERT(data.msl == data_re.msl);
@@ -845,7 +845,7 @@ mlx5_mr_create_primary(void *pd,
* Must be unlocked before calling rte_free() because
* mlx5_mr_mem_event_free_cb() can be called inside.
*/
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
return entry->lkey;
}
/*
@@ -912,7 +912,7 @@ mlx5_mr_create_primary(void *pd,
* mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
* through mlx5_alloc_verbs_buf().
*/
- share_cache->reg_mr_cb(pd, (void *)data.start, len, &mr->pmd_mr);
+ mlx5_os_reg_mr(pd, (void *)data.start, len, &mr->pmd_mr);
if (mr->pmd_mr.obj == NULL) {
DRV_LOG(DEBUG, "Fail to create an MR for address (%p)",
(void *)addr);
@@ -948,7 +948,7 @@ mlx5_mr_create_primary(void *pd,
* calling rte_free() because mlx5_mr_mem_event_free_cb() can be called
* inside.
*/
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
return UINT32_MAX;
}
@@ -1139,9 +1139,6 @@ mlx5_mr_release_cache(struct mlx5_mr_share_cache *share_cache)
int
mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket)
{
- /* Set the reg_mr and dereg_mr callback functions */
- mlx5_os_set_reg_mr_cb(&share_cache->reg_mr_cb,
- &share_cache->dereg_mr_cb);
rte_rwlock_init(&share_cache->rwlock);
rte_rwlock_init(&share_cache->mprwlock);
/* Initialize B-tree and allocate memory for global MR cache table. */
@@ -1189,8 +1186,7 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
* Pointer to MR structure on success, NULL otherwise.
*/
struct mlx5_mr *
-mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
- mlx5_reg_mr_t reg_mr_cb)
+mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
{
struct mlx5_mr *mr = NULL;
@@ -1199,7 +1195,7 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
RTE_CACHE_LINE_SIZE, socket_id);
if (mr == NULL)
return NULL;
- reg_mr_cb(pd, (void *)addr, len, &mr->pmd_mr);
+ mlx5_os_reg_mr(pd, (void *)addr, len, &mr->pmd_mr);
if (mr->pmd_mr.obj == NULL) {
DRV_LOG(WARNING,
"Fail to create MR for address (%p)",
@@ -1624,14 +1620,13 @@ mlx5_mempool_reg_create(struct rte_mempool *mp, unsigned int mrs_n,
* Whether @p mpr owns its MRs exclusively, i.e. they are not shared.
*/
static void
-mlx5_mempool_reg_destroy(struct mlx5_mr_share_cache *share_cache,
- struct mlx5_mempool_reg *mpr, bool standalone)
+mlx5_mempool_reg_destroy(struct mlx5_mempool_reg *mpr, bool standalone)
{
if (standalone) {
unsigned int i;
for (i = 0; i < mpr->mrs_n; i++)
- share_cache->dereg_mr_cb(&mpr->mrs[i].pmd_mr);
+ mlx5_os_dereg_mr(&mpr->mrs[i].pmd_mr);
mlx5_free(mpr->mrs);
}
mlx5_free(mpr);
@@ -1748,7 +1743,7 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
const struct mlx5_range *range = &ranges[i];
size_t len = range->end - range->start;
- if (share_cache->reg_mr_cb(pd, (void *)range->start, len,
+ if (mlx5_os_reg_mr(pd, (void *)range->start, len,
&mr->pmd_mr) < 0) {
DRV_LOG(ERR,
"Failed to create an MR in PD %p for address range "
@@ -1763,7 +1758,7 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
mp->name);
}
if (i != ranges_n) {
- mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
+ mlx5_mempool_reg_destroy(new_mpr, true);
rte_errno = EINVAL;
goto exit;
}
@@ -1785,13 +1780,13 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
if (mpr != NULL) {
DRV_LOG(DEBUG, "Mempool %s is already registered for PD %p",
mp->name, pd);
- mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
+ mlx5_mempool_reg_destroy(new_mpr, true);
rte_errno = EEXIST;
goto exit;
} else if (old_mpr != NULL) {
DRV_LOG(DEBUG, "Mempool %s registration for PD %p updated for external memory",
mp->name, pd);
- mlx5_mempool_reg_destroy(share_cache, old_mpr, standalone);
+ mlx5_mempool_reg_destroy(old_mpr, standalone);
}
exit:
free(ranges);
@@ -1860,7 +1855,7 @@ mlx5_mr_mempool_unregister_primary(struct mlx5_mr_share_cache *share_cache,
rte_errno = ENOENT;
return -1;
}
- mlx5_mempool_reg_destroy(share_cache, mpr, standalone);
+ mlx5_mempool_reg_destroy(mpr, standalone);
return 0;
}
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index 00f3d832c3..5fb931a1b5 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -32,13 +32,6 @@ struct mlx5_pmd_mr {
struct mlx5_devx_obj *mkey; /* devx mkey object. */
};
-/**
- * mr operations typedef
- */
-typedef int (*mlx5_reg_mr_t)(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr);
-typedef void (*mlx5_dereg_mr_t)(struct mlx5_pmd_mr *pmd_mr);
-
/* Memory Region object. */
struct mlx5_mr {
LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
@@ -88,8 +81,6 @@ struct __rte_packed_begin mlx5_mr_share_cache {
struct mlx5_mr_list mr_list; /* Registered MR list. */
struct mlx5_mr_list mr_free_list; /* Freed MR list. */
struct mlx5_mempool_reg_list mempool_reg_list; /* Mempool database. */
- mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
- mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
} __rte_packed_end;
/* Multi-Packet RQ buffer header. */
@@ -233,9 +224,8 @@ struct mlx5_mr *
mlx5_mr_lookup_list(struct mlx5_mr_share_cache *share_cache,
struct mr_cache_entry *entry, uintptr_t addr);
struct mlx5_mr *
-mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
- mlx5_reg_mr_t reg_mr_cb);
-void mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
+mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id);
+void mlx5_mr_free(struct mlx5_mr *mr);
__rte_internal
uint32_t
mlx5_mr_create(struct mlx5_common_device *cdev,
@@ -246,19 +236,13 @@ __rte_internal
uint32_t
mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr);
-/* mlx5_common_verbs.c */
-
__rte_internal
int
-mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr);
+mlx5_os_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr);
__rte_internal
void
-mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
-
-__rte_internal
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
__rte_internal
struct mlx5_pmd_mr *
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index fb2bbae578..c790c9a4ae 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -377,7 +377,8 @@ mlx5_os_umem_dereg(void *pumem)
* @return
* 0 on successful registration, -1 otherwise
*/
-static int
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_reg_mr)
+int
mlx5_os_reg_mr(void *pd,
void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr)
{
@@ -425,7 +426,8 @@ mlx5_os_reg_mr(void *pd,
* @param[in] pmd_mr
* Pointer to PMD mr object
*/
-static void
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_dereg_mr)
+void
mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
{
if (!pmd_mr)
@@ -437,23 +439,6 @@ mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
memset(pmd_mr, 0, sizeof(*pmd_mr));
}
-/**
- * Set the reg_mr and dereg_mr callbacks.
- *
- * @param[out] reg_mr_cb
- * Pointer to reg_mr func
- * @param[out] dereg_mr_cb
- * Pointer to dereg_mr func
- *
- */
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_set_reg_mr_cb)
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
-{
- *reg_mr_cb = mlx5_os_reg_mr;
- *dereg_mr_cb = mlx5_os_dereg_mr;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
struct mlx5_pmd_mr *
mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index e5325c6150..1361dab630 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -117,7 +117,7 @@ mlx5_compress_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
if (qp->opaque_mr.obj != NULL) {
void *opaq = qp->opaque_mr.addr;
- mlx5_common_verbs_dereg_mr(&qp->opaque_mr);
+ mlx5_os_dereg_mr(&qp->opaque_mr);
rte_free(opaq);
}
mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
@@ -199,7 +199,7 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
qp->priv = priv;
qp->ops = (struct rte_comp_op **)RTE_ALIGN((uintptr_t)(qp + 1),
RTE_CACHE_LINE_SIZE);
- if (mlx5_common_verbs_reg_mr(priv->cdev->pd, opaq_buf, qp->entries_n *
+ if (mlx5_os_reg_mr(priv->cdev->pd, opaq_buf, qp->entries_n *
sizeof(union mlx5_gga_compress_opaque),
&qp->opaque_mr) != 0) {
rte_free(opaq_buf);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f9f127e9e6..93a2bb2c78 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -40,8 +40,6 @@ struct mlx5_crypto_priv {
TAILQ_ENTRY(mlx5_crypto_priv) next;
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
struct rte_cryptodev *crypto_dev;
- mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
- mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
struct mlx5_uar uar; /* User Access Region. */
uint32_t max_segs_num; /* Maximum supported data segs. */
uint32_t max_klm_num; /* Maximum supported klm. */
diff --git a/drivers/crypto/mlx5/mlx5_crypto_gcm.c b/drivers/crypto/mlx5/mlx5_crypto_gcm.c
index 89f32c7722..1a2600655a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto_gcm.c
+++ b/drivers/crypto/mlx5/mlx5_crypto_gcm.c
@@ -219,7 +219,6 @@ mlx5_crypto_gcm_mkey_klm_update(struct mlx5_crypto_priv *priv,
static int
mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
{
- struct mlx5_crypto_priv *priv = dev->data->dev_private;
struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
if (qp->umr_qp_obj.qp != NULL)
@@ -231,7 +230,7 @@ mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
if (qp->mr.obj != NULL) {
void *opaq = qp->mr.addr;
- priv->dereg_mr_cb(&qp->mr);
+ mlx5_os_dereg_mr(&qp->mr);
rte_free(opaq);
}
mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
@@ -363,7 +362,7 @@ mlx5_crypto_gcm_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
rte_errno = ENOMEM;
goto err;
}
- if (priv->reg_mr_cb(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) {
+ if (mlx5_os_reg_mr(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) {
rte_free(mr_buf);
DRV_LOG(ERR, "Failed to register opaque MR.");
rte_errno = ENOMEM;
@@ -1186,7 +1185,6 @@ mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
/* Override AES-GCM specified ops. */
dev_ops->sym_session_configure = mlx5_crypto_sym_gcm_session_configure;
- mlx5_os_set_reg_mr_cb(&priv->reg_mr_cb, &priv->dereg_mr_cb);
dev_ops->queue_pair_setup = mlx5_crypto_gcm_qp_setup;
dev_ops->queue_pair_release = mlx5_crypto_gcm_qp_release;
if (mlx5_crypto_is_ipsec_opt(priv)) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index bd6ef35b53..a4d5392e8f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -2706,8 +2706,7 @@ int mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh,
int mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
struct mlx5_aso_ct_pools_mng *ct_mng,
uint32_t nb_queues);
-int mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_aso_ct_pools_mng *ct_mng);
+int mlx5_aso_ct_queue_uninit(struct mlx5_aso_ct_pools_mng *ct_mng);
int
mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,
void *uar, uint16_t log_desc_n);
diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c
index 5e2a81ef9c..cd84ab1966 100644
--- a/drivers/net/mlx5/mlx5_flow_aso.c
+++ b/drivers/net/mlx5/mlx5_flow_aso.c
@@ -19,17 +19,15 @@
/**
* Free MR resources.
*
- * @param[in] cdev
- * Pointer to the mlx5 common device.
* @param[in] mr
* MR to free.
*/
static void
-mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr)
+mlx5_aso_dereg_mr(struct mlx5_pmd_mr *mr)
{
void *addr = mr->addr;
- cdev->mr_scache.dereg_mr_cb(mr);
+ mlx5_os_dereg_mr(mr);
mlx5_free(addr);
memset(mr, 0, sizeof(*mr));
}
@@ -59,7 +57,7 @@ mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length,
DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
return -1;
}
- ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr);
+ ret = mlx5_os_reg_mr(cdev->pd, mr->addr, length, mr);
if (ret) {
DRV_LOG(ERR, "Failed to create direct Mkey.");
mlx5_free(mr->addr);
@@ -362,7 +360,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq,
sh->tx_uar.obj,
MLX5_ASO_QUEUE_LOG_DESC)) {
- mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
return -1;
}
mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
@@ -399,14 +397,14 @@ mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
switch (aso_opc_mod) {
case ASO_OPC_MOD_FLOW_HIT:
- mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
sq = &sh->aso_age_mng->aso_sq;
break;
case ASO_OPC_MOD_POLICER:
mlx5_aso_mtr_queue_uninit(sh, NULL, &sh->mtrmng->pools_mng);
break;
case ASO_OPC_MOD_CONNECTION_TRACKING:
- mlx5_aso_ct_queue_uninit(sh, sh->ct_mng);
+ mlx5_aso_ct_queue_uninit(sh->ct_mng);
break;
default:
DRV_LOG(ERR, "Unknown ASO operation mode");
@@ -1147,15 +1145,14 @@ __mlx5_aso_ct_get_pool(struct mlx5_dev_ctx_shared *sh,
}
int
-mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_aso_ct_pools_mng *ct_mng)
+mlx5_aso_ct_queue_uninit(struct mlx5_aso_ct_pools_mng *ct_mng)
{
uint32_t i;
/* 64B per object for query. */
for (i = 0; i < ct_mng->nb_sq; i++) {
if (ct_mng->aso_sqs[i].mr.addr)
- mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
+ mlx5_aso_dereg_mr(&ct_mng->aso_sqs[i].mr);
mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
}
return 0;
@@ -1197,7 +1194,7 @@ mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
error:
do {
if (ct_mng->aso_sqs[i].mr.addr)
- mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
+ mlx5_aso_dereg_mr(&ct_mng->aso_sqs[i].mr);
mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
} while (i--);
ct_mng->nb_sq = 0;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index b6bb9f12a6..7cc601d681 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -11086,12 +11086,9 @@ flow_hw_create_nic_ctrl_tables(struct rte_eth_dev *dev, struct rte_flow_error *e
}
static void
-flow_hw_ct_mng_destroy(struct rte_eth_dev *dev,
- struct mlx5_aso_ct_pools_mng *ct_mng)
+flow_hw_ct_mng_destroy(struct mlx5_aso_ct_pools_mng *ct_mng)
{
- struct mlx5_priv *priv = dev->data->dev_private;
-
- mlx5_aso_ct_queue_uninit(priv->sh, ct_mng);
+ mlx5_aso_ct_queue_uninit(ct_mng);
mlx5_free(ct_mng);
}
@@ -11230,7 +11227,7 @@ mlx5_flow_ct_init(struct rte_eth_dev *dev,
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ flow_hw_ct_mng_destroy(priv->ct_mng);
priv->ct_mng = NULL;
}
return ret;
@@ -11804,7 +11801,7 @@ __mlx5_flow_hw_resource_release(struct rte_eth_dev *dev, bool ctx_close)
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ flow_hw_ct_mng_destroy(priv->ct_mng);
priv->ct_mng = NULL;
}
mlx5_flow_quota_destroy(dev);
diff --git a/drivers/net/mlx5/mlx5_flow_quota.c b/drivers/net/mlx5/mlx5_flow_quota.c
index d94167d0b0..b661bd376e 100644
--- a/drivers/net/mlx5/mlx5_flow_quota.c
+++ b/drivers/net/mlx5/mlx5_flow_quota.c
@@ -412,12 +412,11 @@ mlx5_quota_alloc_sq(struct mlx5_priv *priv)
static void
mlx5_quota_destroy_read_buf(struct mlx5_priv *priv)
{
- struct mlx5_dev_ctx_shared *sh = priv->sh;
struct mlx5_quota_ctx *qctx = &priv->quota_ctx;
if (qctx->mr.lkey) {
void *addr = qctx->mr.addr;
- sh->cdev->mr_scache.dereg_mr_cb(&qctx->mr);
+ mlx5_os_dereg_mr(&qctx->mr);
mlx5_free(addr);
}
if (qctx->read_buf)
@@ -446,8 +445,7 @@ mlx5_quota_alloc_read_buf(struct mlx5_priv *priv)
DRV_LOG(DEBUG, "QUOTA: failed to allocate MTR ASO READ buffer [1]");
return -ENOMEM;
}
- ret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, buf,
- rd_buf_size, &qctx->mr);
+ ret = mlx5_os_reg_mr(sh->cdev->pd, buf, rd_buf_size, &qctx->mr);
if (ret) {
DRV_LOG(DEBUG, "QUOTA: failed to register MTR ASO READ MR");
return -errno;
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 1b6acb7a3b..d0c4ead71b 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -259,12 +259,11 @@ mlx5_hws_aging_check(struct mlx5_priv *priv, struct mlx5_hws_cnt_pool *cpool)
}
static void
-mlx5_hws_cnt_raw_data_free(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_hws_cnt_raw_data_mng *mng)
+mlx5_hws_cnt_raw_data_free(struct mlx5_hws_cnt_raw_data_mng *mng)
{
if (mng == NULL)
return;
- sh->cdev->mr_scache.dereg_mr_cb(&mng->mr);
+ mlx5_os_dereg_mr(&mng->mr);
mlx5_free(mng->raw);
mlx5_free(mng);
}
@@ -296,8 +295,7 @@ mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n,
NULL, "failed to allocate raw counters memory");
goto error;
}
- ret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, mng->raw, sz,
- &mng->mr);
+ ret = mlx5_os_reg_mr(sh->cdev->pd, mng->raw, sz, &mng->mr);
if (ret) {
rte_flow_error_set(error, errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -306,7 +304,7 @@ mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n,
}
return mng;
error:
- mlx5_hws_cnt_raw_data_free(sh, mng);
+ mlx5_hws_cnt_raw_data_free(mng);
return NULL;
}
@@ -639,8 +637,7 @@ mlx5_hws_cnt_pool_dcs_alloc(struct mlx5_dev_ctx_shared *sh,
}
static void
-mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_hws_cnt_pool *cpool)
+mlx5_hws_cnt_pool_dcs_free(struct mlx5_hws_cnt_pool *cpool)
{
uint32_t idx;
@@ -649,7 +646,7 @@ mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,
for (idx = 0; idx < MLX5_HWS_CNT_DCS_NUM; idx++)
mlx5_devx_cmd_destroy(cpool->dcs_mng.dcs[idx].obj);
if (cpool->raw_mng) {
- mlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);
+ mlx5_hws_cnt_raw_data_free(cpool->raw_mng);
cpool->raw_mng = NULL;
}
}
@@ -842,8 +839,8 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
}
mlx5_hws_cnt_pool_action_destroy(cpool);
if (cpool->cfg.host_cpool == NULL) {
- mlx5_hws_cnt_pool_dcs_free(sh, cpool);
- mlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);
+ mlx5_hws_cnt_pool_dcs_free(cpool);
+ mlx5_hws_cnt_raw_data_free(cpool->raw_mng);
}
mlx5_free((void *)cpool->cfg.name);
mlx5_hws_cnt_pool_deinit(cpool);
--
2.54.0
^ permalink raw reply related
* [PATCH v9 06/10] net/mlx5: support selective Rx
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Gregory Etelson, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Selective Rx may save some PCI bandwidth.
Implement selective Rx in the (quite slow) scalar SPRQ Rx path
mlx5_rx_burst() where the performance impact
of the added condition branches is acceptable.
Other Rx functions do not support this feature.
When using selective Rx, mlx5_rx_burst will be selected.
A null Memory Region (MR) is always allocated
at shared device context initialization.
The selective Rx capability is not advertised
if this special MR allocation fails.
For each Rx segment configured with a NULL mempool,
a "null mbuf" is created.
It is a fake mbuf allocated outside any mempool,
used as a placeholder in the Rx ring.
The null MR lkey is used in the WQE for these segments
so the NIC writes received data to a discard buffer.
The mbuf data room size is resolved from the first segment having a pool.
For null segments, the buffer length is from the last seen pool,
so that the WQE stride size remains consistent.
In mlx5_rx_burst, discarded segments are not chained
into the packet mbuf list, NB_SEGS is decremented accordingly,
and no replacement buffer is allocated.
A separate data_seg_len accumulator tracks the total length
of delivered segments only.
The packet length is adjusted to reflect only the data
actually delivered to the application.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
doc/guides/nics/features/mlx5.ini | 1 +
doc/guides/nics/mlx5.rst | 86 +++++++++---
doc/guides/rel_notes/release_26_07.rst | 4 +
drivers/net/mlx5/mlx5.c | 7 +
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_ethdev.c | 25 ++++
drivers/net/mlx5/mlx5_rx.c | 187 +++++++++++++++----------
drivers/net/mlx5/mlx5_rx.h | 1 +
drivers/net/mlx5/mlx5_rxq.c | 95 +++++++++----
drivers/net/mlx5/mlx5_trigger.c | 64 +++++++--
10 files changed, 330 insertions(+), 141 deletions(-)
diff --git a/doc/guides/nics/features/mlx5.ini b/doc/guides/nics/features/mlx5.ini
index 3b3eda28b8..ae8c83057b 100644
--- a/doc/guides/nics/features/mlx5.ini
+++ b/doc/guides/nics/features/mlx5.ini
@@ -16,6 +16,7 @@ Burst mode info = Y
Power mgmt address monitor = Y
MTU update = Y
Buffer split on Rx = Y
+Selective Rx = Y
Scattered Rx = Y
LRO = Y
TSO = Y
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 00bfb31370..afbf040e66 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -84,6 +84,9 @@ The Rx / Tx data path use different techniques to offer the best performance.
with :ref:`multi-packet Rx queues (MPRQ) <mlx5_mprq_params>`.
This feature is disabled by default.
+- Some PCI bandwidth is saved by receiving partial packets
+ with :ref:`selective Rx <mlx5_selective_rx>`.
+
More details about Rx implementations and their configurations are provided
in the chapter about :ref:`mlx5_rx_functions`.
@@ -879,6 +882,8 @@ MLX5 supports various methods to report statistics:
Basic port statistics can be queried using ``rte_eth_stats_get()``.
The received and sent statistics are through SW only
and counts the number of packets received or sent successfully by the PMD.
+In the case of :ref:`selective Rx <mlx5_selective_rx>`,
+the ``ibytes`` counter matches segments delivered, not the skipped ones.
The ``imissed`` counter is the amount of packets that could not be delivered
to SW because a queue was full.
Packets not received due to congestion in the bus or on the NIC
@@ -992,25 +997,26 @@ These configurations may also have an impact on the behavior:
.. table:: Rx burst functions
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- || Function Name || Parameters to Enable || Scatter|| Error Recovery || CQE || Large|| Shared |
- | | | | || comp|| MTU | RxQ |
- +===================+========================+=========+=================+======+=======+=========+
- | rx_burst | rx_vec_en=0 | Yes | Yes | Yes | Yes | No |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_vec | rx_vec_en=1 (default) | No | if CQE comp off | Yes | No | No |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_mprq || mprq_en=1 | No | Yes | Yes | Yes | No |
- | || RxQs >= rxqs_min_mprq | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_mprq_vec || rx_vec_en=1 (default) | No | if CQE comp off | Yes | Yes | No |
- | || mprq_en=1 | | | | | |
- | || RxQs >= rxqs_min_mprq | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst | at least one Rx queue | Yes | Yes | Yes | Yes | Yes |
- | (out of order) | on the device | | | | | |
- | | is shared | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ || Function|| Parameters to Enable || Scatter|| Selec-|| Error || CQE || Large|| Shared|
+ || Name | | || tive || Recovery|| comp|| MTU || RxQ |
+ +==========+=======================+=========+========+==========+======+=======+========+
+ | rx_burst | rx_vec_en=0 | Yes | Yes | Yes | Yes | Yes | No |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _vec | rx_vec_en=1 (default) | No | No || if CQE | Yes | No | No |
+ | | | | || comp off| | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _mprq || mprq_en=1 | No | No | Yes | Yes | Yes | No |
+ | || RxQs >= rxqs_min_mprq| | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _mprq_vec|| rx_vec_en=1 (default)| No | No || if CQE | Yes | Yes | No |
+ | || mprq_en=1 | | || comp off| | | |
+ | || RxQs >= rxqs_min_mprq| | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ || _out_of || at least one Rx queue| Yes | No | Yes | Yes | Yes | Yes |
+ || _order || on the device | | | | | | |
+ | || is shared | | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
Rx/Tx Tuning
@@ -1105,13 +1111,14 @@ Rx interrupt X
:ref:`Rx threshold <mlx5_rx_threshold>` X X
:ref:`Rx drop delay <mlx5_drop>` X X
:ref:`Rx timestamp <mlx5_rx_timstp>` X X
+:ref:`buffer split <mlx5_buf_split>` X X
+:ref:`selective Rx <mlx5_selective_rx>` X
+:ref:`multi-segment <mlx5_multiseg>` X X
:ref:`Tx scheduling <mlx5_tx_sched>` X
:ref:`Tx rate limit <mlx5_rate_limit>` X
:ref:`Tx inline <mlx5_tx_inline>` X X
:ref:`Tx fast free <mlx5_tx_fast_free>` X X
:ref:`Tx affinity <mlx5_aggregated>` X
-:ref:`buffer split <mlx5_buf_split>` X X
-:ref:`multi-segment <mlx5_multiseg>` X X
promiscuous X X
multicast promiscuous X X
multiple MAC addresses X
@@ -2248,13 +2255,50 @@ OFED 5.1-2
DPDK 20.11
========= ==========
+Runtime configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+The offload flag ``RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT`` is required.
+
+When calling ``rte_eth_rx_queue_setup()``,
+the input ``rte_eth_rxconf::rx_seg`` defines the configuration of the segments,
+mainly offset and length.
+
Limitations
^^^^^^^^^^^
+#. Splitting per protocol header is not supported.
+
#. Buffer split offload is supported with regular Rx burst routine only,
no MPRQ feature or vectorized code can be engaged.
+.. _mlx5_selective_rx:
+
+Selective Rx
+~~~~~~~~~~~~
+
+Some PCI bandwidth can be saved
+by :ref:`skipping some parts of Rx data <nic_features_selective_rx>`.
+It is enabled when using :ref:`buffer split <mlx5_buf_split>`
+and configuring no mempool in some segments to discard.
+
+Runtime configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+The offload flag ``RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT`` is required.
+
+When calling ``rte_eth_rx_queue_setup()``,
+the segment to discard (``rte_eth_rxconf::rx_seg::split``)
+is marked by the absence of mempool (``mp = NULL``).
+
+Limitations
+^^^^^^^^^^^
+
+#. Selective Rx is supported with regular Rx burst routine only,
+ no MPRQ feature or vectorized code can be engaged.
+
+
.. _mlx5_multiseg:
Multi-Segment Scatter/Gather
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index 46a8fe2cc1..0ac9816a85 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -103,6 +103,10 @@ New Features
* Added support for transmitting LLDP packets based on mbuf packet type.
* Implemented AVX2 context descriptor transmit paths.
+* **Updated NVIDIA mlx5 ethernet driver.**
+
+ * Added support for selective Rx in scalar SPRQ Rx path.
+
* **Updated PCAP ethernet driver.**
* Added support for VLAN insertion and stripping.
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f190654756..61c26d1206 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1975,6 +1975,9 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
/* Init counter pool list header and lock. */
LIST_INIT(&sh->hws_cpool_list);
rte_spinlock_init(&sh->cpool_lock);
+ sh->null_mr = mlx5_os_alloc_null_mr(sh->cdev->dev, sh->cdev->pd);
+ if (!sh->null_mr)
+ DRV_LOG(DEBUG, "Fail to initialize NULL MR, selective Rx is disabled.");
exit:
pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
return sh;
@@ -2139,6 +2142,10 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL);
pthread_mutex_destroy(&sh->txpp.mutex);
mlx5_lwm_unset(sh);
+ if (sh->null_mr) {
+ mlx5_os_free_null_mr(sh->null_mr);
+ sh->null_mr = NULL;
+ }
mlx5_physical_device_destroy(sh->phdev);
mlx5_free(sh);
return;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 92a00cfaa8..bd6ef35b53 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1674,6 +1674,7 @@ struct mlx5_dev_ctx_shared {
rte_spinlock_t cpool_lock;
LIST_HEAD(hws_cpool_list, mlx5_hws_cnt_pool) hws_cpool_list; /* Count pool list. */
struct mlx5_dev_registers registers;
+ struct mlx5_pmd_mr *null_mr;
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index a29cdeeb50..7b7536fa1e 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -381,6 +381,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
info->rx_seg_capa.multi_pools = !priv->config.mprq.enabled;
info->rx_seg_capa.offset_allowed = !priv->config.mprq.enabled;
info->rx_seg_capa.offset_align_log2 = 0;
+ info->rx_seg_capa.selective_rx = !!priv->sh->null_mr;
info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
info->rx_queue_offload_capa);
info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
@@ -708,6 +709,25 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
return -rte_errno;
}
+static bool
+mlx5_selective_rx_enabled(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ for (uint32_t q = 0; q < priv->rxqs_n; ++q) {
+ struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, q);
+
+ if (rxq_ctrl == NULL || rxq_ctrl->is_hairpin)
+ continue;
+ for (uint16_t s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
+ if (rxq_ctrl->rxq.rxseg[s].mp == NULL)
+ return true;
+ }
+ }
+
+ return false;
+}
+
/**
* Configure the RX function to use.
*
@@ -723,6 +743,11 @@ mlx5_select_rx_function(struct rte_eth_dev *dev)
eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
MLX5_ASSERT(dev != NULL);
+ if (mlx5_selective_rx_enabled(dev)) {
+ DRV_LOG(DEBUG, "port %u forced to scalar SPRQ Rx (selective Rx configured)",
+ dev->data->port_id);
+ return rx_pkt_burst;
+ }
if (mlx5_shared_rq_enabled(dev)) {
rx_pkt_burst = mlx5_rx_burst_out_of_order;
DRV_LOG(DEBUG, "port %u forced to use SPRQ"
diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index 185bfd4fff..9812bc7929 100644
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -486,7 +486,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
rxq->wqes)[i];
addr = rte_pktmbuf_mtod(buf, uintptr_t);
byte_count = DATA_LEN(buf);
- lkey = mlx5_rx_mb2mr(rxq, buf);
+ lkey = buf->pool ? mlx5_rx_mb2mr(rxq, buf) : rxq->sh->null_mr->lkey;
}
/* scat->addr must be able to store a pointer. */
MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
@@ -1044,11 +1044,14 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
const unsigned int sges_n = rxq->sges_n;
struct rte_mbuf *pkt = NULL;
struct rte_mbuf *seg = NULL;
+ struct rte_mbuf *tail = NULL;
volatile struct mlx5_cqe *cqe =
&(*rxq->cqes)[rxq->cq_ci & cqe_mask];
+ volatile struct mlx5_mini_cqe8 *mcqe = NULL;
unsigned int i = 0;
unsigned int rq_ci = rxq->rq_ci << sges_n;
int len = 0; /* keep its value across iterations. */
+ uint32_t data_seg_len = 0;
while (pkts_n) {
uint16_t skip_cnt;
@@ -1056,105 +1059,137 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
volatile struct mlx5_wqe_data_seg *wqe =
&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
struct rte_mbuf *rep = (*rxq->elts)[idx];
- volatile struct mlx5_mini_cqe8 *mcqe = NULL;
- if (pkt)
- NEXT(seg) = rep;
+ if (pkt) {
+ if (rep->pool)
+ NEXT(tail) = rep;
+ else
+ --NB_SEGS(pkt);
+ }
seg = rep;
rte_prefetch0(seg);
rte_prefetch0(cqe);
rte_prefetch0(wqe);
- /* Allocate the buf from the same pool. */
- rep = rte_mbuf_raw_alloc(seg->pool);
- if (unlikely(rep == NULL)) {
- ++rxq->stats.rx_nombuf;
- if (!pkt) {
- /*
- * no buffers before we even started,
- * bail out silently.
- */
- break;
- }
- while (pkt != seg) {
- MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
- rep = NEXT(pkt);
- NEXT(pkt) = NULL;
- NB_SEGS(pkt) = 1;
- rte_mbuf_raw_free(pkt);
- pkt = rep;
- }
- rq_ci >>= sges_n;
- ++rq_ci;
- rq_ci <<= sges_n;
- break;
- }
- if (!pkt) {
- cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
- len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
- &mcqe, &skip_cnt, false, NULL);
- if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
- /* We drop packets with non-critical errors */
- rte_mbuf_raw_free(rep);
- if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
- rq_ci = rxq->rq_ci << sges_n;
+ if (seg->pool) {
+ /* Allocate the buf from the same pool. */
+ rep = rte_mbuf_raw_alloc(seg->pool);
+ if (unlikely(rep == NULL)) {
+ ++rxq->stats.rx_nombuf;
+ if (!pkt) {
+ /*
+ * no buffers before we even started,
+ * bail out silently.
+ */
break;
}
- /* Skip specified amount of error CQEs packets */
+ while (pkt != seg) {
+ MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
+ rep = NEXT(pkt);
+ NEXT(pkt) = NULL;
+ NB_SEGS(pkt) = 1;
+ rte_mbuf_raw_free(pkt);
+ pkt = rep;
+ }
rq_ci >>= sges_n;
- rq_ci += skip_cnt;
+ ++rq_ci;
rq_ci <<= sges_n;
- MLX5_ASSERT(!pkt);
- continue;
- }
- if (len == 0) {
- rte_mbuf_raw_free(rep);
break;
}
- pkt = seg;
- MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
- pkt->ol_flags &= RTE_MBUF_F_EXTERNAL;
- if (rxq->cqe_comp_layout && mcqe)
- cqe = &rxq->title_cqe;
- rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
- if (rxq->crc_present)
- len -= RTE_ETHER_CRC_LEN;
- PKT_LEN(pkt) = len;
- if (cqe->lro_num_seg > 1) {
- mlx5_lro_update_hdr
- (rte_pktmbuf_mtod(pkt, uint8_t *), cqe,
- mcqe, rxq, len);
- pkt->ol_flags |= RTE_MBUF_F_RX_LRO;
- pkt->tso_segsz = len / cqe->lro_num_seg;
+ }
+ if (!pkt) { /* new packet */
+ if (len == 0) { /* no CQE polled yet */
+ mcqe = NULL;
+ cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
+ len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
+ &mcqe, &skip_cnt, false, NULL);
+ if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
+ /* We drop packets with non-critical errors */
+ if (seg->pool)
+ rte_mbuf_raw_free(rep);
+ if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
+ rq_ci = rxq->rq_ci << sges_n;
+ break;
+ }
+ /* Skip specified amount of error CQEs packets */
+ rq_ci >>= sges_n;
+ rq_ci += skip_cnt;
+ rq_ci <<= sges_n;
+ MLX5_ASSERT(!pkt);
+ len = 0;
+ continue;
+ }
+ if (len == 0) {
+ if (seg->pool)
+ rte_mbuf_raw_free(rep);
+ break;
+ }
+ MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
+ if (rxq->crc_present)
+ len -= RTE_ETHER_CRC_LEN;
+ }
+ if (seg->pool) { /* first real segment */
+ pkt = seg;
+ pkt->ol_flags &= RTE_MBUF_F_EXTERNAL;
+ if (rxq->cqe_comp_layout && mcqe)
+ cqe = &rxq->title_cqe;
+ rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
+ PKT_LEN(pkt) = len;
+ if (cqe->lro_num_seg > 1) {
+ mlx5_lro_update_hdr
+ (rte_pktmbuf_mtod(pkt, uint8_t *), cqe,
+ mcqe, rxq, len);
+ pkt->ol_flags |= RTE_MBUF_F_RX_LRO;
+ pkt->tso_segsz = len / cqe->lro_num_seg;
+ }
}
}
- DATA_LEN(rep) = DATA_LEN(seg);
- PKT_LEN(rep) = PKT_LEN(seg);
- SET_DATA_OFF(rep, DATA_OFF(seg));
- PORT(rep) = PORT(seg);
- (*rxq->elts)[idx] = rep;
- /*
- * Fill NIC descriptor with the new buffer. The lkey and size
- * of the buffers are already known, only the buffer address
- * changes.
- */
- wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
- /* If there's only one MR, no need to replace LKey in WQE. */
- if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
- wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
- if (len > DATA_LEN(seg)) {
+ if (seg->pool) { /* real segment: replenish WQE */
+ tail = seg;
+ DATA_LEN(rep) = DATA_LEN(seg);
+ PKT_LEN(rep) = PKT_LEN(seg);
+ SET_DATA_OFF(rep, DATA_OFF(seg));
+ PORT(rep) = PORT(seg);
+ (*rxq->elts)[idx] = rep;
+ /*
+ * Fill NIC descriptor with the new buffer. The lkey and size
+ * of the buffers are already known, only the buffer address
+ * changes.
+ */
+ wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+ wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+ }
+ if (len > DATA_LEN(seg)) { /* more data: move to next segment */
+ if (seg->pool)
+ data_seg_len += DATA_LEN(seg);
len -= DATA_LEN(seg);
- ++NB_SEGS(pkt);
+ if (pkt)
+ ++NB_SEGS(pkt);
++rq_ci;
continue;
}
- DATA_LEN(seg) = len;
+ if (seg->pool) { /* last segment */
+ DATA_LEN(seg) = len;
+ data_seg_len += len;
+ }
+ if (unlikely(!pkt)) { /* no real segment found, skip packet */
+ len = 0;
+ rq_ci >>= sges_n;
+ ++rq_ci;
+ rq_ci <<= sges_n;
+ continue;
+ }
+ PKT_LEN(pkt) = RTE_MIN(PKT_LEN(pkt), data_seg_len);
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment bytes counter. */
rxq->stats.ibytes += PKT_LEN(pkt);
#endif
+ data_seg_len = 0;
/* Return packet. */
*(pkts++) = pkt;
pkt = NULL;
+ len = 0;
--pkts_n;
++i;
/* Align consumer index to the next stride. */
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 01b563d981..cd48ee37ef 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -96,6 +96,7 @@ struct mlx5_eth_rxseg {
uint16_t length; /**< Segment data length, configures split point. */
uint16_t offset; /**< Data offset from beginning of mbuf data buffer. */
uint32_t reserved; /**< Reserved field. */
+ struct rte_mbuf *null_mbuf; /**< For selective Rx. */
};
/* RX queue descriptor. */
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 48d982a8c2..25dba7f4d9 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -151,26 +151,30 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
struct mlx5_eth_rxseg *seg = &rxq_ctrl->rxq.rxseg[i % sges_n];
struct rte_mbuf *buf;
- buf = rte_pktmbuf_alloc(seg->mp);
- if (buf == NULL) {
- if (rxq_ctrl->share_group == 0)
- DRV_LOG(ERR, "port %u queue %u empty mbuf pool",
- RXQ_PORT_ID(rxq_ctrl),
- rxq_ctrl->rxq.idx);
- else
- DRV_LOG(ERR, "share group %u queue %u empty mbuf pool",
- rxq_ctrl->share_group,
- rxq_ctrl->share_qid);
- rte_errno = ENOMEM;
- goto error;
+ if (seg->mp) {
+ buf = rte_pktmbuf_alloc(seg->mp);
+ if (buf == NULL) {
+ if (rxq_ctrl->share_group == 0)
+ DRV_LOG(ERR, "port %u queue %u empty mbuf pool",
+ RXQ_PORT_ID(rxq_ctrl),
+ rxq_ctrl->rxq.idx);
+ else
+ DRV_LOG(ERR, "share group %u queue %u empty mbuf pool",
+ rxq_ctrl->share_group,
+ rxq_ctrl->share_qid);
+ rte_errno = ENOMEM;
+ goto error;
+ }
+ /* Only vectored Rx routines rely on headroom size. */
+ MLX5_ASSERT(!has_vec_support ||
+ DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
+ /* Buffer is supposed to be empty. */
+ MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0);
+ MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0);
+ MLX5_ASSERT(!buf->next);
+ } else {
+ buf = seg->null_mbuf;
}
- /* Only vectored Rx routines rely on headroom size. */
- MLX5_ASSERT(!has_vec_support ||
- DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
- /* Buffer is supposed to be empty. */
- MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0);
- MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0);
- MLX5_ASSERT(!buf->next);
SET_DATA_OFF(buf, seg->offset);
PORT(buf) = rxq_ctrl->rxq.port_id;
DATA_LEN(buf) = seg->length;
@@ -324,10 +328,14 @@ rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
rxq->rq_pi = elts_ci;
}
for (i = 0; i != q_n; ++i) {
- if ((*rxq->elts)[i] != NULL)
+ if ((*rxq->elts)[i] != NULL && (*rxq->elts)[i]->pool != NULL)
rte_pktmbuf_free_seg((*rxq->elts)[i]);
(*rxq->elts)[i] = NULL;
}
+ for (i = 0; i < rxq->rxseg_n; i++) {
+ mlx5_free(rxq->rxseg[i].null_mbuf);
+ rxq->rxseg[i].null_mbuf = NULL;
+ }
}
/**
@@ -1815,7 +1823,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
int ret;
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *tmpl;
- unsigned int mb_len = rte_pktmbuf_data_room_size(rx_seg[0].mp);
+ struct rte_mempool *first_mp = NULL;
+ struct rte_mempool *last_mp = NULL;
+ unsigned int mb_len;
struct mlx5_port_config *config = &priv->config;
uint64_t offloads = conf->offloads |
dev->data->dev_conf.rxmode.offloads;
@@ -1827,7 +1837,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int non_scatter_min_mbuf_size = max_rx_pktlen +
RTE_PKTMBUF_HEADROOM;
unsigned int max_lro_size = 0;
- unsigned int first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
+ unsigned int first_mb_free_size;
uint32_t mprq_log_actual_stride_num = 0;
uint32_t mprq_log_actual_stride_size = 0;
bool rx_seg_en = n_seg != 1 || rx_seg[0].offset || rx_seg[0].length;
@@ -1845,6 +1855,21 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
const struct rte_eth_rxseg_split *qs_seg = rx_seg;
unsigned int tail_len;
+ /* Find first segment with a mempool. */
+ for (uint16_t seg = 0; seg < n_seg; seg++) {
+ if (rx_seg[seg].mp != NULL) {
+ first_mp = rx_seg[seg].mp;
+ break;
+ }
+ }
+ if (first_mp == NULL) {
+ DRV_LOG(ERR, "port %u Rx queue %u has no mempool", dev->data->port_id, idx);
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ mb_len = rte_pktmbuf_data_room_size(first_mp);
+ first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
+
if (mprq_en) {
/* Trim the number of descs needed. */
desc >>= mprq_log_actual_stride_num;
@@ -1884,35 +1909,44 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
do {
struct mlx5_eth_rxseg *hw_seg =
&tmpl->rxq.rxseg[tmpl->rxq.rxseg_n];
- uint32_t buf_len, offset, seg_len;
+ uint32_t buf_len = 0, offset, seg_len;
/*
* For the buffers beyond descriptions offset is zero,
* the first buffer contains head room.
*/
- buf_len = rte_pktmbuf_data_room_size(qs_seg->mp);
+ if (qs_seg->mp != NULL) {
+ last_mp = qs_seg->mp;
+ buf_len = rte_pktmbuf_data_room_size(qs_seg->mp);
+ } else if (last_mp != NULL) {
+ buf_len = rte_pktmbuf_data_room_size(last_mp);
+ } else {
+ buf_len = mb_len;
+ }
offset = (tmpl->rxq.rxseg_n >= n_seg ? 0 : qs_seg->offset) +
(tmpl->rxq.rxseg_n ? 0 : RTE_PKTMBUF_HEADROOM);
/*
* For the buffers beyond descriptions the length is
* pool buffer length, zero lengths are replaced with
- * pool buffer length either.
+ * pool buffer length for real segments,
+ * or remaining packet length for discard segments.
*/
seg_len = tmpl->rxq.rxseg_n >= n_seg ? buf_len :
qs_seg->length ?
qs_seg->length :
- (buf_len - offset);
+ qs_seg->mp != NULL ?
+ (buf_len - offset) : tail_len;
/* Check is done in long int, now overflows. */
- if (buf_len < seg_len + offset) {
+ if (qs_seg->mp != NULL && buf_len < seg_len + offset) {
DRV_LOG(ERR, "port %u Rx queue %u: Split offset/length "
"%u/%u can't be satisfied",
dev->data->port_id, idx,
- qs_seg->length, qs_seg->offset);
+ qs_seg->offset, qs_seg->length);
rte_errno = EINVAL;
goto error;
}
if (seg_len > tail_len)
- seg_len = buf_len - offset;
+ seg_len = qs_seg->mp != NULL ? buf_len - offset : tail_len;
if (++tmpl->rxq.rxseg_n > MLX5_MAX_RXQ_NSEG) {
DRV_LOG(ERR,
"port %u too many SGEs (%u) needed to handle"
@@ -2077,7 +2111,8 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save port ID. */
tmpl->rxq.port_id = dev->data->port_id;
tmpl->sh = priv->sh;
- tmpl->rxq.mp = rx_seg[0].mp;
+ tmpl->rxq.sh = priv->sh;
+ tmpl->rxq.mp = first_mp;
tmpl->rxq.elts_n = log2above(desc);
tmpl->rxq.rq_repl_thresh = MLX5_VPMD_RXQ_RPLNSH_THRESH(desc_n);
tmpl->rxq.elts = (struct rte_mbuf *(*)[])(tmpl + 1);
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index a070aaecfd..ac966c51b4 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -116,6 +116,27 @@ mlx5_txq_start(struct rte_eth_dev *dev)
return -rte_errno;
}
+static struct rte_mbuf *
+mlx5_alloc_null_mbuf(uint32_t data_len)
+{
+ size_t alloc_size = sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM +
+ rte_align32pow2(data_len);
+ struct rte_mbuf *m;
+
+ m = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, 0, SOCKET_ID_ANY);
+ if (m == NULL)
+ return NULL;
+ m->buf_addr = RTE_PTR_ADD(m, sizeof(*m));
+ m->buf_len = alloc_size - sizeof(*m);
+ rte_mbuf_iova_set(m, rte_mem_virt2iova(m->buf_addr));
+ m->data_off = RTE_PKTMBUF_HEADROOM;
+ m->refcnt = 1;
+ m->nb_segs = 1;
+ m->port = RTE_MBUF_PORT_INVALID;
+ m->pool = NULL;
+ return m;
+}
+
/**
* Register Rx queue mempools and fill the Rx queue cache.
* This function tolerates repeated mempool registration.
@@ -130,7 +151,8 @@ static int
mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
{
struct rte_mempool *mp;
- uint32_t s;
+ struct mlx5_eth_rxseg *seg;
+ uint16_t s;
int ret = 0;
mlx5_mr_flush_local_cache(&rxq_ctrl->rxq.mr_ctrl);
@@ -139,21 +161,35 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
return mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
rxq_ctrl->rxq.mprq_mp);
for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
- bool is_extmem;
-
- mp = rxq_ctrl->rxq.rxseg[s].mp;
- is_extmem = (rte_pktmbuf_priv_flags(mp) &
- RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
- ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp,
- is_extmem);
- if (ret < 0 && rte_errno != EEXIST)
- return ret;
- ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
- mp);
- if (ret < 0)
- return ret;
+ seg = &rxq_ctrl->rxq.rxseg[s];
+ mp = seg->mp;
+ if (mp) { /* Regular segment */
+ bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
+ RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
+ ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp, is_extmem);
+ if (ret < 0 && rte_errno != EEXIST)
+ goto error;
+ ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl, mp);
+ if (ret < 0)
+ goto error;
+ } else { /* NULL segment used in selective Rx */
+ seg->null_mbuf = mlx5_alloc_null_mbuf(seg->length);
+ if (seg->null_mbuf == NULL) {
+ rte_errno = ENOMEM;
+ ret = -rte_errno;
+ goto error;
+ }
+ }
}
return 0;
+
+error:
+ while (s-- > 0) {
+ seg = &rxq_ctrl->rxq.rxseg[s];
+ mlx5_free(seg->null_mbuf);
+ seg->null_mbuf = NULL;
+ }
+ return ret;
}
/**
--
2.54.0
^ permalink raw reply related
* [PATCH v9 05/10] net/mlx5: fix Rx split segment counter type
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
In the API, rx_nseg and max_nseg are uint16_t.
In mlx5, MLX5_MAX_RXQ_NSEG is 32.
So there is no reason to have rxseg_n as uint32_t.
Reduce the fields to uint16_t and move them to avoid struct holes.
Fixes: 9f209b59c8b0 ("net/mlx5: support Rx buffer split description")
Fixes: 572c9d4bda08 ("net/mlx5: fix shared Rx queue segment configuration match")
Cc: stable@dpdk.org
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/net/mlx5/mlx5_rx.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index dffab3955b..01b563d981 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -164,9 +164,9 @@ struct __rte_cache_aligned mlx5_rxq_data {
uint64_t flow_meta_mask;
int32_t flow_meta_offset;
uint32_t flow_meta_port_mask;
- uint32_t rxseg_n; /* Number of split segment descriptions. */
struct mlx5_eth_rxseg rxseg[MLX5_MAX_RXQ_NSEG];
/* Buffer split segment descriptions - sizes, offsets, pools. */
+ uint16_t rxseg_n; /* Number of split segment descriptions. */
uint16_t rq_win_cnt; /* Number of packets in the sliding window data. */
uint16_t rq_win_idx_mask; /* Sliding window index wrapping mask. */
uint16_t rq_win_idx; /* Index of the first element in sliding window. */
@@ -191,9 +191,9 @@ struct mlx5_rxq_ctrl {
unsigned int irq:1; /* Whether IRQ is enabled. */
uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
uint32_t wqn; /* WQ number. */
- uint32_t rxseg_n; /* Number of split segment descriptions. */
struct rte_eth_rxseg_split rxseg[MLX5_MAX_RXQ_NSEG];
/* Saved original buffer split segment configuration. */
+ uint16_t rxseg_n; /* Number of split segment descriptions. */
uint16_t dump_file_n; /* Number of dump files. */
};
--
2.54.0
^ permalink raw reply related
* [PATCH v9 04/10] common/mlx5: add null MR functions
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Gregory Etelson, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Add functions to allocate and free a null Memory Region (MR)
using ibverbs on Linux.
There is no implementation for DevX on Windows.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/common/mlx5/linux/mlx5_common_verbs.c | 35 +++++++++++++++++++
drivers/common/mlx5/mlx5_common_mr.h | 9 +++++
drivers/common/mlx5/windows/mlx5_common_os.c | 16 +++++++++
3 files changed, 60 insertions(+)
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index 2322d9d033..6d44e1f566 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -161,3 +161,38 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
*reg_mr_cb = mlx5_common_verbs_reg_mr;
*dereg_mr_cb = mlx5_common_verbs_dereg_mr;
}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
+{
+ struct ibv_mr *ibv_mr;
+ struct mlx5_pmd_mr *null_mr;
+
+ null_mr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*null_mr), 0, dev->numa_node);
+ if (!null_mr)
+ return NULL;
+ ibv_mr = mlx5_glue->alloc_null_mr(pd);
+ if (!ibv_mr) {
+ mlx5_free(null_mr);
+ return NULL;
+ }
+ *null_mr = (struct mlx5_pmd_mr) {
+ .lkey = rte_cpu_to_be_32(ibv_mr->lkey),
+ .addr = ibv_mr->addr,
+ .len = ibv_mr->length,
+ .obj = (void *)ibv_mr,
+ };
+ return null_mr;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_free_null_mr)
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr)
+{
+ if (!null_mr)
+ return;
+ if (null_mr->obj)
+ claim_zero(mlx5_glue->dereg_mr(null_mr->obj));
+ mlx5_free(null_mr);
+}
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index cf7c685e9b..00f3d832c3 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -21,6 +21,8 @@
#include "mlx5_common_mp.h"
#include "mlx5_common_defs.h"
+struct rte_device;
+
/* mlx5 PMD MR struct. */
struct mlx5_pmd_mr {
uint32_t lkey;
@@ -258,6 +260,13 @@ __rte_internal
void
mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+__rte_internal
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd);
+__rte_internal
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr);
+
__rte_internal
int
mlx5_mr_mempool_register(struct mlx5_common_device *cdev,
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index a3033f5305..fb2bbae578 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -454,6 +454,22 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
*dereg_mr_cb = mlx5_os_dereg_mr;
}
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
+{
+ RTE_SET_USED(dev);
+ RTE_SET_USED(pd);
+ return NULL;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_free_null_mr)
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr)
+{
+ RTE_SET_USED(null_mr);
+}
+
/*
* In Windows, no need to wrap the MR, no known issue for it in kernel.
* Use the regular function to create direct MR.
--
2.54.0
^ permalink raw reply related
* [PATCH v9 03/10] app/testpmd: support selective Rx
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Gregory Etelson, Aman Singh
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Add support for selective Rx using existing rxpkts and mbuf-size
command line parameters.
When a segment is specified with rxpkts and a matching 0 mbuf-size
on PMDs supporting selective Rx,
testpmd set the mempool of the segment to NULL,
meaning the segment won't be received.
Example usage to receive only Ethernet header and 64 bytes at offset 128:
--rxpkts=14,114,64,0 --mbuf-size=256,0,256,0
This creates segments:
- [0-13]: 14 bytes with mempool (received)
- [14-127]: 114 bytes with NULL mempool (discarded)
- [128-191]: 64 bytes with mempool (received)
- [192-max]: remaining bytes with NULL mempool (discarded)
If the first segment has no mempool,
there will be no mempool created with the index 0.
That's why the lookup of the first mempool is now achieved
in the new function mbuf_pool_find_first(socket)
instead of mbuf_pool_find(socket, index 0)
Note: RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT is required for this feature
and is checked at ethdev API level.
This check is removed from testpmd to allow negative testing of the API.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
app/test-pmd/cmdline.c | 2 +-
app/test-pmd/parameters.c | 5 +--
app/test-pmd/testpmd.c | 48 +++++++++++++--------
app/test-pmd/testpmd.h | 16 +++++++
doc/guides/testpmd_app_ug/run_app.rst | 16 +++++++
doc/guides/testpmd_app_ug/testpmd_funcs.rst | 3 +-
6 files changed, 66 insertions(+), 24 deletions(-)
diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index cc9c462498..3c39e27aa8 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -3076,7 +3076,7 @@ cmd_setup_rxtx_queue_parsed(
if (!numa_support || socket_id == NUMA_NO_CONFIG)
socket_id = port->socket_id;
- mp = mbuf_pool_find(socket_id, 0);
+ mp = mbuf_pool_find_first(socket_id);
if (mp == NULL) {
fprintf(stderr,
"Failed to setup RX queue: No mempool allocation on the socket %d\n",
diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index ecbd618f00..337d8fc8ac 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -1170,10 +1170,9 @@ launch_args_parse(int argc, char** argv)
rte_exit(EXIT_FAILURE,
"bad mbuf-size\n");
for (i = 0; i < nb_segs; i++) {
- if (mb_sz[i] <= 0 || mb_sz[i] > 0xFFFF)
+ if (mb_sz[i] > 0xFFFF)
rte_exit(EXIT_FAILURE,
- "mbuf-size should be "
- "> 0 and < 65536\n");
+ "mbuf-size should be < 65536\n");
mbuf_data_size[i] = (uint16_t) mb_sz[i];
}
mbuf_data_size_n = nb_segs;
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index a9b35f530a..fcd8a90967 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -1806,19 +1806,25 @@ init_config(void)
uint8_t i, j;
for (i = 0; i < num_sockets; i++)
- for (j = 0; j < mbuf_data_size_n; j++)
+ for (j = 0; j < mbuf_data_size_n; j++) {
+ if (mbuf_data_size[j] == 0)
+ continue;
mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
mbuf_pool_create(mbuf_data_size[j],
nb_mbuf_per_pool,
socket_ids[i], j);
+ }
} else {
uint8_t i;
- for (i = 0; i < mbuf_data_size_n; i++)
+ for (i = 0; i < mbuf_data_size_n; i++) {
+ if (mbuf_data_size[i] == 0)
+ continue;
mempools[i] = mbuf_pool_create
(mbuf_data_size[i],
nb_mbuf_per_pool,
SOCKET_ID_ANY, i);
+ }
}
init_port_config();
@@ -1831,11 +1837,11 @@ init_config(void)
* Records which Mbuf pool to use by each logical core, if needed.
*/
for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
- mbp = mbuf_pool_find(
- rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
+ mbp = mbuf_pool_find_first(
+ rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
if (mbp == NULL)
- mbp = mbuf_pool_find(0, 0);
+ mbp = mbuf_pool_find_first(0);
fwd_lcores[lc_id]->mbp = mbp;
#ifdef RTE_LIB_GSO
/* initialize GSO context */
@@ -2744,31 +2750,35 @@ rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
uint32_t prev_hdrs = 0;
int ret;
- if ((rx_pkt_nb_segs > 1) &&
- (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ if (multi_rx_mempool == 0 &&
+ (rx_pkt_nb_segs > 1 || mbuf_data_size_n > 1)) {
+ unsigned int nb_segs = RTE_MAX(rx_pkt_nb_segs, (uint8_t)mbuf_data_size_n);
+
/* multi-segment configuration */
- for (i = 0; i < rx_pkt_nb_segs; i++) {
+ for (i = 0; i < nb_segs; i++) {
struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
/*
* Use last valid pool for the segments with number
* exceeding the pool index.
*/
mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
- mpx = mbuf_pool_find(socket_id, mp_n);
- /* Handle zero as mbuf data buffer size. */
rx_seg->offset = i < rx_pkt_nb_offs ?
rx_pkt_seg_offsets[i] : 0;
- rx_seg->mp = mpx ? mpx : mp;
+ if (mbuf_data_size[mp_n] == 0) {
+ rx_seg->mp = NULL;
+ } else {
+ mpx = mbuf_pool_find(socket_id, mp_n);
+ rx_seg->mp = mpx ? mpx : mp;
+ }
if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) {
rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs;
prev_hdrs |= rx_seg->proto_hdr;
} else {
- rx_seg->length = rx_pkt_seg_lengths[i] ?
- rx_pkt_seg_lengths[i] :
- mbuf_data_size[mp_n];
+ rx_seg->length = i < rx_pkt_nb_segs ?
+ rx_pkt_seg_lengths[i] : 0;
}
}
- rx_conf->rx_nseg = rx_pkt_nb_segs;
+ rx_conf->rx_nseg = nb_segs;
rx_conf->rx_seg = rx_useg;
rx_conf->rx_mempools = NULL;
rx_conf->rx_nmempool = 0;
@@ -3126,8 +3136,8 @@ start_port(portid_t pid)
if ((numa_support) &&
(rxring_numa[pi] != NUMA_NO_CONFIG)) {
struct rte_mempool * mp =
- mbuf_pool_find
- (rxring_numa[pi], 0);
+ mbuf_pool_find_first
+ (rxring_numa[pi]);
if (mp == NULL) {
fprintf(stderr,
"Failed to setup RX queue: No mempool allocation on the socket %d\n",
@@ -3142,9 +3152,9 @@ start_port(portid_t pid)
mp);
} else {
struct rte_mempool *mp =
- mbuf_pool_find
+ mbuf_pool_find_first
((numa_support ? port->socket_id :
- (unsigned int)SOCKET_ID_ANY), 0);
+ (unsigned int)SOCKET_ID_ANY));
if (mp == NULL) {
fprintf(stderr,
"Failed to setup RX queue: No mempool allocation on the socket %d\n",
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 1a54535470..3d4b36d668 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -895,6 +895,22 @@ mbuf_pool_find(unsigned int sock_id, uint16_t idx)
return rte_mempool_lookup((const char *)pool_name);
}
+static inline struct rte_mempool *
+mbuf_pool_find_first(unsigned int sock_id)
+{
+ struct rte_mempool *mp;
+ uint16_t idx;
+
+ for (idx = 0; idx < mbuf_data_size_n; idx++) {
+ if (mbuf_data_size[idx] == 0) /* no mempool with this index */
+ continue;
+ mp = mbuf_pool_find(sock_id, idx);
+ if (mp != NULL)
+ return mp;
+ }
+ return NULL;
+}
+
static inline uint16_t
common_fwd_stream_receive(struct fwd_stream *fs, struct rte_mbuf **burst,
unsigned int nb_pkts)
diff --git a/doc/guides/testpmd_app_ug/run_app.rst b/doc/guides/testpmd_app_ug/run_app.rst
index 1a4a4b6c12..d654484546 100644
--- a/doc/guides/testpmd_app_ug/run_app.rst
+++ b/doc/guides/testpmd_app_ug/run_app.rst
@@ -127,6 +127,7 @@ The command line options are:
The default value is 2048. If multiple mbuf-size values are specified the
extra memory pools will be created for allocating mbufs to receive packets
with buffer splitting features.
+ A value of 0 indicates a discarded segment in buffer split.
* ``--total-num-mbufs=N``
@@ -372,6 +373,21 @@ The command line options are:
Optionally the multiple memory pools can be specified with --mbuf-size
command line parameter and the mbufs to receive will be allocated
sequentially from these extra memory pools.
+ A length of 0 means maximum length: rest of the segment
+ or all remaining packet data in case of a discard segment.
+
+ To receive only the Ethernet header (14 bytes)
+ and a 64-byte segment starting at offset 128,
+ while discarding the rest::
+
+ --rxpkts=14,114,64,0 --mbuf-size=256,0,256,0
+
+ This configuration will:
+
+ * Receive 14 bytes (Ethernet header)
+ * Discard 114 bytes (NULL mempool segment)
+ * Receive 64 bytes
+ * Discard remaining bytes (NULL mempool segment, length=0)
* ``--txpkts=X[,Y]``
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index d50921258a..f0f2b0758b 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -850,7 +850,8 @@ mbuf for remaining segments will be allocated from the last valid pool).
testpmd> set rxpkts (x[,y]*)
Where x[,y]* represents a CSV list of values, without white space. Zero value
-means to use the corresponding memory pool data buffer size.
+means to use the corresponding memory pool data buffer size,
+or to discard all remaining packet data for a discard segment (mbuf-size=0).
set rxhdrs
~~~~~~~~~~
--
2.54.0
^ permalink raw reply related
* [PATCH v9 02/10] ethdev: introduce selective Rx
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Gregory Etelson, Andrew Rybchenko, Aman Singh
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Receiving an entire packet is not always needed.
The Rx performance can be improved by receiving only partial data
and safely discard the rest of the packet data,
because it reduces the PCI bandwidth and the memory consumption.
Selective Rx allows an application to receive
only pre-configured packet segments and discard the rest.
For example:
- Deliver the first N bytes only.
- Deliver the last N bytes only.
- Deliver N1 bytes from offset Off1 and N2 bytes from offset Off2.
Selective Rx is implemented on top of the Rx buffer split API:
- rte_eth_rxseg_split uses the null mempool for segments
that should be discarded.
- the PMD does not create mbuf segments if no data read.
For example: Deliver Ethernet header only
Rx queue segments configuration:
struct rte_eth_rxseg_split split[2] = {
{
.mp = <some mempool>,
.length = sizeof(struct rte_ether_hdr)
},
{
.mp = NULL, /* discard data */
.length = 0 /* default to buffer size */
}
};
Received mbuf:
pkt_len = sizeof(struct rte_ether_hdr);
data_len = sizeof(struct rte_ether_hdr);
next = NULL; /* The next segment did not deliver data */
After selective Rx, the mbuf packet length reflects only the data
that was actually received,
and can be less than the original wire packet length.
A PMD activates the selective Rx capability by setting
the rte_eth_rxseg_capa.selective_rx bit.
This new capability bit is inserted in a bitmap hole
of the struct rte_eth_rxseg_capa,
but it needs to be ignored in the ABI check as libabigail sees a change.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
Reviewed-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
---
app/test-pmd/config.c | 1 +
devtools/libabigail.abignore | 7 +++++++
doc/guides/nics/features.rst | 14 ++++++++++++++
doc/guides/nics/features/default.ini | 1 +
doc/guides/rel_notes/release_26_07.rst | 7 +++++++
lib/ethdev/rte_ethdev.c | 24 ++++++++++++++++--------
lib/ethdev/rte_ethdev.h | 17 +++++++++++++++--
7 files changed, 61 insertions(+), 10 deletions(-)
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 55d1c6d696..9d457ca88e 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -925,6 +925,7 @@ port_infos_display(portid_t port_id)
print_bool_capa("\tBuffer offset", dev_info.rx_seg_capa.offset_allowed);
printf("\tOffset alignment: %u\n",
RTE_BIT32(dev_info.rx_seg_capa.offset_align_log2));
+ print_bool_capa("\tSelective Rx", dev_info.rx_seg_capa.selective_rx);
}
if (dev_info.max_vfs)
diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
index 21b8cd6113..2a0efd718e 100644
--- a/devtools/libabigail.abignore
+++ b/devtools/libabigail.abignore
@@ -33,3 +33,10 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Temporary exceptions till next major ABI version ;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; Ignore new bit selective_rx in rte_eth_rxseg_capa bitmap hole
+[suppress_type]
+ name = rte_eth_rxseg_capa
+ type_kind = struct
+ has_size_change = no
+ has_data_member_inserted_at = 6
diff --git a/doc/guides/nics/features.rst b/doc/guides/nics/features.rst
index a075c057ec..26357036ca 100644
--- a/doc/guides/nics/features.rst
+++ b/doc/guides/nics/features.rst
@@ -199,6 +199,20 @@ Scatters the packets being received on specified boundaries to segmented mbufs.
* **[related] API**: ``rte_eth_rx_queue_setup()``, ``rte_eth_buffer_split_get_supported_hdr_ptypes()``.
+.. _nic_features_selective_rx:
+
+Selective Rx
+------------
+
+Discards some segments of buffer split on Rx.
+
+* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT``.
+* **[uses] rte_eth_rxconf**: ``rx_seg.mp = NULL`` to discard segments.
+* **[provides] rte_eth_dev_info**: ``rx_offload_capa:RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT``.
+* **[provides] rte_eth_dev_info**: ``rx_seg_capa.selective_rx``.
+* **[related] API**: ``rte_eth_rx_queue_setup()``.
+
+
.. _nic_features_lro:
LRO
diff --git a/doc/guides/nics/features/default.ini b/doc/guides/nics/features/default.ini
index e50514d750..8303a530c1 100644
--- a/doc/guides/nics/features/default.ini
+++ b/doc/guides/nics/features/default.ini
@@ -25,6 +25,7 @@ Burst mode info =
Power mgmt address monitor =
MTU update =
Buffer split on Rx =
+Selective Rx =
Scattered Rx =
LRO =
TSO =
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index d2563ac503..46a8fe2cc1 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -87,6 +87,13 @@ New Features
Added no-IOMMU mode for devices without or not enabling IOMMU/SVA.
+* **Added selective Rx in ethdev API.**
+
+ Some parts of packets may be discarded in Rx
+ by configuring a split of packets received in a queue,
+ and assigning no mempool to some configuration segments.
+ This is a driver capability advertised in the ``selective_rx`` bit.
+
* **Added LinkData sxe2 ethernet driver.**
Added network driver for the LinkData network adapters.
diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c
index ce0407b67f..9efeaf77cb 100644
--- a/lib/ethdev/rte_ethdev.c
+++ b/lib/ethdev/rte_ethdev.c
@@ -2129,7 +2129,7 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
const struct rte_eth_dev_info *dev_info)
{
const struct rte_eth_rxseg_capa *seg_capa = &dev_info->rx_seg_capa;
- struct rte_mempool *mp_first;
+ struct rte_mempool *mp_first = NULL;
uint32_t offset_mask;
uint16_t seg_idx;
int ret = 0;
@@ -2148,7 +2148,6 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
* Check the sizes and offsets against buffer sizes
* for each segment specified in extended configuration.
*/
- mp_first = rx_seg[0].mp;
offset_mask = RTE_BIT32(seg_capa->offset_align_log2) - 1;
ptypes = NULL;
@@ -2160,13 +2159,17 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
uint32_t offset = rx_seg[seg_idx].offset;
uint32_t proto_hdr = rx_seg[seg_idx].proto_hdr;
- if (mpl == NULL) {
- RTE_ETHDEV_LOG_LINE(ERR, "null mempool pointer");
- ret = -EINVAL;
- goto out;
+ if (mpl == NULL) { /* discarded segment */
+ if (seg_capa->selective_rx == 0) { /* not supported */
+ RTE_ETHDEV_LOG_LINE(ERR, "null mempool pointer");
+ ret = -EINVAL;
+ goto out;
+ }
+ continue; /* next checks are not relevant if no mempool */
}
- if (seg_idx != 0 && mp_first != mpl &&
- seg_capa->multi_pools == 0) {
+ if (mp_first == NULL)
+ mp_first = mpl;
+ if (mp_first != mpl && seg_capa->multi_pools == 0) {
RTE_ETHDEV_LOG_LINE(ERR, "Receiving to multiple pools is not supported");
ret = -ENOTSUP;
goto out;
@@ -2233,6 +2236,11 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
if (ret != 0)
goto out;
}
+ if (mp_first == NULL) {
+ RTE_ETHDEV_LOG_LINE(ERR, "At least one Rx segment must have a mempool");
+ ret = -EINVAL;
+ goto out;
+ }
out:
free(ptypes);
return ret;
diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h
index dedbc05554..ee400b386f 100644
--- a/lib/ethdev/rte_ethdev.h
+++ b/lib/ethdev/rte_ethdev.h
@@ -1073,6 +1073,7 @@ struct rte_eth_txmode {
* - The first network buffer will be allocated from the memory pool,
* specified in the first array element, the second buffer, from the
* pool in the second element, and so on.
+ * If the pool is NULL, the segment will be discarded, i.e. not received.
*
* - The proto_hdrs in the elements define the split position of
* received packets.
@@ -1090,7 +1091,8 @@ struct rte_eth_txmode {
*
* - If the length in the segment description element is zero
* the actual buffer size will be deduced from the appropriate
- * memory pool properties.
+ * memory pool properties, or from the remaining packet length
+ * in case of no memory pool to discard the end of the packet.
*
* - If there is not enough elements to describe the buffer for entire
* packet of maximal length the following parameters will be used
@@ -1121,7 +1123,15 @@ struct rte_eth_txmode {
* The rest will be put into the last valid pool.
*/
struct rte_eth_rxseg_split {
- struct rte_mempool *mp; /**< Memory pool to allocate segment from. */
+ /**
+ * Memory pool to allocate segment from.
+ *
+ * NULL means discarded segment.
+ * Length of discarded segment is not reflected in mbuf packet length
+ * and not accounted in ibytes statistics.
+ * @see rte_eth_rxseg_capa::selective_rx
+ */
+ struct rte_mempool *mp;
uint16_t length; /**< Segment data length, configures split point. */
uint16_t offset; /**< Data offset from beginning of mbuf data buffer. */
/**
@@ -1752,12 +1762,15 @@ struct rte_eth_switch_info {
* @b EXPERIMENTAL: this structure may change without prior notice.
*
* Ethernet device Rx buffer segmentation capabilities.
+ *
+ * @see rte_eth_rxseg_split
*/
struct rte_eth_rxseg_capa {
__extension__
uint32_t multi_pools:1; /**< Supports receiving to multiple pools.*/
uint32_t offset_allowed:1; /**< Supports buffer offsets. */
uint32_t offset_align_log2:4; /**< Required offset alignment. */
+ uint32_t selective_rx:1; /**< Supports discarding segment. */
uint16_t max_nseg; /**< Maximum amount of segments to split. */
uint16_t reserved; /**< Reserved field. */
};
--
2.54.0
^ permalink raw reply related
* [PATCH v9 01/10] app/testpmd: print Rx split capabilities
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Aman Singh
In-Reply-To: <20260605233456.3017423-1-thomas@monjalon.net>
The capabilities from rte_eth_rxseg_capa are added
to the command "show port info".
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
app/test-pmd/config.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index c950793aaf..55d1c6d696 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -790,6 +790,12 @@ rss_offload_types_display(uint64_t offload_types, uint16_t char_num_per_line)
printf("\n");
}
+static void
+print_bool_capa(const char *label, int value)
+{
+ printf("%s: %s\n", label, value ? "supported" : "not supported");
+}
+
void
port_infos_display(portid_t port_id)
{
@@ -911,6 +917,16 @@ port_infos_display(portid_t port_id)
dev_info.max_rx_pktlen);
printf("Maximum configurable size of LRO aggregated packet: %u\n",
dev_info.max_lro_pkt_size);
+
+ printf("Rx split:\n");
+ printf("\tMax segments: %hu\n", dev_info.rx_seg_capa.max_nseg);
+ if (dev_info.rx_seg_capa.max_nseg > 0) {
+ print_bool_capa("\tMulti-pool", dev_info.rx_seg_capa.multi_pools);
+ print_bool_capa("\tBuffer offset", dev_info.rx_seg_capa.offset_allowed);
+ printf("\tOffset alignment: %u\n",
+ RTE_BIT32(dev_info.rx_seg_capa.offset_align_log2));
+ }
+
if (dev_info.max_vfs)
printf("Maximum number of VFs: %u\n", dev_info.max_vfs);
if (dev_info.max_vmdq_pools)
--
2.54.0
^ permalink raw reply related
* [PATCH v9 00/10] selective Rx
From: Thomas Monjalon @ 2026-06-05 23:33 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260202160903.254621-1-getelson@nvidia.com>
This is a new feature in ethdev with tests and mlx5 implementation.
Selective Rx allows to receive partial data,
saving some hardware bandwidth.
v2: rework after Gregory
v3: fix bugs found with AI by Stephen
v4: fix packet type in DTS test
v5: fix mlx5 Rx to handle discarding first segment
v6: fix reindent patch
v7: fix mlx5 CQE error handling + outdated mcqe + redundant assignment
v8: use --mbuf-size 0 in testpmd instead of changing --rxoffs behaviour
v9: fix testpmd and DTS
Gregory Etelson (4):
ethdev: introduce selective Rx
app/testpmd: support selective Rx
common/mlx5: add null MR functions
net/mlx5: support selective Rx
Thomas Monjalon (6):
app/testpmd: print Rx split capabilities
net/mlx5: fix Rx split segment counter type
common/mlx5: remove callbacks for MR registration
dts: fix topology capability comparison
dts: use specific types for Rx/Tx offloads
dts: add selective Rx tests
app/test-pmd/cmdline.c | 2 +-
app/test-pmd/config.c | 17 ++
app/test-pmd/parameters.c | 5 +-
app/test-pmd/testpmd.c | 48 +--
app/test-pmd/testpmd.h | 16 +
devtools/libabigail.abignore | 7 +
doc/guides/nics/features.rst | 14 +
doc/guides/nics/features/default.ini | 1 +
doc/guides/nics/features/mlx5.ini | 1 +
doc/guides/nics/mlx5.rst | 86 ++++--
doc/guides/rel_notes/release_26_07.rst | 11 +
doc/guides/testpmd_app_ug/run_app.rst | 16 +
doc/guides/testpmd_app_ug/testpmd_funcs.rst | 3 +-
drivers/common/mlx5/linux/mlx5_common_verbs.c | 53 ++--
drivers/common/mlx5/mlx5_common.c | 6 +-
drivers/common/mlx5/mlx5_common_mr.c | 37 +--
drivers/common/mlx5/mlx5_common_mr.h | 29 +-
drivers/common/mlx5/windows/mlx5_common_os.c | 31 +-
drivers/compress/mlx5/mlx5_compress.c | 4 +-
drivers/crypto/mlx5/mlx5_crypto.h | 2 -
drivers/crypto/mlx5/mlx5_crypto_gcm.c | 6 +-
drivers/net/mlx5/mlx5.c | 7 +
drivers/net/mlx5/mlx5.h | 4 +-
drivers/net/mlx5/mlx5_ethdev.c | 25 ++
drivers/net/mlx5/mlx5_flow_aso.c | 21 +-
drivers/net/mlx5/mlx5_flow_hw.c | 11 +-
drivers/net/mlx5/mlx5_flow_quota.c | 6 +-
drivers/net/mlx5/mlx5_hws_cnt.c | 19 +-
drivers/net/mlx5/mlx5_rx.c | 187 +++++++-----
drivers/net/mlx5/mlx5_rx.h | 5 +-
drivers/net/mlx5/mlx5_rxq.c | 95 ++++--
drivers/net/mlx5/mlx5_trigger.c | 64 +++-
dts/api/capabilities.py | 2 +
dts/api/testpmd/__init__.py | 17 ++
dts/api/testpmd/config.py | 11 +-
dts/api/testpmd/types.py | 6 +
dts/framework/params/__init__.py | 14 +
dts/framework/params/types.py | 5 +-
dts/framework/testbed_model/capability.py | 10 +-
dts/tests/TestSuite_rx_split.py | 277 ++++++++++++++++++
lib/ethdev/rte_ethdev.c | 24 +-
lib/ethdev/rte_ethdev.h | 17 +-
42 files changed, 921 insertions(+), 301 deletions(-)
create mode 100644 dts/tests/TestSuite_rx_split.py
--
2.54.0
^ permalink raw reply
* Re: [PATCH v8 9/9] dts: add selective Rx tests
From: Thomas Monjalon @ 2026-06-05 23:31 UTC (permalink / raw)
To: Stephen Hemminger; +Cc: dev, Luca Vizzarro, Patrick Robb
In-Reply-To: <20260605142815.1aaf727b@phoenix.local>
Sorry I should have told you that I was working on this today.
I fully agree with the comments you sent as I was fixing it.
I'm going to send a new version which is better reviewed
and properly tested with DTS.
Results
=======
test_suites: PASS
rx_split: PASS
selective_rx_all_discard: PASS
selective_rx_headers: PASS
selective_rx_headers_discard_length: PASS
selective_rx_no_offload: PASS
selective_rx_payload_only: PASS
selective_rx_segment_exceeds_mbuf: PASS
selective_rx_two_segments: PASS
Test Cases Summary
==================
SKIP = 0
PASS = 7
BLOCK = 0
FAIL = 0
ERROR = 0
PASS RATE = 100%
05/06/2026 23:28, Stephen Hemminger:
> AI review found:
> Patch 9 (dts: add selective Rx tests)
>
> selective_rx_out_of_range expects a rejection that never happens, so the
> negative test will fail. It configures a real segment plus an oversized
> discard segment:
>
> rx_segments_length=[ETHER_IP_HDR_LEN, 20000],
> mbuf_size=[256, 0],
>
> and expects start_all_ports() to fail. But an over-range length on a discard
> segment is not rejected anywhere: rte_eth_rx_queue_check_split() does
> "continue" for mp == NULL segments, so it never length-checks them, and
> mlx5_rxq_new() clamps it:
>
> if (seg_len > tail_len)
> seg_len = qs_seg->mp != NULL ? buf_len - offset : tail_len;
>
> The discard seg_len becomes the remaining frame length, the queue is built,
> the port starts, and the test hits its fail().
>
> Clamping an over-long discard to "the rest" is harmless (the bytes are
> discarded anyway), so the cleanest fix is probably to drop or rework this
> test rather than add a rejection path. If rejection is the intended
> contract, it would have to be added for discard segments in patch 2 or
> patch 6 -- a behavior choice, not a correctness requirement.
>
> Minor: expressing a leading discard as --mbuf-size=0,... puts 0 at index 0,
> and testpmd treats mbuf_data_size[0] as the primary pool size elsewhere (the
> max_rx_pkt_len > mbuf_data_size[0] check, the default mbuf_pool_find(socket,
> 0)). Only bites an unusual config, but it is a latent foot-gun.
^ permalink raw reply
* [PATCH 2/2] common/cnxk: fix thread-unsafe NIX telemetry parsing
From: Stephen Hemminger @ 2026-06-05 22:44 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Nithin Dabilpuram, Kiran Kumar K,
Sunil Kumar Kori, Satha Rao, Harman Kalra,
Gowrishankar Muthukrishnan, Jerin Jacob
In-Reply-To: <20260605224514.651081-1-stephen@networkplumber.org>
cnxk_nix_tel_handle_info_x() backs the /cnxk/nix/{rq,cq,sq}/{info,ctx}
telemetry commands and parsed its "<pcidev>,<queue_id>" parameter with
strtok(), which keeps non-reentrant state and races when telemetry
callbacks run on per-connection threads.
Split the parameter with strchr() and parse the queue id with strtoul().
While here, copy the full parameter (the length was capped at
PCI_PRI_STR_SIZE + 1, truncating the id for longer device addresses) and
reject non-numeric or out-of-range ids instead of letting strtol() alias
them to queue 0.
Fixes: af75aac78978 ("common/cnxk: support telemetry for NIX")
Cc: stable@dpdk.org
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/common/cnxk/cnxk_telemetry_nix.c | 80 +++++++++---------------
1 file changed, 30 insertions(+), 50 deletions(-)
diff --git a/drivers/common/cnxk/cnxk_telemetry_nix.c b/drivers/common/cnxk/cnxk_telemetry_nix.c
index abeefafe1e..82a146c139 100644
--- a/drivers/common/cnxk/cnxk_telemetry_nix.c
+++ b/drivers/common/cnxk/cnxk_telemetry_nix.c
@@ -1015,76 +1015,56 @@ cnxk_nix_tel_handle_info_x(const char *cmd, const char *params,
struct plt_tel_data *d)
{
struct nix_tel_node *node;
- char *name, *param;
char buf[1024];
+ char *comma, *end;
+ unsigned long qid;
int rc = -1;
- if (params == NULL || strlen(params) == 0 || !isdigit(*params))
- goto exit;
+ if (params == NULL || !isdigit((unsigned char)params[0]))
+ return -1;
- plt_strlcpy(buf, params, PCI_PRI_STR_SIZE + 1);
- name = strtok(buf, ",");
- if (name == NULL)
- goto exit;
+ plt_strlcpy(buf, params, sizeof(buf)); /* was PCI_PRI_STR_SIZE + 1 */
- param = strtok(NULL, "\0");
+ /* params is "<pcidev_name>,<queue_id>" */
+ comma = strchr(buf, ',');
+ if (comma == NULL || !isdigit((unsigned char)comma[1]))
+ return -1;
+ *comma = '\0';
- node = nix_tel_node_get_by_pcidev_name(name);
- if (!node)
- goto exit;
+ errno = 0;
+ qid = strtoul(comma + 1, &end, 10);
+ if (errno != 0 || (*end != '\0' && *end != ','))
+ return -1;
+
+ node = nix_tel_node_get_by_pcidev_name(buf);
+ if (node == NULL)
+ return -1;
plt_tel_data_start_dict(d);
if (strstr(cmd, "rq")) {
- char *tok = strtok(param, ",");
- int rq;
-
- if (!tok)
- goto exit;
-
- rq = strtol(tok, NULL, 10);
- if ((node->n_rq <= rq) || (rq < 0))
- goto exit;
-
+ if (qid >= node->n_rq)
+ return -1;
if (strstr(cmd, "ctx"))
- rc = cnxk_tel_nix_rq_ctx(node->nix, rq, d);
+ rc = cnxk_tel_nix_rq_ctx(node->nix, qid, d);
else
- rc = cnxk_tel_nix_rq(node->rqs[rq], d);
-
+ rc = cnxk_tel_nix_rq(node->rqs[qid], d);
} else if (strstr(cmd, "cq")) {
- char *tok = strtok(param, ",");
- int cq;
-
- if (!tok)
- goto exit;
-
- cq = strtol(tok, NULL, 10);
- if ((node->n_cq <= cq) || (cq < 0))
- goto exit;
-
+ if (qid >= node->n_cq)
+ return -1;
if (strstr(cmd, "ctx"))
- rc = cnxk_tel_nix_cq_ctx(node->nix, cq, d);
+ rc = cnxk_tel_nix_cq_ctx(node->nix, qid, d);
else
- rc = cnxk_tel_nix_cq(node->cqs[cq], d);
-
+ rc = cnxk_tel_nix_cq(node->cqs[qid], d);
} else if (strstr(cmd, "sq")) {
- char *tok = strtok(param, ",");
- int sq;
-
- if (!tok)
- goto exit;
-
- sq = strtol(tok, NULL, 10);
- if ((node->n_sq <= sq) || (sq < 0))
- goto exit;
-
+ if (qid >= node->n_sq)
+ return -1;
if (strstr(cmd, "ctx"))
- rc = cnxk_tel_nix_sq_ctx(node->nix, sq, d);
+ rc = cnxk_tel_nix_sq_ctx(node->nix, qid, d);
else
- rc = cnxk_tel_nix_sq(node->sqs[sq], d);
+ rc = cnxk_tel_nix_sq(node->sqs[qid], d);
}
-exit:
return rc;
}
--
2.53.0
^ permalink raw reply related
* [PATCH 1/2] net/cnxk: fix telemetry SA info parameter parsing
From: Stephen Hemminger @ 2026-06-05 22:44 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Nithin Dabilpuram, Kiran Kumar K,
Sunil Kumar Kori, Satha Rao, Harman Kalra, Rakesh Kudurumalla
In-Reply-To: <20260605224514.651081-1-stephen@networkplumber.org>
The /cnxk/ipsec/sa_info handler would silently wrap 32 bit value
to 16 bit port id.
An out-of-range port such as 65536 narrowed to a valid port
for the check and then read past the array.
Reject port ids >= RTE_MAX_ETHPORTS before the lookup.
The /cnxk/ipsec/info handler has similar issue with
strtoul().
Rework parse_params() to walk the string with strtoul()/endptr
rather than strtok(), which is not thread safe and races when the
telemetry callbacks run on per-connection threads. This drops the
strdup()/free(), range checks each value against UINT32_MAX, and
passes an unsigned char to isdigit().
Fixes: d74ed1628f7e ("net/cnxk: add SA info telemetry")
Cc: stable@dpdk.org
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/cnxk/cnxk_ethdev_sec_telemetry.c | 50 ++++++++++----------
1 file changed, 24 insertions(+), 26 deletions(-)
diff --git a/drivers/net/cnxk/cnxk_ethdev_sec_telemetry.c b/drivers/net/cnxk/cnxk_ethdev_sec_telemetry.c
index 86c2453c09..0c1533e3d7 100644
--- a/drivers/net/cnxk/cnxk_ethdev_sec_telemetry.c
+++ b/drivers/net/cnxk/cnxk_ethdev_sec_telemetry.c
@@ -211,33 +211,30 @@ copy_inb_sa_10k(struct rte_tel_data *d, uint32_t i, void *sa)
static int
parse_params(const char *params, uint32_t *vals, size_t n_vals)
{
- char dlim[2] = ",";
- char *params_args;
size_t count = 0;
- char *token;
- if (vals == NULL || params == NULL || strlen(params) == 0)
+ if (params == NULL || !isdigit((unsigned char)params[0]))
return -1;
- /* strtok expects char * and param is const char *. Hence on using
- * params as "const char *" compiler throws warning.
- */
- params_args = strdup(params);
- if (params_args == NULL)
- return -1;
+ while (count < n_vals) {
+ char *end;
+ unsigned long v;
- token = strtok(params_args, dlim);
- while (token && isdigit(*token) && count < n_vals) {
- vals[count++] = strtoul(token, NULL, 10);
- token = strtok(NULL, dlim);
- }
+ errno = 0;
+ v = strtoul(params, &end, 10);
+ if (errno != 0 || v > UINT32_MAX)
+ return -EINVAL;
+ vals[count++] = v;
- free(params_args);
+ if (*end == '\0')
+ break;
- if (count < n_vals)
- return -1;
+ if (*end != ',' || !isdigit((unsigned char)end[1]))
+ return -EINVAL;
+ params = end + 1;
+ }
- return 0;
+ return count == n_vals ? 0 : -EINVAL;
}
static int
@@ -252,13 +249,13 @@ ethdev_sec_tel_handle_sa_info(const char *cmd __rte_unused, const char *params,
uint32_t i;
int ret;
- if (params == NULL || strlen(params) == 0 || !isdigit(*params))
- return -EINVAL;
-
if (parse_params(params, vals, RTE_DIM(vals)) < 0)
return -EINVAL;
port_id = vals[0];
+ if (port_id >= RTE_MAX_ETHPORTS)
+ return -EINVAL;
+
sa_idx = vals[1];
if (!rte_eth_dev_is_valid_port(port_id)) {
@@ -320,12 +317,13 @@ ethdev_sec_tel_handle_info(const char *cmd __rte_unused, const char *params,
struct cnxk_eth_sec_sess *eth_sec, *tvar;
struct rte_eth_dev *eth_dev;
struct cnxk_eth_dev *dev;
- uint16_t port_id;
+ unsigned long port_id;
char *end_p;
- if (params == NULL || strlen(params) == 0 || !isdigit(*params))
+ if (params == NULL || !isdigit((unsigned char)*params))
return -EINVAL;
+ errno = 0;
port_id = strtoul(params, &end_p, 0);
if (errno != 0)
return -EINVAL;
@@ -333,8 +331,8 @@ ethdev_sec_tel_handle_info(const char *cmd __rte_unused, const char *params,
if (*end_p != '\0')
plt_err("Extra parameters passed to telemetry, ignoring it");
- if (!rte_eth_dev_is_valid_port(port_id)) {
- plt_err("Invalid port id %u", port_id);
+ if (port_id >= RTE_MAX_ETHPORTS || !rte_eth_dev_is_valid_port(port_id)) {
+ plt_err("Invalid port id %lu", port_id);
return -EINVAL;
}
--
2.53.0
^ permalink raw reply related
* [PATCH 0/2] net/cnxk: harden telemetry parameter parsing
From: Stephen Hemminger @ 2026-06-05 22:44 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
The cnxk telemetry handlers parse their command parameters with strtok(),
which keeps non-reentrant internal state and races when telemetry callbacks
run on per-connection threads. Both handlers also trust the parsed integers
more than they should: values are narrowed to the destination width or
aliased to a valid index before any range check, so an out-of-range port or
queue id can slip through and read past the backing array.
These two patches replace the strtok() walks with strtoul()/endptr parsing,
range-check each value before it is used, and drop the strdup()/free() that
the old SA-info path needed. The NIX handler additionally copies the full
parameter string rather than capping it at PCI_PRI_STR_SIZE + 1, which had
been truncating the queue id for longer device addresses.
Stephen Hemminger (2):
net/cnxk: fix telemetry SA info parameter parsing
common/cnxk: fix thread-unsafe NIX telemetry parsing
drivers/common/cnxk/cnxk_telemetry_nix.c | 80 ++++++++------------
drivers/net/cnxk/cnxk_ethdev_sec_telemetry.c | 50 ++++++------
2 files changed, 54 insertions(+), 76 deletions(-)
--
2.53.0
^ permalink raw reply
* [PATCH v3 6/6] net/gve: reconstruct HW timestamps from DQO
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260605213022.2770893-1-blasko@google.com>
A full 64-bit NIC timestamp is periodically synced via an AdminQ
command and cached in the driver. In the RX datapath, this cached
value is used as a base to expand the 32-bit hardware timestamp into
a full 64-bit value, which is then stored in the mbuf's dynamic
timestamp field.
Signed-off-by: Mark Blasko <blasko@google.com>
Reviewed-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Jasper Tran O'Leary <jtranoleary@google.com>
---
v2:
- Scoped timestamp offload capability advertisement strictly
to DQO queues.
- Predicated capability advertisement directly on memzone
allocation.
- Initialized mbuf_timestamp_offset to -1.
- Added blank line separating release notes.
---
doc/guides/nics/features/gve.ini | 1 +
doc/guides/nics/gve.rst | 20 ++++++++++++++++++++
doc/guides/rel_notes/release_26_07.rst | 4 ++++
drivers/net/gve/base/gve_desc_dqo.h | 8 ++++++--
drivers/net/gve/gve_ethdev.c | 15 ++++++++++++++-
drivers/net/gve/gve_ethdev.h | 25 +++++++++++++++++++++++++
drivers/net/gve/gve_rx_dqo.c | 26 ++++++++++++++++++++++++++
7 files changed, 96 insertions(+), 3 deletions(-)
diff --git a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini
index 89c97fd27a..117ad4fc65 100644
--- a/doc/guides/nics/features/gve.ini
+++ b/doc/guides/nics/features/gve.ini
@@ -13,6 +13,7 @@ RSS hash = Y
RSS key update = Y
RSS reta update = Y
L4 checksum offload = Y
+Timestamp offload = Y
Basic stats = Y
FreeBSD = Y
Linux = Y
diff --git a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst
index be855b645d..b0a02f29bd 100644
--- a/doc/guides/nics/gve.rst
+++ b/doc/guides/nics/gve.rst
@@ -72,6 +72,7 @@ Supported features of the GVE PMD are:
- Tx UDP/TCP/SCTP Checksum
- RSS hash configuration
- RSS redirection table query and update
+- Timestamp offload
Currently, only GQI_QPL and GQI_RDA queue format are supported in PMD.
Jumbo Frame is not supported in PMD for now.
@@ -132,6 +133,25 @@ Security Protocols
- Flow priorities are not supported (must be 0).
- Masking is limited to full matches i.e. ``0x00...0`` or ``0xFF...F``.
+Timestamp Offload
+^^^^^^^^^^^^^^^^^
+
+The driver supports hardware-based packet timestamping on supported
+devices via the standard ``RTE_ETH_RX_OFFLOAD_TIMESTAMP`` offload capability.
+While the ethdev ``.read_clock`` operation works regardless of queue format,
+per-packet RX timestamp offloading requires the DQO queue format.
+
+**Limitations**
+
+- If the driver fails to fetch the NIC hardware clock for 7 consecutive periods,
+ the cached timestamp is marked as stale,
+ and the reconstructed timestamps are no longer propagated to the mbuf.
+- The timestamp reconstruction is only accurate
+ if the time between a packet's reception
+ and the last hardware clock sync is less than approximately 2 seconds.
+ The driver's internal clock sync period is set to respect this limitation.
+
+
Device Reset
^^^^^^^^^^^^
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index 2e449d3ee8..4bd4b9ad93 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -69,6 +69,10 @@ New Features
``rte_eal_init`` and the application is responsible for probing each device,
* ``--auto-probing`` enables the initial bus probing, which is the current default behavior.
+* **Updated Google GVE net driver.**
+
+ * Added hardware timestamping support on DQO queues.
+
* **Added RISC-V vector paths.**
* Increased the default SIMD bitwidth to allow using the vector extension.
diff --git a/drivers/net/gve/base/gve_desc_dqo.h b/drivers/net/gve/base/gve_desc_dqo.h
index 71d9d60bb9..c1534959c2 100644
--- a/drivers/net/gve/base/gve_desc_dqo.h
+++ b/drivers/net/gve/base/gve_desc_dqo.h
@@ -226,7 +226,8 @@ struct gve_rx_compl_desc_dqo {
u8 status_error1;
- __le16 reserved5;
+ u8 reserved5;
+ u8 ts_sub_nsecs_low;
__le16 buf_id; /* Buffer ID which was sent on the buffer queue. */
union {
@@ -237,9 +238,12 @@ struct gve_rx_compl_desc_dqo {
};
__le32 hash;
__le32 reserved6;
- __le64 reserved7;
+ __le32 reserved7;
+ __le32 ts; /* timestamp in nanosecs */
} __packed;
+#define GVE_DQO_RX_HWTSTAMP_VALID 0x1
+
GVE_CHECK_STRUCT_LEN(32, gve_rx_compl_desc_dqo);
/* Ringing the doorbell too often can hurt performance.
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c
index ec9f511a00..e7f4860d2d 100644
--- a/drivers/net/gve/gve_ethdev.c
+++ b/drivers/net/gve/gve_ethdev.c
@@ -214,6 +214,7 @@ static int
gve_dev_configure(struct rte_eth_dev *dev)
{
struct gve_priv *priv = dev->data->dev_private;
+ int err;
if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) {
dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
@@ -223,13 +224,22 @@ gve_dev_configure(struct rte_eth_dev *dev)
if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)
priv->enable_rsc = 1;
+ if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
+ err = rte_mbuf_dyn_rx_timestamp_register(&priv->mbuf_timestamp_offset,
+ &priv->mbuf_timestamp_mask);
+ if (err < 0) {
+ PMD_DRV_LOG(ERR, "Failed to register dynamic timestamp field");
+ return err;
+ }
+ }
+
/* Reset RSS RETA in case number of queues changed. */
if (priv->rss_config.indir) {
struct gve_rss_config update_reta_config;
gve_init_rss_config_from_priv(priv, &update_reta_config);
gve_generate_rss_reta(dev, &update_reta_config);
- int err = gve_adminq_configure_rss(priv, &update_reta_config);
+ err = gve_adminq_configure_rss(priv, &update_reta_config);
if (err)
PMD_DRV_LOG(ERR,
"Could not reconfigure RSS redirection table.");
@@ -821,6 +831,8 @@ gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_RSS_HASH;
+ if (!gve_is_gqi(priv) && priv->nic_ts_report_mz)
+ dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TIMESTAMP;
dev_info->tx_offload_capa =
RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
@@ -1661,6 +1673,7 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
priv->max_nb_txq = max_tx_queues;
priv->max_nb_rxq = max_rx_queues;
+ priv->mbuf_timestamp_offset = -1;
err = gve_init_priv(priv, false);
if (err)
return err;
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index 114531a481..16ba6aa40a 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -260,6 +260,7 @@ struct gve_rx_queue {
struct rte_mbuf **refill_bufs;
uint8_t is_gqi_qpl;
+ bool timestamp_enabled;
};
struct gve_flow {
@@ -369,8 +370,32 @@ struct gve_priv {
RTE_ATOMIC(uint64_t) last_read_nic_timestamp;
RTE_ATOMIC(uint32_t) nic_ts_read_fails;
RTE_ATOMIC(uint8_t) nic_ts_stale;
+
+ int mbuf_timestamp_offset;
+ uint64_t mbuf_timestamp_mask;
};
+/* Expand the hardware timestamp to the full 64 bits of width.
+ *
+ * This algorithm works by using the passed hardware timestamp to generate a
+ * diff relative to the last read of the nic clock. This diff can be positive or
+ * negative, as it is possible that we have read the clock more recently than
+ * the hardware has received this packet. To detect this, we use the high bit of
+ * the diff, and assume that the read is more recent if the high bit is set. In
+ * this case we invert the process.
+ *
+ * Note that this means if the time delta between packet reception and the last
+ * clock read is greater than ~2 seconds, this will provide invalid results.
+ */
+static inline uint64_t
+gve_reconstruct_ts(uint64_t last_sync, uint32_t ts)
+{
+ uint32_t low = (uint32_t)last_sync;
+ int32_t diff = (int32_t)(ts - low);
+
+ return last_sync + diff;
+}
+
static inline bool
gve_is_gqi(struct gve_priv *priv)
{
diff --git a/drivers/net/gve/gve_rx_dqo.c b/drivers/net/gve/gve_rx_dqo.c
index 8035aee572..cc343f3fd8 100644
--- a/drivers/net/gve/gve_rx_dqo.c
+++ b/drivers/net/gve/gve_rx_dqo.c
@@ -160,6 +160,8 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
volatile struct gve_rx_compl_desc_dqo *rx_desc;
struct gve_rx_queue *rxq;
+ uint64_t last_sync = 0;
+ struct gve_priv *priv;
struct rte_mbuf *rxm;
uint16_t rx_buf_id;
uint16_t pkt_len;
@@ -171,6 +173,15 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
nb_rx = 0;
rxq = rx_queue;
rx_id = rxq->rx_tail;
+ priv = rxq->hw;
+
+ if (rxq->timestamp_enabled &&
+ !rte_atomic_load_explicit(&priv->nic_ts_stale,
+ rte_memory_order_acquire)) {
+ last_sync =
+ rte_atomic_load_explicit(&priv->last_read_nic_timestamp,
+ rte_memory_order_relaxed);
+ }
while (nb_rx < nb_pkts) {
rx_desc = &rxq->compl_ring[rx_id];
@@ -208,6 +219,16 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
gve_parse_csum_ol_flags(rxm, rx_desc);
rxm->hash.rss = rte_le_to_cpu_32(rx_desc->hash);
+ if (last_sync != 0 &&
+ (rx_desc->ts_sub_nsecs_low & GVE_DQO_RX_HWTSTAMP_VALID) &&
+ priv->mbuf_timestamp_offset >= 0) {
+ uint32_t ts = rte_le_to_cpu_32(rx_desc->ts);
+ uint64_t full_ts = gve_reconstruct_ts(last_sync, ts);
+
+ *RTE_MBUF_DYNFIELD(rxm, priv->mbuf_timestamp_offset, uint64_t *) = full_ts;
+ rxm->ol_flags |= priv->mbuf_timestamp_mask;
+ }
+
rx_pkts[nb_rx++] = rxm;
bytes += pkt_len;
}
@@ -320,6 +341,11 @@ gve_rx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
return -ENOMEM;
}
+ /* Setup hardware timestamping if enabled */
+ if ((conf->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) ||
+ (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP))
+ rxq->timestamp_enabled = true;
+
/* check free_thresh here */
free_thresh = conf->rx_free_thresh ?
conf->rx_free_thresh : GVE_DEFAULT_RX_FREE_THRESH;
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [PATCH v3 5/6] net/gve: support read clock ethdev op
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260605213022.2770893-1-blasko@google.com>
Implement the read_clock operation in eth_dev_ops. The function calls
the AdminQ command to fetch the current NIC timestamp synchronously,
updates the cached timestamp used for reconstruction, and returns the
full 64-bit value.
Signed-off-by: Mark Blasko <blasko@google.com>
Reviewed-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Jasper Tran O'Leary <jtranoleary@google.com>
---
v3:
- Add mutex lock to protect shared NIC timestamp memzone access.
- Fix missing read_clock assignment to DQO queue ops table
(accidental omission in v2).
v2:
- Scoped read_clock ethdev operation strictly to DQO queues.
---
drivers/net/gve/gve_ethdev.c | 38 ++++++++++++++++++++++++++++++++++++
drivers/net/gve/gve_ethdev.h | 1 +
2 files changed, 39 insertions(+)
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c
index 5d4f1e4ae8..ec9f511a00 100644
--- a/drivers/net/gve/gve_ethdev.c
+++ b/drivers/net/gve/gve_ethdev.c
@@ -463,11 +463,13 @@ gve_read_nic_clock(void *arg)
if (!priv || !priv->nic_ts_report_mz)
return;
+ pthread_mutex_lock(&priv->nic_ts_lock);
memset(priv->nic_ts_report, 0, sizeof(struct gve_nic_ts_report));
err = gve_adminq_report_nic_timestamp(priv, priv->nic_ts_report_mz->iova);
if (err == 0) {
ts = be64_to_cpu(priv->nic_ts_report->nic_timestamp);
+ pthread_mutex_unlock(&priv->nic_ts_lock);
rte_atomic_store_explicit(&priv->last_read_nic_timestamp, ts,
rte_memory_order_relaxed);
PMD_DRV_LOG(DEBUG, "Fetched NIC Timestamp: %" PRIu64, ts);
@@ -476,6 +478,7 @@ gve_read_nic_clock(void *arg)
rte_atomic_store_explicit(&priv->nic_ts_stale, 0,
rte_memory_order_release);
} else {
+ pthread_mutex_unlock(&priv->nic_ts_lock);
PMD_DRV_LOG(ERR, "Failed to read NIC clock, AQ err: %d", err);
fails = rte_atomic_fetch_add_explicit(&priv->nic_ts_read_fails, 1,
rte_memory_order_relaxed) + 1;
@@ -699,6 +702,7 @@ gve_dev_close(struct rte_eth_dev *dev)
gve_teardown_flow_subsystem(priv);
pthread_mutex_destroy(&priv->flow_rule_lock);
+ pthread_mutex_destroy(&priv->nic_ts_lock);
gve_free_queues(dev);
gve_teardown_device_resources(priv);
@@ -1271,6 +1275,38 @@ gve_flow_ops_get(struct rte_eth_dev *dev, const struct rte_flow_ops **ops)
return 0;
}
+static int
+gve_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
+{
+ struct gve_priv *priv = dev->data->dev_private;
+ uint64_t ts;
+ int err;
+
+ if (!priv->nic_timestamp_supported)
+ return -EOPNOTSUPP;
+
+ if (!priv->nic_ts_report_mz)
+ return -EIO;
+
+ pthread_mutex_lock(&priv->nic_ts_lock);
+ err = gve_adminq_report_nic_timestamp(priv, priv->nic_ts_report_mz->iova);
+ if (err != 0) {
+ pthread_mutex_unlock(&priv->nic_ts_lock);
+ return err;
+ }
+
+ ts = be64_to_cpu(priv->nic_ts_report->nic_timestamp);
+ pthread_mutex_unlock(&priv->nic_ts_lock);
+ *clock = ts;
+
+ /* Update the cached value */
+ rte_atomic_store_explicit(&priv->last_read_nic_timestamp, ts, rte_memory_order_relaxed);
+ rte_atomic_store_explicit(&priv->nic_ts_read_fails, 0, rte_memory_order_relaxed);
+ rte_atomic_store_explicit(&priv->nic_ts_stale, 0, rte_memory_order_release);
+
+ return 0;
+}
+
static const struct eth_dev_ops gve_eth_dev_ops = {
.dev_configure = gve_dev_configure,
.dev_start = gve_dev_start,
@@ -1325,6 +1361,7 @@ static const struct eth_dev_ops gve_eth_dev_ops_dqo = {
.rss_hash_conf_get = gve_rss_hash_conf_get,
.reta_update = gve_rss_reta_update,
.reta_query = gve_rss_reta_query,
+ .read_clock = gve_read_clock,
};
static int
@@ -1643,6 +1680,7 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
pthread_mutexattr_init(&mutexattr);
pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(&priv->flow_rule_lock, &mutexattr);
+ pthread_mutex_init(&priv->nic_ts_lock, &mutexattr);
pthread_mutexattr_destroy(&mutexattr);
return 0;
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index 7e6f24e910..114531a481 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -365,6 +365,7 @@ struct gve_priv {
bool nic_timestamp_supported;
const struct rte_memzone *nic_ts_report_mz;
struct gve_nic_ts_report *nic_ts_report;
+ pthread_mutex_t nic_ts_lock;
RTE_ATOMIC(uint64_t) last_read_nic_timestamp;
RTE_ATOMIC(uint32_t) nic_ts_read_fails;
RTE_ATOMIC(uint8_t) nic_ts_stale;
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [PATCH v3 4/6] net/gve: add periodic NIC clock synchronization
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260605213022.2770893-1-blasko@google.com>
Introduce a mechanism to periodically fetch the NIC hardware timestamp
using the GVE_ADMINQ_REPORT_NIC_TIMESTAMP AdminQ command. The
synchronization runs every 250ms using rte_alarm. If the read fails,
the alarm is still rescheduled. After 7 consecutive failures, the
timestamp is marked as stale, indicating to the RX path that
reconstructed timestamps may be unreliable.
Atomics exist because of the potential for async callers (introduced
here) and async callers (introduced later in the RX datapath) accessing
the cached state.
Signed-off-by: Mark Blasko <blasko@google.com>
Reviewed-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Jasper Tran O'Leary <jtranoleary@google.com>
---
v2:
- Removed redundant void* casts.
- Handled alarm reschedule failures by marking timestamp stale.
- Added transient error logging on memzone allocation failure.
---
drivers/net/gve/gve_ethdev.c | 106 +++++++++++++++++++++++++++++++++++
drivers/net/gve/gve_ethdev.h | 9 +++
2 files changed, 115 insertions(+)
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c
index 476b2c311f..5d4f1e4ae8 100644
--- a/drivers/net/gve/gve_ethdev.c
+++ b/drivers/net/gve/gve_ethdev.c
@@ -452,6 +452,86 @@ gve_dev_start(struct rte_eth_dev *dev)
return 0;
}
+static void
+gve_read_nic_clock(void *arg)
+{
+ struct gve_priv *priv = arg;
+ uint32_t fails;
+ uint64_t ts;
+ int err;
+
+ if (!priv || !priv->nic_ts_report_mz)
+ return;
+
+ memset(priv->nic_ts_report, 0, sizeof(struct gve_nic_ts_report));
+
+ err = gve_adminq_report_nic_timestamp(priv, priv->nic_ts_report_mz->iova);
+ if (err == 0) {
+ ts = be64_to_cpu(priv->nic_ts_report->nic_timestamp);
+ rte_atomic_store_explicit(&priv->last_read_nic_timestamp, ts,
+ rte_memory_order_relaxed);
+ PMD_DRV_LOG(DEBUG, "Fetched NIC Timestamp: %" PRIu64, ts);
+ rte_atomic_store_explicit(&priv->nic_ts_read_fails, 0,
+ rte_memory_order_relaxed);
+ rte_atomic_store_explicit(&priv->nic_ts_stale, 0,
+ rte_memory_order_release);
+ } else {
+ PMD_DRV_LOG(ERR, "Failed to read NIC clock, AQ err: %d", err);
+ fails = rte_atomic_fetch_add_explicit(&priv->nic_ts_read_fails, 1,
+ rte_memory_order_relaxed) + 1;
+ if (fails >= GVE_NIC_CLOCK_READ_MAX_FAILS) {
+ if (!rte_atomic_load_explicit(&priv->nic_ts_stale,
+ rte_memory_order_relaxed))
+ PMD_DRV_LOG(ERR,
+ "NIC timestamping marked as stale after %u consecutive failures",
+ GVE_NIC_CLOCK_READ_MAX_FAILS);
+ rte_atomic_store_explicit(&priv->nic_ts_stale, 1,
+ rte_memory_order_release);
+ }
+ }
+
+ /* Reschedule the alarm for the next interval */
+ if (priv->nic_ts_report_mz) {
+ err = rte_eal_alarm_set(GVE_NIC_CLOCK_READ_PERIOD_MS * 1000,
+ gve_read_nic_clock, priv);
+ if (err < 0) {
+ PMD_DRV_LOG(ERR, "Failed to reschedule NIC clock read alarm, ret=%d", err);
+ rte_atomic_store_explicit(&priv->nic_ts_stale, 1,
+ rte_memory_order_release);
+ }
+ }
+}
+
+static int
+gve_alloc_nic_ts_report(struct gve_priv *priv)
+{
+ char z_name[RTE_MEMZONE_NAMESIZE];
+
+ snprintf(z_name, sizeof(z_name), "gve_%s_nic_ts_report",
+ priv->pci_dev->device.name);
+ priv->nic_ts_report_mz = rte_memzone_reserve_aligned(z_name,
+ sizeof(struct gve_nic_ts_report), rte_socket_id(),
+ RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE);
+
+ if (!priv->nic_ts_report_mz) {
+ PMD_DRV_LOG(ERR, "Failed to allocate memzone for NIC TS report");
+ return -ENOMEM;
+ }
+ priv->nic_ts_report = priv->nic_ts_report_mz->addr;
+ rte_atomic_store_explicit(&priv->nic_ts_read_fails, 0, rte_memory_order_relaxed);
+ return 0;
+}
+
+static void
+gve_free_nic_ts_report(struct gve_priv *priv)
+{
+ if (priv->nic_ts_report_mz) {
+ rte_memzone_free(priv->nic_ts_report_mz);
+ priv->nic_ts_report_mz = NULL;
+ priv->nic_ts_report = NULL;
+ }
+}
+
static int
gve_dev_stop(struct rte_eth_dev *dev)
{
@@ -576,6 +656,7 @@ static void
gve_teardown_device_resources(struct gve_priv *priv)
{
int err;
+ int ret;
/* Tell device its resources are being freed */
if (gve_get_device_resources_ok(priv)) {
@@ -586,6 +667,13 @@ gve_teardown_device_resources(struct gve_priv *priv)
err);
}
+ if (priv->nic_ts_report_mz) {
+ ret = rte_eal_alarm_cancel(gve_read_nic_clock, priv);
+ if (ret < 0)
+ PMD_DRV_LOG(ERR, "Failed to cancel NIC clock sync alarm, ret=%d", ret);
+ gve_free_nic_ts_report(priv);
+ }
+
gve_free_ptype_lut_dqo(priv);
gve_free_counter_array(priv);
gve_free_irq_db(priv);
@@ -1252,6 +1340,23 @@ pci_dev_msix_vec_count(struct rte_pci_device *pdev)
return 0;
}
+static void
+gve_setup_nic_timestamp(struct gve_priv *priv)
+{
+ int err;
+
+ if (!priv->nic_timestamp_supported)
+ return;
+
+ rte_atomic_store_explicit(&priv->nic_ts_read_fails, 0, rte_memory_order_relaxed);
+ rte_atomic_store_explicit(&priv->nic_ts_stale, 1, rte_memory_order_relaxed);
+ err = gve_alloc_nic_ts_report(priv);
+ if (err == 0)
+ gve_read_nic_clock(priv);
+ else
+ PMD_DRV_LOG(ERR, "Failed to allocate memory for NIC timestamping subsystem. Please reset device to retry.");
+}
+
static int
gve_setup_device_resources(struct gve_priv *priv)
{
@@ -1307,6 +1412,7 @@ gve_setup_device_resources(struct gve_priv *priv)
goto free_ptype_lut;
}
}
+ gve_setup_nic_timestamp(priv);
gve_set_device_resources_ok(priv);
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index b67f82c263..7e6f24e910 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -12,6 +12,7 @@
#include <rte_pci.h>
#include <pthread.h>
#include <rte_bitmap.h>
+#include <rte_memzone.h>
#include "base/gve.h"
@@ -39,6 +40,9 @@
#define GVE_RSS_HASH_KEY_SIZE 40
#define GVE_RSS_INDIR_SIZE 128
+#define GVE_NIC_CLOCK_READ_PERIOD_MS 250
+#define GVE_NIC_CLOCK_READ_MAX_FAILS 7
+
#define GVE_TX_CKSUM_OFFLOAD_MASK ( \
RTE_MBUF_F_TX_L4_MASK | \
RTE_MBUF_F_TX_TCP_SEG)
@@ -359,6 +363,11 @@ struct gve_priv {
/* HW Timestamping Fields */
bool nic_timestamp_supported;
+ const struct rte_memzone *nic_ts_report_mz;
+ struct gve_nic_ts_report *nic_ts_report;
+ RTE_ATOMIC(uint64_t) last_read_nic_timestamp;
+ RTE_ATOMIC(uint32_t) nic_ts_read_fails;
+ RTE_ATOMIC(uint8_t) nic_ts_stale;
};
static inline bool
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [PATCH v3 3/6] net/gve: add AdminQ command for NIC timestamps
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260605213022.2770893-1-blasko@google.com>
Introduce the necessary definitions and functions for the
GVE_ADMINQ_REPORT_NIC_TIMESTAMP AdminQ command.
Signed-off-by: Mark Blasko <blasko@google.com>
Reviewed-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Jasper Tran O'Leary <jtranoleary@google.com>
---
v2:
- Added adminq timestamp counter reset to gve_adminq_alloc.
---
drivers/net/gve/base/gve_adminq.c | 20 ++++++++++++++++++++
drivers/net/gve/base/gve_adminq.h | 20 ++++++++++++++++++++
drivers/net/gve/gve_ethdev.h | 1 +
3 files changed, 41 insertions(+)
diff --git a/drivers/net/gve/base/gve_adminq.c b/drivers/net/gve/base/gve_adminq.c
index 1ced1e442e..2b25c7f390 100644
--- a/drivers/net/gve/base/gve_adminq.c
+++ b/drivers/net/gve/base/gve_adminq.c
@@ -263,6 +263,8 @@ int gve_adminq_alloc(struct gve_priv *priv)
priv->adminq_get_ptype_map_cnt = 0;
priv->adminq_cfg_flow_rule_cnt = 0;
+ priv->adminq_report_nic_timestamp_cnt = 0;
+
pthread_mutexattr_init(&mutexattr);
pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(&priv->adminq_lock, &mutexattr);
@@ -522,6 +524,10 @@ static int gve_adminq_issue_cmd(struct gve_priv *priv,
case GVE_ADMINQ_CONFIGURE_FLOW_RULE:
priv->adminq_cfg_flow_rule_cnt++;
break;
+ case GVE_ADMINQ_REPORT_NIC_TIMESTAMP:
+ priv->adminq_report_nic_timestamp_cnt++;
+ break;
+
default:
PMD_DRV_LOG(ERR, "unknown AQ command opcode %d", opcode);
}
@@ -636,6 +642,20 @@ int gve_adminq_reset_flow_rules(struct gve_priv *priv)
return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
}
+int gve_adminq_report_nic_timestamp(struct gve_priv *priv, dma_addr_t nic_ts_report_addr)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_NIC_TIMESTAMP);
+ cmd.report_nic_timestamp = (struct gve_adminq_report_nic_timestamp) {
+ .nic_ts_report_len = cpu_to_be64(sizeof(struct gve_nic_ts_report)),
+ .nic_timestamp_addr = cpu_to_be64(nic_ts_report_addr),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
/* The device specifies that the management vector can either be the first irq
* or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
* the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
diff --git a/drivers/net/gve/base/gve_adminq.h b/drivers/net/gve/base/gve_adminq.h
index eaee5649f2..954be39fbf 100644
--- a/drivers/net/gve/base/gve_adminq.h
+++ b/drivers/net/gve/base/gve_adminq.h
@@ -26,6 +26,7 @@ enum gve_adminq_opcodes {
GVE_ADMINQ_REPORT_LINK_SPEED = 0xD,
GVE_ADMINQ_GET_PTYPE_MAP = 0xE,
GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY = 0xF,
+ GVE_ADMINQ_REPORT_NIC_TIMESTAMP = 0x11,
/* For commands that are larger than 56 bytes */
GVE_ADMINQ_EXTENDED_COMMAND = 0xFF,
};
@@ -373,6 +374,23 @@ struct gve_stats_report {
GVE_CHECK_STRUCT_LEN(8, gve_stats_report);
+struct gve_adminq_report_nic_timestamp {
+ __be64 nic_ts_report_len;
+ __be64 nic_timestamp_addr;
+};
+
+GVE_CHECK_STRUCT_LEN(16, gve_adminq_report_nic_timestamp);
+
+struct gve_nic_ts_report {
+ __be64 nic_timestamp; /* NIC clock in nanoseconds */
+ __be64 pre_cycles; /* System cycle counter before NIC clock read */
+ __be64 post_cycles; /* System cycle counter after NIC clock read */
+ __be64 reserved3;
+ __be64 reserved4;
+};
+
+GVE_CHECK_STRUCT_LEN(40, gve_nic_ts_report);
+
/* Numbers of gve tx/rx stats in stats report. */
#define GVE_TX_STATS_REPORT_NUM 6
#define GVE_RX_STATS_REPORT_NUM 2
@@ -490,6 +508,7 @@ union gve_adminq_command {
struct gve_adminq_verify_driver_compatibility
verify_driver_compatibility;
struct gve_adminq_extended_command extended_command;
+ struct gve_adminq_report_nic_timestamp report_nic_timestamp;
};
};
u8 reserved[64];
@@ -537,5 +556,6 @@ int gve_adminq_add_flow_rule(struct gve_priv *priv,
struct gve_flow_rule_params *rule, u32 loc);
int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc);
int gve_adminq_reset_flow_rules(struct gve_priv *priv);
+int gve_adminq_report_nic_timestamp(struct gve_priv *priv, dma_addr_t nic_ts_report_addr);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index b9b4688367..b67f82c263 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -328,6 +328,7 @@ struct gve_priv {
uint32_t adminq_get_ptype_map_cnt;
uint32_t adminq_verify_driver_compatibility_cnt;
uint32_t adminq_cfg_flow_rule_cnt;
+ uint32_t adminq_report_nic_timestamp_cnt;
volatile uint32_t state_flags;
/* Gvnic device link speed from hypervisor. */
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [PATCH v3 2/6] net/gve: add device option support for HW timestamps
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260605213022.2770893-1-blasko@google.com>
Introduce the necessary definitions and functions for the device
option flag (GVE_DEV_OPT_ID_NIC_TIMESTAMP) to detect hardware
timestamping support in the gvnic device.
Signed-off-by: Mark Blasko <blasko@google.com>
Reviewed-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Jasper Tran O'Leary <jtranoleary@google.com>
---
drivers/net/gve/base/gve_adminq.c | 41 ++++++++++++++++++++++++++-----
drivers/net/gve/base/gve_adminq.h | 9 +++++++
drivers/net/gve/gve_ethdev.h | 3 +++
3 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/drivers/net/gve/base/gve_adminq.c b/drivers/net/gve/base/gve_adminq.c
index 743ab8e7ae..1ced1e442e 100644
--- a/drivers/net/gve/base/gve_adminq.c
+++ b/drivers/net/gve/base/gve_adminq.c
@@ -38,7 +38,8 @@ void gve_parse_device_option(struct gve_priv *priv,
struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_flow_steering **dev_op_flow_steering,
struct gve_device_option_modify_ring **dev_op_modify_ring,
- struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
+ struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
+ struct gve_device_option_nic_timestamp **dev_op_nic_timestamp)
{
u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
u16 option_length = be16_to_cpu(option->option_length);
@@ -168,6 +169,24 @@ void gve_parse_device_option(struct gve_priv *priv,
}
*dev_op_jumbo_frames = RTE_PTR_ADD(option, sizeof(*option));
break;
+ case GVE_DEV_OPT_ID_NIC_TIMESTAMP:
+ if (option_length < sizeof(**dev_op_nic_timestamp) ||
+ req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP) {
+ PMD_DRV_LOG(WARNING, GVE_DEVICE_OPTION_ERROR_FMT,
+ "Nic Timestamp",
+ (int)sizeof(**dev_op_nic_timestamp),
+ GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP,
+ option_length, req_feat_mask);
+ break;
+ }
+
+ if (option_length > sizeof(**dev_op_nic_timestamp)) {
+ PMD_DRV_LOG(WARNING,
+ GVE_DEVICE_OPTION_TOO_BIG_FMT,
+ "Nic Timestamp");
+ }
+ *dev_op_nic_timestamp = RTE_PTR_ADD(option, sizeof(*option));
+ break;
default:
/* If we don't recognize the option just continue
* without doing anything.
@@ -186,7 +205,8 @@ gve_process_device_options(struct gve_priv *priv,
struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_flow_steering **dev_op_flow_steering,
struct gve_device_option_modify_ring **dev_op_modify_ring,
- struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
+ struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
+ struct gve_device_option_nic_timestamp **dev_op_nic_timestamp)
{
const int num_options = be16_to_cpu(descriptor->num_device_options);
struct gve_device_option *dev_opt;
@@ -207,7 +227,8 @@ gve_process_device_options(struct gve_priv *priv,
gve_parse_device_option(priv, dev_opt,
dev_op_gqi_rda, dev_op_gqi_qpl,
dev_op_dqo_rda, dev_op_flow_steering,
- dev_op_modify_ring, dev_op_jumbo_frames);
+ dev_op_modify_ring, dev_op_jumbo_frames,
+ dev_op_nic_timestamp);
dev_opt = next_opt;
}
@@ -920,7 +941,8 @@ static void gve_enable_supported_features(struct gve_priv *priv,
u32 supported_features_mask,
const struct gve_device_option_flow_steering *dev_op_flow_steering,
const struct gve_device_option_modify_ring *dev_op_modify_ring,
- const struct gve_device_option_jumbo_frames *dev_op_jumbo_frames)
+ const struct gve_device_option_jumbo_frames *dev_op_jumbo_frames,
+ const struct gve_device_option_nic_timestamp *dev_op_nic_timestamp)
{
if (dev_op_flow_steering &&
(supported_features_mask & GVE_SUP_FLOW_STEERING_MASK) &&
@@ -947,6 +969,11 @@ static void gve_enable_supported_features(struct gve_priv *priv,
PMD_DRV_LOG(INFO, "JUMBO FRAMES device option enabled.");
priv->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
}
+ if (dev_op_nic_timestamp &&
+ (supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK)) {
+ PMD_DRV_LOG(INFO, "NIC TIMESTAMP device option enabled.");
+ priv->nic_timestamp_supported = true;
+ }
}
int gve_adminq_describe_device(struct gve_priv *priv)
@@ -954,6 +981,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
struct gve_device_option_flow_steering *dev_op_flow_steering = NULL;
+ struct gve_device_option_nic_timestamp *dev_op_nic_timestamp = NULL;
struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
@@ -983,7 +1011,8 @@ int gve_adminq_describe_device(struct gve_priv *priv)
&dev_op_gqi_qpl, &dev_op_dqo_rda,
&dev_op_flow_steering,
&dev_op_modify_ring,
- &dev_op_jumbo_frames);
+ &dev_op_jumbo_frames,
+ &dev_op_nic_timestamp);
if (err)
goto free_device_descriptor;
@@ -1038,7 +1067,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
gve_enable_supported_features(priv, supported_features_mask,
dev_op_flow_steering, dev_op_modify_ring,
- dev_op_jumbo_frames);
+ dev_op_jumbo_frames, dev_op_nic_timestamp);
free_device_descriptor:
gve_free_dma_mem(&descriptor_dma_mem);
diff --git a/drivers/net/gve/base/gve_adminq.h b/drivers/net/gve/base/gve_adminq.h
index d8e5e6a352..eaee5649f2 100644
--- a/drivers/net/gve/base/gve_adminq.h
+++ b/drivers/net/gve/base/gve_adminq.h
@@ -153,6 +153,12 @@ struct gve_device_option_jumbo_frames {
GVE_CHECK_STRUCT_LEN(8, gve_device_option_jumbo_frames);
+struct gve_device_option_nic_timestamp {
+ __be32 supported_features_mask;
+};
+
+GVE_CHECK_STRUCT_LEN(4, gve_device_option_nic_timestamp);
+
/* Terminology:
*
* RDA - Raw DMA Addressing - Buffers associated with SKBs are directly DMA
@@ -169,6 +175,7 @@ enum gve_dev_opt_id {
GVE_DEV_OPT_ID_MODIFY_RING = 0x6,
GVE_DEV_OPT_ID_JUMBO_FRAMES = 0x8,
GVE_DEV_OPT_ID_FLOW_STEERING = 0xb,
+ GVE_DEV_OPT_ID_NIC_TIMESTAMP = 0xd,
};
enum gve_dev_opt_req_feat_mask {
@@ -179,12 +186,14 @@ enum gve_dev_opt_req_feat_mask {
GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING = 0x0,
GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING = 0x0,
GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES = 0x0,
+ GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP = 0x0,
};
enum gve_sup_feature_mask {
GVE_SUP_MODIFY_RING_MASK = 1 << 0,
GVE_SUP_JUMBO_FRAMES_MASK = 1 << 2,
GVE_SUP_FLOW_STEERING_MASK = 1 << 5,
+ GVE_SUP_NIC_TIMESTAMP_MASK = 1 << 8,
};
#define GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING 0x0
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index 524e48e723..b9b4688367 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -355,6 +355,9 @@ struct gve_priv {
void *avail_flow_rule_bmp_mem; /* Backing memory for the bitmap */
pthread_mutex_t flow_rule_lock; /* Lock for bitmap and tailq access */
TAILQ_HEAD(, gve_flow) active_flows;
+
+ /* HW Timestamping Fields */
+ bool nic_timestamp_supported;
};
static inline bool
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [PATCH v3 1/6] net/gve: add thread safety to admin queue
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260605213022.2770893-1-blasko@google.com>
Introduce a pthread_mutex to protect the admin queue operations.
Locking was added around gve_adminq_execute_cmd and the batch
queue creation/destruction functions.
Signed-off-by: Mark Blasko <blasko@google.com>
Reviewed-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Jasper Tran O'Leary <jtranoleary@google.com>
---
v2:
- Dropped ROBUST mutex attribute.
---
.mailmap | 1 +
drivers/net/gve/base/gve_adminq.c | 67 +++++++++++++++++++++++++------
drivers/net/gve/gve_ethdev.h | 1 +
3 files changed, 56 insertions(+), 13 deletions(-)
diff --git a/.mailmap b/.mailmap
index e052b85213..1b10cfca35 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1016,6 +1016,7 @@ Mario Carrillo <mario.alfredo.c.arevalo@intel.com>
Mário Kuka <kuka@cesnet.cz>
Mariusz Drost <mariuszx.drost@intel.com>
Mark Asselstine <mark.asselstine@windriver.com>
+Mark Blasko <blasko@google.com>
Mark Bloch <mbloch@nvidia.com> <markb@mellanox.com>
Mark Gillott <mgillott@vyatta.att-mail.com>
Mark Kavanagh <mark.b.kavanagh@intel.com>
diff --git a/drivers/net/gve/base/gve_adminq.c b/drivers/net/gve/base/gve_adminq.c
index 9c5316fb00..743ab8e7ae 100644
--- a/drivers/net/gve/base/gve_adminq.c
+++ b/drivers/net/gve/base/gve_adminq.c
@@ -216,6 +216,7 @@ gve_process_device_options(struct gve_priv *priv,
int gve_adminq_alloc(struct gve_priv *priv)
{
+ pthread_mutexattr_t mutexattr;
uint8_t pci_rev_id;
priv->adminq = gve_alloc_dma_mem(&priv->adminq_dma_mem, PAGE_SIZE);
@@ -241,6 +242,11 @@ int gve_adminq_alloc(struct gve_priv *priv)
priv->adminq_get_ptype_map_cnt = 0;
priv->adminq_cfg_flow_rule_cnt = 0;
+ pthread_mutexattr_init(&mutexattr);
+ pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
+ pthread_mutex_init(&priv->adminq_lock, &mutexattr);
+ pthread_mutexattr_destroy(&mutexattr);
+
/* Setup Admin queue with the device */
rte_pci_read_config(priv->pci_dev, &pci_rev_id, sizeof(pci_rev_id),
RTE_PCI_REVISION_ID);
@@ -304,6 +310,7 @@ void gve_adminq_free(struct gve_priv *priv)
return;
gve_adminq_release(priv);
gve_free_dma_mem(&priv->adminq_dma_mem);
+ pthread_mutex_destroy(&priv->adminq_lock);
gve_clear_admin_queue_ok(priv);
}
@@ -418,7 +425,10 @@ static int gve_adminq_issue_cmd(struct gve_priv *priv,
(tail & priv->adminq_mask)) {
int err;
- /* Flush existing commands to make room. */
+ /* Flush existing commands to make room.
+ * Note: This kicks the doorbell for all staged commands.
+ * Any failure here means we failed after attempting to kick.
+ */
err = gve_adminq_kick_and_wait(priv);
if (err)
return err;
@@ -509,17 +519,24 @@ static int gve_adminq_execute_cmd(struct gve_priv *priv,
u32 tail, head;
int err;
+ pthread_mutex_lock(&priv->adminq_lock);
tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
head = priv->adminq_prod_cnt;
- if (tail != head)
+ if (tail != head) {
/* This is not a valid path */
- return -EINVAL;
+ err = -EINVAL;
+ goto unlock_and_return;
+ }
err = gve_adminq_issue_cmd(priv, cmd_orig);
if (err)
- return err;
+ goto unlock_and_return;
- return gve_adminq_kick_and_wait(priv);
+ err = gve_adminq_kick_and_wait(priv);
+
+unlock_and_return:
+ pthread_mutex_unlock(&priv->adminq_lock);
+ return err;
}
static int gve_adminq_execute_extended_cmd(struct gve_priv *priv, u32 opcode,
@@ -693,13 +710,19 @@ int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues)
int err;
u32 i;
+ pthread_mutex_lock(&priv->adminq_lock);
+
for (i = 0; i < num_queues; i++) {
err = gve_adminq_create_tx_queue(priv, i);
if (err)
- return err;
+ goto unlock_and_return;
}
- return gve_adminq_kick_and_wait(priv);
+ err = gve_adminq_kick_and_wait(priv);
+
+unlock_and_return:
+ pthread_mutex_unlock(&priv->adminq_lock);
+ return err;
}
static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
@@ -747,13 +770,19 @@ int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
int err;
u32 i;
+ pthread_mutex_lock(&priv->adminq_lock);
+
for (i = 0; i < num_queues; i++) {
err = gve_adminq_create_rx_queue(priv, i);
if (err)
- return err;
+ goto unlock_and_return;
}
- return gve_adminq_kick_and_wait(priv);
+ err = gve_adminq_kick_and_wait(priv);
+
+unlock_and_return:
+ pthread_mutex_unlock(&priv->adminq_lock);
+ return err;
}
static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
@@ -779,13 +808,19 @@ int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues)
int err;
u32 i;
+ pthread_mutex_lock(&priv->adminq_lock);
+
for (i = 0; i < num_queues; i++) {
err = gve_adminq_destroy_tx_queue(priv, i);
if (err)
- return err;
+ goto unlock_and_return;
}
- return gve_adminq_kick_and_wait(priv);
+ err = gve_adminq_kick_and_wait(priv);
+
+unlock_and_return:
+ pthread_mutex_unlock(&priv->adminq_lock);
+ return err;
}
static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
@@ -811,13 +846,19 @@ int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
int err;
u32 i;
+ pthread_mutex_lock(&priv->adminq_lock);
+
for (i = 0; i < num_queues; i++) {
err = gve_adminq_destroy_rx_queue(priv, i);
if (err)
- return err;
+ goto unlock_and_return;
}
- return gve_adminq_kick_and_wait(priv);
+ err = gve_adminq_kick_and_wait(priv);
+
+unlock_and_return:
+ pthread_mutex_unlock(&priv->adminq_lock);
+ return err;
}
static int gve_set_desc_cnt(struct gve_priv *priv,
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index 0577f03974..524e48e723 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -339,6 +339,7 @@ struct gve_priv {
struct gve_tx_queue **txqs;
struct gve_rx_queue **rxqs;
+ pthread_mutex_t adminq_lock; /* Protects AdminQ command execution */
uint32_t stats_report_len;
const struct rte_memzone *stats_report_mem;
uint16_t stats_start_idx; /* start index of array of stats written by NIC */
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply related
* [PATCH v3 0/6] net/gve: add hardware timestamping support
From: Mark Blasko @ 2026-06-05 21:29 UTC (permalink / raw)
To: stephen; +Cc: dev, joshwash, jtranoleary, blasko
In-Reply-To: <20260515231936.3296603-1-blasko@google.com>
This patch series introduces support for GVE hardware timestamping on DQO
queues. To support concurrent access, a mutex lock is introduced to protect
admin queue operations. A mechanism is then added to periodically synchronize
the NIC clock via AdminQ, and support is introduced for the read_clock ethdev
operation. Finally, the RX datapath is updated to reconstruct full 64-bit
timestamps from the 32-bit values in DQO descriptors.
---
v3:
- Patch 5:
- Add mutex lock to protect shared NIC timestamp memzone access.
- Fix missing read_clock assignment to DQO queue ops table
(accidental omission in v2).
v2:
- Patch 1: Dropped ROBUST mutex attribute.
- Patch 3: Added adminq timestamp counter reset to gve_adminq_alloc.
- Patch 4:
- Removed redundant void* casts.
- Handled alarm reschedule failures by marking timestamp stale.
- Added transient error logging on memzone allocation failure.
- Patch 5: Scoped read_clock ethdev operation strictly to DQO queues.
- Patch 6:
- Scoped timestamp offload capability advertisement strictly to
DQO queues.
- Predicated capability advertisement directly on memzone
allocation.
- Initialized mbuf_timestamp_offset to -1.
- Added blank line separating release notes.
---
Mark Blasko (6):
net/gve: add thread safety to admin queue
net/gve: add device option support for HW timestamps
net/gve: add AdminQ command for NIC timestamps
net/gve: add periodic NIC clock synchronization
net/gve: support read clock ethdev op
net/gve: reconstruct HW timestamps from DQO
.mailmap | 1 +
doc/guides/nics/features/gve.ini | 1 +
doc/guides/nics/gve.rst | 20 ++++
doc/guides/rel_notes/release_26_07.rst | 4 +
drivers/net/gve/base/gve_adminq.c | 128 +++++++++++++++++---
drivers/net/gve/base/gve_adminq.h | 29 +++++
drivers/net/gve/base/gve_desc_dqo.h | 8 +-
drivers/net/gve/gve_ethdev.c | 159 ++++++++++++++++++++++++-
drivers/net/gve/gve_ethdev.h | 40 +++++++
drivers/net/gve/gve_rx_dqo.c | 26 ++++
10 files changed, 394 insertions(+), 22 deletions(-)
--
2.54.0.1032.g2f8565e1d1-goog
^ permalink raw reply
* Re: [PATCH v8 9/9] dts: add selective Rx tests
From: Stephen Hemminger @ 2026-06-05 21:28 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev, Luca Vizzarro, Patrick Robb
In-Reply-To: <20260604193324.1996141-10-thomas@monjalon.net>
On Thu, 4 Jun 2026 21:31:01 +0200
Thomas Monjalon <thomas@monjalon.net> wrote:
> Add TestSuite_rx_split with 7 test cases:
> - 3 positive: headers only, payload only, two non-contiguous segments
> - 4 negative: missing offload flag, out-of-range, overlap, all-discard
>
> Add selective Rx capability detection via testpmd "show port info".
>
> The test suite could be completed later for the basic buffer split
> configuration based on offsets or protocols.
>
> Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
> ---
AI review found:
Patch 9 (dts: add selective Rx tests)
selective_rx_out_of_range expects a rejection that never happens, so the
negative test will fail. It configures a real segment plus an oversized
discard segment:
rx_segments_length=[ETHER_IP_HDR_LEN, 20000],
mbuf_size=[256, 0],
and expects start_all_ports() to fail. But an over-range length on a discard
segment is not rejected anywhere: rte_eth_rx_queue_check_split() does
"continue" for mp == NULL segments, so it never length-checks them, and
mlx5_rxq_new() clamps it:
if (seg_len > tail_len)
seg_len = qs_seg->mp != NULL ? buf_len - offset : tail_len;
The discard seg_len becomes the remaining frame length, the queue is built,
the port starts, and the test hits its fail().
Clamping an over-long discard to "the rest" is harmless (the bytes are
discarded anyway), so the cleanest fix is probably to drop or rework this
test rather than add a rejection path. If rejection is the intended
contract, it would have to be added for discard segments in patch 2 or
patch 6 -- a behavior choice, not a correctness requirement.
Minor: expressing a leading discard as --mbuf-size=0,... puts 0 at index 0,
and testpmd treats mbuf_data_size[0] as the primary pool size elsewhere (the
max_rx_pkt_len > mbuf_data_size[0] check, the default mbuf_pool_find(socket,
0)). Only bites an unusual config, but it is a latent foot-gun.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox