* [PATCH v7 04/20] net/sxe2: support L2 filtering and MAC config
From: liujie5 @ 2026-06-03 2:21 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260603022150.1140722-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
- Support primary/secondary MAC address setup.
- Enable L2 broadcast/multicast filter bits.
- Add multicast address update logic.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 198 ++++++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 17 +
drivers/net/sxe2/sxe2_drv_cmd.h | 87 ++++
drivers/net/sxe2/sxe2_ethdev.c | 70 ++-
drivers/net/sxe2/sxe2_ethdev.h | 43 +-
drivers/net/sxe2/sxe2_filter.c | 782 ++++++++++++++++++++++++++++++
drivers/net/sxe2/sxe2_filter.h | 98 ++++
drivers/net/sxe2/sxe2_mac.c | 432 +++++++++++++++++
drivers/net/sxe2/sxe2_mac.h | 34 ++
drivers/net/sxe2/sxe2_txrx_poll.c | 49 ++
11 files changed, 1805 insertions(+), 6 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index b14b5120c1..b661e3cbf4 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -61,4 +61,5 @@ sources += files(
'sxe2_txrx.c',
'sxe2_txrx_vec.c',
'sxe2_mac.c',
+ 'sxe2_filter.c',
)
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index 07eeb7f38c..1fa9ad718e 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -343,3 +343,201 @@ int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
l_end:
return ret;
}
+
+int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_promisc_filter_cfg_req promisc_filter_cfg_req = {0};
+
+ promisc_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ promisc_filter_cfg_req.is_add = set;
+ promisc_filter_cfg_req.type = SXE2_PROMISC_FILTER_TYPE_PROMISC;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_PROMISC_CFG,
+ &promisc_filter_cfg_req,
+ sizeof(promisc_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "promic config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_promisc_filter_cfg_req promisc_filter_cfg_req = {0};
+
+ promisc_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ promisc_filter_cfg_req.is_add = set;
+ promisc_filter_cfg_req.type = SXE2_PROMISC_FILTER_TYPE_ALLMULTI;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_ALLMULTI_CFG,
+ &promisc_filter_cfg_req,
+ sizeof(promisc_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "allmulti config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add)
+{
+ int32_t ret = 0;
+ int32_t i;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_mac_filter_cfg_req mac_filter_cfg_req = {0};
+
+ mac_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ for (i = 0; i < SXE2_ETH_ALEN; i++)
+ mac_filter_cfg_req.addr[i] = addr->addr_bytes[i];
+ mac_filter_cfg_req.is_add = add;
+ mac_filter_cfg_req.type = SXE2_MAC_FILTER_TYPE_UC;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_ADDR_UC,
+ &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "uc config query failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_mc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add)
+{
+ int32_t ret = 0;
+ int32_t i;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_mac_filter_cfg_req mac_filter_cfg_req = {0};
+
+ mac_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ for (i = 0; i < SXE2_ETH_ALEN; i++)
+ mac_filter_cfg_req.addr[i] = addr->addr_bytes[i];
+
+ mac_filter_cfg_req.is_add = add;
+ mac_filter_cfg_req.type = SXE2_MAC_FILTER_TYPE_MC;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_ADDR_MC,
+ &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "mac config query failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_config_query(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_vlan_cfg_query_resp vlan_cfg_query_resp = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_CFG_QUERY,
+ NULL, 0,
+ &vlan_cfg_query_resp,
+ sizeof(vlan_cfg_query_resp));
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config query failed, ret=%d", ret);
+
+ adapter->filter_ctxt.vlan_info.port_vlan_exist = vlan_cfg_query_resp.port_vlan_exist;
+ adapter->filter_ctxt.vlan_info.is_switchdev = vlan_cfg_query_resp.is_switchdev;
+
+
+ adapter->filter_ctxt.vlan_info.tpid = vlan_cfg_query_resp.tpid;
+ adapter->filter_ctxt.vlan_info.vid = vlan_cfg_query_resp.vid;
+
+ adapter->filter_ctxt.vlan_info.outer_insert = vlan_cfg_query_resp.outer_insert;
+ adapter->filter_ctxt.vlan_info.outer_strip = vlan_cfg_query_resp.outer_strip;
+ adapter->filter_ctxt.vlan_info.inner_insert = vlan_cfg_query_resp.inner_insert;
+ adapter->filter_ctxt.vlan_info.inner_strip = vlan_cfg_query_resp.inner_strip;
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_filter_id_config(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool on)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_vlan_filter_cfg_req vlan_filter_cfg_req = {0};
+
+ vlan_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ vlan_filter_cfg_req.tpid_id = vlan->tpid;
+ vlan_filter_cfg_req.vlan_id = vlan->vid;
+ vlan_filter_cfg_req.prio = vlan->prio;
+ vlan_filter_cfg_req.is_add = on;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_FILTER_ADD_DEL,
+ &vlan_filter_cfg_req, sizeof(vlan_filter_cfg_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_insert_strip_cfg(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_vlan_offload_cfg_req vlan_offload_cfg_req = {0};
+
+ vlan_offload_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ vlan_offload_cfg_req.tpid = adapter->filter_ctxt.vlan_info.tpid;
+ vlan_offload_cfg_req.outer_insert = adapter->filter_ctxt.vlan_info.outer_insert;
+ vlan_offload_cfg_req.outer_strip = adapter->filter_ctxt.vlan_info.outer_strip;
+ vlan_offload_cfg_req.inner_insert = adapter->filter_ctxt.vlan_info.inner_insert;
+ vlan_offload_cfg_req.inner_strip = adapter->filter_ctxt.vlan_info.inner_strip;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_OFFLOAD_CFG,
+ &vlan_offload_cfg_req,
+ sizeof(vlan_offload_cfg_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config query failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_vlan_filter_switch_req vlan_filter_switch_req = {0};
+
+ vlan_filter_switch_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ vlan_filter_switch_req.is_oper_enable = on;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_FILTER_SWITCH,
+ &vlan_filter_switch_req,
+ sizeof(vlan_filter_switch_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config filter failed, ret=%d", ret);
+
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index cda676ed97..c93bc2b0c9 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -36,4 +36,21 @@ int32_t sxe2_drv_txq_ctxt_cfg(struct sxe2_adapter *adapter,
int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
+int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+
+int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set);
+
+int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
+
+int32_t sxe2_drv_mc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
+
+int32_t sxe2_drv_vlan_config_query(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_vlan_filter_id_config(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool on);
+
+int32_t sxe2_drv_vlan_insert_strip_cfg(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on);
+
#endif /* SXE2_CMD_CHNL_H */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index a0f08b5184..6399f22c5c 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -233,6 +233,93 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_link_info_resp {
uint8_t rsv[3];
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_info {
+ uint8_t is_switchdev;
+ uint8_t master;
+ uint8_t representor;
+ uint8_t port_name_type;
+ uint32_t ctrl_num;
+ uint32_t pf_num;
+ uint32_t vf_num;
+ uint32_t mpesw_owner;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vlan_cfg_query_resp {
+ uint16_t vsi_id;
+ uint8_t port_vlan_exist;
+ uint8_t is_switchdev;
+ uint16_t tpid;
+ uint16_t vid;
+ uint8_t outer_insert;
+ uint8_t outer_strip;
+ uint8_t inner_insert;
+ uint8_t inner_strip;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vlan_offload_cfg_req {
+ uint16_t vsi_id;
+ uint16_t tpid;
+ uint8_t outer_insert;
+ uint8_t outer_strip;
+ uint8_t inner_insert;
+ uint8_t inner_strip;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_port_vlan_cfg_req {
+ uint16_t vsi_id;
+ uint16_t tpid;
+ uint16_t vid;
+ uint8_t prio;
+ uint8_t rsv;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_mac_filter_cfg_req {
+ uint16_t vsi_id;
+ uint8_t addr[SXE2_ETH_ALEN];
+ uint8_t type;
+ uint8_t is_add;
+ uint8_t rsv[2];
+} __rte_packed_end;
+
+enum sxe2_promisc_filter_type {
+ SXE2_PROMISC_FILTER_TYPE_PROMISC = 0,
+ SXE2_PROMISC_FILTER_TYPE_ALLMULTI,
+ SXE2_PROMISC_FILTER_TYPE_MAX,
+};
+
+enum sxe2_mac_filter_type {
+ SXE2_MAC_FILTER_TYPE_UC = 0,
+ SXE2_MAC_FILTER_TYPE_MC,
+ SXE2_MAC_FILTER_TYPE_MAX,
+};
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_promisc_filter_cfg_req {
+ uint16_t vsi_id;
+ uint8_t type;
+ uint8_t is_add;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_srcvsi_ext_cfg_req {
+ uint16_t vsi_id;
+ uint16_t srcvsi_list[SXE2_SRCVSI_PRUNE_MAX_NUM];
+ uint8_t srcvsi_cnt;
+ uint8_t is_add;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_vlan_filter_cfg_req {
+ uint16_t vsi_id;
+ uint16_t vlan_id;
+ uint16_t tpid_id;
+ uint8_t prio;
+ uint8_t is_add;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_vlan_filter_switch_req {
+ uint16_t vsi_id;
+ uint8_t is_oper_enable;
+ uint8_t rsv;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 01552a8202..9b117f097e 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -111,8 +111,20 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.tx_burst_mode_get = sxe2_tx_burst_mode_get,
.tx_done_cleanup = sxe2_tx_done_cleanup,
+ .promiscuous_enable = sxe2_promisc_enable,
+ .promiscuous_disable = sxe2_promisc_disable,
+ .allmulticast_enable = sxe2_allmulti_enable,
+ .allmulticast_disable = sxe2_allmulti_disable,
+
+ .mac_addr_add = sxe2_mac_addr_add,
+ .mac_addr_remove = sxe2_mac_addr_del,
+ .mac_addr_set = sxe2_mac_addr_set,
+ .set_mc_addr_list = sxe2_set_mc_addr_list,
.mtu_set = sxe2_mtu_set,
.buffer_split_supported_hdr_ptypes_get = sxe2_buffer_split_supported_hdr_ptypes_get,
+
+ .vlan_filter_set = sxe2_dev_vlan_filter_set,
+ .vlan_offload_set = sxe2_dev_vlan_offload_set,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -123,6 +135,13 @@ static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
+ ret = sxe2_vlan_default_cfg(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init vlan, ret=%d", ret);
+ goto end;
+ }
+
+end:
return ret;
}
@@ -138,6 +157,8 @@ static int32_t sxe2_dev_stop(struct rte_eth_dev *dev)
sxe2_txqs_all_stop(dev);
sxe2_rxqs_all_stop(dev);
+ (void)sxe2_filter_rule_stop(dev);
+
dev->data->dev_started = 0;
adapter->started = 0;
l_end:
@@ -165,16 +186,23 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
goto l_end;
}
+ ret = sxe2_filter_rule_start(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to add all mc addr to fw.");
+ goto l_end;
+ }
+
ret = sxe2_queues_start(dev);
if (ret) {
PMD_LOG_ERR(INIT, "enable queues failed");
- goto l_end;
+ goto l_start_queues_err;
}
dev->data->dev_started = 1;
adapter->started = 1;
goto l_end;
-
+l_start_queues_err:
+ (void)sxe2_filter_rule_stop(dev);
l_end:
return ret;
}
@@ -194,6 +222,7 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->rx_offload_capa =
+ RTE_ETH_RX_OFFLOAD_VLAN_STRIP |
RTE_ETH_RX_OFFLOAD_KEEP_CRC |
RTE_ETH_RX_OFFLOAD_SCATTER |
RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
@@ -202,9 +231,15 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
RTE_ETH_RX_OFFLOAD_SCTP_CKSUM |
RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM |
RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT |
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ RTE_ETH_RX_OFFLOAD_QINQ_STRIP |
+#endif
+ RTE_ETH_RX_OFFLOAD_VLAN_EXTEND |
RTE_ETH_RX_OFFLOAD_TCP_LRO;
dev_info->tx_offload_capa =
+ RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
+ RTE_ETH_TX_OFFLOAD_QINQ_INSERT |
RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE |
RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
@@ -428,6 +463,12 @@ static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
{
int32_t ret = 0;
+ ret = sxe2_filter_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize l2 filter, ret:%d", ret);
+ goto l_end;
+ }
+
ret = sxe2_link_update_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
@@ -439,12 +480,37 @@ static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
PMD_LOG_ERR(INIT, "Failed to set mtu, ret=%d", ret);
goto l_end;
}
+
+ ret = sxe2_mac_addr_init(dev);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to initialize mac address, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_mac_default_cfg(dev);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to configure default mac address, ret:%d", ret);
+ goto l_err;
+ }
+
+ ret = sxe2_vlan_cfg_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize vlan config, ret:%d", ret);
+ goto l_err;
+ }
+ goto l_end;
+
+l_err:
+ sxe2_mac_addr_uinit(dev);
+ (void)sxe2_filter_uinit(dev);
l_end:
return ret;
}
static void sxe2_eth_uinit(struct rte_eth_dev *dev __rte_unused)
{
+ sxe2_mac_addr_uinit(dev);
+ (void)sxe2_filter_uinit(dev);
}
static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index c188b41a0e..34a4a45e4f 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -15,9 +15,11 @@
#include "sxe2_common.h"
#include "sxe2_vsi.h"
-#include "sxe2_queue.h"
#include "sxe2_irq.h"
+#include "sxe2_queue.h"
+#include "sxe2_mac.h"
#include "sxe2_osal.h"
+#include "sxe2_filter.h"
struct sxe2_link_msg {
uint32_t speed;
@@ -35,7 +37,7 @@ enum sxe2_fnav_tunnel_flag_type {
#define SXE2_FRAME_SIZE_MAX 9832
#define SXE2_VLAN_TAG_SIZE 4
#define SXE2_ETH_OVERHEAD \
- (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + SXE2_VLAN_TAG_SIZE)
+ (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 2 * SXE2_VLAN_TAG_SIZE)
#define SXE2_ETH_MAX_LEN (RTE_ETHER_MTU + SXE2_ETH_OVERHEAD)
#ifdef SXE2_TEST
@@ -267,6 +269,27 @@ struct sxe2_link_context {
uint32_t speed;
};
+struct sxe2_filter_context {
+ rte_spinlock_t filter_lock;
+ struct sxe2_vlan_info vlan_info;
+ struct sxe2_uc_filter_list_head uc_list;
+ struct sxe2_mc_filter_list_head mc_list;
+ struct sxe2_vlan_filter_list_head vlan_list;
+ uint8_t uc_num;
+ uint8_t mc_num;
+ uint8_t vlan_num;
+ uint8_t rsv;
+ uint32_t hw_promisc_flags;
+ uint32_t cur_promisc_flags;
+
+ bool hw_uplink_config;
+ bool cur_uplink_config;
+ bool hw_repr_config;
+ bool cur_repr_config;
+ bool hw_l2_config;
+ bool cur_l2_config;
+};
+
struct sxe2_adapter {
struct sxe2_common_device *cdev;
struct sxe2_dev_info dev_info;
@@ -276,10 +299,14 @@ struct sxe2_adapter {
struct sxe2_irq_context irq_ctxt;
struct sxe2_queue_context q_ctxt;
struct sxe2_vsi_context vsi_ctxt;
+ struct sxe2_filter_context filter_ctxt;
struct sxe2_link_context link_ctxt;
struct sxe2_devargs devargs;
- uint16_t dev_port_id;
- uint64_t cap_flags;
+ struct sxe2_switchdev_info switchdev_info;
+ bool rule_started;
+ bool flow_isolated;
+ uint16_t dev_port_id;
+ uint64_t cap_flags;
enum sxe2_dev_type dev_type;
uint32_t ptype_tbl[SXE2_MAX_PTYPE_NUM];
struct rte_ether_addr mac_addr;
@@ -318,4 +345,12 @@ int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev);
void sxe2_dev_pci_map_uinit(struct rte_eth_dev *dev);
+static inline bool
+sxe2_dev_port_vlan_check(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *ad = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ return ad->filter_ctxt.vlan_info.port_vlan_exist;
+}
+
#endif /* SXE2_ETHDEV_H */
diff --git a/drivers/net/sxe2/sxe2_filter.c b/drivers/net/sxe2/sxe2_filter.c
new file mode 100644
index 0000000000..b2a726f77e
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_filter.c
@@ -0,0 +1,782 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_os.h>
+#include <rte_tailq.h>
+#include "sxe2_osal.h"
+#include "sxe2_mac.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+
+static struct sxe2_mac_filter *sxe2_uc_filter_find(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *macaddr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ struct sxe2_mac_filter *entry = NULL;
+ struct sxe2_mac_filter *next_entry = NULL;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.uc_list, next, next_entry) {
+ if (rte_is_same_ether_addr(macaddr, &entry->mac_addr)) {
+ filter = entry;
+ break;
+ }
+ }
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ return filter;
+}
+
+int32_t sxe2_uc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ bool hw_config = false;
+ int32_t ret = 0;
+
+ filter = sxe2_uc_filter_find(adapter, mac_addr);
+ if (filter) {
+ if (default_config && !filter->default_config)
+ filter->default_config = true;
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter already exists.");
+ goto l_end;
+ }
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add hw uc addr in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw uc addr in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw uc addr in switchdev mode");
+ } else {
+ ret = sxe2_drv_uc_config(adapter, mac_addr, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add uc rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ hw_config = true;
+ }
+
+ filter = rte_zmalloc("sxe2_uc_filter",
+ sizeof(struct sxe2_mac_filter), 0);
+ if (!filter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to allocate memory");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ filter->hw_config = hw_config;
+ filter->default_config = default_config;
+ rte_ether_addr_copy(mac_addr, &filter->mac_addr);
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_INSERT_TAIL(&adapter->filter_ctxt.uc_list, filter, next);
+ adapter->filter_ctxt.uc_num++;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ PMD_DEV_LOG_INFO(adapter, DRV, "add mac rule, mac num %u.", adapter->filter_ctxt.uc_num);
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_uc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ int32_t ret = -1;
+
+ filter = sxe2_uc_filter_find(adapter, mac_addr);
+ if (!filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter not exists.");
+ ret = 0;
+ goto l_end;
+ }
+ if (filter->hw_config) {
+ ret = sxe2_drv_uc_config(adapter, mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mac rule");
+ if (ret == -EPERM)
+ goto l_free;
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ PMD_DEV_LOG_INFO(adapter, DRV, "remove mac rule, uc num %u.", adapter->filter_ctxt.uc_num);
+ ret = 0;
+
+l_free:
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_REMOVE(&adapter->filter_ctxt.uc_list, filter, next);
+ adapter->filter_ctxt.uc_num--;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+ rte_free(filter);
+ filter = NULL;
+l_end:
+ return ret;
+}
+
+void sxe2_uc_filter_clear(struct sxe2_adapter *adapter, bool default_config)
+{
+ struct sxe2_mac_filter *entry;
+ struct sxe2_mac_filter *next_entry;
+
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.uc_list, next, next_entry) {
+ if (entry->default_config && !default_config)
+ continue;
+
+ if (sxe2_uc_filter_del(adapter, &entry->mac_addr))
+ PMD_DEV_LOG_ERR(adapter, DRV, "This MAC filter delete fail.");
+ }
+}
+
+static struct sxe2_mac_filter *sxe2_mc_filter_find(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *macaddr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ struct sxe2_mac_filter *entry = NULL;
+ struct sxe2_mac_filter *next_entry = NULL;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.mc_list, next, next_entry) {
+ if (rte_is_same_ether_addr(macaddr, &entry->mac_addr)) {
+ filter = entry;
+ break;
+ }
+ }
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ return filter;
+}
+
+int32_t sxe2_mc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ bool hw_config = false;
+ int32_t ret = 0;
+
+ filter = sxe2_mc_filter_find(adapter, mac_addr);
+ if (filter) {
+ if (default_config && !filter->default_config)
+ filter->default_config = true;
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter already exists.");
+ goto l_end;
+ }
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add hw mc addr in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw mc addr in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw mc addr in switchdev mode");
+ } else {
+ ret = sxe2_drv_mc_config(adapter, mac_addr, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add mac rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ hw_config = true;
+ }
+
+ filter = rte_zmalloc("sxe2_mc_filter",
+ sizeof(struct sxe2_mac_filter), 0);
+ if (!filter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to allocate memory");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ filter->hw_config = hw_config;
+ filter->default_config = default_config;
+ rte_ether_addr_copy(mac_addr, &filter->mac_addr);
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_INSERT_TAIL(&adapter->filter_ctxt.mc_list, filter, next);
+ adapter->filter_ctxt.mc_num++;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ PMD_DEV_LOG_INFO(adapter, DRV, "add mc rule, mc num %u.", adapter->filter_ctxt.mc_num);
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_mc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ int32_t ret = -1;
+
+ filter = sxe2_mc_filter_find(adapter, mac_addr);
+ if (!filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter not exists.");
+ ret = 0;
+ goto l_end;
+ }
+
+ if (filter->hw_config) {
+ ret = sxe2_drv_mc_config(adapter, mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mc rule");
+ if (ret == -EPERM)
+ goto l_free;
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ PMD_DEV_LOG_INFO(adapter, DRV, "remove mc rule, mc num %u.", adapter->filter_ctxt.mc_num);
+ ret = 0;
+
+l_free:
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_REMOVE(&adapter->filter_ctxt.mc_list, filter, next);
+ adapter->filter_ctxt.mc_num--;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+ rte_free(filter);
+ filter = NULL;
+l_end:
+ return ret;
+}
+
+void sxe2_mc_filter_clear(struct sxe2_adapter *adapter, bool default_config)
+{
+ struct sxe2_mac_filter *entry;
+ struct sxe2_mac_filter *next_entry;
+
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.mc_list, next, next_entry) {
+ if (entry->default_config && !default_config)
+ continue;
+ if (sxe2_mc_filter_del(adapter, &entry->mac_addr))
+ PMD_DEV_LOG_ERR(adapter, DRV, "This MAC filter delete fail.");
+ }
+}
+
+static struct sxe2_vlan_filter *sxe2_vlan_filter_find(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan)
+{
+ struct sxe2_vlan_filter *f;
+ struct sxe2_vlan_filter *save_f = NULL;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_FOREACH(f, &adapter->filter_ctxt.vlan_list, next)
+ {
+ if (vlan->tpid == f->vlan_info.tpid &&
+ vlan->vid == f->vlan_info.vid) {
+ save_f = f;
+ break;
+ }
+ }
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ return save_f;
+}
+
+int32_t sxe2_vlan_filter_add(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool default_config)
+{
+ struct sxe2_vlan_filter *filter = NULL;
+ bool hw_config = false;
+ int32_t ret = 0;
+
+ if (!vlan || vlan->vid > RTE_ETHER_MAX_VLAN_ID) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "This vlan filter is invalid.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ filter = sxe2_vlan_filter_find(adapter, vlan);
+ if (filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This vlan filter already exists.");
+ ret = 0;
+ goto l_end;
+ }
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add vlan in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan in switchdev mode");
+ } else {
+ ret = sxe2_drv_vlan_filter_id_config(adapter, vlan, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ hw_config = true;
+ }
+
+ filter = rte_zmalloc("sxe2_vlan_filter", sizeof(*filter), 0);
+ if (!filter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to allocate memory");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ filter->hw_config = hw_config;
+ filter->default_config = default_config;
+
+ filter->vlan_info.tpid = vlan->tpid;
+ filter->vlan_info.vid = vlan->vid;
+ filter->vlan_info.prio = vlan->prio;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_INSERT_TAIL(&adapter->filter_ctxt.vlan_list, filter, next);
+ adapter->filter_ctxt.vlan_num++;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_vlan_filter_del(struct sxe2_adapter *adapter, struct sxe2_vlan *vlan)
+{
+ struct sxe2_vlan_filter *filter = NULL;
+ int32_t ret = -1;
+
+ if (!vlan || vlan->vid > RTE_ETHER_MAX_VLAN_ID) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This vlan filter is invalid.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ filter = sxe2_vlan_filter_find(adapter, vlan);
+ if (!filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This vlan filter not exists.");
+ ret = 0;
+ goto l_end;
+ }
+
+ if (filter->hw_config) {
+ ret = sxe2_drv_vlan_filter_id_config(adapter, vlan, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete vlan rule");
+ if (ret == -EPERM)
+ goto l_free;
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ ret = 0;
+
+l_free:
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_REMOVE(&adapter->filter_ctxt.vlan_list, filter, next);
+ adapter->filter_ctxt.vlan_num--;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+ rte_free(filter);
+ filter = NULL;
+l_end:
+ return ret;
+}
+
+void sxe2_vlan_filters_clear(struct sxe2_adapter *adapter, bool default_config)
+{
+ int32_t ret = 0;
+ struct sxe2_vlan_filter *v_f;
+ void *temp;
+
+ if (adapter->filter_ctxt.vlan_num == 0)
+ return;
+
+ RTE_TAILQ_FOREACH_SAFE(v_f, &adapter->filter_ctxt.vlan_list, next, temp)
+ {
+ if (v_f->default_config && !default_config)
+ continue;
+ ret = sxe2_vlan_filter_del(adapter, &v_f->vlan_info);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "This vlan filter delete fail.");
+ }
+}
+
+int32_t sxe2_vlan_filter_ctrl(struct sxe2_adapter *adapter, bool flag)
+{
+ struct sxe2_vlan_info *vlan_info = &adapter->filter_ctxt.vlan_info;
+ int32_t ret = 0;
+
+ if (vlan_info->filter_on == flag)
+ goto l_end;
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add vlan filter ctrl in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan filter ctrl in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan filter ctrl in switchdev mode");
+ } else {
+ ret = sxe2_drv_vlan_filter_switch(adapter, flag);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan filter ctrl");
+ goto l_end;
+ }
+ vlan_info->hw_filter_on = flag;
+ }
+ vlan_info->filter_on = flag;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_promisc_add(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot enable promiscuous in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable promiscuous in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable promiscuous in switchdev mode");
+ } else if (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC)) {
+ ret = sxe2_drv_promisc_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg promiscuous, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC;
+ }
+ adapter->filter_ctxt.cur_promisc_flags |= SXE2_PROMISC;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_promisc_del(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated &&
+ (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC)) {
+ ret = sxe2_drv_promisc_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg promiscuous, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC;
+ }
+
+ adapter->filter_ctxt.cur_promisc_flags &= ~SXE2_PROMISC;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_allmulti_add(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot enable allmulticast in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable allmulticast in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable allmulticast in switchdev mode");
+ } else if (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST)) {
+ ret = sxe2_drv_allmulti_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg allmulticast, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC_MULTICAST;
+ }
+ adapter->filter_ctxt.cur_promisc_flags |= SXE2_PROMISC_MULTICAST;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_allmulti_del(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated &&
+ (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST)) {
+ ret = sxe2_drv_allmulti_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg allmulticast, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC_MULTICAST;
+ }
+
+ adapter->filter_ctxt.cur_promisc_flags &= ~SXE2_PROMISC_MULTICAST;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_all_filter_hw_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_mac_filter *mac_entry;
+ struct sxe2_mac_filter *next_mac_entry;
+ struct sxe2_vlan_filter *vlan_entry;
+ struct sxe2_vlan_filter *next_vlan_entry;
+
+ if (adapter->filter_ctxt.uc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.uc_list, next,
+ next_mac_entry) {
+ if (mac_entry->hw_config) {
+ ret = sxe2_drv_uc_config(adapter, &mac_entry->mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mac rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = false;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.mc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.mc_list, next,
+ next_mac_entry) {
+ if (mac_entry->hw_config) {
+ ret = sxe2_drv_mc_config(adapter, &mac_entry->mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mc rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = false;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(vlan_entry, &adapter->filter_ctxt.vlan_list, next,
+ next_vlan_entry) {
+ if (vlan_entry->hw_config) {
+ ret = sxe2_drv_vlan_filter_id_config(adapter,
+ &vlan_entry->vlan_info, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete vlan rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ vlan_entry->hw_config = false;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_info.hw_filter_on) {
+ ret = sxe2_drv_vlan_filter_switch(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete vlan rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ adapter->filter_ctxt.vlan_info.hw_filter_on = false;
+ }
+
+ if (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC) {
+ ret = sxe2_drv_promisc_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg promiscuous, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC;
+ }
+
+ if (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST) {
+ ret = sxe2_drv_allmulti_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg allmulticast, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC_MULTICAST;
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_all_filter_hw_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_mac_filter *mac_entry;
+ struct sxe2_mac_filter *next_mac_entry;
+ struct sxe2_vlan_filter *vlan_entry;
+ struct sxe2_vlan_filter *next_vlan_entry;
+
+ if (adapter->filter_ctxt.uc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.uc_list, next,
+ next_mac_entry) {
+ if (!mac_entry->hw_config) {
+ ret = sxe2_drv_uc_config(adapter, &mac_entry->mac_addr,
+ true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add uc rule, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = true;
+ ret = 0;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.mc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.mc_list, next,
+ next_mac_entry) {
+ if (!mac_entry->hw_config) {
+ ret = sxe2_drv_mc_config(adapter, &mac_entry->mac_addr, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add mc rule, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = true;
+ ret = 0;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(vlan_entry, &adapter->filter_ctxt.vlan_list, next,
+ next_vlan_entry) {
+ if (!vlan_entry->hw_config) {
+ ret = sxe2_drv_vlan_filter_id_config(adapter,
+ &vlan_entry->vlan_info, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add vlan rule, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ vlan_entry->hw_config = true;
+ ret = 0;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_info.filter_on) {
+ if (!(adapter->filter_ctxt.vlan_info.hw_filter_on)) {
+ ret = sxe2_drv_vlan_filter_switch(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add vlan ctrl, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ adapter->filter_ctxt.vlan_info.hw_filter_on = true;
+ ret = 0;
+ }
+ }
+
+ if ((adapter->filter_ctxt.cur_promisc_flags & SXE2_PROMISC) &&
+ (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC))) {
+ ret = sxe2_drv_promisc_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to set promisc, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC;
+ ret = 0;
+ }
+
+ if ((adapter->filter_ctxt.cur_promisc_flags & SXE2_PROMISC_MULTICAST) &&
+ (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST))) {
+ ret = sxe2_drv_allmulti_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to set allmulti, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC_MULTICAST;
+ ret = 0;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated && !adapter->switchdev_info.is_switchdev &&
+ adapter->rule_started) {
+ adapter->filter_ctxt.cur_l2_config = true;
+ } else {
+ adapter->filter_ctxt.cur_l2_config = false;
+ }
+
+ if (adapter->filter_ctxt.cur_l2_config !=
+ adapter->filter_ctxt.hw_l2_config) {
+ if (adapter->filter_ctxt.cur_l2_config) {
+ ret = sxe2_all_filter_hw_set(adapter);
+ if (!ret)
+ adapter->filter_ctxt.hw_l2_config = true;
+ } else {
+ ret = sxe2_all_filter_hw_clear(adapter);
+ if (!ret)
+ adapter->filter_ctxt.hw_l2_config = false;
+ }
+ }
+ return ret;
+}
+
+int32_t sxe2_filter_rule_stop(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ adapter->rule_started = 0;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ return ret;
+}
+
+int32_t sxe2_filter_rule_start(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ adapter->rule_started = 1;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ return ret;
+}
+
+int32_t sxe2_filter_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ rte_spinlock_init(&adapter->filter_ctxt.filter_lock);
+
+ TAILQ_INIT(&adapter->filter_ctxt.uc_list);
+ adapter->filter_ctxt.uc_num = 0;
+
+ TAILQ_INIT(&adapter->filter_ctxt.mc_list);
+ adapter->filter_ctxt.mc_num = 0;
+
+ TAILQ_INIT(&adapter->filter_ctxt.vlan_list);
+ adapter->filter_ctxt.vlan_num = 0;
+ return 0;
+}
+
+int32_t sxe2_filter_uinit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ sxe2_uc_filter_clear(adapter, true);
+ adapter->filter_ctxt.uc_num = 0;
+
+ sxe2_mc_filter_clear(adapter, true);
+ adapter->filter_ctxt.mc_num = 0;
+
+ sxe2_vlan_filters_clear(adapter, true);
+ adapter->filter_ctxt.vlan_num = 0;
+ return 0;
+}
diff --git a/drivers/net/sxe2/sxe2_filter.h b/drivers/net/sxe2/sxe2_filter.h
new file mode 100644
index 0000000000..6262e8c845
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_filter.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FILTER_H__
+#define __SXE2_FILTER_H__
+#include <ethdev_driver.h>
+
+#define SXE2_PROMISC (1UL << 0UL)
+#define SXE2_PROMISC_MULTICAST (1UL << 1UL)
+
+struct sxe2_vlan_info {
+ uint8_t port_vlan_exist;
+ uint8_t is_switchdev;
+ uint16_t max_cnt;
+ uint16_t cnt;
+
+ bool filter_on;
+ bool hw_filter_on;
+
+ uint16_t tpid;
+ uint16_t vid;
+
+ uint8_t outer_insert;
+ uint8_t outer_strip;
+ uint8_t inner_insert;
+ uint8_t inner_strip;
+};
+
+struct sxe2_vlan {
+ uint16_t tpid;
+ uint16_t vid;
+ uint8_t prio;
+};
+
+struct sxe2_vlan_filter {
+ TAILQ_ENTRY(sxe2_vlan_filter) next;
+ bool hw_config;
+ bool default_config;
+ struct sxe2_vlan vlan_info;
+};
+
+TAILQ_HEAD(sxe2_vlan_filter_list_head, sxe2_vlan_filter);
+
+struct sxe2_mac_filter {
+ TAILQ_ENTRY(sxe2_mac_filter) next;
+ bool hw_config;
+ bool default_config;
+ struct rte_ether_addr mac_addr;
+};
+
+TAILQ_HEAD(sxe2_uc_filter_list_head, sxe2_mac_filter);
+TAILQ_HEAD(sxe2_mc_filter_list_head, sxe2_mac_filter);
+
+int32_t sxe2_uc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config);
+
+int32_t sxe2_uc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr);
+
+void sxe2_uc_filter_clear(struct sxe2_adapter *adapter, bool default_config);
+
+int32_t sxe2_mc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config);
+
+int32_t sxe2_mc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr);
+
+void sxe2_mc_filter_clear(struct sxe2_adapter *adapter, bool default_config);
+
+int32_t sxe2_vlan_filter_add(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool default_config);
+
+int32_t sxe2_vlan_filter_del(struct sxe2_adapter *adapter, struct sxe2_vlan *vlan);
+
+void sxe2_vlan_filters_clear(struct sxe2_adapter *adapter, bool default_config);
+
+int32_t sxe2_vlan_filter_ctrl(struct sxe2_adapter *adapter, bool flag);
+
+int32_t sxe2_promisc_add(struct sxe2_adapter *adapter);
+
+int32_t sxe2_promisc_del(struct sxe2_adapter *adapter);
+
+int32_t sxe2_allmulti_add(struct sxe2_adapter *adapter);
+
+int32_t sxe2_allmulti_del(struct sxe2_adapter *adapter);
+
+int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter);
+
+int32_t sxe2_filter_rule_stop(struct rte_eth_dev *dev);
+
+int32_t sxe2_filter_rule_start(struct rte_eth_dev *dev);
+
+int32_t sxe2_filter_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_filter_uinit(struct rte_eth_dev *dev);
+
+#endif /* __SXE2_FILTER_H__ */
diff --git a/drivers/net/sxe2/sxe2_mac.c b/drivers/net/sxe2/sxe2_mac.c
index 3c2f909002..d94936a742 100644
--- a/drivers/net/sxe2/sxe2_mac.c
+++ b/drivers/net/sxe2/sxe2_mac.c
@@ -10,6 +10,438 @@
#include "sxe2_cmd_chnl.h"
#include "sxe2_host_regs.h"
+int32_t sxe2_mac_default_cfg(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret;
+ struct rte_ether_addr broadcast = {
+ .addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff} };
+ struct rte_ether_addr mac_addr;
+
+ rte_ether_addr_copy((struct rte_ether_addr *)
+ adapter->dev_info.mac.perm_addr, &mac_addr);
+ ret = sxe2_uc_filter_add(adapter, &mac_addr, true);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add default MAC filter");
+ goto l_end;
+ }
+
+ rte_ether_addr_copy(&broadcast, &mac_addr);
+ ret = sxe2_mc_filter_add(adapter, &mac_addr, true);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add broadcast MAC filter");
+ goto l_end;
+ }
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_mac_addr_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = -1;
+ PMD_INIT_FUNC_TRACE();
+
+ if (!rte_is_unicast_ether_addr
+ ((struct rte_ether_addr *)adapter->dev_info.mac.perm_addr)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Invalid MAC address");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ dev->data->mac_addrs = rte_zmalloc("sxe2_mac_adds",
+ sizeof(struct rte_ether_addr) * SXE2_NUM_MACADDR_MAX, 0);
+ if (!dev->data->mac_addrs) {
+ PMD_LOG_ERR(DRV, "Failed to allocate memory to store mac address");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ rte_ether_addr_copy((struct rte_ether_addr *)adapter->dev_info.mac.perm_addr,
+ &dev->data->mac_addrs[0]);
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+void sxe2_mac_addr_uinit(struct rte_eth_dev *dev)
+{
+ PMD_INIT_FUNC_TRACE();
+ if (dev != NULL && dev->data->mac_addrs != NULL) {
+ rte_free(dev->data->mac_addrs);
+ dev->data->mac_addrs = NULL;
+ }
+}
+
+int32_t sxe2_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
+ __rte_unused uint32_t index, __rte_unused uint32_t pool)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = -1;
+
+ if (rte_is_zero_ether_addr(mac_addr)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Invalid MAC Address");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (rte_is_multicast_ether_addr(mac_addr))
+ ret = sxe2_mc_filter_add(adapter, mac_addr, true);
+ else
+ ret = sxe2_uc_filter_add(adapter, mac_addr, false);
+
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add MAC filter");
+
+l_end:
+ return ret;
+}
+
+void sxe2_mac_addr_del(struct rte_eth_dev *dev, uint32_t index)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_ether_addr *mac_addr = &dev->data->mac_addrs[index];
+ int32_t ret = -1;
+
+ if (rte_is_multicast_ether_addr(mac_addr))
+ ret = sxe2_mc_filter_del(adapter, mac_addr);
+ else
+ ret = sxe2_uc_filter_del(adapter, mac_addr);
+
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to remove MAC filter");
+}
+
+int32_t sxe2_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ struct rte_ether_addr *old_addr = (struct rte_ether_addr *)&adapter->dev_info.mac.perm_addr;
+ struct rte_ether_addr temp_addr;
+
+ if (rte_is_same_ether_addr(old_addr, mac_addr))
+ goto l_end;
+
+ if (rte_is_multicast_ether_addr(mac_addr)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set multicast addr");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_uc_filter_del(adapter, old_addr);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to remove MAC filter");
+ goto l_end;
+ }
+
+ rte_ether_addr_copy(old_addr, &temp_addr);
+
+ rte_ether_addr_copy(mac_addr, old_addr);
+
+ ret = sxe2_uc_filter_add(adapter, mac_addr, true);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add MAC filter");
+ rte_ether_addr_copy(&temp_addr, old_addr);
+ (void)sxe2_uc_filter_add(adapter, old_addr, true);
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_set_mc_addr_list(struct rte_eth_dev *dev,
+ struct rte_ether_addr *mc_addrs,
+ uint32_t mc_addrs_num)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ uint32_t i;
+ const uint8_t *mac;
+
+ if (mc_addrs_num > SXE2_NUM_MACADDR_MAX) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Too many multicast MAC addresses, ");
+ ret = -1;
+ goto l_end;
+ }
+
+ sxe2_mc_filter_clear(adapter, false);
+
+ for (i = 0; i < mc_addrs_num; i++) {
+ if (!rte_is_multicast_ether_addr(&mc_addrs[i])) {
+ mac = mc_addrs[i].addr_bytes;
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Invalid mac: %02x:%02x:%02x:%02x:%02x:%02x",
+ mac[0], mac[1], mac[2], mac[3], mac[4],
+ mac[5]);
+ ret = -EINVAL;
+ goto add_err;
+ }
+
+ ret = sxe2_mc_filter_add(adapter, &mc_addrs[i], false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to remove old multicast MAC filter list");
+ goto add_err;
+ }
+ }
+ goto l_end;
+add_err:
+ sxe2_mc_filter_clear(adapter, false);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int32_t on)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vlan vlan = {
+ .tpid = RTE_ETHER_TYPE_VLAN,
+ .vid = vlan_id,
+ .prio = 0
+ };
+ int32_t ret = 0;
+
+ if (sxe2_dev_port_vlan_check(dev)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Filter not supported with Port VLAN");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (vlan_id == 0)
+ goto l_end;
+
+ if (on) {
+ ret = sxe2_vlan_filter_add(adapter, &vlan, false);
+ if (ret < 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan filter");
+ goto l_end;
+ }
+ } else {
+ ret = sxe2_vlan_filter_del(adapter, &vlan);
+ if (ret < 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to remove vlan filter");
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_vlan_offload_set(struct rte_eth_dev *dev, int32_t mask)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
+ struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
+ struct sxe2_vlan_info new_info = adapter->filter_ctxt.vlan_info;
+ bool port_vlan = new_info.port_vlan_exist;
+
+ uint8_t out_strip_mask = SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021Q |
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021AD |
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_QINQ1;
+
+ if (txmode->offloads & RTE_ETH_TX_OFFLOAD_QINQ_INSERT) {
+ if (!(txmode->offloads & RTE_ETH_TX_OFFLOAD_VLAN_INSERT)) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "VLAN INSERT must be enabled when QinQ INSERT is enabled");
+ return -EINVAL;
+ }
+ if (port_vlan) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "QINQ INSERT not supported with Port VLAN");
+ return -EINVAL;
+ }
+ }
+
+ if (mask & RTE_ETH_QINQ_STRIP_MASK) {
+ if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP) {
+ if (port_vlan) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "QinQ strip not supported with Port VLAN");
+ return -EINVAL;
+ }
+ new_info.inner_strip = SXE2_VSI_TSR_ID_VLAN;
+ } else {
+ new_info.inner_strip = 0;
+ }
+ }
+
+ if (mask & RTE_ETH_VLAN_STRIP_MASK) {
+ if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) {
+ new_info.outer_strip =
+ port_vlan ? 0 : out_strip_mask;
+ new_info.inner_strip =
+ port_vlan ? new_info.inner_strip : new_info.inner_strip;
+ } else {
+ if (new_info.inner_strip != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Must disable QinQ strip before disabling VLAN strip");
+ return -EINVAL;
+ }
+ new_info.outer_strip = 0;
+ }
+ }
+
+ if (mask & (RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_QINQ_STRIP_MASK)) {
+ struct sxe2_vlan_info old_info = adapter->filter_ctxt.vlan_info;
+ adapter->filter_ctxt.vlan_info = new_info;
+
+ ret = sxe2_drv_vlan_insert_strip_cfg(adapter);
+ if (ret) {
+ adapter->filter_ctxt.vlan_info = old_info;
+ return ret;
+ }
+ }
+ if (mask & RTE_ETH_VLAN_FILTER_MASK) {
+ if (adapter->filter_ctxt.vlan_info.port_vlan_exist) {
+ ret = 0;
+ PMD_DEV_LOG_INFO(adapter, INIT, "vlan filter is not support when port vlan is enabled");
+ goto l_end;
+ }
+
+ ret = sxe2_vlan_filter_ctrl(adapter,
+ !!(rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER));
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "sxe2_drv_vlan_filter_switch failed ret:%d", ret);
+ goto l_end;
+ }
+ }
+
+ PMD_DEV_LOG_DEBUG(adapter, DRV,
+ "mask:0x%x rx mode offload:0x%" PRIx64 " vlan offload set done",
+ mask, rxmode->offloads);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_vlan_filter_zero(struct sxe2_adapter *adapter)
+{
+ struct sxe2_vlan vlan;
+ int32_t ret;
+ uint16_t tpids[] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, RTE_ETHER_TYPE_QINQ1};
+ uint8_t i;
+
+ vlan = (struct sxe2_vlan){0, 0, 0};
+ ret = sxe2_vlan_filter_add(adapter, &vlan, true);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add VLAN ID 0");
+ goto l_end;
+ }
+
+ for (i = 0; i < RTE_DIM(tpids); i++) {
+ vlan = (struct sxe2_vlan){tpids[i], 0, 0};
+ ret = sxe2_vlan_filter_add(adapter, &vlan, true);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add VLAN ID 0 when tpid:0x%x",
+ tpids[i]);
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_vlan_cfg_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_drv_vlan_config_query(adapter);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to query vlan config, ret=%d", ret);
+ goto l_end;
+ }
+
+ if (!sxe2_dev_port_vlan_check(dev))
+ adapter->filter_ctxt.vlan_info.outer_insert =
+ SXE2_DPDK_OFFLOAD_OUTER_INSERT_8021Q |
+ SXE2_DPDK_OFFLOAD_INSERT_ENABLE;
+ else
+ adapter->filter_ctxt.vlan_info.outer_insert = 0;
+
+ adapter->filter_ctxt.vlan_info.inner_insert =
+ SXE2_DPDK_OFFLOAD_INNER_INSERT_QINQ1 | SXE2_DPDK_OFFLOAD_INSERT_ENABLE;
+
+ if (!sxe2_dev_port_vlan_check(dev)) {
+ ret = sxe2_vlan_filter_zero(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan filter switch:0 "
+ "for port:%d", adapter->port_idx);
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_vlan_default_cfg(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ ret = sxe2_dev_vlan_offload_set(dev, RTE_ETH_VLAN_STRIP_MASK |
+ RTE_ETH_QINQ_STRIP_MASK |
+ RTE_ETH_VLAN_FILTER_MASK);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cfg vlan offload, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_promisc_enable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_promisc_add(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to enable promisc, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_promisc_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_promisc_del(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to disable promisc, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_allmulti_enable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_allmulti_add(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to enable allmulti, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_allmulti_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_allmulti_del(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to disable allmulti, ret:%d", ret);
+
+ return ret;
+}
+
int32_t sxe2_link_update_init(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
diff --git a/drivers/net/sxe2/sxe2_mac.h b/drivers/net/sxe2/sxe2_mac.h
index f2f3edaeff..55fd1829a0 100644
--- a/drivers/net/sxe2/sxe2_mac.h
+++ b/drivers/net/sxe2/sxe2_mac.h
@@ -43,6 +43,40 @@ struct sxe2_mac_mc_list {
int32_t sxe2_link_update_init(struct rte_eth_dev *dev);
+int32_t sxe2_mac_default_cfg(struct rte_eth_dev *dev);
+
+int32_t sxe2_vlan_cfg_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_mac_addr_init(struct rte_eth_dev *dev);
+
+void sxe2_mac_addr_uinit(struct rte_eth_dev *dev);
+
+int32_t sxe2_mac_addr_add(struct rte_eth_dev *dev,
+ struct rte_ether_addr *mac_addr,
+ __rte_unused uint32_t index, __rte_unused uint32_t pool);
+
+void sxe2_mac_addr_del(struct rte_eth_dev *dev, uint32_t index);
+
+int32_t sxe2_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
+
+int32_t sxe2_set_mc_addr_list(struct rte_eth_dev *dev,
+ struct rte_ether_addr *mc_addrs,
+ uint32_t mc_addrs_num);
+
+int32_t sxe2_promisc_enable(struct rte_eth_dev *dev);
+
+int32_t sxe2_promisc_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_allmulti_enable(struct rte_eth_dev *dev);
+
+int32_t sxe2_allmulti_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int32_t on);
+
+int32_t sxe2_dev_vlan_offload_set(struct rte_eth_dev *dev, int32_t mask);
+
+int32_t sxe2_vlan_default_cfg(struct rte_eth_dev *dev);
+
int32_t sxe2_link_update(struct rte_eth_dev *dev, __rte_unused int32_t wait_to_complete);
int32_t sxe2_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index b9d34afb31..21d5c38725 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -660,6 +660,53 @@ sxe2_rx_desc_error_para(__rte_unused struct sxe2_rx_queue *rxq,
return flags;
}
+static inline void sxe2_rx_desc_vlan_para_fill(struct rte_mbuf *mbuf,
+ union sxe2_rx_desc *desc)
+{
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_L2TAG1_P_MASK)) {
+ mbuf->vlan_tci = 0;
+ } else {
+ mbuf->ol_flags |= (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
+ mbuf->vlan_tci = rte_le_to_cpu_16(desc->wb.l2tag1);
+ PMD_LOG_DEBUG(RX, "Rx desc mbuf vlan, vlan_tci:%u",
+ mbuf->vlan_tci);
+ }
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (0 == (rte_le_to_cpu_32(desc->wb.status_lrocnt_fdpf_id) &
+ SXE2_RX_DESC_EXT_STATUS_L2TAG2P_MASK)) {
+ mbuf->vlan_tci_outer = 0;
+ } else {
+ mbuf->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED | RTE_MBUF_F_RX_QINQ |
+ RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN;
+ mbuf->vlan_tci_outer = mbuf->vlan_tci;
+ mbuf->vlan_tci = rte_le_to_cpu_16(desc->wb.l2tag2_2nd);
+ PMD_LOG_DEBUG(RX, "Rx desc out vlan, l2tag2_1st:%u l2tag2_2nd:%u.",
+ rte_le_to_cpu_16(desc->wb.l2tag2_1st),
+ rte_le_to_cpu_16(desc->wb.l2tag2_2nd));
+ }
+#endif
+}
+
+static inline void
+sxe2_rx_desc_filter_para_fill(struct sxe2_rx_queue *rxq __rte_unused,
+ struct rte_mbuf *mbuf, union sxe2_rx_desc *desc)
+{
+ if (SXE2_RX_DESC_STATUS_RSS_VLD_MASK &
+ rte_le_to_cpu_64(desc->wb.status_err_ptype_len)) {
+ mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+ mbuf->hash.rss = rte_le_to_cpu_32(desc->wb.filter_status);
+ PMD_LOG_DEBUG(RX, "rss id:%u", mbuf->hash.rss);
+ }
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (SXE2_RX_DESC_FD_VLD_MASK & desc->wb.rxdid_src) {
+ mbuf->ol_flags |= (RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
+ mbuf->hash.fdir.hi = rte_le_to_cpu_32(desc->wb.fd_filter_id);
+ PMD_LOG_DEBUG(RX, "fdir id:%u", mbuf->hash.fdir.hi);
+ }
+#endif
+}
+
static __rte_always_inline void
sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf,
union sxe2_rx_desc *rxd)
@@ -673,6 +720,8 @@ sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf
mbuf->packet_type = ptype_tbl[SXE2_RX_DESC_PTYPE_VAL_GET(qword1)];
pkt_flags = sxe2_rx_desc_error_para(rxq, rxd);
+ sxe2_rx_desc_vlan_para_fill(mbuf, rxd);
+ sxe2_rx_desc_filter_para_fill(rxq, mbuf, rxd);
mbuf->ol_flags |= pkt_flags;
}
--
2.52.0
^ permalink raw reply related
* [PATCH v7 02/20] net/sxe2: add AVX2 vector data path for Rx and Tx
From: liujie5 @ 2026-06-03 2:21 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260603022150.1140722-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Added AVX256 vectorized versions of Rx and Tx data path functions to
improve packet processing performance.
The vector path uses AVX2 SIMD instructions to process multiple
descriptors per loop, significantly reducing the per-packet overhead.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
net/sxe2: support AVX512 vectorized path for Rx and Tx
---
drivers/net/sxe2/meson.build | 9 +
drivers/net/sxe2/sxe2_txrx.c | 38 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 12 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 776 ++++++++++++++++++++++++++
4 files changed, 830 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 7bd0d8120c..c225dd7cd8 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -39,6 +39,15 @@ if arch_subdir == 'x86'
c_args: avx512_args)
objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
endif
+ sxe2_avx2_lib = static_library('sxe2_avx2_lib',
+ 'sxe2_txrx_vec_avx2.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev,
+ static_rte_bus_pci],
+ include_directories: includes,
+ c_args: [cflags, '-mavx2'])
+ objs += sxe2_avx2_lib.extract_objects('sxe2_txrx_vec_avx2.c')
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index aa1c474088..eaf95259a5 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -167,8 +167,14 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
#endif
}
- if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
- tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+ if (((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX2;
+
+ if ((0 == (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK)))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
@@ -197,6 +203,13 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
}
#endif
+ } else if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX2) {
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2_simple;
+ }
} else {
if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
@@ -231,6 +244,10 @@ static const struct {
{ sxe2_tx_pkts_vec_avx512_simple,
"Vector AVX512 Simple" },
#endif
+ { sxe2_tx_pkts_vec_avx2,
+ "Vector AVX2" },
+ { sxe2_tx_pkts_vec_avx2_simple,
+ "Vector AVX2 Simple" },
{ sxe2_tx_pkts_vec_sse,
"Vector SSE" },
{ sxe2_tx_pkts_vec_sse_simple,
@@ -330,7 +347,13 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
#endif
}
- if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX2;
+
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
#endif
@@ -354,6 +377,11 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
else
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
#endif
+ } else if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX2) {
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2;
} else {
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
}
@@ -381,6 +409,10 @@ static const struct {
{ sxe2_rx_pkts_scattered_vec_avx512_offload,
"Offload Vector AVX512 Scattered" },
#endif
+ { sxe2_rx_pkts_scattered_vec_avx2,
+ "Vector AVX2 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx2_offload,
+ "Offload Vector AVX2 Scattered" },
{ sxe2_rx_pkts_scattered_vec_sse_offload,
"Vector SSE Scattered" },
#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index af7c8d12b2..369777606f 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,19 +11,21 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
- SXE2_RX_MODE_VEC_AVX512)
+ SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
- SXE2_TX_MODE_VEC_AVX512)
+ SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -68,6 +70,12 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
new file mode 100644
index 0000000000..72b09850b6
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
@@ -0,0 +1,776 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static inline void
+sxe2_tx_desc_fill_one_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m256i desc_group0;
+ __m256i desc_group1;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ if (((uint64_t)desc & 0x1F) != 0 && pkts_num != 0) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+
+ while (pkts_num > 3) {
+ desc3_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group1 = _mm256_set_epi64x(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]));
+
+ desc_group0 = _mm256_set_epi64x(desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc + 2), desc_group1);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc), desc_group0);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx2 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx2(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+ }
+
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num = sxe2_tx_pkts_vec_avx2_batch(txq,
+ tx_pkts + tx_done_num, tx_need_num, with_offloads);
+
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx2(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read), dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ return;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ?
+ (rxq->ring_depth - 1) : (rxq->realloc_start - 1);
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx2(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m256i mbufs6_7, mbufs4_5, mbufs2_3, mbufs0_1;
+ uint32_t bit_num;
+ uint16_t done_num;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+ done_num = 0;
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx2(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m256i crc_adjust =
+ _mm256_set_epi16(0, 0, 0, -rxq->crc_len,
+ 0, -rxq->crc_len, 0,
+ 0, 0, 0, 0,
+ -rxq->crc_len, 0, -rxq->crc_len, 0, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+ const __m256i rvp_shuf_mask =
+ _mm256_set_epi8(7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 8, 0, 10, 2,
+ 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ if (split_rxe_flags) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs6_7 = _mm256_shuffle_epi8(descs6_7, rvp_shuf_mask);
+ mbufs4_5 = _mm256_shuffle_epi8(descs4_5, rvp_shuf_mask);
+
+ mbufs6_7 = _mm256_add_epi16(mbufs6_7, crc_adjust);
+ mbufs4_5 = _mm256_add_epi16(mbufs4_5, crc_adjust);
+
+ const __m256i ptype_mask = _mm256_set1_epi32(SXE2_RX_DESC_PTYPE_MASK);
+
+ const __m256i staterrs4_7 = _mm256_unpackhi_epi32(descs6_7, descs4_5);
+
+ __m256i ptypes4_7 = _mm256_and_si256(staterrs4_7, ptype_mask);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes4_7, 9);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes4_7, 1);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_7, 11);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_7, 3);
+
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype7], 4);
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype6], 0);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype5], 4);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype4], 0);
+
+ mbufs2_3 = _mm256_shuffle_epi8(descs2_3, rvp_shuf_mask);
+ mbufs0_1 = _mm256_shuffle_epi8(descs0_1, rvp_shuf_mask);
+
+ mbufs2_3 = _mm256_add_epi16(mbufs2_3, crc_adjust);
+ mbufs0_1 = _mm256_add_epi16(mbufs0_1, crc_adjust);
+
+ const __m256i staterrs0_3 = _mm256_unpackhi_epi32(descs2_3, descs0_1);
+
+ __m256i ptypes0_3 = _mm256_and_si256(staterrs0_3, ptype_mask);
+
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes0_3, 9);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes0_3, 1);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_3, 11);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_3, 3);
+
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype3], 4);
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype2], 0);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype1], 4);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype0], 0);
+
+ __m256i staterrs0_7 = _mm256_unpacklo_epi64(staterrs4_7, staterrs0_3);
+
+ __m256i stu_len0_7 = _mm256_unpackhi_epi64(staterrs4_7, staterrs0_3);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0x00001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+ const __m256i vlan_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 8), 4);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 4), 4);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 4);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(mbuf_flags, 4), 4);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] =
+ _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] =
+ _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags != NULL) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 = _mm256_and_si256(staterrs0_7,
+ umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+
+ __m128i umbcast_bits_128 = _mm_packs_epi32
+ (_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256
+ (umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags != NULL) {
+ const __m256i eop_rxe_mask = _mm256_set1_epi32
+ (SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m128i eop_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits = _mm_or_si128
+ (not_eop_bits,
+ _mm_srli_epi16
+ (_mm_and_si128(eop_rxe_bits, rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((1 == (rxq->processing_idx & 1)) && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ split_rxe_flags64[0] == 0 && split_rxe_flags64[1] == 0 &&
+ split_rxe_flags64[2] == 0 && split_rxe_flags64[3] == 0) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num =
+ sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM,
+ do_offload);
+ done_num += once_num;
+ nb_pkts -= once_num;
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num, nb_pkts, do_offload);
+l_end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
--
2.52.0
^ permalink raw reply related
* [PATCH v7 01/20] net/sxe2: support AVX512 vectorized path for Rx and Tx
From: liujie5 @ 2026-06-03 2:21 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260603022150.1140722-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add AVX512 vector data path for Rx and Tx burst functions.
The decision to use AVX512 is based on:
1. CPU hardware flags (AVX512F, AVX512BW).
2. Compiler support (CC_AVX512_SUPPORT).
3. Max SIMD bitwidth configuration.
Performance shows approximately X% improvement in small packet
forwarding scenarios.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 24 +
drivers/net/sxe2/sxe2_drv_cmd.h | 80 +--
drivers/net/sxe2/sxe2_ethdev.c | 2 +-
drivers/net/sxe2/sxe2_txrx.c | 92 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 18 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 ++++++++++++++++++++++++
7 files changed, 1099 insertions(+), 60 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 6b2eb75b0e..7bd0d8120c 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -15,6 +15,30 @@ includes += include_directories('../../common/sxe2')
if arch_subdir == 'x86'
sources += files('sxe2_txrx_vec_sse.c')
+
+ sxe2_avx512_cpu_support =(
+ cc.get_define('__AVX512F__', args: machine_args) != '' and
+ cc.get_define('__AVX512BW__', args: machine_args) != '')
+
+ sxe2_avx512_cc_support = (
+ not machine_args.contains('-mno-avx512f') and
+ cc.has_argument('-mavx512f') and
+ cc.has_argument('-mavx512bw'))
+
+ if sxe2_avx512_cpu_support == true or sxe2_avx512_cc_support == true
+ cflags += ['-DCC_AVX512_SUPPORT']
+ avx512_args = [cflags, '-mavx512f', '-mavx512bw']
+ if cc.has_argument('-march=skylake-avx512')
+ avx512_args += '-march=skylake-avx512'
+ endif
+ sxe2_avx512_lib = static_library('sxe2_avx512_lib', 'sxe2_txrx_vec_avx512.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev, static_rte_bus_pci],
+ include_directories: includes,
+ c_args: avx512_args)
+ objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
+ endif
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index bba6476c2e..ccc9c20ef4 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -67,20 +67,20 @@ enum sxe2_dev_type {
SXE2_DEV_T_MAX,
};
-struct sxe2_drv_queue_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_caps {
uint16_t queues_cnt;
uint16_t base_idx_in_pf;
-};
+} __rte_packed_end;
-struct sxe2_drv_msix_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_msix_caps {
uint16_t msix_vectors_cnt;
uint16_t base_idx_in_func;
-};
+} __rte_packed_end;
-struct sxe2_drv_rss_hash_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rss_hash_caps {
uint16_t hash_key_size;
uint16_t lut_key_size;
-};
+} __rte_packed_end;
enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_BOTH = 0,
@@ -89,18 +89,18 @@ enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_MAX,
};
-struct sxe2_drv_vsi_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_caps {
uint16_t func_id;
uint16_t dpdk_vsi_id;
uint16_t kernel_vsi_id;
uint16_t vsi_type;
-};
+} __rte_packed_end;
-struct sxe2_drv_representor_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_representor_caps {
uint16_t cnt_repr_vf;
uint8_t rsv[2];
struct sxe2_drv_vsi_caps repr_vf_id[256];
-};
+} __rte_packed_end;
enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_NOTSET = 0,
@@ -111,25 +111,25 @@ enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_UNKNOWN,
};
-struct sxe2_switchdev_mode_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_mode_info {
uint8_t pf_id;
uint8_t is_switchdev;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_switchdev_cpvsi_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_cpvsi_info {
uint16_t cp_vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_txsch_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_txsch_caps {
uint8_t layer_cap;
uint8_t tm_mid_node_num;
uint8_t prio_num;
uint8_t rev;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_caps_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_caps_resp {
struct sxe2_drv_queue_caps queue_caps;
struct sxe2_drv_msix_caps msix_caps;
struct sxe2_drv_rss_hash_caps rss_hash_caps;
@@ -141,24 +141,24 @@ struct sxe2_drv_dev_caps_resp {
uint8_t dev_type;
uint8_t rev;
uint32_t cap_flags;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_info_resp {
uint64_t dsn;
uint16_t vsi_id;
uint8_t rsv[2];
uint8_t mac_addr[SXE2_ETH_ALEN];
uint8_t rsv2[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_fw_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_fw_info_resp {
uint8_t main_version_id;
uint8_t sub_version_id;
uint8_t fix_version_id;
uint8_t build_id;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_ctxt {
uint64_t dma_addr;
uint32_t max_lro_size;
uint32_t split_type_mask;
@@ -170,62 +170,62 @@ struct sxe2_drv_rxq_ctxt {
uint8_t keep_crc_en;
uint8_t split_en;
uint8_t desc_size;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
uint16_t max_frame_size;
uint8_t rsv[2];
struct sxe2_drv_rxq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_ctxt {
uint64_t dma_addr;
uint32_t sched_mode;
uint16_t queue_id;
uint16_t depth;
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
struct sxe2_drv_txq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_q_switch_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_q_switch_req {
uint16_t q_idx;
uint16_t vsi_id;
uint8_t is_enable;
uint8_t sched_mode;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_create_req_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_create_req_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_free_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_free_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 8d66e5d8c5..e0f7002138 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -891,7 +891,7 @@ static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
static int32_t sxe2_parse_eth_devargs(struct rte_device *dev,
struct rte_eth_devargs *eth_da)
{
- int ret = 0;
+ int32_t ret = 0;
if (dev->devargs == NULL)
return 0;
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 8d17535301..aa1c474088 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -157,6 +157,19 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
tx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
+#endif
+ }
+ if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
if (ret != 0)
@@ -172,14 +185,25 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
tx_mode_flags = adapter->q_ctxt.tx_mode_flags;
}
-#ifdef RTE_ARCH_X86
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
- if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
- dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ dev->tx_pkt_prepare = NULL;
+#ifdef RTE_ARCH_X86
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
+ }
+#endif
} else {
- dev->tx_pkt_prepare = NULL;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ }
}
} else {
#endif
@@ -201,8 +225,16 @@ static const struct {
} sxe2_tx_burst_infos[] = {
{ sxe2_tx_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
- { sxe2_tx_pkts_vec_sse, "Vector SSE" },
- { sxe2_tx_pkts_vec_sse_simple, "Vector SSE Simple" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_tx_pkts_vec_avx512,
+ "Vector AVX512" },
+ { sxe2_tx_pkts_vec_avx512_simple,
+ "Vector AVX512 Simple" },
+#endif
+ { sxe2_tx_pkts_vec_sse,
+ "Vector SSE" },
+ { sxe2_tx_pkts_vec_sse_simple,
+ "Vector SSE Simple" },
#endif
};
@@ -288,6 +320,20 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
rx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
+#endif
+ }
+ if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
+ rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
if (ret != 0)
@@ -301,7 +347,16 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
#ifdef RTE_ARCH_X86
if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
- dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
+#endif
+ } else {
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ }
return;
}
#endif
@@ -315,19 +370,30 @@ static const struct {
eth_rx_burst_t rx_burst;
const char *info;
} sxe2_rx_burst_infos[] = {
- { sxe2_rx_pkts_scattered, "Scalar Scattered" },
- { sxe2_rx_pkts_scattered_split, "Scalar Scattered split" },
+ { sxe2_rx_pkts_scattered,
+ "Scalar Scattered" },
+ { sxe2_rx_pkts_scattered_split,
+ "Scalar Scattered split" },
#ifdef RTE_ARCH_X86
- { sxe2_rx_pkts_scattered_vec_sse_offload, "Vector SSE Scattered" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_rx_pkts_scattered_vec_avx512,
+ "Vector AVX512 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx512_offload,
+ "Offload Vector AVX512 Scattered" },
+#endif
+ { sxe2_rx_pkts_scattered_vec_sse_offload,
+ "Vector SSE Scattered" },
#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
- __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode)
+ __rte_unused uint16_t queue_id,
+ struct rte_eth_burst_mode *mode)
{
eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
int32_t ret = -EINVAL;
uint32_t i, size;
+
size = RTE_DIM(sxe2_rx_burst_infos);
for (i = 0; i < size; ++i) {
if (pkt_burst == sxe2_rx_burst_infos[i].rx_burst) {
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index 8df4954d86..cf004f5eb2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -165,16 +165,54 @@ static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
return;
}
i = txq->next_dd - (txq->rs_thresh - 1);
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
+#ifdef CC_AVX512_SUPPORT
+ struct rte_eth_dev *dev;
+ struct sxe2_tx_buffer_vec *buffer_vec;
+
+ dev = &rte_eth_devices[txq->port_id];
+
+ if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
+ dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
+ buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for ( ; i < txq->next_use; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ } else {
+#endif
+ buffer = txq->buffer_ring;
+ buffer = txq->buffer_ring;
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer[i].mbuf);
+ buffer[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
buffer[i].mbuf = NULL;
}
}
- i = 0;
+#ifdef CC_AVX512_SUPPORT
}
+#endif
+
for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 04ff4d96a5..af7c8d12b2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,15 +11,19 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
- SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE)
+ SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
+ SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
- SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE)
+ SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
+ SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -54,6 +58,16 @@ uint16_t sxe2_tx_pkts_vec_sse(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_
uint16_t sxe2_tx_pkts_vec_sse_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_sse_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_ctx_offload(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
new file mode 100644
index 0000000000..2aec8037dd
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -0,0 +1,897 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_TEST
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_queue *txq)
+{
+ struct sxe2_tx_buffer_vec *buffer;
+ struct rte_mbuf *mbuf;
+ struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
+ struct rte_mempool *mp;
+ struct rte_mempool_cache *cache;
+ void **cache_objs;
+ uint32_t copied;
+ uint32_t i;
+ int32_t ret;
+ uint16_t rs_thresh;
+ uint16_t free_num;
+
+ if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
+ (txq->desc_ring[txq->next_dd].wb.dd &
+ rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
+ ret = 0;
+ goto l_end;
+ }
+
+ rs_thresh = txq->rs_thresh;
+
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += txq->next_dd - (rs_thresh - 1);
+
+ if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
+ (rs_thresh & 31) == 0) {
+ mp = buffer[0].mbuf->pool;
+ cache = rte_mempool_default_cache(mp, rte_lcore_id());
+
+ if (cache == NULL || cache->len)
+ goto normal;
+
+ if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer, rs_thresh);
+ goto done;
+ }
+ cache_objs = &cache->objs[cache->len];
+
+ copied = 0;
+ while (copied < rs_thresh) {
+ const __m512i objs0 = _mm512_loadu_si512(&buffer[copied]);
+ const __m512i objs1 = _mm512_loadu_si512(&buffer[copied + 8]);
+ const __m512i objs2 = _mm512_loadu_si512(&buffer[copied + 16]);
+ const __m512i objs3 = _mm512_loadu_si512(&buffer[copied + 24]);
+
+ _mm512_storeu_si512(&cache_objs[copied], objs0);
+ _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
+ _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
+ _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
+ copied += 32;
+ }
+ cache->len += rs_thresh;
+
+ if (cache->len >= cache->flushthresh) {
+ (void)rte_mempool_ops_enqueue_bulk(mp,
+ &cache->objs[cache->size], cache->len - cache->size);
+ cache->len = cache->size;
+ }
+ goto done;
+ }
+
+normal:
+ mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
+
+ if (likely(mbuf)) {
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+
+ if (likely(mbuf)) {
+ if (likely(mbuf->pool == mbuf_free_arr[0]->pool)) {
+ mbuf_free_arr[free_num] = mbuf;
+ free_num++;
+ } else {
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+ }
+ }
+ }
+
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+ } else {
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+ if (mbuf != NULL)
+ rte_mempool_put(mbuf->pool, mbuf);
+ }
+ }
+
+done:
+ txq->desc_free_num += txq->rs_thresh;
+ txq->next_dd += txq->rs_thresh;
+ if (txq->next_dd >= txq->ring_depth)
+ txq->next_dd = txq->rs_thresh - 1;
+ ret = rs_thresh;
+
+l_end:
+ return ret;
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline
+void sxe2_tx_desc_fill_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m512i desc_group;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ while (pkts_num > 3) {
+ desc3_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group =
+ _mm512_set_epi64(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]),
+ desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, desc), desc_group);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx512(desc, *pkts, desc_cmd, with_offloads);
+
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline void
+sxe2_tx_pkts_mbuf_fill_avx512(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ uint16_t i;
+
+ for (i = 0; i < nb_pkts; ++i)
+ buffer[i].mbuf = tx_pkts[i];
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer_vec *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec_avx512(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx512 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += next_use;
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx512(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = txq->desc_ring;
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ }
+
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num =
+ sxe2_tx_pkts_vec_avx512_batch(txq, tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx512(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read),
+ dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ? (rxq->ring_depth - 1) :
+ (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m512i mbufs4_7;
+ __m512i mbufs0_3;
+ __m256i mbufs6_7;
+ __m256i mbufs4_5;
+ __m256i mbufs2_3;
+ __m256i mbufs0_1;
+ uint32_t bit_num = 0;
+ uint16_t done_num = 0;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx512(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) & SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m512i crc_adjust =
+ _mm512_set4_epi32(0, -rxq->crc_len, -rxq->crc_len, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+
+ const __m512i rvp_shuf_mask =
+ _mm512_set4_epi32((7 << 24) | (6 << 16) | (5 << 8) | 4,
+ (3 << 24) | (2 << 16) | (13 << 8) | 12,
+ (0xFFU << 24) | (0xFF << 16) | (13 << 8) | 12,
+ 0xFFFFFFFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 8, 0, 10, 2, 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ const __m512i descs4_7 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs4_5), descs6_7, 1);
+ const __m512i descs0_3 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs0_1), descs2_3, 1);
+
+ if (split_rxe_flags != NULL) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs4_7 = _mm512_shuffle_epi8(descs4_7, rvp_shuf_mask);
+ mbufs0_3 = _mm512_shuffle_epi8(descs0_3, rvp_shuf_mask);
+
+ mbufs4_7 = _mm512_add_epi32(mbufs4_7, crc_adjust);
+ mbufs0_3 = _mm512_add_epi32(mbufs0_3, crc_adjust);
+
+ const __m512i ptype_mask = _mm512_set1_epi64(SXE2_RX_FLEX_DESC_PTYPE_M <<
+ SXE2_RX_FLEX_DESC_PTYPE_S);
+
+ __m512i ptypes4_7 = _mm512_and_si512(descs4_7, ptype_mask);
+ __m512i ptypes0_3 = _mm512_and_si512(descs0_3, ptype_mask);
+
+ const __m256i ptypes6_7 = _mm512_extracti64x4_epi64(ptypes4_7, 1);
+ const __m256i ptypes4_5 = _mm512_extracti64x4_epi64(ptypes4_7, 0);
+ const __m256i ptypes2_3 = _mm512_extracti64x4_epi64(ptypes0_3, 1);
+ const __m256i ptypes0_1 = _mm512_extracti64x4_epi64(ptypes0_3, 0);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 13);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 5);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 13);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 5);
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 13);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 5);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 13);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 5);
+
+ const __m512i ptype_mask4_7 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype7],
+ 0, 0, 0, ptype_tbl[ptype6],
+ 0, 0, 0, ptype_tbl[ptype5],
+ 0, 0, 0, ptype_tbl[ptype4]);
+ const __m512i ptype_mask0_3 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype3],
+ 0, 0, 0, ptype_tbl[ptype2],
+ 0, 0, 0, ptype_tbl[ptype1],
+ 0, 0, 0, ptype_tbl[ptype0]);
+
+ mbufs4_7 = _mm512_or_si512(mbufs4_7, ptype_mask4_7);
+ mbufs0_3 = _mm512_or_si512(mbufs0_3, ptype_mask0_3);
+
+ mbufs6_7 = _mm512_extracti64x4_epi64(mbufs4_7, 1);
+ mbufs4_5 = _mm512_extracti64x4_epi64(mbufs4_7, 0);
+ mbufs2_3 = _mm512_extracti64x4_epi64(mbufs0_3, 1);
+ mbufs0_1 = _mm512_extracti64x4_epi64(mbufs0_3, 0);
+
+ const __m512i staterr_per_mask =
+ _mm512_set_epi32(0x17, 0x1F, 0x07, 0x0F,
+ 0x13, 0x1B, 0x03, 0x0B,
+ 0x16, 0x1E, 0x06, 0x0E,
+ 0x12, 0x1A, 0x02, 0x0A);
+ __m512i qw1_0_7 = _mm512_permutex2var_epi32(descs4_7,
+ staterr_per_mask,
+ descs0_3);
+
+ __m256i staterrs0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 0);
+
+ __m256i stu_len0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 1);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0xC0001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0,
+ 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+
+ const __m256i vlan_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 8), 0x04);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 4), 0x04);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(mbuf_flags, 4), 0x04);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] = _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 =
+ _mm256_and_si256(staterrs0_7, umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+ __m128i umbcast_bits_128 =
+ _mm_packs_epi32(_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256(umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags) {
+ const __m256i eop_rxe_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+ const __m128i eop_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits =
+ _mm_or_si128(not_eop_bits,
+ _mm_srli_epi16(_mm_and_si128(eop_rxe_bits,
+ rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((rxq->processing_idx & 1) == 1 && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ !split_rxe_flags64[0] && !split_rxe_flags64[1] &&
+ !split_rxe_flags64[2] && !split_rxe_flags64[3]) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num = 0;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue, rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM, offload);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue,
+ rx_pkts + done_num, nb_pkts, offload);
+
+end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
+
+#endif
--
2.52.0
^ permalink raw reply related
* [PATCH v7 00/20]net/sxe2: added Linkdata sxe2 ethernet driver
From: liujie5 @ 2026-06-03 2:21 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260602155240.1002602-21-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch set implements core functionality for the SXE2 PMD,
including basic driver framework, data path setup, and advanced
offload features (VLAN, RSS,TM, PTP etc.).
V7:
- Addressed AI comments
Jie Liu (20):
net/sxe2: support AVX512 vectorized path for Rx and Tx
net/sxe2: add AVX2 vector data path for Rx and Tx
drivers: add supported packet types get callback
net/sxe2: support L2 filtering and MAC config
drivers: support RSS feature
net/sxe2: support TM hierarchy and shaping
net/sxe2: support IPsec inline protocol offload
net/sxe2: support statistics and multi-process
drivers: interrupt handling
net/sxe2: add NEON vec Rx/Tx burst functions
drivers: add support for VF representors
net/sxe2: add support for custom UDP tunnel ports
net/sxe2: support firmware version reading
net/sxe2: implement get monitor address
common/sxe2: add shared SFP module definitions
net/sxe2: support SFP module info and EEPROM access
net/sxe2: implement private dump info
net/sxe2: add mbuf validation in Tx debug mode
drivers: add testpmd commands for private features
net/sxe2: update sxe2 feature matrix docs
doc/guides/nics/features/sxe2.ini | 64 +
drivers/common/sxe2/sxe2_common.c | 156 ++
drivers/common/sxe2/sxe2_common.h | 4 +
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++
drivers/common/sxe2/sxe2_ioctl_chnl.c | 178 +-
drivers/common/sxe2/sxe2_ioctl_chnl_func.h | 18 +
drivers/common/sxe2/sxe2_msg.h | 118 ++
drivers/common/sxe2/sxe2_ptype.h | 1793 ++++++++++++++++++
drivers/net/sxe2/meson.build | 56 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 1587 +++++++++++++++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 139 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 521 +++++-
drivers/net/sxe2/sxe2_dump.c | 304 +++
drivers/net/sxe2/sxe2_dump.h | 12 +
drivers/net/sxe2/sxe2_ethdev.c | 1531 +++++++++++++++-
drivers/net/sxe2/sxe2_ethdev.h | 112 +-
drivers/net/sxe2/sxe2_ethdev_repr.c | 610 ++++++
drivers/net/sxe2/sxe2_ethdev_repr.h | 32 +
drivers/net/sxe2/sxe2_filter.c | 895 +++++++++
drivers/net/sxe2/sxe2_filter.h | 100 +
drivers/net/sxe2/sxe2_flow.c | 1391 ++++++++++++++
drivers/net/sxe2/sxe2_flow.h | 30 +
drivers/net/sxe2/sxe2_flow_define.h | 144 ++
drivers/net/sxe2/sxe2_flow_parse_action.c | 1182 ++++++++++++
drivers/net/sxe2/sxe2_flow_parse_action.h | 23 +
drivers/net/sxe2/sxe2_flow_parse_engine.c | 106 ++
drivers/net/sxe2/sxe2_flow_parse_engine.h | 13 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 1935 ++++++++++++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 46 +
drivers/net/sxe2/sxe2_ipsec.c | 1565 ++++++++++++++++
drivers/net/sxe2/sxe2_ipsec.h | 254 +++
drivers/net/sxe2/sxe2_irq.c | 1025 +++++++++++
drivers/net/sxe2/sxe2_irq.h | 25 +
drivers/net/sxe2/sxe2_mac.c | 535 ++++++
drivers/net/sxe2/sxe2_mac.h | 84 +
drivers/net/sxe2/sxe2_mp.c | 414 +++++
drivers/net/sxe2/sxe2_mp.h | 67 +
drivers/net/sxe2/sxe2_queue.c | 17 +-
drivers/net/sxe2/sxe2_rss.c | 584 ++++++
drivers/net/sxe2/sxe2_rss.h | 81 +
drivers/net/sxe2/sxe2_rx.c | 38 +
drivers/net/sxe2/sxe2_rx.h | 2 +
drivers/net/sxe2/sxe2_security.c | 335 ++++
drivers/net/sxe2/sxe2_security.h | 77 +
drivers/net/sxe2/sxe2_stats.c | 591 ++++++
drivers/net/sxe2/sxe2_stats.h | 39 +
drivers/net/sxe2/sxe2_switchdev.c | 332 ++++
drivers/net/sxe2/sxe2_switchdev.h | 33 +
drivers/net/sxe2/sxe2_testpmd.c | 733 ++++++++
drivers/net/sxe2/sxe2_testpmd_lib.c | 969 ++++++++++
drivers/net/sxe2/sxe2_testpmd_lib.h | 142 ++
drivers/net/sxe2/sxe2_tm.c | 1169 ++++++++++++
drivers/net/sxe2/sxe2_tm.h | 78 +
drivers/net/sxe2/sxe2_tx.c | 7 +
drivers/net/sxe2/sxe2_txrx.c | 176 +-
drivers/net/sxe2/sxe2_txrx.h | 4 +
drivers/net/sxe2/sxe2_txrx_check_mbuf.c | 595 ++++++
drivers/net/sxe2/sxe2_txrx_check_mbuf.h | 38 +
drivers/net/sxe2/sxe2_txrx_poll.c | 243 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 38 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 776 ++++++++
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 +++++++++
drivers/net/sxe2/sxe2_txrx_vec_common.h | 1 +
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 721 ++++++++
drivers/net/sxe2/sxe2_vsi.c | 146 ++
drivers/net/sxe2/sxe2_vsi.h | 12 +-
drivers/net/sxe2/sxe2vf_regs.h | 85 +
68 files changed, 26583 insertions(+), 124 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/common/sxe2/sxe2_msg.h
create mode 100644 drivers/common/sxe2/sxe2_ptype.h
create mode 100644 drivers/net/sxe2/sxe2_dump.c
create mode 100644 drivers/net/sxe2/sxe2_dump.h
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.c
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.h
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
create mode 100644 drivers/net/sxe2/sxe2_flow.c
create mode 100644 drivers/net/sxe2/sxe2_flow.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.h
create mode 100644 drivers/net/sxe2/sxe2_ipsec.c
create mode 100644 drivers/net/sxe2/sxe2_ipsec.h
create mode 100644 drivers/net/sxe2/sxe2_irq.c
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
create mode 100644 drivers/net/sxe2/sxe2_mp.c
create mode 100644 drivers/net/sxe2/sxe2_mp.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
create mode 100644 drivers/net/sxe2/sxe2_security.c
create mode 100644 drivers/net/sxe2/sxe2_security.h
create mode 100644 drivers/net/sxe2/sxe2_stats.c
create mode 100644 drivers/net/sxe2/sxe2_stats.h
create mode 100644 drivers/net/sxe2/sxe2_switchdev.c
create mode 100644 drivers/net/sxe2/sxe2_switchdev.h
create mode 100644 drivers/net/sxe2/sxe2_testpmd.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.h
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
create mode 100644 drivers/net/sxe2/sxe2vf_regs.h
--
2.52.0
^ permalink raw reply
* Re: [PATCH v2] ethdev: add buffer size parameter to rte_eth_dev_get_name_by_port()
From: Stephen Hemminger @ 2026-06-03 2:20 UTC (permalink / raw)
To: dev
Cc: Andrew Rybchenko, Reshma Pattan, Aman Singh, Naga Harish K S V,
Bruce Richardson, Kishore Padmanabha, Ajit Khaparde,
Nithin Dabilpuram, Kiran Kumar K, Sunil Kumar Kori, Satha Rao,
Harman Kalra, Anatoly Burakov, Cristian Dumitrescu,
Thomas Monjalon
In-Reply-To: <20260529000748.275863-1-stephen@networkplumber.org>
On Thu, 28 May 2026 17:07:48 -0700
Stephen Hemminger <stephen@networkplumber.org> wrote:
> rte_eth_dev_get_name_by_port() uses strcpy() into a caller-provided
> buffer with no bounds checking. A mistaken caller that supplies a
> buffer smaller than RTE_ETH_NAME_MAX_LEN can overflow the buffer.
>
> Add a size parameter and replace strcpy() with strlcpy(), returning
> -ERANGE if the name is truncated. The copy is now performed under
> the ethdev shared data lock so that the name cannot be mutated by
> another process between reading and copying.
>
> The previous ABI is preserved via symbol versioning (DPDK_26) with a
> thin wrapper that delegates to the new implementation with
> RTE_ETH_NAME_MAX_LEN. The versioned symbol will be removed in 26.11.
>
> Update all in-tree callers to pass sizeof(name) as the new parameter.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> Reviewed-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> ---
Since function versioning breaks LTO (see Bugzilla).
Holding off on this until 26.11 where versioning not needed.
^ permalink raw reply
* [PATCH] eal: fix function versioning with LTO
From: Stephen Hemminger @ 2026-06-02 22:57 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, stable, David Marchand
When using function versioning and building with Link Time Optimization,
the compiler does not see the __asm__ annotation of symbols and
therefore thinks there are two versions of the same symbol.
The fix is to use compiler symver attribute on the function which
was added in GCC 10. Keep the older method for backward compatibility
with older compilers.
Bugzilla ID: 1949
Fixes: e30e194c4d06 ("eal: rework function versioning macros")
Cc: stable@dpdk.org
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/eal/common/eal_export.h | 46 +++++++++++++++++++++++++++----------
1 file changed, 34 insertions(+), 12 deletions(-)
diff --git a/lib/eal/common/eal_export.h b/lib/eal/common/eal_export.h
index 7e47d34064..1318e6278d 100644
--- a/lib/eal/common/eal_export.h
+++ b/lib/eal/common/eal_export.h
@@ -30,39 +30,61 @@
* new functionality, behavior, etc. When that occurs, it is desirable to
* allow for backwards compatibility for a time with older binaries that are
* dynamically linked to the dpdk.
+ *
+ * RTE_VERSION_SYMBOL
+ * Create a symbol version table entry binding symbol <name>@DPDK_<ver> to the internal
+ * function name <name>_v<ver>.
+ *
+ * RTE_VERSION_EXPERIMENTAL_SYMBOL similar to RTE_VERSION_SYMBOL but for experimental API symbols.
+ * This is mainly used for keeping compatibility for symbols that get promoted to stable ABI.
+ *
+ * RTE_DEFAULT_SYMBOL
+ * Create a symbol version entry instructing the linker to bind references to
+ * symbol <name> to the internal symbol <name>_v<ver>.
*/
#ifdef RTE_BUILD_SHARED_LIB
-/*
- * Create a symbol version table entry binding symbol <name>@DPDK_<ver> to the internal
- * function name <name>_v<ver>.
- */
+/* Prefer the compiler method of versioning which uses attributes */
+#if __has_attribute(symver)
+
+#define RTE_VERSION_SYMBOL(ver, type, name, args) VERSIONING_WARN \
+ __attribute__((__symver__(RTE_STR(name) "@DPDK_" RTE_STR(ver)))) \
+ type name ## _v ## ver args; \
+ type name ## _v ## ver args
+
+#define RTE_VERSION_EXPERIMENTAL_SYMBOL(type, name, args) VERSIONING_WARN \
+ __attribute__((__symver__(RTE_STR(name) "@EXPERIMENTAL"))) \
+ type name ## _exp args; \
+ type name ## _exp args
+
+#define RTE_DEFAULT_SYMBOL(ver, type, name, args) VERSIONING_WARN \
+ __attribute__((__symver__(RTE_STR(name) "@@DPDK_" RTE_STR(ver)))) \
+ type name ## _v ## ver args; \
+ type name ## _v ## ver args
+#else /* !__has_attribute(symver) */
+
+/* Use asm tag to create symbol table entry */
#define RTE_VERSION_SYMBOL(ver, type, name, args) VERSIONING_WARN \
__asm__(".symver " RTE_STR(name) "_v" RTE_STR(ver) ", " RTE_STR(name) "@DPDK_" RTE_STR(ver)); \
__rte_used type name ## _v ## ver args; \
type name ## _v ## ver args
-/*
- * Similar to RTE_VERSION_SYMBOL but for experimental API symbols.
- * This is mainly used for keeping compatibility for symbols that get promoted to stable ABI.
- */
#define RTE_VERSION_EXPERIMENTAL_SYMBOL(type, name, args) VERSIONING_WARN \
__asm__(".symver " RTE_STR(name) "_exp, " RTE_STR(name) "@EXPERIMENTAL") \
__rte_used type name ## _exp args; \
type name ## _exp args
-/*
- * Create a symbol version entry instructing the linker to bind references to
- * symbol <name> to the internal symbol <name>_v<ver>.
- */
#define RTE_DEFAULT_SYMBOL(ver, type, name, args) VERSIONING_WARN \
__asm__(".symver " RTE_STR(name) "_v" RTE_STR(ver) ", " RTE_STR(name) "@@DPDK_" RTE_STR(ver)); \
__rte_used type name ## _v ## ver args; \
type name ## _v ## ver args
+#endif /* __has_attribute(symver) */
+
#else /* !RTE_BUILD_SHARED_LIB */
+/* static library does not have versioned symbols */
#define RTE_VERSION_SYMBOL(ver, type, name, args) VERSIONING_WARN \
type name ## _v ## ver args; \
type name ## _v ## ver args
--
2.53.0
^ permalink raw reply related
* [DPDK/core Bug 1949] function versioning is incompatible with LTO
From: bugzilla @ 2026-06-02 22:30 UTC (permalink / raw)
To: dev
http://bugs.dpdk.org/show_bug.cgi?id=1949
Bug ID: 1949
Summary: function versioning is incompatible with LTO
Product: DPDK
Version: 25.07
Hardware: All
OS: All
Status: UNCONFIRMED
Severity: minor
Priority: Normal
Component: core
Assignee: dev@dpdk.org
Reporter: stephen@networkplumber.org
Target Milestone: ---
The default-version symver (@@) fails to assemble when build is using LTO.
Building a shared library with LTO fails when a function uses
RTE_DEFAULT_SYMBOL.
I introduces a new versioned symbol rte_eth_dev_get_name_by_port (ethdev).
This fails when doing LTO build with
invalid attempt to declare external version name as default
in symbol `rte_eth_dev_get_name_by_port@@DPDK_27'
Cause: the versioning macros emit a file-scope __asm__(".symver
…@@DPDK_<ver>"). Top-level asm is opaque to LTO, so GCC may place the directive
in a different ltrans partition than the _v<ver> definition. The implementation
symbol is then external in that partition, and gas rejects binding a default
version (@@) to an external symbol. Only the @@ (default) line fails; the
single-@ compat line is legal on an external symbol.
Fix: use the GCC symver function attribute (GCC ≥ 10) instead of top-level asm,
gated on __has_attribute(symver); retain the asm path as fallback. Clang lacks
the attribute and stays on the asm path (LTO + versioning remains unsupported
there).
Marking severity as minor since no current in-tree code uses versioned symbols.
But it is a trap waiting to happen.
--
You are receiving this mail because:
You are the assignee for the bug.
^ permalink raw reply
* [PATCH v6 10/10] dts: add selective Rx tests
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Luca Vizzarro, Patrick Robb
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
Add TestSuite_rx_split with 7 test cases:
- 3 positive: headers only, payload only, two non-contiguous segments
- 4 negative: missing offload flag, out-of-range, overlap, all-discard
Add selective Rx capability detection via testpmd "show port info".
The test suite could be completed later for the basic buffer split
configuration based on offsets or protocols.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/api/capabilities.py | 2 +
dts/api/testpmd/__init__.py | 17 ++
dts/api/testpmd/types.py | 6 +
dts/framework/testbed_model/capability.py | 2 +
dts/tests/TestSuite_rx_split.py | 262 ++++++++++++++++++++++
5 files changed, 289 insertions(+)
create mode 100644 dts/tests/TestSuite_rx_split.py
diff --git a/dts/api/capabilities.py b/dts/api/capabilities.py
index 09bc538523..b0c1d81d36 100644
--- a/dts/api/capabilities.py
+++ b/dts/api/capabilities.py
@@ -136,6 +136,8 @@ class NicCapability(IntEnum):
#: Device supports all VLAN capabilities.
PORT_RX_OFFLOAD_VLAN = auto()
QUEUE_RX_OFFLOAD_VLAN = auto()
+ #: Device supports selective Rx.
+ SELECTIVE_RX = auto()
#: Device supports Rx queue setup after device started.
RUNTIME_RX_QUEUE_SETUP = auto()
#: Device supports Tx queue setup after device started.
diff --git a/dts/api/testpmd/__init__.py b/dts/api/testpmd/__init__.py
index e9187440bb..6973a64573 100644
--- a/dts/api/testpmd/__init__.py
+++ b/dts/api/testpmd/__init__.py
@@ -1409,6 +1409,23 @@ def get_capabilities_show_port_info(
self.ports[0].device_capabilities,
)
+ def get_capabilities_selective_rx(
+ self,
+ supported_capabilities: MutableSet["NicCapability"],
+ unsupported_capabilities: MutableSet["NicCapability"],
+ ) -> None:
+ """Get selective Rx capability from show port info.
+
+ Args:
+ supported_capabilities: Supported capabilities will be added to this set.
+ unsupported_capabilities: Unsupported capabilities will be added to this set.
+ """
+ port_info = self.show_port_info(self.ports[0].id)
+ if port_info.selective_rx:
+ supported_capabilities.add(NicCapability.SELECTIVE_RX)
+ else:
+ unsupported_capabilities.add(NicCapability.SELECTIVE_RX)
+
def get_capabilities_mcast_filtering(
self,
supported_capabilities: MutableSet["NicCapability"],
diff --git a/dts/api/testpmd/types.py b/dts/api/testpmd/types.py
index 0d322aece2..6f1eaf47cc 100644
--- a/dts/api/testpmd/types.py
+++ b/dts/api/testpmd/types.py
@@ -614,6 +614,12 @@ def _validate(info: str) -> str | None:
metadata=VLANOffloadFlag.make_parser(),
)
+ #: Selective Rx support
+ selective_rx: bool = field(
+ default=False,
+ metadata=TextParser.find(r"Selective Rx: supported"),
+ )
+
#: Maximum size of RX buffer
max_rx_bufsize: int | None = field(
default=None, metadata=TextParser.find_int(r"Maximum size of RX buffer: (\d+)")
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 96e1cd449f..b10799ea4b 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -324,6 +324,8 @@ def mapping(cap: NicCapability) -> TestPmdNicCapability:
| NicCapability.FLOW_SHARED_OBJECT_KEEP
):
return (TestPmd.get_capabilities_show_port_info, None)
+ case NicCapability.SELECTIVE_RX:
+ return (TestPmd.get_capabilities_selective_rx, None)
case NicCapability.MCAST_FILTERING:
return (TestPmd.get_capabilities_mcast_filtering, None)
case NicCapability.FLOW_CTRL:
diff --git a/dts/tests/TestSuite_rx_split.py b/dts/tests/TestSuite_rx_split.py
new file mode 100644
index 0000000000..e12fe1a828
--- /dev/null
+++ b/dts/tests/TestSuite_rx_split.py
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 NVIDIA Corporation & Affiliates
+
+"""Rx split test suite.
+
+Test configuring a packet split on Rx,
+and discarding some segments (selective Rx) at NIC level.
+"""
+
+from typing import Any
+
+from scapy.layers.inet import IP
+from scapy.layers.l2 import Ether
+from scapy.packet import Packet, Raw
+
+from api.capabilities import (
+ NicCapability,
+ requires_nic_capability,
+)
+from api.packet import send_packet_and_capture
+from api.test import fail, verify
+from api.testpmd import TestPmd
+from api.testpmd.config import SimpleForwardingModes
+from api.testpmd.types import RxOffloadCapability, TxOffloadCapability
+from framework.exception import InteractiveCommandExecutionError
+from framework.test_suite import TestSuite, func_test
+
+PAYLOAD = bytes(range(256))
+ETHER_HDR_LEN = len(Ether())
+IP_HDR_LEN = len(IP())
+ETHER_IP_HDR_LEN = ETHER_HDR_LEN + IP_HDR_LEN
+
+
+@requires_nic_capability(NicCapability.PORT_RX_OFFLOAD_BUFFER_SPLIT)
+@requires_nic_capability(NicCapability.SELECTIVE_RX)
+class TestRxSplit(TestSuite):
+ """Rx split test suite.
+
+ Configure testpmd with various Rx segment offset/length combinations
+ and verify that only the requested portions of the packet are received
+ and forwarded.
+ """
+
+ def _create_testpmd(self, **kwargs: Any) -> TestPmd:
+ """Create a TestPmd instance with defaults overridden by kwargs."""
+ defaults: dict[str, Any] = {
+ "forward_mode": SimpleForwardingModes.mac,
+ "rx_offloads": RxOffloadCapability.BUFFER_SPLIT,
+ "enable_scatter": True,
+ }
+ return TestPmd(**{**defaults, **kwargs})
+
+ def _build_packet(self) -> Packet:
+ """Build a test packet with an incrementing byte pattern payload."""
+ return Ether() / IP() / Raw(load=PAYLOAD)
+
+ def _send_and_verify(
+ self,
+ testpmd: TestPmd,
+ packet: Packet,
+ expected_bytes: bytes,
+ ) -> None:
+ """Clear stats, send a packet, and verify received content and stats.
+
+ Args:
+ testpmd: The running testpmd instance.
+ packet: The packet to send.
+ expected_bytes: Expected raw bytes of the received packet.
+ """
+ expected_len = len(expected_bytes)
+ testpmd.clear_port_stats_all(verify=False)
+
+ received = send_packet_and_capture(packet)
+ verify(
+ len(received) > 0,
+ "Did not receive any packets.",
+ )
+
+ recv_bytes = bytes(received[0])
+ verify(
+ len(recv_bytes) == expected_len,
+ f"Expected packet length {expected_len}, got {len(recv_bytes)}.",
+ )
+ verify(
+ recv_bytes == expected_bytes,
+ "Received packet content does not match expected bytes.",
+ )
+
+ all_stats, _ = testpmd.show_port_stats_all()
+ total_rx_packets = sum(s.rx_packets for s in all_stats)
+ total_rx_bytes = sum(s.rx_bytes for s in all_stats)
+ verify(
+ total_rx_packets == 1,
+ f"Expected 1 Rx packet, got {total_rx_packets}.",
+ )
+ verify(
+ total_rx_bytes == expected_len,
+ f"Expected {expected_len} Rx bytes, got {total_rx_bytes}.",
+ )
+
+ @func_test
+ def selective_rx_headers(self) -> None:
+ """Keep only the Ethernet + IP headers, discard the payload.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts and buffer split enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet has Ether + IP headers only.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[0],
+ rx_segments_length=[ETHER_IP_HDR_LEN],
+ ) as testpmd:
+ testpmd.start()
+ packet = self._build_packet()
+ expected = bytes(packet)[:ETHER_IP_HDR_LEN]
+ self._send_and_verify(testpmd, packet, expected)
+
+ @func_test
+ def selective_rx_payload_only(self) -> None:
+ """Skip the Ethernet + IP headers, keep only the payload.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts and buffer split enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet is matching the original payload.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[ETHER_IP_HDR_LEN],
+ rx_segments_length=[len(PAYLOAD)],
+ ) as testpmd:
+ testpmd.start()
+ self._send_and_verify(testpmd, self._build_packet(), PAYLOAD)
+
+ @func_test
+ def selective_rx_two_segments(self) -> None:
+ """Keep the IP header and the middle of the payload, skip the rest.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts, buffer split
+ and multi-segment Tx enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet is matching the IP header and middle of payload.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ payload_offset = 100
+ payload_length = 100
+ with self._create_testpmd(
+ tx_offloads=TxOffloadCapability.MULTI_SEGS,
+ rx_segments_offsets=[ETHER_HDR_LEN, ETHER_IP_HDR_LEN + payload_offset],
+ rx_segments_length=[IP_HDR_LEN, payload_length],
+ ) as testpmd:
+ testpmd.start()
+ packet = self._build_packet()
+ raw = bytes(packet)
+ payload_start = ETHER_IP_HDR_LEN + payload_offset
+ expected = (
+ raw[ETHER_HDR_LEN:ETHER_IP_HDR_LEN]
+ + raw[payload_start : payload_start + payload_length]
+ )
+ self._send_and_verify(testpmd, packet, expected)
+
+ @func_test
+ def selective_rx_no_offload(self) -> None:
+ """Configure selective Rx with buffer split disabled.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts, buffer split
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_offloads=None,
+ rx_segments_offsets=[0],
+ rx_segments_length=[ETHER_IP_HDR_LEN],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with buffer split disabled.")
+ except InteractiveCommandExecutionError:
+ pass
+
+ @func_test
+ def selective_rx_offset_out_of_range(self) -> None:
+ """Configure selective Rx with an offset beyond max_rx_pktlen.
+
+ Steps:
+ Start testpmd with rxoffs too big, buffer split enabled,
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[20000],
+ rx_segments_length=[100],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with out-of-range offset.")
+ except InteractiveCommandExecutionError:
+ pass
+
+ @func_test
+ def selective_rx_overlap(self) -> None:
+ """Configure selective Rx with overlapping segments.
+
+ Steps:
+ Start testpmd with overlapping rxoffs/rxpkts, buffer split enabled,
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[0, 10],
+ rx_segments_length=[64, 64],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with overlapping segments.")
+ except InteractiveCommandExecutionError:
+ pass
+
+ @func_test
+ def selective_rx_all_discard(self) -> None:
+ """Configure selective Rx with only discard segment.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts=0 (null segment), buffer split enabled,
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[0],
+ rx_segments_length=[0],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with only discard segment.")
+ except InteractiveCommandExecutionError:
+ pass
--
2.54.0
^ permalink raw reply related
* [PATCH v6 09/10] dts: fix topology capability comparison
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Luca Vizzarro, Patrick Robb,
Juraj Linkeš, Dean Marx, Jeremy Spewock
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
TopologyCapability.__gt__() was delegating to __lt__(),
which caused infinite recursion when "other" is not a TopologyCapability:
other.__lt__(self) returns NotImplemented,
Python retries with self.__gt__(other),
and the cycle repeats.
dts/framework/testbed_model/capability.py", line 579, in __gt__
return other < self
^^^^^^^^^^^^
RecursionError: maximum recursion depth exceeded
Similarly, __le__() was delegating to "not __gt__()",
which returns True for non-comparable types instead of False.
Fix both by checking is_comparable_with() first
and comparing topology_type directly, consistent with __lt__().
Fixes: 039256daa8bf ("dts: add topology capability")
Cc: stable@dpdk.org
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/framework/testbed_model/capability.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 960370fc72..96e1cd449f 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -574,7 +574,9 @@ def __gt__(self, other: Any) -> bool:
Returns:
:data:`True` if the instance's topology type is more complex than the compared object's.
"""
- return other < self
+ if not self.is_comparable_with(other):
+ return False
+ return self.topology_type > other.topology_type
def __le__(self, other: Any) -> bool:
"""Compare the :attr:`~TopologyCapability.topology_type`s.
@@ -586,7 +588,9 @@ def __le__(self, other: Any) -> bool:
:data:`True` if the instance's topology type is less complex or equal than
the compared object's.
"""
- return not self > other
+ if not self.is_comparable_with(other):
+ return False
+ return self.topology_type <= other.topology_type
def __hash__(self):
"""Each instance is identified by :attr:`topology_type`."""
--
2.54.0
^ permalink raw reply related
* [PATCH v6 08/10] common/mlx5: remove callbacks for MR registration
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Dariusz Sosnowski, Viacheslav Ovsiienko,
Bing Zhao, Ori Kam, Suanming Mou, Matan Azrad, Fan Zhang,
Ashish Gupta
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
The functions register/unregister for a Memory Region (MR)
were not called directly.
There are only 2 implementations for Linux and Windows,
no need of handling this difference with function pointers.
The callback pointers are replaced with direct calls
and link time decision based on the Operating System.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/common/mlx5/linux/mlx5_common_verbs.c | 26 +++----------
drivers/common/mlx5/mlx5_common.c | 6 +--
drivers/common/mlx5/mlx5_common_mr.c | 37 ++++++++-----------
drivers/common/mlx5/mlx5_common_mr.h | 26 +++----------
drivers/common/mlx5/windows/mlx5_common_os.c | 23 ++----------
drivers/compress/mlx5/mlx5_compress.c | 4 +-
drivers/crypto/mlx5/mlx5_crypto.h | 2 -
drivers/crypto/mlx5/mlx5_crypto_gcm.c | 6 +--
drivers/net/mlx5/mlx5.h | 3 +-
drivers/net/mlx5/mlx5_flow_aso.c | 21 +++++------
drivers/net/mlx5/mlx5_flow_hw.c | 11 ++----
drivers/net/mlx5/mlx5_flow_quota.c | 6 +--
drivers/net/mlx5/mlx5_hws_cnt.c | 19 ++++------
13 files changed, 61 insertions(+), 129 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index 6d44e1f566..5e23c5844d 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -106,10 +106,10 @@ mlx5_set_context_attr(struct rte_device *dev, struct ibv_context *ctx)
* @return
* 0 on successful registration, -1 otherwise
*/
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_common_verbs_reg_mr)
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_reg_mr)
int
-mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr)
+mlx5_os_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr)
{
struct ibv_mr *ibv_mr;
@@ -136,9 +136,9 @@ mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
* pmd_mr struct set with lkey, address, length and pointer to mr object
*
*/
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_common_verbs_dereg_mr)
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_dereg_mr)
void
-mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
+mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
{
if (pmd_mr && pmd_mr->obj != NULL) {
claim_zero(mlx5_glue->dereg_mr(pmd_mr->obj));
@@ -146,22 +146,6 @@ mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
}
}
-/**
- * Set the reg_mr and dereg_mr callbacks.
- *
- * @param[out] reg_mr_cb
- * Pointer to reg_mr func
- * @param[out] dereg_mr_cb
- * Pointer to dereg_mr func
- */
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_set_reg_mr_cb)
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
-{
- *reg_mr_cb = mlx5_common_verbs_reg_mr;
- *dereg_mr_cb = mlx5_common_verbs_dereg_mr;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
struct mlx5_pmd_mr *
mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index f71dbe4637..87de6d0ff0 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -1135,7 +1135,7 @@ mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
return -1;
}
mr = mlx5_create_mr_ext(dev->pd, (uintptr_t)addr, len,
- SOCKET_ID_ANY, dev->mr_scache.reg_mr_cb);
+ SOCKET_ID_ANY);
if (!mr) {
DRV_LOG(WARNING, "Device %s unable to DMA map", rte_dev->name);
rte_errno = EINVAL;
@@ -1165,7 +1165,7 @@ mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
ret = mlx5_mr_expand_cache(&dev->mr_scache, size,
rte_dev->numa_node);
if (ret < 0) {
- mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+ mlx5_mr_free(mr);
rte_errno = ret;
return -1;
}
@@ -1221,7 +1221,7 @@ mlx5_common_dev_dma_unmap(struct rte_device *rte_dev, void *addr,
}
LIST_REMOVE(mr, mr);
DRV_LOG(DEBUG, "MR(%p) is removed from list.", (void *)mr);
- mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+ mlx5_mr_free(mr);
mlx5_mr_rebuild_cache(&dev->mr_scache);
/*
* No explicit wmb is needed after updating dev_gen due to
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index 64ffc7f4ea..aa2d5e88a4 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -492,12 +492,12 @@ mlx5_mr_lookup_cache(struct mlx5_mr_share_cache *share_cache,
* Pointer to MR to free.
*/
void
-mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb)
+mlx5_mr_free(struct mlx5_mr *mr)
{
if (mr == NULL)
return;
DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
- dereg_mr_cb(&mr->pmd_mr);
+ mlx5_os_dereg_mr(&mr->pmd_mr);
rte_bitmap_free(mr->ms_bmp);
mlx5_free(mr);
}
@@ -545,7 +545,7 @@ mlx5_mr_garbage_collect(struct mlx5_mr_share_cache *share_cache)
struct mlx5_mr *mr = mr_next;
mr_next = LIST_NEXT(mr, mr);
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
}
}
@@ -821,7 +821,7 @@ mlx5_mr_create_primary(void *pd,
data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
data.end = data.start + msl->page_sz;
rte_mcfg_mem_read_unlock();
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
goto alloc_resources;
}
MLX5_ASSERT(data.msl == data_re.msl);
@@ -845,7 +845,7 @@ mlx5_mr_create_primary(void *pd,
* Must be unlocked before calling rte_free() because
* mlx5_mr_mem_event_free_cb() can be called inside.
*/
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
return entry->lkey;
}
/*
@@ -912,7 +912,7 @@ mlx5_mr_create_primary(void *pd,
* mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
* through mlx5_alloc_verbs_buf().
*/
- share_cache->reg_mr_cb(pd, (void *)data.start, len, &mr->pmd_mr);
+ mlx5_os_reg_mr(pd, (void *)data.start, len, &mr->pmd_mr);
if (mr->pmd_mr.obj == NULL) {
DRV_LOG(DEBUG, "Fail to create an MR for address (%p)",
(void *)addr);
@@ -948,7 +948,7 @@ mlx5_mr_create_primary(void *pd,
* calling rte_free() because mlx5_mr_mem_event_free_cb() can be called
* inside.
*/
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
return UINT32_MAX;
}
@@ -1139,9 +1139,6 @@ mlx5_mr_release_cache(struct mlx5_mr_share_cache *share_cache)
int
mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket)
{
- /* Set the reg_mr and dereg_mr callback functions */
- mlx5_os_set_reg_mr_cb(&share_cache->reg_mr_cb,
- &share_cache->dereg_mr_cb);
rte_rwlock_init(&share_cache->rwlock);
rte_rwlock_init(&share_cache->mprwlock);
/* Initialize B-tree and allocate memory for global MR cache table. */
@@ -1189,8 +1186,7 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
* Pointer to MR structure on success, NULL otherwise.
*/
struct mlx5_mr *
-mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
- mlx5_reg_mr_t reg_mr_cb)
+mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
{
struct mlx5_mr *mr = NULL;
@@ -1199,7 +1195,7 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
RTE_CACHE_LINE_SIZE, socket_id);
if (mr == NULL)
return NULL;
- reg_mr_cb(pd, (void *)addr, len, &mr->pmd_mr);
+ mlx5_os_reg_mr(pd, (void *)addr, len, &mr->pmd_mr);
if (mr->pmd_mr.obj == NULL) {
DRV_LOG(WARNING,
"Fail to create MR for address (%p)",
@@ -1624,14 +1620,13 @@ mlx5_mempool_reg_create(struct rte_mempool *mp, unsigned int mrs_n,
* Whether @p mpr owns its MRs exclusively, i.e. they are not shared.
*/
static void
-mlx5_mempool_reg_destroy(struct mlx5_mr_share_cache *share_cache,
- struct mlx5_mempool_reg *mpr, bool standalone)
+mlx5_mempool_reg_destroy(struct mlx5_mempool_reg *mpr, bool standalone)
{
if (standalone) {
unsigned int i;
for (i = 0; i < mpr->mrs_n; i++)
- share_cache->dereg_mr_cb(&mpr->mrs[i].pmd_mr);
+ mlx5_os_dereg_mr(&mpr->mrs[i].pmd_mr);
mlx5_free(mpr->mrs);
}
mlx5_free(mpr);
@@ -1748,7 +1743,7 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
const struct mlx5_range *range = &ranges[i];
size_t len = range->end - range->start;
- if (share_cache->reg_mr_cb(pd, (void *)range->start, len,
+ if (mlx5_os_reg_mr(pd, (void *)range->start, len,
&mr->pmd_mr) < 0) {
DRV_LOG(ERR,
"Failed to create an MR in PD %p for address range "
@@ -1763,7 +1758,7 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
mp->name);
}
if (i != ranges_n) {
- mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
+ mlx5_mempool_reg_destroy(new_mpr, true);
rte_errno = EINVAL;
goto exit;
}
@@ -1785,13 +1780,13 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
if (mpr != NULL) {
DRV_LOG(DEBUG, "Mempool %s is already registered for PD %p",
mp->name, pd);
- mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
+ mlx5_mempool_reg_destroy(new_mpr, true);
rte_errno = EEXIST;
goto exit;
} else if (old_mpr != NULL) {
DRV_LOG(DEBUG, "Mempool %s registration for PD %p updated for external memory",
mp->name, pd);
- mlx5_mempool_reg_destroy(share_cache, old_mpr, standalone);
+ mlx5_mempool_reg_destroy(old_mpr, standalone);
}
exit:
free(ranges);
@@ -1860,7 +1855,7 @@ mlx5_mr_mempool_unregister_primary(struct mlx5_mr_share_cache *share_cache,
rte_errno = ENOENT;
return -1;
}
- mlx5_mempool_reg_destroy(share_cache, mpr, standalone);
+ mlx5_mempool_reg_destroy(mpr, standalone);
return 0;
}
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index 00f3d832c3..5fb931a1b5 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -32,13 +32,6 @@ struct mlx5_pmd_mr {
struct mlx5_devx_obj *mkey; /* devx mkey object. */
};
-/**
- * mr operations typedef
- */
-typedef int (*mlx5_reg_mr_t)(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr);
-typedef void (*mlx5_dereg_mr_t)(struct mlx5_pmd_mr *pmd_mr);
-
/* Memory Region object. */
struct mlx5_mr {
LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
@@ -88,8 +81,6 @@ struct __rte_packed_begin mlx5_mr_share_cache {
struct mlx5_mr_list mr_list; /* Registered MR list. */
struct mlx5_mr_list mr_free_list; /* Freed MR list. */
struct mlx5_mempool_reg_list mempool_reg_list; /* Mempool database. */
- mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
- mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
} __rte_packed_end;
/* Multi-Packet RQ buffer header. */
@@ -233,9 +224,8 @@ struct mlx5_mr *
mlx5_mr_lookup_list(struct mlx5_mr_share_cache *share_cache,
struct mr_cache_entry *entry, uintptr_t addr);
struct mlx5_mr *
-mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
- mlx5_reg_mr_t reg_mr_cb);
-void mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
+mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id);
+void mlx5_mr_free(struct mlx5_mr *mr);
__rte_internal
uint32_t
mlx5_mr_create(struct mlx5_common_device *cdev,
@@ -246,19 +236,13 @@ __rte_internal
uint32_t
mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr);
-/* mlx5_common_verbs.c */
-
__rte_internal
int
-mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr);
+mlx5_os_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr);
__rte_internal
void
-mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
-
-__rte_internal
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
__rte_internal
struct mlx5_pmd_mr *
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index 692517a9bf..bf1b654da3 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -377,7 +377,8 @@ mlx5_os_umem_dereg(void *pumem)
* @return
* 0 on successful registration, -1 otherwise
*/
-static int
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_reg_mr)
+int
mlx5_os_reg_mr(void *pd,
void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr)
{
@@ -425,7 +426,8 @@ mlx5_os_reg_mr(void *pd,
* @param[in] pmd_mr
* Pointer to PMD mr object
*/
-static void
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_dereg_mr)
+void
mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
{
if (!pmd_mr)
@@ -437,23 +439,6 @@ mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
memset(pmd_mr, 0, sizeof(*pmd_mr));
}
-/**
- * Set the reg_mr and dereg_mr callbacks.
- *
- * @param[out] reg_mr_cb
- * Pointer to reg_mr func
- * @param[out] dereg_mr_cb
- * Pointer to dereg_mr func
- *
- */
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_set_reg_mr_cb)
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
-{
- *reg_mr_cb = mlx5_os_reg_mr;
- *dereg_mr_cb = mlx5_os_dereg_mr;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
struct mlx5_pmd_mr *
mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index e5325c6150..1361dab630 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -117,7 +117,7 @@ mlx5_compress_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
if (qp->opaque_mr.obj != NULL) {
void *opaq = qp->opaque_mr.addr;
- mlx5_common_verbs_dereg_mr(&qp->opaque_mr);
+ mlx5_os_dereg_mr(&qp->opaque_mr);
rte_free(opaq);
}
mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
@@ -199,7 +199,7 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
qp->priv = priv;
qp->ops = (struct rte_comp_op **)RTE_ALIGN((uintptr_t)(qp + 1),
RTE_CACHE_LINE_SIZE);
- if (mlx5_common_verbs_reg_mr(priv->cdev->pd, opaq_buf, qp->entries_n *
+ if (mlx5_os_reg_mr(priv->cdev->pd, opaq_buf, qp->entries_n *
sizeof(union mlx5_gga_compress_opaque),
&qp->opaque_mr) != 0) {
rte_free(opaq_buf);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f9f127e9e6..93a2bb2c78 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -40,8 +40,6 @@ struct mlx5_crypto_priv {
TAILQ_ENTRY(mlx5_crypto_priv) next;
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
struct rte_cryptodev *crypto_dev;
- mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
- mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
struct mlx5_uar uar; /* User Access Region. */
uint32_t max_segs_num; /* Maximum supported data segs. */
uint32_t max_klm_num; /* Maximum supported klm. */
diff --git a/drivers/crypto/mlx5/mlx5_crypto_gcm.c b/drivers/crypto/mlx5/mlx5_crypto_gcm.c
index 89f32c7722..1a2600655a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto_gcm.c
+++ b/drivers/crypto/mlx5/mlx5_crypto_gcm.c
@@ -219,7 +219,6 @@ mlx5_crypto_gcm_mkey_klm_update(struct mlx5_crypto_priv *priv,
static int
mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
{
- struct mlx5_crypto_priv *priv = dev->data->dev_private;
struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
if (qp->umr_qp_obj.qp != NULL)
@@ -231,7 +230,7 @@ mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
if (qp->mr.obj != NULL) {
void *opaq = qp->mr.addr;
- priv->dereg_mr_cb(&qp->mr);
+ mlx5_os_dereg_mr(&qp->mr);
rte_free(opaq);
}
mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
@@ -363,7 +362,7 @@ mlx5_crypto_gcm_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
rte_errno = ENOMEM;
goto err;
}
- if (priv->reg_mr_cb(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) {
+ if (mlx5_os_reg_mr(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) {
rte_free(mr_buf);
DRV_LOG(ERR, "Failed to register opaque MR.");
rte_errno = ENOMEM;
@@ -1186,7 +1185,6 @@ mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
/* Override AES-GCM specified ops. */
dev_ops->sym_session_configure = mlx5_crypto_sym_gcm_session_configure;
- mlx5_os_set_reg_mr_cb(&priv->reg_mr_cb, &priv->dereg_mr_cb);
dev_ops->queue_pair_setup = mlx5_crypto_gcm_qp_setup;
dev_ops->queue_pair_release = mlx5_crypto_gcm_qp_release;
if (mlx5_crypto_is_ipsec_opt(priv)) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index bd6ef35b53..a4d5392e8f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -2706,8 +2706,7 @@ int mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh,
int mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
struct mlx5_aso_ct_pools_mng *ct_mng,
uint32_t nb_queues);
-int mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_aso_ct_pools_mng *ct_mng);
+int mlx5_aso_ct_queue_uninit(struct mlx5_aso_ct_pools_mng *ct_mng);
int
mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,
void *uar, uint16_t log_desc_n);
diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c
index 5e2a81ef9c..cd84ab1966 100644
--- a/drivers/net/mlx5/mlx5_flow_aso.c
+++ b/drivers/net/mlx5/mlx5_flow_aso.c
@@ -19,17 +19,15 @@
/**
* Free MR resources.
*
- * @param[in] cdev
- * Pointer to the mlx5 common device.
* @param[in] mr
* MR to free.
*/
static void
-mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr)
+mlx5_aso_dereg_mr(struct mlx5_pmd_mr *mr)
{
void *addr = mr->addr;
- cdev->mr_scache.dereg_mr_cb(mr);
+ mlx5_os_dereg_mr(mr);
mlx5_free(addr);
memset(mr, 0, sizeof(*mr));
}
@@ -59,7 +57,7 @@ mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length,
DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
return -1;
}
- ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr);
+ ret = mlx5_os_reg_mr(cdev->pd, mr->addr, length, mr);
if (ret) {
DRV_LOG(ERR, "Failed to create direct Mkey.");
mlx5_free(mr->addr);
@@ -362,7 +360,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq,
sh->tx_uar.obj,
MLX5_ASO_QUEUE_LOG_DESC)) {
- mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
return -1;
}
mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
@@ -399,14 +397,14 @@ mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
switch (aso_opc_mod) {
case ASO_OPC_MOD_FLOW_HIT:
- mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
sq = &sh->aso_age_mng->aso_sq;
break;
case ASO_OPC_MOD_POLICER:
mlx5_aso_mtr_queue_uninit(sh, NULL, &sh->mtrmng->pools_mng);
break;
case ASO_OPC_MOD_CONNECTION_TRACKING:
- mlx5_aso_ct_queue_uninit(sh, sh->ct_mng);
+ mlx5_aso_ct_queue_uninit(sh->ct_mng);
break;
default:
DRV_LOG(ERR, "Unknown ASO operation mode");
@@ -1147,15 +1145,14 @@ __mlx5_aso_ct_get_pool(struct mlx5_dev_ctx_shared *sh,
}
int
-mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_aso_ct_pools_mng *ct_mng)
+mlx5_aso_ct_queue_uninit(struct mlx5_aso_ct_pools_mng *ct_mng)
{
uint32_t i;
/* 64B per object for query. */
for (i = 0; i < ct_mng->nb_sq; i++) {
if (ct_mng->aso_sqs[i].mr.addr)
- mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
+ mlx5_aso_dereg_mr(&ct_mng->aso_sqs[i].mr);
mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
}
return 0;
@@ -1197,7 +1194,7 @@ mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
error:
do {
if (ct_mng->aso_sqs[i].mr.addr)
- mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
+ mlx5_aso_dereg_mr(&ct_mng->aso_sqs[i].mr);
mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
} while (i--);
ct_mng->nb_sq = 0;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index b6bb9f12a6..7cc601d681 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -11086,12 +11086,9 @@ flow_hw_create_nic_ctrl_tables(struct rte_eth_dev *dev, struct rte_flow_error *e
}
static void
-flow_hw_ct_mng_destroy(struct rte_eth_dev *dev,
- struct mlx5_aso_ct_pools_mng *ct_mng)
+flow_hw_ct_mng_destroy(struct mlx5_aso_ct_pools_mng *ct_mng)
{
- struct mlx5_priv *priv = dev->data->dev_private;
-
- mlx5_aso_ct_queue_uninit(priv->sh, ct_mng);
+ mlx5_aso_ct_queue_uninit(ct_mng);
mlx5_free(ct_mng);
}
@@ -11230,7 +11227,7 @@ mlx5_flow_ct_init(struct rte_eth_dev *dev,
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ flow_hw_ct_mng_destroy(priv->ct_mng);
priv->ct_mng = NULL;
}
return ret;
@@ -11804,7 +11801,7 @@ __mlx5_flow_hw_resource_release(struct rte_eth_dev *dev, bool ctx_close)
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ flow_hw_ct_mng_destroy(priv->ct_mng);
priv->ct_mng = NULL;
}
mlx5_flow_quota_destroy(dev);
diff --git a/drivers/net/mlx5/mlx5_flow_quota.c b/drivers/net/mlx5/mlx5_flow_quota.c
index d94167d0b0..b661bd376e 100644
--- a/drivers/net/mlx5/mlx5_flow_quota.c
+++ b/drivers/net/mlx5/mlx5_flow_quota.c
@@ -412,12 +412,11 @@ mlx5_quota_alloc_sq(struct mlx5_priv *priv)
static void
mlx5_quota_destroy_read_buf(struct mlx5_priv *priv)
{
- struct mlx5_dev_ctx_shared *sh = priv->sh;
struct mlx5_quota_ctx *qctx = &priv->quota_ctx;
if (qctx->mr.lkey) {
void *addr = qctx->mr.addr;
- sh->cdev->mr_scache.dereg_mr_cb(&qctx->mr);
+ mlx5_os_dereg_mr(&qctx->mr);
mlx5_free(addr);
}
if (qctx->read_buf)
@@ -446,8 +445,7 @@ mlx5_quota_alloc_read_buf(struct mlx5_priv *priv)
DRV_LOG(DEBUG, "QUOTA: failed to allocate MTR ASO READ buffer [1]");
return -ENOMEM;
}
- ret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, buf,
- rd_buf_size, &qctx->mr);
+ ret = mlx5_os_reg_mr(sh->cdev->pd, buf, rd_buf_size, &qctx->mr);
if (ret) {
DRV_LOG(DEBUG, "QUOTA: failed to register MTR ASO READ MR");
return -errno;
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 1b6acb7a3b..d0c4ead71b 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -259,12 +259,11 @@ mlx5_hws_aging_check(struct mlx5_priv *priv, struct mlx5_hws_cnt_pool *cpool)
}
static void
-mlx5_hws_cnt_raw_data_free(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_hws_cnt_raw_data_mng *mng)
+mlx5_hws_cnt_raw_data_free(struct mlx5_hws_cnt_raw_data_mng *mng)
{
if (mng == NULL)
return;
- sh->cdev->mr_scache.dereg_mr_cb(&mng->mr);
+ mlx5_os_dereg_mr(&mng->mr);
mlx5_free(mng->raw);
mlx5_free(mng);
}
@@ -296,8 +295,7 @@ mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n,
NULL, "failed to allocate raw counters memory");
goto error;
}
- ret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, mng->raw, sz,
- &mng->mr);
+ ret = mlx5_os_reg_mr(sh->cdev->pd, mng->raw, sz, &mng->mr);
if (ret) {
rte_flow_error_set(error, errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -306,7 +304,7 @@ mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n,
}
return mng;
error:
- mlx5_hws_cnt_raw_data_free(sh, mng);
+ mlx5_hws_cnt_raw_data_free(mng);
return NULL;
}
@@ -639,8 +637,7 @@ mlx5_hws_cnt_pool_dcs_alloc(struct mlx5_dev_ctx_shared *sh,
}
static void
-mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_hws_cnt_pool *cpool)
+mlx5_hws_cnt_pool_dcs_free(struct mlx5_hws_cnt_pool *cpool)
{
uint32_t idx;
@@ -649,7 +646,7 @@ mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,
for (idx = 0; idx < MLX5_HWS_CNT_DCS_NUM; idx++)
mlx5_devx_cmd_destroy(cpool->dcs_mng.dcs[idx].obj);
if (cpool->raw_mng) {
- mlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);
+ mlx5_hws_cnt_raw_data_free(cpool->raw_mng);
cpool->raw_mng = NULL;
}
}
@@ -842,8 +839,8 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
}
mlx5_hws_cnt_pool_action_destroy(cpool);
if (cpool->cfg.host_cpool == NULL) {
- mlx5_hws_cnt_pool_dcs_free(sh, cpool);
- mlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);
+ mlx5_hws_cnt_pool_dcs_free(cpool);
+ mlx5_hws_cnt_raw_data_free(cpool->raw_mng);
}
mlx5_free((void *)cpool->cfg.name);
mlx5_hws_cnt_pool_deinit(cpool);
--
2.54.0
^ permalink raw reply related
* [PATCH v6 07/10] net/mlx5: reindent previous changes
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Dariusz Sosnowski, Viacheslav Ovsiienko,
Bing Zhao, Ori Kam, Suanming Mou, Matan Azrad
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
Fix indent which was left untouched to help reviews.
This must be squashed before merging.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/net/mlx5/mlx5_rx.c | 162 ++++++++++++++++----------------
drivers/net/mlx5/mlx5_rxq.c | 32 +++----
drivers/net/mlx5/mlx5_trigger.c | 18 ++--
3 files changed, 105 insertions(+), 107 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index 6aeab8d423..b8c3b27db6 100644
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -1071,106 +1071,106 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
rte_prefetch0(cqe);
rte_prefetch0(wqe);
if (seg->pool) {
- /* Allocate the buf from the same pool. */
- rep = rte_mbuf_raw_alloc(seg->pool);
- if (unlikely(rep == NULL)) {
- ++rxq->stats.rx_nombuf;
- if (!pkt) {
- /*
- * no buffers before we even started,
- * bail out silently.
- */
- break;
- }
- while (pkt != seg) {
- MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
- rep = NEXT(pkt);
- NEXT(pkt) = NULL;
- NB_SEGS(pkt) = 1;
- rte_mbuf_raw_free(pkt);
- pkt = rep;
- }
- rq_ci >>= sges_n;
- ++rq_ci;
- rq_ci <<= sges_n;
- break;
- }
- }
- if (!pkt) {
- if (len == 0) { /* no CQE polled yet */
- cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
- len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
- &mcqe, &skip_cnt, false, NULL);
- if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
- /* We drop packets with non-critical errors */
- if (seg->pool)
- rte_mbuf_raw_free(rep);
- if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
- rq_ci = rxq->rq_ci << sges_n;
+ /* Allocate the buf from the same pool. */
+ rep = rte_mbuf_raw_alloc(seg->pool);
+ if (unlikely(rep == NULL)) {
+ ++rxq->stats.rx_nombuf;
+ if (!pkt) {
+ /*
+ * no buffers before we even started,
+ * bail out silently.
+ */
break;
}
- /* Skip specified amount of error CQEs packets */
+ while (pkt != seg) {
+ MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
+ rep = NEXT(pkt);
+ NEXT(pkt) = NULL;
+ NB_SEGS(pkt) = 1;
+ rte_mbuf_raw_free(pkt);
+ pkt = rep;
+ }
rq_ci >>= sges_n;
- rq_ci += skip_cnt;
+ ++rq_ci;
rq_ci <<= sges_n;
- MLX5_ASSERT(!pkt);
- continue;
- }
- if (len == 0) {
- if (seg->pool)
- rte_mbuf_raw_free(rep);
break;
}
- MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
- if (rxq->crc_present)
- len -= RTE_ETHER_CRC_LEN;
}
- if (seg->pool) { /* first real segment */
- pkt = seg;
- tail = seg;
- pkt->ol_flags &= RTE_MBUF_F_EXTERNAL;
- if (rxq->cqe_comp_layout && mcqe)
- cqe = &rxq->title_cqe;
- rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
- PKT_LEN(pkt) = len;
- if (cqe->lro_num_seg > 1) {
- mlx5_lro_update_hdr
- (rte_pktmbuf_mtod(pkt, uint8_t *), cqe,
- mcqe, rxq, len);
- pkt->ol_flags |= RTE_MBUF_F_RX_LRO;
- pkt->tso_segsz = len / cqe->lro_num_seg;
+ if (!pkt) { /* new packet */
+ if (len == 0) { /* no CQE polled yet */
+ cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
+ len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
+ &mcqe, &skip_cnt, false, NULL);
+ if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
+ /* We drop packets with non-critical errors */
+ if (seg->pool)
+ rte_mbuf_raw_free(rep);
+ if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
+ rq_ci = rxq->rq_ci << sges_n;
+ break;
+ }
+ /* Skip specified amount of error CQEs packets */
+ rq_ci >>= sges_n;
+ rq_ci += skip_cnt;
+ rq_ci <<= sges_n;
+ MLX5_ASSERT(!pkt);
+ continue;
+ }
+ if (len == 0) {
+ if (seg->pool)
+ rte_mbuf_raw_free(rep);
+ break;
+ }
+ MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
+ if (rxq->crc_present)
+ len -= RTE_ETHER_CRC_LEN;
+ }
+ if (seg->pool) { /* first real segment */
+ pkt = seg;
+ tail = seg;
+ pkt->ol_flags &= RTE_MBUF_F_EXTERNAL;
+ if (rxq->cqe_comp_layout && mcqe)
+ cqe = &rxq->title_cqe;
+ rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
+ PKT_LEN(pkt) = len;
+ if (cqe->lro_num_seg > 1) {
+ mlx5_lro_update_hdr
+ (rte_pktmbuf_mtod(pkt, uint8_t *), cqe,
+ mcqe, rxq, len);
+ pkt->ol_flags |= RTE_MBUF_F_RX_LRO;
+ pkt->tso_segsz = len / cqe->lro_num_seg;
+ }
}
- }
}
if (seg->pool) { /* real segment: replenish WQE */
- tail = seg;
- DATA_LEN(rep) = DATA_LEN(seg);
- PKT_LEN(rep) = PKT_LEN(seg);
- SET_DATA_OFF(rep, DATA_OFF(seg));
- PORT(rep) = PORT(seg);
- (*rxq->elts)[idx] = rep;
- /*
- * Fill NIC descriptor with the new buffer. The lkey and size
- * of the buffers are already known, only the buffer address
- * changes.
- */
- wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
- /* If there's only one MR, no need to replace LKey in WQE. */
- if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
- wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+ tail = seg;
+ DATA_LEN(rep) = DATA_LEN(seg);
+ PKT_LEN(rep) = PKT_LEN(seg);
+ SET_DATA_OFF(rep, DATA_OFF(seg));
+ PORT(rep) = PORT(seg);
+ (*rxq->elts)[idx] = rep;
+ /*
+ * Fill NIC descriptor with the new buffer. The lkey and size
+ * of the buffers are already known, only the buffer address
+ * changes.
+ */
+ wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+ wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
}
- if (len > DATA_LEN(seg)) {
+ if (len > DATA_LEN(seg)) { /* more data: move to next segment */
if (seg->pool)
data_seg_len += DATA_LEN(seg);
len -= DATA_LEN(seg);
if (pkt)
- ++NB_SEGS(pkt);
+ ++NB_SEGS(pkt);
++rq_ci;
continue;
}
if (seg->pool) { /* last segment */
- DATA_LEN(seg) = len;
- data_seg_len += len;
+ DATA_LEN(seg) = len;
+ data_seg_len += len;
}
if (unlikely(!pkt)) { /* no real segment found, skip packet */
len = 0;
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 3fae189fa4..6ca29f7543 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -152,22 +152,22 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
struct rte_mbuf *buf;
if (seg->mp) {
- buf = rte_pktmbuf_alloc(seg->mp);
- if (buf == NULL) {
- if (rxq_ctrl->share_group == 0)
- DRV_LOG(ERR, "port %u queue %u empty mbuf pool",
- RXQ_PORT_ID(rxq_ctrl),
- rxq_ctrl->rxq.idx);
- else
- DRV_LOG(ERR, "share group %u queue %u empty mbuf pool",
- rxq_ctrl->share_group,
- rxq_ctrl->share_qid);
- rte_errno = ENOMEM;
- goto error;
- }
- /* Only vectored Rx routines rely on headroom size. */
- MLX5_ASSERT(!has_vec_support ||
- DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
+ buf = rte_pktmbuf_alloc(seg->mp);
+ if (buf == NULL) {
+ if (rxq_ctrl->share_group == 0)
+ DRV_LOG(ERR, "port %u queue %u empty mbuf pool",
+ RXQ_PORT_ID(rxq_ctrl),
+ rxq_ctrl->rxq.idx);
+ else
+ DRV_LOG(ERR, "share group %u queue %u empty mbuf pool",
+ rxq_ctrl->share_group,
+ rxq_ctrl->share_qid);
+ rte_errno = ENOMEM;
+ goto error;
+ }
+ /* Only vectored Rx routines rely on headroom size. */
+ MLX5_ASSERT(!has_vec_support ||
+ DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
} else {
buf = seg->null_mbuf;
}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 5b04d9a234..ac966c51b4 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -164,16 +164,14 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
seg = &rxq_ctrl->rxq.rxseg[s];
mp = seg->mp;
if (mp) { /* Regular segment */
- bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
- RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
- ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp,
- is_extmem);
- if (ret < 0 && rte_errno != EEXIST)
- goto error;
- ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
- mp);
- if (ret < 0)
- goto error;
+ bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
+ RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
+ ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp, is_extmem);
+ if (ret < 0 && rte_errno != EEXIST)
+ goto error;
+ ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl, mp);
+ if (ret < 0)
+ goto error;
} else { /* NULL segment used in selective Rx */
seg->null_mbuf = mlx5_alloc_null_mbuf(seg->length);
if (seg->null_mbuf == NULL) {
--
2.54.0
^ permalink raw reply related
* [PATCH v6 06/10] net/mlx5: support selective Rx
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Gregory Etelson, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Selective Rx may save some PCI bandwidth.
Implement selective Rx in the (quite slow) scalar SPRQ Rx path
mlx5_rx_burst() where the performance impact
of the added condition branches is acceptable.
Other Rx functions do not support this feature.
When using selective Rx, mlx5_rx_burst will be selected.
A null Memory Region (MR) is always allocated
at shared device context initialization.
The selective Rx capability is not advertised
if this special MR allocation fails.
For each Rx segment configured with a NULL mempool,
a "null mbuf" is created.
It is a fake mbuf allocated outside any mempool,
used as a placeholder in the Rx ring.
The null MR lkey is used in the WQE for these segments
so the NIC writes received data to a discard buffer.
The mbuf data room size is resolved from the first segment having a pool.
For null segments, the buffer length is from the last seen pool,
so that the WQE stride size remains consistent.
In mlx5_rx_burst, discarded segments are not chained
into the packet mbuf list, NB_SEGS is decremented accordingly,
and no replacement buffer is allocated.
A separate data_seg_len accumulator tracks the total length
of delivered segments only.
The packet length is adjusted to reflect only the data
actually delivered to the application.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
doc/guides/nics/features/mlx5.ini | 1 +
doc/guides/nics/mlx5.rst | 86 +++++++++++++++++++-------
doc/guides/rel_notes/release_26_07.rst | 4 ++
drivers/net/mlx5/mlx5.c | 7 +++
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_ethdev.c | 25 ++++++++
drivers/net/mlx5/mlx5_rx.c | 48 +++++++++++---
drivers/net/mlx5/mlx5_rx.h | 1 +
drivers/net/mlx5/mlx5_rxq.c | 45 ++++++++++++--
drivers/net/mlx5/mlx5_trigger.c | 52 +++++++++++++---
10 files changed, 229 insertions(+), 41 deletions(-)
diff --git a/doc/guides/nics/features/mlx5.ini b/doc/guides/nics/features/mlx5.ini
index 3b3eda28b8..ae8c83057b 100644
--- a/doc/guides/nics/features/mlx5.ini
+++ b/doc/guides/nics/features/mlx5.ini
@@ -16,6 +16,7 @@ Burst mode info = Y
Power mgmt address monitor = Y
MTU update = Y
Buffer split on Rx = Y
+Selective Rx = Y
Scattered Rx = Y
LRO = Y
TSO = Y
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 00bfb31370..afbf040e66 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -84,6 +84,9 @@ The Rx / Tx data path use different techniques to offer the best performance.
with :ref:`multi-packet Rx queues (MPRQ) <mlx5_mprq_params>`.
This feature is disabled by default.
+- Some PCI bandwidth is saved by receiving partial packets
+ with :ref:`selective Rx <mlx5_selective_rx>`.
+
More details about Rx implementations and their configurations are provided
in the chapter about :ref:`mlx5_rx_functions`.
@@ -879,6 +882,8 @@ MLX5 supports various methods to report statistics:
Basic port statistics can be queried using ``rte_eth_stats_get()``.
The received and sent statistics are through SW only
and counts the number of packets received or sent successfully by the PMD.
+In the case of :ref:`selective Rx <mlx5_selective_rx>`,
+the ``ibytes`` counter matches segments delivered, not the skipped ones.
The ``imissed`` counter is the amount of packets that could not be delivered
to SW because a queue was full.
Packets not received due to congestion in the bus or on the NIC
@@ -992,25 +997,26 @@ These configurations may also have an impact on the behavior:
.. table:: Rx burst functions
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- || Function Name || Parameters to Enable || Scatter|| Error Recovery || CQE || Large|| Shared |
- | | | | || comp|| MTU | RxQ |
- +===================+========================+=========+=================+======+=======+=========+
- | rx_burst | rx_vec_en=0 | Yes | Yes | Yes | Yes | No |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_vec | rx_vec_en=1 (default) | No | if CQE comp off | Yes | No | No |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_mprq || mprq_en=1 | No | Yes | Yes | Yes | No |
- | || RxQs >= rxqs_min_mprq | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_mprq_vec || rx_vec_en=1 (default) | No | if CQE comp off | Yes | Yes | No |
- | || mprq_en=1 | | | | | |
- | || RxQs >= rxqs_min_mprq | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst | at least one Rx queue | Yes | Yes | Yes | Yes | Yes |
- | (out of order) | on the device | | | | | |
- | | is shared | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ || Function|| Parameters to Enable || Scatter|| Selec-|| Error || CQE || Large|| Shared|
+ || Name | | || tive || Recovery|| comp|| MTU || RxQ |
+ +==========+=======================+=========+========+==========+======+=======+========+
+ | rx_burst | rx_vec_en=0 | Yes | Yes | Yes | Yes | Yes | No |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _vec | rx_vec_en=1 (default) | No | No || if CQE | Yes | No | No |
+ | | | | || comp off| | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _mprq || mprq_en=1 | No | No | Yes | Yes | Yes | No |
+ | || RxQs >= rxqs_min_mprq| | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _mprq_vec|| rx_vec_en=1 (default)| No | No || if CQE | Yes | Yes | No |
+ | || mprq_en=1 | | || comp off| | | |
+ | || RxQs >= rxqs_min_mprq| | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ || _out_of || at least one Rx queue| Yes | No | Yes | Yes | Yes | Yes |
+ || _order || on the device | | | | | | |
+ | || is shared | | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
Rx/Tx Tuning
@@ -1105,13 +1111,14 @@ Rx interrupt X
:ref:`Rx threshold <mlx5_rx_threshold>` X X
:ref:`Rx drop delay <mlx5_drop>` X X
:ref:`Rx timestamp <mlx5_rx_timstp>` X X
+:ref:`buffer split <mlx5_buf_split>` X X
+:ref:`selective Rx <mlx5_selective_rx>` X
+:ref:`multi-segment <mlx5_multiseg>` X X
:ref:`Tx scheduling <mlx5_tx_sched>` X
:ref:`Tx rate limit <mlx5_rate_limit>` X
:ref:`Tx inline <mlx5_tx_inline>` X X
:ref:`Tx fast free <mlx5_tx_fast_free>` X X
:ref:`Tx affinity <mlx5_aggregated>` X
-:ref:`buffer split <mlx5_buf_split>` X X
-:ref:`multi-segment <mlx5_multiseg>` X X
promiscuous X X
multicast promiscuous X X
multiple MAC addresses X
@@ -2248,13 +2255,50 @@ OFED 5.1-2
DPDK 20.11
========= ==========
+Runtime configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+The offload flag ``RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT`` is required.
+
+When calling ``rte_eth_rx_queue_setup()``,
+the input ``rte_eth_rxconf::rx_seg`` defines the configuration of the segments,
+mainly offset and length.
+
Limitations
^^^^^^^^^^^
+#. Splitting per protocol header is not supported.
+
#. Buffer split offload is supported with regular Rx burst routine only,
no MPRQ feature or vectorized code can be engaged.
+.. _mlx5_selective_rx:
+
+Selective Rx
+~~~~~~~~~~~~
+
+Some PCI bandwidth can be saved
+by :ref:`skipping some parts of Rx data <nic_features_selective_rx>`.
+It is enabled when using :ref:`buffer split <mlx5_buf_split>`
+and configuring no mempool in some segments to discard.
+
+Runtime configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+The offload flag ``RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT`` is required.
+
+When calling ``rte_eth_rx_queue_setup()``,
+the segment to discard (``rte_eth_rxconf::rx_seg::split``)
+is marked by the absence of mempool (``mp = NULL``).
+
+Limitations
+^^^^^^^^^^^
+
+#. Selective Rx is supported with regular Rx burst routine only,
+ no MPRQ feature or vectorized code can be engaged.
+
+
.. _mlx5_multiseg:
Multi-Segment Scatter/Gather
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index 90a7948c1d..3532792229 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -97,6 +97,10 @@ New Features
* Added support for transmitting LLDP packets based on mbuf packet type.
* Implemented AVX2 context descriptor transmit paths.
+* **Updated NVIDIA mlx5 ethernet driver.**
+
+ * Added support for selective Rx in scalar SPRQ Rx path.
+
* **Updated PCAP ethernet driver.**
* Added support for VLAN insertion and stripping.
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f190654756..61c26d1206 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1975,6 +1975,9 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
/* Init counter pool list header and lock. */
LIST_INIT(&sh->hws_cpool_list);
rte_spinlock_init(&sh->cpool_lock);
+ sh->null_mr = mlx5_os_alloc_null_mr(sh->cdev->dev, sh->cdev->pd);
+ if (!sh->null_mr)
+ DRV_LOG(DEBUG, "Fail to initialize NULL MR, selective Rx is disabled.");
exit:
pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
return sh;
@@ -2139,6 +2142,10 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL);
pthread_mutex_destroy(&sh->txpp.mutex);
mlx5_lwm_unset(sh);
+ if (sh->null_mr) {
+ mlx5_os_free_null_mr(sh->null_mr);
+ sh->null_mr = NULL;
+ }
mlx5_physical_device_destroy(sh->phdev);
mlx5_free(sh);
return;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 92a00cfaa8..bd6ef35b53 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1674,6 +1674,7 @@ struct mlx5_dev_ctx_shared {
rte_spinlock_t cpool_lock;
LIST_HEAD(hws_cpool_list, mlx5_hws_cnt_pool) hws_cpool_list; /* Count pool list. */
struct mlx5_dev_registers registers;
+ struct mlx5_pmd_mr *null_mr;
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index a29cdeeb50..7b7536fa1e 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -381,6 +381,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
info->rx_seg_capa.multi_pools = !priv->config.mprq.enabled;
info->rx_seg_capa.offset_allowed = !priv->config.mprq.enabled;
info->rx_seg_capa.offset_align_log2 = 0;
+ info->rx_seg_capa.selective_rx = !!priv->sh->null_mr;
info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
info->rx_queue_offload_capa);
info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
@@ -708,6 +709,25 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
return -rte_errno;
}
+static bool
+mlx5_selective_rx_enabled(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ for (uint32_t q = 0; q < priv->rxqs_n; ++q) {
+ struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, q);
+
+ if (rxq_ctrl == NULL || rxq_ctrl->is_hairpin)
+ continue;
+ for (uint16_t s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
+ if (rxq_ctrl->rxq.rxseg[s].mp == NULL)
+ return true;
+ }
+ }
+
+ return false;
+}
+
/**
* Configure the RX function to use.
*
@@ -723,6 +743,11 @@ mlx5_select_rx_function(struct rte_eth_dev *dev)
eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
MLX5_ASSERT(dev != NULL);
+ if (mlx5_selective_rx_enabled(dev)) {
+ DRV_LOG(DEBUG, "port %u forced to scalar SPRQ Rx (selective Rx configured)",
+ dev->data->port_id);
+ return rx_pkt_burst;
+ }
if (mlx5_shared_rq_enabled(dev)) {
rx_pkt_burst = mlx5_rx_burst_out_of_order;
DRV_LOG(DEBUG, "port %u forced to use SPRQ"
diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index 185bfd4fff..6aeab8d423 100644
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -486,7 +486,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
rxq->wqes)[i];
addr = rte_pktmbuf_mtod(buf, uintptr_t);
byte_count = DATA_LEN(buf);
- lkey = mlx5_rx_mb2mr(rxq, buf);
+ lkey = buf->pool ? mlx5_rx_mb2mr(rxq, buf) : rxq->sh->null_mr->lkey;
}
/* scat->addr must be able to store a pointer. */
MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
@@ -1044,11 +1044,14 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
const unsigned int sges_n = rxq->sges_n;
struct rte_mbuf *pkt = NULL;
struct rte_mbuf *seg = NULL;
+ struct rte_mbuf *tail = NULL;
volatile struct mlx5_cqe *cqe =
&(*rxq->cqes)[rxq->cq_ci & cqe_mask];
+ volatile struct mlx5_mini_cqe8 *mcqe = NULL;
unsigned int i = 0;
unsigned int rq_ci = rxq->rq_ci << sges_n;
int len = 0; /* keep its value across iterations. */
+ uint32_t data_seg_len = 0;
while (pkts_n) {
uint16_t skip_cnt;
@@ -1056,14 +1059,18 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
volatile struct mlx5_wqe_data_seg *wqe =
&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
struct rte_mbuf *rep = (*rxq->elts)[idx];
- volatile struct mlx5_mini_cqe8 *mcqe = NULL;
- if (pkt)
- NEXT(seg) = rep;
+ if (pkt) {
+ if (rep->pool)
+ NEXT(tail) = rep;
+ else
+ --NB_SEGS(pkt);
+ }
seg = rep;
rte_prefetch0(seg);
rte_prefetch0(cqe);
rte_prefetch0(wqe);
+ if (seg->pool) {
/* Allocate the buf from the same pool. */
rep = rte_mbuf_raw_alloc(seg->pool);
if (unlikely(rep == NULL)) {
@@ -1088,12 +1095,15 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
rq_ci <<= sges_n;
break;
}
+ }
if (!pkt) {
+ if (len == 0) { /* no CQE polled yet */
cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
&mcqe, &skip_cnt, false, NULL);
if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
/* We drop packets with non-critical errors */
+ if (seg->pool)
rte_mbuf_raw_free(rep);
if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
rq_ci = rxq->rq_ci << sges_n;
@@ -1107,17 +1117,21 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
continue;
}
if (len == 0) {
+ if (seg->pool)
rte_mbuf_raw_free(rep);
break;
}
- pkt = seg;
MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
+ if (rxq->crc_present)
+ len -= RTE_ETHER_CRC_LEN;
+ }
+ if (seg->pool) { /* first real segment */
+ pkt = seg;
+ tail = seg;
pkt->ol_flags &= RTE_MBUF_F_EXTERNAL;
if (rxq->cqe_comp_layout && mcqe)
cqe = &rxq->title_cqe;
rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
- if (rxq->crc_present)
- len -= RTE_ETHER_CRC_LEN;
PKT_LEN(pkt) = len;
if (cqe->lro_num_seg > 1) {
mlx5_lro_update_hdr
@@ -1127,6 +1141,9 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
pkt->tso_segsz = len / cqe->lro_num_seg;
}
}
+ }
+ if (seg->pool) { /* real segment: replenish WQE */
+ tail = seg;
DATA_LEN(rep) = DATA_LEN(seg);
PKT_LEN(rep) = PKT_LEN(seg);
SET_DATA_OFF(rep, DATA_OFF(seg));
@@ -1141,20 +1158,37 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
/* If there's only one MR, no need to replace LKey in WQE. */
if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+ }
if (len > DATA_LEN(seg)) {
+ if (seg->pool)
+ data_seg_len += DATA_LEN(seg);
len -= DATA_LEN(seg);
+ if (pkt)
++NB_SEGS(pkt);
++rq_ci;
continue;
}
+ if (seg->pool) { /* last segment */
DATA_LEN(seg) = len;
+ data_seg_len += len;
+ }
+ if (unlikely(!pkt)) { /* no real segment found, skip packet */
+ len = 0;
+ rq_ci >>= sges_n;
+ ++rq_ci;
+ rq_ci <<= sges_n;
+ continue;
+ }
+ PKT_LEN(pkt) = RTE_MIN(PKT_LEN(pkt), data_seg_len);
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment bytes counter. */
rxq->stats.ibytes += PKT_LEN(pkt);
#endif
+ data_seg_len = 0;
/* Return packet. */
*(pkts++) = pkt;
pkt = NULL;
+ len = 0;
--pkts_n;
++i;
/* Align consumer index to the next stride. */
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 01b563d981..cd48ee37ef 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -96,6 +96,7 @@ struct mlx5_eth_rxseg {
uint16_t length; /**< Segment data length, configures split point. */
uint16_t offset; /**< Data offset from beginning of mbuf data buffer. */
uint32_t reserved; /**< Reserved field. */
+ struct rte_mbuf *null_mbuf; /**< For selective Rx. */
};
/* RX queue descriptor. */
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 48d982a8c2..3fae189fa4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -151,6 +151,7 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
struct mlx5_eth_rxseg *seg = &rxq_ctrl->rxq.rxseg[i % sges_n];
struct rte_mbuf *buf;
+ if (seg->mp) {
buf = rte_pktmbuf_alloc(seg->mp);
if (buf == NULL) {
if (rxq_ctrl->share_group == 0)
@@ -167,6 +168,9 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
/* Only vectored Rx routines rely on headroom size. */
MLX5_ASSERT(!has_vec_support ||
DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
+ } else {
+ buf = seg->null_mbuf;
+ }
/* Buffer is supposed to be empty. */
MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0);
MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0);
@@ -324,10 +328,14 @@ rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
rxq->rq_pi = elts_ci;
}
for (i = 0; i != q_n; ++i) {
- if ((*rxq->elts)[i] != NULL)
+ if ((*rxq->elts)[i] != NULL && (*rxq->elts)[i]->pool != NULL)
rte_pktmbuf_free_seg((*rxq->elts)[i]);
(*rxq->elts)[i] = NULL;
}
+ for (i = 0; i < rxq->rxseg_n; i++) {
+ mlx5_free(rxq->rxseg[i].null_mbuf);
+ rxq->rxseg[i].null_mbuf = NULL;
+ }
}
/**
@@ -1815,7 +1823,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
int ret;
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *tmpl;
- unsigned int mb_len = rte_pktmbuf_data_room_size(rx_seg[0].mp);
+ struct rte_mempool *first_mp = NULL;
+ struct rte_mempool *last_mp = NULL;
+ unsigned int mb_len;
struct mlx5_port_config *config = &priv->config;
uint64_t offloads = conf->offloads |
dev->data->dev_conf.rxmode.offloads;
@@ -1827,7 +1837,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int non_scatter_min_mbuf_size = max_rx_pktlen +
RTE_PKTMBUF_HEADROOM;
unsigned int max_lro_size = 0;
- unsigned int first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
+ unsigned int first_mb_free_size;
uint32_t mprq_log_actual_stride_num = 0;
uint32_t mprq_log_actual_stride_size = 0;
bool rx_seg_en = n_seg != 1 || rx_seg[0].offset || rx_seg[0].length;
@@ -1845,6 +1855,21 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
const struct rte_eth_rxseg_split *qs_seg = rx_seg;
unsigned int tail_len;
+ /* Find first segment with a mempool. */
+ for (uint16_t seg = 0; seg < n_seg; seg++) {
+ if (rx_seg[seg].mp != NULL) {
+ first_mp = rx_seg[seg].mp;
+ break;
+ }
+ }
+ if (first_mp == NULL) {
+ DRV_LOG(ERR, "port %u Rx queue %u has no mempool", dev->data->port_id, idx);
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ mb_len = rte_pktmbuf_data_room_size(first_mp);
+ first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
+
if (mprq_en) {
/* Trim the number of descs needed. */
desc >>= mprq_log_actual_stride_num;
@@ -1884,13 +1909,20 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
do {
struct mlx5_eth_rxseg *hw_seg =
&tmpl->rxq.rxseg[tmpl->rxq.rxseg_n];
- uint32_t buf_len, offset, seg_len;
+ uint32_t buf_len = 0, offset, seg_len;
/*
* For the buffers beyond descriptions offset is zero,
* the first buffer contains head room.
*/
- buf_len = rte_pktmbuf_data_room_size(qs_seg->mp);
+ if (qs_seg->mp != NULL) {
+ last_mp = qs_seg->mp;
+ buf_len = rte_pktmbuf_data_room_size(qs_seg->mp);
+ } else if (last_mp != NULL) {
+ buf_len = rte_pktmbuf_data_room_size(last_mp);
+ } else {
+ buf_len = mb_len;
+ }
offset = (tmpl->rxq.rxseg_n >= n_seg ? 0 : qs_seg->offset) +
(tmpl->rxq.rxseg_n ? 0 : RTE_PKTMBUF_HEADROOM);
/*
@@ -2077,7 +2109,8 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save port ID. */
tmpl->rxq.port_id = dev->data->port_id;
tmpl->sh = priv->sh;
- tmpl->rxq.mp = rx_seg[0].mp;
+ tmpl->rxq.sh = priv->sh;
+ tmpl->rxq.mp = first_mp;
tmpl->rxq.elts_n = log2above(desc);
tmpl->rxq.rq_repl_thresh = MLX5_VPMD_RXQ_RPLNSH_THRESH(desc_n);
tmpl->rxq.elts = (struct rte_mbuf *(*)[])(tmpl + 1);
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index a070aaecfd..5b04d9a234 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -116,6 +116,27 @@ mlx5_txq_start(struct rte_eth_dev *dev)
return -rte_errno;
}
+static struct rte_mbuf *
+mlx5_alloc_null_mbuf(uint32_t data_len)
+{
+ size_t alloc_size = sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM +
+ rte_align32pow2(data_len);
+ struct rte_mbuf *m;
+
+ m = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, 0, SOCKET_ID_ANY);
+ if (m == NULL)
+ return NULL;
+ m->buf_addr = RTE_PTR_ADD(m, sizeof(*m));
+ m->buf_len = alloc_size - sizeof(*m);
+ rte_mbuf_iova_set(m, rte_mem_virt2iova(m->buf_addr));
+ m->data_off = RTE_PKTMBUF_HEADROOM;
+ m->refcnt = 1;
+ m->nb_segs = 1;
+ m->port = RTE_MBUF_PORT_INVALID;
+ m->pool = NULL;
+ return m;
+}
+
/**
* Register Rx queue mempools and fill the Rx queue cache.
* This function tolerates repeated mempool registration.
@@ -130,7 +151,8 @@ static int
mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
{
struct rte_mempool *mp;
- uint32_t s;
+ struct mlx5_eth_rxseg *seg;
+ uint16_t s;
int ret = 0;
mlx5_mr_flush_local_cache(&rxq_ctrl->rxq.mr_ctrl);
@@ -139,21 +161,37 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
return mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
rxq_ctrl->rxq.mprq_mp);
for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
- bool is_extmem;
-
- mp = rxq_ctrl->rxq.rxseg[s].mp;
- is_extmem = (rte_pktmbuf_priv_flags(mp) &
+ seg = &rxq_ctrl->rxq.rxseg[s];
+ mp = seg->mp;
+ if (mp) { /* Regular segment */
+ bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp,
is_extmem);
if (ret < 0 && rte_errno != EEXIST)
- return ret;
+ goto error;
ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
mp);
if (ret < 0)
- return ret;
+ goto error;
+ } else { /* NULL segment used in selective Rx */
+ seg->null_mbuf = mlx5_alloc_null_mbuf(seg->length);
+ if (seg->null_mbuf == NULL) {
+ rte_errno = ENOMEM;
+ ret = -rte_errno;
+ goto error;
+ }
+ }
}
return 0;
+
+error:
+ while (s-- > 0) {
+ seg = &rxq_ctrl->rxq.rxseg[s];
+ mlx5_free(seg->null_mbuf);
+ seg->null_mbuf = NULL;
+ }
+ return ret;
}
/**
--
2.54.0
^ permalink raw reply related
* [PATCH v6 05/10] net/mlx5: fix Rx split segment counter type
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
In the API, rx_nseg and max_nseg are uint16_t.
In mlx5, MLX5_MAX_RXQ_NSEG is 32.
So there is no reason to have rxseg_n as uint32_t.
Reduce the fields to uint16_t and move them to avoid struct holes.
Fixes: 9f209b59c8b0 ("net/mlx5: support Rx buffer split description")
Fixes: 572c9d4bda08 ("net/mlx5: fix shared Rx queue segment configuration match")
Cc: stable@dpdk.org
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/net/mlx5/mlx5_rx.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index dffab3955b..01b563d981 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -164,9 +164,9 @@ struct __rte_cache_aligned mlx5_rxq_data {
uint64_t flow_meta_mask;
int32_t flow_meta_offset;
uint32_t flow_meta_port_mask;
- uint32_t rxseg_n; /* Number of split segment descriptions. */
struct mlx5_eth_rxseg rxseg[MLX5_MAX_RXQ_NSEG];
/* Buffer split segment descriptions - sizes, offsets, pools. */
+ uint16_t rxseg_n; /* Number of split segment descriptions. */
uint16_t rq_win_cnt; /* Number of packets in the sliding window data. */
uint16_t rq_win_idx_mask; /* Sliding window index wrapping mask. */
uint16_t rq_win_idx; /* Index of the first element in sliding window. */
@@ -191,9 +191,9 @@ struct mlx5_rxq_ctrl {
unsigned int irq:1; /* Whether IRQ is enabled. */
uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
uint32_t wqn; /* WQ number. */
- uint32_t rxseg_n; /* Number of split segment descriptions. */
struct rte_eth_rxseg_split rxseg[MLX5_MAX_RXQ_NSEG];
/* Saved original buffer split segment configuration. */
+ uint16_t rxseg_n; /* Number of split segment descriptions. */
uint16_t dump_file_n; /* Number of dump files. */
};
--
2.54.0
^ permalink raw reply related
* [PATCH v6 04/10] common/mlx5: add null MR functions
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Gregory Etelson, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Add functions to allocate and free a null Memory Region (MR)
using ibverbs on Linux.
There is no implementation for DevX on Windows.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/common/mlx5/linux/mlx5_common_verbs.c | 35 +++++++++++++++++++
drivers/common/mlx5/mlx5_common_mr.h | 9 +++++
drivers/common/mlx5/windows/mlx5_common_os.c | 16 +++++++++
3 files changed, 60 insertions(+)
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index 2322d9d033..6d44e1f566 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -161,3 +161,38 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
*reg_mr_cb = mlx5_common_verbs_reg_mr;
*dereg_mr_cb = mlx5_common_verbs_dereg_mr;
}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
+{
+ struct ibv_mr *ibv_mr;
+ struct mlx5_pmd_mr *null_mr;
+
+ null_mr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*null_mr), 0, dev->numa_node);
+ if (!null_mr)
+ return NULL;
+ ibv_mr = mlx5_glue->alloc_null_mr(pd);
+ if (!ibv_mr) {
+ mlx5_free(null_mr);
+ return NULL;
+ }
+ *null_mr = (struct mlx5_pmd_mr) {
+ .lkey = rte_cpu_to_be_32(ibv_mr->lkey),
+ .addr = ibv_mr->addr,
+ .len = ibv_mr->length,
+ .obj = (void *)ibv_mr,
+ };
+ return null_mr;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_free_null_mr)
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr)
+{
+ if (!null_mr)
+ return;
+ if (null_mr->obj)
+ claim_zero(mlx5_glue->dereg_mr(null_mr->obj));
+ mlx5_free(null_mr);
+}
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index cf7c685e9b..00f3d832c3 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -21,6 +21,8 @@
#include "mlx5_common_mp.h"
#include "mlx5_common_defs.h"
+struct rte_device;
+
/* mlx5 PMD MR struct. */
struct mlx5_pmd_mr {
uint32_t lkey;
@@ -258,6 +260,13 @@ __rte_internal
void
mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+__rte_internal
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd);
+__rte_internal
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr);
+
__rte_internal
int
mlx5_mr_mempool_register(struct mlx5_common_device *cdev,
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index 16fcc5f9fc..692517a9bf 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -454,6 +454,22 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
*dereg_mr_cb = mlx5_os_dereg_mr;
}
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
+{
+ RTE_SET_USED(dev);
+ RTE_SET_USED(pd);
+ return NULL;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_free_null_mr)
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr)
+{
+ RTE_SET_USED(null_mr);
+}
+
/*
* In Windows, no need to wrap the MR, no known issue for it in kernel.
* Use the regular function to create direct MR.
--
2.54.0
^ permalink raw reply related
* [PATCH v6 03/10] app/testpmd: support selective Rx
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Gregory Etelson, Aman Singh
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Add support for selective Rx using existing rxoffs and rxpkts
command line parameters.
When both rxoffs and rxpkts are specified
on PMDs supporting selective Rx, testpmd automatically:
1. Inserts segments with NULL mempool
for gaps between configured segments to discard unwanted data.
2. Adds a trailing segment with NULL mempool
to cover any remaining data up to the max packet length.
Example usage to receive only Ethernet header and a segment at offset 128:
--rxoffs=0,128 --rxpkts=14,64
This creates segments:
- [0-13]: 14 bytes with mempool (received)
- [14-127]: 114 bytes with NULL mempool (discarded)
- [128-191]: 64 bytes with mempool (received)
- [192-max]: remaining bytes with NULL mempool (discarded)
Note: RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT is required for this feature
and is checked at ethdev API level.
This check is removed from testpmd to allow negative testing of the API.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
---
app/test-pmd/testpmd.c | 71 ++++++++++++++++++++++-----
doc/guides/testpmd_app_ug/run_app.rst | 20 ++++++++
2 files changed, 80 insertions(+), 11 deletions(-)
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index a9b35f530a..d14341d3ff 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -2731,6 +2731,16 @@ port_is_started(portid_t port_id)
return 1;
}
+static struct rte_eth_rxseg_split *
+next_rx_seg(union rte_eth_rxseg *segs, uint16_t *idx)
+{
+ if (*idx >= MAX_SEGS_BUFFER_SPLIT) {
+ fprintf(stderr, "Too many segments (max %u)\n", MAX_SEGS_BUFFER_SPLIT);
+ return NULL;
+ }
+ return &segs[(*idx)++].split;
+}
+
/* Configure the Rx with optional split. */
int
rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
@@ -2744,31 +2754,70 @@ rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
uint32_t prev_hdrs = 0;
int ret;
- if ((rx_pkt_nb_segs > 1) &&
- (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ if (rx_pkt_nb_segs > 1 || rx_pkt_nb_offs > 0) {
+ struct rte_eth_dev_info dev_info;
+ uint16_t seg_idx = 0;
+ uint16_t next_offset = 0;
+
+ ret = rte_eth_dev_info_get(port_id, &dev_info);
+ if (ret != 0)
+ return ret;
+
/* multi-segment configuration */
for (i = 0; i < rx_pkt_nb_segs; i++) {
- struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
- /*
- * Use last valid pool for the segments with number
- * exceeding the pool index.
- */
+ struct rte_eth_rxseg_split *rx_seg;
+ uint16_t seg_offset;
+
+ if (i < rx_pkt_nb_offs)
+ seg_offset = rx_pkt_seg_offsets[i];
+ else
+ seg_offset = rx_pkt_nb_offs > 0 ? next_offset : 0;
+
+ /* Insert selective Rx discard segment if there's a gap */
+ if (seg_offset > next_offset) {
+ rx_seg = next_rx_seg(rx_useg, &seg_idx);
+ if (rx_seg == NULL)
+ return -EINVAL;
+ rx_seg->offset = next_offset;
+ rx_seg->length = seg_offset - next_offset;
+ rx_seg->mp = NULL;
+ next_offset = seg_offset;
+ }
+
+ rx_seg = next_rx_seg(rx_useg, &seg_idx);
+ if (rx_seg == NULL)
+ return -EINVAL;
mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
mpx = mbuf_pool_find(socket_id, mp_n);
- /* Handle zero as mbuf data buffer size. */
- rx_seg->offset = i < rx_pkt_nb_offs ?
- rx_pkt_seg_offsets[i] : 0;
+ rx_seg->offset = seg_offset;
rx_seg->mp = mpx ? mpx : mp;
if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) {
rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs;
prev_hdrs |= rx_seg->proto_hdr;
+ } else if (rx_pkt_nb_offs > 0 && rx_pkt_seg_lengths[i] == 0) {
+ /* Insert fake discard segment if explicitly requested */
+ rx_seg->mp = NULL;
+ rx_seg->length = 0;
} else {
rx_seg->length = rx_pkt_seg_lengths[i] ?
rx_pkt_seg_lengths[i] :
mbuf_data_size[mp_n];
}
+
+ next_offset = seg_offset + rx_seg->length;
}
- rx_conf->rx_nseg = rx_pkt_nb_segs;
+
+ /* Add trailing selective Rx discard segment up to max packet length */
+ if (rx_pkt_nb_offs > 0 && next_offset < dev_info.max_rx_pktlen) {
+ struct rte_eth_rxseg_split *rx_seg = next_rx_seg(rx_useg, &seg_idx);
+ if (rx_seg == NULL)
+ return -EINVAL;
+ rx_seg->offset = next_offset;
+ rx_seg->length = dev_info.max_rx_pktlen - next_offset;
+ rx_seg->mp = NULL;
+ }
+
+ rx_conf->rx_nseg = seg_idx;
rx_conf->rx_seg = rx_useg;
rx_conf->rx_mempools = NULL;
rx_conf->rx_nmempool = 0;
diff --git a/doc/guides/testpmd_app_ug/run_app.rst b/doc/guides/testpmd_app_ug/run_app.rst
index 1a4a4b6c12..b59991ed89 100644
--- a/doc/guides/testpmd_app_ug/run_app.rst
+++ b/doc/guides/testpmd_app_ug/run_app.rst
@@ -364,6 +364,11 @@ The command line options are:
feature is engaged. Affects only the queues configured
with split offloads (currently BUFFER_SPLIT is supported only).
+ When used with ``--rxpkts`` on PMDs supporting selective Rx,
+ enables receiving only specific packet segments and discarding the rest.
+ Gaps between configured segments and any trailing data up to the max packet length
+ are automatically filled with NULL mempool segments (data is discarded).
+
* ``--rxpkts=X[,Y]``
Set the length of segments to scatter packets on receiving if split
@@ -373,6 +378,21 @@ The command line options are:
command line parameter and the mbufs to receive will be allocated
sequentially from these extra memory pools.
+ Note: ``--rxoffs`` is required to enable selective Rx in testpmd.
+ To receive only the first N bytes, use ``--rxoffs=0 --rxpkts=N``.
+
+ To receive only the Ethernet header (14 bytes at offset 0) and
+ a 64-byte segment starting at offset 128, while discarding the rest::
+
+ --rxoffs=0,128 --rxpkts=14,64
+
+ This configuration will:
+
+ * Receive 14 bytes at offset 0 (Ethernet header)
+ * Discard bytes 14-127 (inserted NULL mempool segment)
+ * Receive 64 bytes at offset 128
+ * Discard remaining bytes (inserted NULL mempool segment)
+
* ``--txpkts=X[,Y]``
Set TX segment sizes or total packet length. Valid for ``tx-only``
--
2.54.0
^ permalink raw reply related
* [PATCH v6 02/10] ethdev: introduce selective Rx
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Gregory Etelson, Andrew Rybchenko, Aman Singh
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Receiving an entire packet is not always needed.
The Rx performance can be improved by receiving only partial data
and safely discard the rest of the packet data,
because it reduces the PCI bandwidth and the memory consumption.
Selective Rx allows an application to receive
only pre-configured packet segments and discard the rest.
For example:
- Deliver the first N bytes only.
- Deliver the last N bytes only.
- Deliver N1 bytes from offset Off1 and N2 bytes from offset Off2.
Selective Rx is implemented on top of the Rx buffer split API:
- rte_eth_rxseg_split uses the null mempool for segments
that should be discarded.
- the PMD does not create mbuf segments if no data read.
For example: Deliver Ethernet header only
Rx queue segments configuration:
struct rte_eth_rxseg_split split[2] = {
{
.mp = <some mempool>,
.length = sizeof(struct rte_ether_hdr)
},
{
.mp = NULL, /* discard data */
.length = 0 /* default to buffer size */
}
};
Received mbuf:
pkt_len = sizeof(struct rte_ether_hdr);
data_len = sizeof(struct rte_ether_hdr);
next = NULL; /* The next segment did not deliver data */
After selective Rx, the mbuf packet length reflects only the data
that was actually received,
and can be less than the original wire packet length.
A PMD activates the selective Rx capability by setting
the rte_eth_rxseg_capa.selective_rx bit.
This new capability bit is inserted in a bitmap hole
of the struct rte_eth_rxseg_capa,
but it needs to be ignored in the ABI check as libabigail sees a change.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
Reviewed-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
---
app/test-pmd/config.c | 1 +
devtools/libabigail.abignore | 7 +++++++
doc/guides/nics/features.rst | 14 ++++++++++++++
doc/guides/nics/features/default.ini | 1 +
doc/guides/rel_notes/release_26_07.rst | 7 +++++++
lib/ethdev/rte_ethdev.c | 24 ++++++++++++++++--------
lib/ethdev/rte_ethdev.h | 14 +++++++++++++-
7 files changed, 59 insertions(+), 9 deletions(-)
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 55d1c6d696..9d457ca88e 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -925,6 +925,7 @@ port_infos_display(portid_t port_id)
print_bool_capa("\tBuffer offset", dev_info.rx_seg_capa.offset_allowed);
printf("\tOffset alignment: %u\n",
RTE_BIT32(dev_info.rx_seg_capa.offset_align_log2));
+ print_bool_capa("\tSelective Rx", dev_info.rx_seg_capa.selective_rx);
}
if (dev_info.max_vfs)
diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
index 21b8cd6113..2a0efd718e 100644
--- a/devtools/libabigail.abignore
+++ b/devtools/libabigail.abignore
@@ -33,3 +33,10 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Temporary exceptions till next major ABI version ;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; Ignore new bit selective_rx in rte_eth_rxseg_capa bitmap hole
+[suppress_type]
+ name = rte_eth_rxseg_capa
+ type_kind = struct
+ has_size_change = no
+ has_data_member_inserted_at = 6
diff --git a/doc/guides/nics/features.rst b/doc/guides/nics/features.rst
index a075c057ec..26357036ca 100644
--- a/doc/guides/nics/features.rst
+++ b/doc/guides/nics/features.rst
@@ -199,6 +199,20 @@ Scatters the packets being received on specified boundaries to segmented mbufs.
* **[related] API**: ``rte_eth_rx_queue_setup()``, ``rte_eth_buffer_split_get_supported_hdr_ptypes()``.
+.. _nic_features_selective_rx:
+
+Selective Rx
+------------
+
+Discards some segments of buffer split on Rx.
+
+* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT``.
+* **[uses] rte_eth_rxconf**: ``rx_seg.mp = NULL`` to discard segments.
+* **[provides] rte_eth_dev_info**: ``rx_offload_capa:RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT``.
+* **[provides] rte_eth_dev_info**: ``rx_seg_capa.selective_rx``.
+* **[related] API**: ``rte_eth_rx_queue_setup()``.
+
+
.. _nic_features_lro:
LRO
diff --git a/doc/guides/nics/features/default.ini b/doc/guides/nics/features/default.ini
index e50514d750..8303a530c1 100644
--- a/doc/guides/nics/features/default.ini
+++ b/doc/guides/nics/features/default.ini
@@ -25,6 +25,7 @@ Burst mode info =
Power mgmt address monitor =
MTU update =
Buffer split on Rx =
+Selective Rx =
Scattered Rx =
LRO =
TSO =
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index b8a3e2ced9..90a7948c1d 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -81,6 +81,13 @@ New Features
Added no-IOMMU mode for devices without or not enabling IOMMU/SVA.
+* **Added selective Rx in ethdev API.**
+
+ Some parts of packets may be discarded in Rx
+ by configuring a split of packets received in a queue,
+ and assigning no mempool to some configuration segments.
+ This is a driver capability advertised in the ``selective_rx`` bit.
+
* **Added LinkData sxe2 ethernet driver.**
Added network driver for the LinkData network adapters.
diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c
index d0273e3f7b..67d609820c 100644
--- a/lib/ethdev/rte_ethdev.c
+++ b/lib/ethdev/rte_ethdev.c
@@ -2129,7 +2129,7 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
const struct rte_eth_dev_info *dev_info)
{
const struct rte_eth_rxseg_capa *seg_capa = &dev_info->rx_seg_capa;
- struct rte_mempool *mp_first;
+ struct rte_mempool *mp_first = NULL;
uint32_t offset_mask;
uint16_t seg_idx;
int ret = 0;
@@ -2148,7 +2148,6 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
* Check the sizes and offsets against buffer sizes
* for each segment specified in extended configuration.
*/
- mp_first = rx_seg[0].mp;
offset_mask = RTE_BIT32(seg_capa->offset_align_log2) - 1;
ptypes = NULL;
@@ -2160,13 +2159,17 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
uint32_t offset = rx_seg[seg_idx].offset;
uint32_t proto_hdr = rx_seg[seg_idx].proto_hdr;
- if (mpl == NULL) {
- RTE_ETHDEV_LOG_LINE(ERR, "null mempool pointer");
- ret = -EINVAL;
- goto out;
+ if (mpl == NULL) { /* discarded segment */
+ if (seg_capa->selective_rx == 0) { /* not supported */
+ RTE_ETHDEV_LOG_LINE(ERR, "null mempool pointer");
+ ret = -EINVAL;
+ goto out;
+ }
+ continue; /* next checks are not relevant if no mempool */
}
- if (seg_idx != 0 && mp_first != mpl &&
- seg_capa->multi_pools == 0) {
+ if (mp_first == NULL)
+ mp_first = mpl;
+ if (mp_first != mpl && seg_capa->multi_pools == 0) {
RTE_ETHDEV_LOG_LINE(ERR, "Receiving to multiple pools is not supported");
ret = -ENOTSUP;
goto out;
@@ -2233,6 +2236,11 @@ rte_eth_rx_queue_check_split(uint16_t port_id,
if (ret != 0)
goto out;
}
+ if (mp_first == NULL) {
+ RTE_ETHDEV_LOG_LINE(ERR, "At least one Rx segment must have a mempool");
+ ret = -EINVAL;
+ goto out;
+ }
out:
free(ptypes);
return ret;
diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h
index dedbc05554..d7e1560828 100644
--- a/lib/ethdev/rte_ethdev.h
+++ b/lib/ethdev/rte_ethdev.h
@@ -1073,6 +1073,7 @@ struct rte_eth_txmode {
* - The first network buffer will be allocated from the memory pool,
* specified in the first array element, the second buffer, from the
* pool in the second element, and so on.
+ * If the pool is NULL, the segment will be discarded, i.e. not received.
*
* - The proto_hdrs in the elements define the split position of
* received packets.
@@ -1121,7 +1122,15 @@ struct rte_eth_txmode {
* The rest will be put into the last valid pool.
*/
struct rte_eth_rxseg_split {
- struct rte_mempool *mp; /**< Memory pool to allocate segment from. */
+ /**
+ * Memory pool to allocate segment from.
+ *
+ * NULL means discarded segment.
+ * Length of discarded segment is not reflected in mbuf packet length
+ * and not accounted in ibytes statistics.
+ * @see rte_eth_rxseg_capa::selective_rx
+ */
+ struct rte_mempool *mp;
uint16_t length; /**< Segment data length, configures split point. */
uint16_t offset; /**< Data offset from beginning of mbuf data buffer. */
/**
@@ -1752,12 +1761,15 @@ struct rte_eth_switch_info {
* @b EXPERIMENTAL: this structure may change without prior notice.
*
* Ethernet device Rx buffer segmentation capabilities.
+ *
+ * @see rte_eth_rxseg_split
*/
struct rte_eth_rxseg_capa {
__extension__
uint32_t multi_pools:1; /**< Supports receiving to multiple pools.*/
uint32_t offset_allowed:1; /**< Supports buffer offsets. */
uint32_t offset_align_log2:4; /**< Required offset alignment. */
+ uint32_t selective_rx:1; /**< Supports discarding segment. */
uint16_t max_nseg; /**< Maximum amount of segments to split. */
uint16_t reserved; /**< Reserved field. */
};
--
2.54.0
^ permalink raw reply related
* [PATCH v6 01/10] app/testpmd: print Rx split capabilities
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Aman Singh
In-Reply-To: <20260602215022.3698662-1-thomas@monjalon.net>
The capabilities from rte_eth_rxseg_capa are added
to the command "show port info".
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
app/test-pmd/config.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index c950793aaf..55d1c6d696 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -790,6 +790,12 @@ rss_offload_types_display(uint64_t offload_types, uint16_t char_num_per_line)
printf("\n");
}
+static void
+print_bool_capa(const char *label, int value)
+{
+ printf("%s: %s\n", label, value ? "supported" : "not supported");
+}
+
void
port_infos_display(portid_t port_id)
{
@@ -911,6 +917,16 @@ port_infos_display(portid_t port_id)
dev_info.max_rx_pktlen);
printf("Maximum configurable size of LRO aggregated packet: %u\n",
dev_info.max_lro_pkt_size);
+
+ printf("Rx split:\n");
+ printf("\tMax segments: %hu\n", dev_info.rx_seg_capa.max_nseg);
+ if (dev_info.rx_seg_capa.max_nseg > 0) {
+ print_bool_capa("\tMulti-pool", dev_info.rx_seg_capa.multi_pools);
+ print_bool_capa("\tBuffer offset", dev_info.rx_seg_capa.offset_allowed);
+ printf("\tOffset alignment: %u\n",
+ RTE_BIT32(dev_info.rx_seg_capa.offset_align_log2));
+ }
+
if (dev_info.max_vfs)
printf("Maximum number of VFs: %u\n", dev_info.max_vfs);
if (dev_info.max_vmdq_pools)
--
2.54.0
^ permalink raw reply related
* [PATCH v6 00/10] selective Rx
From: Thomas Monjalon @ 2026-06-02 21:49 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260202160903.254621-1-getelson@nvidia.com>
This is a new feature in ethdev with tests and mlx5 implementation.
Selective Rx allows to receive partial data,
saving some hardware bandwidth.
Note 1: mlx5 support patch is not correctly indented
to make review easier. An indent patch follows to be squashed.
Note 2: DTS patch is an attempt to test the feature on day 1,
it is not mandatory if it is blocking the merge.
v2: rework after Gregory
v3: fix bugs found with AI by Stephen
v4: fix packet type in DTS test
v5: fix mlx5 Rx to handle discarding first segment
v6: fix reindent patch
Gregory Etelson (4):
ethdev: introduce selective Rx
app/testpmd: support selective Rx
common/mlx5: add null MR functions
net/mlx5: support selective Rx
Thomas Monjalon (6):
app/testpmd: print Rx split capabilities
net/mlx5: fix Rx split segment counter type
net/mlx5: reindent previous changes
common/mlx5: remove callbacks for MR registration
dts: fix topology capability comparison
dts: add selective Rx tests
app/test-pmd/config.c | 17 ++
app/test-pmd/testpmd.c | 71 ++++-
devtools/libabigail.abignore | 7 +
doc/guides/nics/features.rst | 14 +
doc/guides/nics/features/default.ini | 1 +
doc/guides/nics/features/mlx5.ini | 1 +
doc/guides/nics/mlx5.rst | 86 ++++--
doc/guides/rel_notes/release_26_07.rst | 11 +
doc/guides/testpmd_app_ug/run_app.rst | 20 ++
drivers/common/mlx5/linux/mlx5_common_verbs.c | 53 ++--
drivers/common/mlx5/mlx5_common.c | 6 +-
drivers/common/mlx5/mlx5_common_mr.c | 37 ++-
drivers/common/mlx5/mlx5_common_mr.h | 29 +-
drivers/common/mlx5/windows/mlx5_common_os.c | 31 ++-
drivers/compress/mlx5/mlx5_compress.c | 4 +-
drivers/crypto/mlx5/mlx5_crypto.h | 2 -
drivers/crypto/mlx5/mlx5_crypto_gcm.c | 6 +-
drivers/net/mlx5/mlx5.c | 7 +
drivers/net/mlx5/mlx5.h | 4 +-
drivers/net/mlx5/mlx5_ethdev.c | 25 ++
drivers/net/mlx5/mlx5_flow_aso.c | 21 +-
drivers/net/mlx5/mlx5_flow_hw.c | 11 +-
drivers/net/mlx5/mlx5_flow_quota.c | 6 +-
drivers/net/mlx5/mlx5_hws_cnt.c | 19 +-
drivers/net/mlx5/mlx5_rx.c | 186 ++++++++-----
drivers/net/mlx5/mlx5_rx.h | 5 +-
drivers/net/mlx5/mlx5_rxq.c | 75 +++--
drivers/net/mlx5/mlx5_trigger.c | 64 ++++-
dts/api/capabilities.py | 2 +
dts/api/testpmd/__init__.py | 17 ++
dts/api/testpmd/types.py | 6 +
dts/framework/testbed_model/capability.py | 10 +-
dts/tests/TestSuite_rx_split.py | 262 ++++++++++++++++++
lib/ethdev/rte_ethdev.c | 24 +-
lib/ethdev/rte_ethdev.h | 14 +-
35 files changed, 880 insertions(+), 274 deletions(-)
create mode 100644 dts/tests/TestSuite_rx_split.py
--
2.54.0
^ permalink raw reply
* [PATCH v5 10/10] dts: add selective Rx tests
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Luca Vizzarro, Patrick Robb
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
Add TestSuite_rx_split with 7 test cases:
- 3 positive: headers only, payload only, two non-contiguous segments
- 4 negative: missing offload flag, out-of-range, overlap, all-discard
Add selective Rx capability detection via testpmd "show port info".
The test suite could be completed later for the basic buffer split
configuration based on offsets or protocols.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/api/capabilities.py | 2 +
dts/api/testpmd/__init__.py | 17 ++
dts/api/testpmd/types.py | 6 +
dts/framework/testbed_model/capability.py | 2 +
dts/tests/TestSuite_rx_split.py | 262 ++++++++++++++++++++++
5 files changed, 289 insertions(+)
create mode 100644 dts/tests/TestSuite_rx_split.py
diff --git a/dts/api/capabilities.py b/dts/api/capabilities.py
index 09bc538523..b0c1d81d36 100644
--- a/dts/api/capabilities.py
+++ b/dts/api/capabilities.py
@@ -136,6 +136,8 @@ class NicCapability(IntEnum):
#: Device supports all VLAN capabilities.
PORT_RX_OFFLOAD_VLAN = auto()
QUEUE_RX_OFFLOAD_VLAN = auto()
+ #: Device supports selective Rx.
+ SELECTIVE_RX = auto()
#: Device supports Rx queue setup after device started.
RUNTIME_RX_QUEUE_SETUP = auto()
#: Device supports Tx queue setup after device started.
diff --git a/dts/api/testpmd/__init__.py b/dts/api/testpmd/__init__.py
index e9187440bb..6973a64573 100644
--- a/dts/api/testpmd/__init__.py
+++ b/dts/api/testpmd/__init__.py
@@ -1409,6 +1409,23 @@ def get_capabilities_show_port_info(
self.ports[0].device_capabilities,
)
+ def get_capabilities_selective_rx(
+ self,
+ supported_capabilities: MutableSet["NicCapability"],
+ unsupported_capabilities: MutableSet["NicCapability"],
+ ) -> None:
+ """Get selective Rx capability from show port info.
+
+ Args:
+ supported_capabilities: Supported capabilities will be added to this set.
+ unsupported_capabilities: Unsupported capabilities will be added to this set.
+ """
+ port_info = self.show_port_info(self.ports[0].id)
+ if port_info.selective_rx:
+ supported_capabilities.add(NicCapability.SELECTIVE_RX)
+ else:
+ unsupported_capabilities.add(NicCapability.SELECTIVE_RX)
+
def get_capabilities_mcast_filtering(
self,
supported_capabilities: MutableSet["NicCapability"],
diff --git a/dts/api/testpmd/types.py b/dts/api/testpmd/types.py
index 0d322aece2..6f1eaf47cc 100644
--- a/dts/api/testpmd/types.py
+++ b/dts/api/testpmd/types.py
@@ -614,6 +614,12 @@ def _validate(info: str) -> str | None:
metadata=VLANOffloadFlag.make_parser(),
)
+ #: Selective Rx support
+ selective_rx: bool = field(
+ default=False,
+ metadata=TextParser.find(r"Selective Rx: supported"),
+ )
+
#: Maximum size of RX buffer
max_rx_bufsize: int | None = field(
default=None, metadata=TextParser.find_int(r"Maximum size of RX buffer: (\d+)")
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 96e1cd449f..b10799ea4b 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -324,6 +324,8 @@ def mapping(cap: NicCapability) -> TestPmdNicCapability:
| NicCapability.FLOW_SHARED_OBJECT_KEEP
):
return (TestPmd.get_capabilities_show_port_info, None)
+ case NicCapability.SELECTIVE_RX:
+ return (TestPmd.get_capabilities_selective_rx, None)
case NicCapability.MCAST_FILTERING:
return (TestPmd.get_capabilities_mcast_filtering, None)
case NicCapability.FLOW_CTRL:
diff --git a/dts/tests/TestSuite_rx_split.py b/dts/tests/TestSuite_rx_split.py
new file mode 100644
index 0000000000..e12fe1a828
--- /dev/null
+++ b/dts/tests/TestSuite_rx_split.py
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 NVIDIA Corporation & Affiliates
+
+"""Rx split test suite.
+
+Test configuring a packet split on Rx,
+and discarding some segments (selective Rx) at NIC level.
+"""
+
+from typing import Any
+
+from scapy.layers.inet import IP
+from scapy.layers.l2 import Ether
+from scapy.packet import Packet, Raw
+
+from api.capabilities import (
+ NicCapability,
+ requires_nic_capability,
+)
+from api.packet import send_packet_and_capture
+from api.test import fail, verify
+from api.testpmd import TestPmd
+from api.testpmd.config import SimpleForwardingModes
+from api.testpmd.types import RxOffloadCapability, TxOffloadCapability
+from framework.exception import InteractiveCommandExecutionError
+from framework.test_suite import TestSuite, func_test
+
+PAYLOAD = bytes(range(256))
+ETHER_HDR_LEN = len(Ether())
+IP_HDR_LEN = len(IP())
+ETHER_IP_HDR_LEN = ETHER_HDR_LEN + IP_HDR_LEN
+
+
+@requires_nic_capability(NicCapability.PORT_RX_OFFLOAD_BUFFER_SPLIT)
+@requires_nic_capability(NicCapability.SELECTIVE_RX)
+class TestRxSplit(TestSuite):
+ """Rx split test suite.
+
+ Configure testpmd with various Rx segment offset/length combinations
+ and verify that only the requested portions of the packet are received
+ and forwarded.
+ """
+
+ def _create_testpmd(self, **kwargs: Any) -> TestPmd:
+ """Create a TestPmd instance with defaults overridden by kwargs."""
+ defaults: dict[str, Any] = {
+ "forward_mode": SimpleForwardingModes.mac,
+ "rx_offloads": RxOffloadCapability.BUFFER_SPLIT,
+ "enable_scatter": True,
+ }
+ return TestPmd(**{**defaults, **kwargs})
+
+ def _build_packet(self) -> Packet:
+ """Build a test packet with an incrementing byte pattern payload."""
+ return Ether() / IP() / Raw(load=PAYLOAD)
+
+ def _send_and_verify(
+ self,
+ testpmd: TestPmd,
+ packet: Packet,
+ expected_bytes: bytes,
+ ) -> None:
+ """Clear stats, send a packet, and verify received content and stats.
+
+ Args:
+ testpmd: The running testpmd instance.
+ packet: The packet to send.
+ expected_bytes: Expected raw bytes of the received packet.
+ """
+ expected_len = len(expected_bytes)
+ testpmd.clear_port_stats_all(verify=False)
+
+ received = send_packet_and_capture(packet)
+ verify(
+ len(received) > 0,
+ "Did not receive any packets.",
+ )
+
+ recv_bytes = bytes(received[0])
+ verify(
+ len(recv_bytes) == expected_len,
+ f"Expected packet length {expected_len}, got {len(recv_bytes)}.",
+ )
+ verify(
+ recv_bytes == expected_bytes,
+ "Received packet content does not match expected bytes.",
+ )
+
+ all_stats, _ = testpmd.show_port_stats_all()
+ total_rx_packets = sum(s.rx_packets for s in all_stats)
+ total_rx_bytes = sum(s.rx_bytes for s in all_stats)
+ verify(
+ total_rx_packets == 1,
+ f"Expected 1 Rx packet, got {total_rx_packets}.",
+ )
+ verify(
+ total_rx_bytes == expected_len,
+ f"Expected {expected_len} Rx bytes, got {total_rx_bytes}.",
+ )
+
+ @func_test
+ def selective_rx_headers(self) -> None:
+ """Keep only the Ethernet + IP headers, discard the payload.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts and buffer split enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet has Ether + IP headers only.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[0],
+ rx_segments_length=[ETHER_IP_HDR_LEN],
+ ) as testpmd:
+ testpmd.start()
+ packet = self._build_packet()
+ expected = bytes(packet)[:ETHER_IP_HDR_LEN]
+ self._send_and_verify(testpmd, packet, expected)
+
+ @func_test
+ def selective_rx_payload_only(self) -> None:
+ """Skip the Ethernet + IP headers, keep only the payload.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts and buffer split enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet is matching the original payload.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[ETHER_IP_HDR_LEN],
+ rx_segments_length=[len(PAYLOAD)],
+ ) as testpmd:
+ testpmd.start()
+ self._send_and_verify(testpmd, self._build_packet(), PAYLOAD)
+
+ @func_test
+ def selective_rx_two_segments(self) -> None:
+ """Keep the IP header and the middle of the payload, skip the rest.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts, buffer split
+ and multi-segment Tx enabled.
+ Send an Ether/IP/payload packet.
+
+ Verify:
+ Received packet is matching the IP header and middle of payload.
+ Port stats show expected rx_packets and rx_bytes.
+ """
+ payload_offset = 100
+ payload_length = 100
+ with self._create_testpmd(
+ tx_offloads=TxOffloadCapability.MULTI_SEGS,
+ rx_segments_offsets=[ETHER_HDR_LEN, ETHER_IP_HDR_LEN + payload_offset],
+ rx_segments_length=[IP_HDR_LEN, payload_length],
+ ) as testpmd:
+ testpmd.start()
+ packet = self._build_packet()
+ raw = bytes(packet)
+ payload_start = ETHER_IP_HDR_LEN + payload_offset
+ expected = (
+ raw[ETHER_HDR_LEN:ETHER_IP_HDR_LEN]
+ + raw[payload_start : payload_start + payload_length]
+ )
+ self._send_and_verify(testpmd, packet, expected)
+
+ @func_test
+ def selective_rx_no_offload(self) -> None:
+ """Configure selective Rx with buffer split disabled.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts, buffer split
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_offloads=None,
+ rx_segments_offsets=[0],
+ rx_segments_length=[ETHER_IP_HDR_LEN],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with buffer split disabled.")
+ except InteractiveCommandExecutionError:
+ pass
+
+ @func_test
+ def selective_rx_offset_out_of_range(self) -> None:
+ """Configure selective Rx with an offset beyond max_rx_pktlen.
+
+ Steps:
+ Start testpmd with rxoffs too big, buffer split enabled,
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[20000],
+ rx_segments_length=[100],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with out-of-range offset.")
+ except InteractiveCommandExecutionError:
+ pass
+
+ @func_test
+ def selective_rx_overlap(self) -> None:
+ """Configure selective Rx with overlapping segments.
+
+ Steps:
+ Start testpmd with overlapping rxoffs/rxpkts, buffer split enabled,
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[0, 10],
+ rx_segments_length=[64, 64],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with overlapping segments.")
+ except InteractiveCommandExecutionError:
+ pass
+
+ @func_test
+ def selective_rx_all_discard(self) -> None:
+ """Configure selective Rx with only discard segment.
+
+ Steps:
+ Start testpmd with rxoffs/rxpkts=0 (null segment), buffer split enabled,
+ and device start disabled.
+ Attempt to start ports.
+
+ Verify:
+ Queue configuration fails.
+ """
+ with self._create_testpmd(
+ rx_segments_offsets=[0],
+ rx_segments_length=[0],
+ disable_device_start=True,
+ ) as testpmd:
+ try:
+ testpmd.start_all_ports()
+ fail("Expected configuration to fail with only discard segment.")
+ except InteractiveCommandExecutionError:
+ pass
--
2.54.0
^ permalink raw reply related
* [PATCH v5 09/10] dts: fix topology capability comparison
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Luca Vizzarro, Patrick Robb, Dean Marx,
Jeremy Spewock, Juraj Linkeš
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
TopologyCapability.__gt__() was delegating to __lt__(),
which caused infinite recursion when "other" is not a TopologyCapability:
other.__lt__(self) returns NotImplemented,
Python retries with self.__gt__(other),
and the cycle repeats.
dts/framework/testbed_model/capability.py", line 579, in __gt__
return other < self
^^^^^^^^^^^^
RecursionError: maximum recursion depth exceeded
Similarly, __le__() was delegating to "not __gt__()",
which returns True for non-comparable types instead of False.
Fix both by checking is_comparable_with() first
and comparing topology_type directly, consistent with __lt__().
Fixes: 039256daa8bf ("dts: add topology capability")
Cc: stable@dpdk.org
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
dts/framework/testbed_model/capability.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 960370fc72..96e1cd449f 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -574,7 +574,9 @@ def __gt__(self, other: Any) -> bool:
Returns:
:data:`True` if the instance's topology type is more complex than the compared object's.
"""
- return other < self
+ if not self.is_comparable_with(other):
+ return False
+ return self.topology_type > other.topology_type
def __le__(self, other: Any) -> bool:
"""Compare the :attr:`~TopologyCapability.topology_type`s.
@@ -586,7 +588,9 @@ def __le__(self, other: Any) -> bool:
:data:`True` if the instance's topology type is less complex or equal than
the compared object's.
"""
- return not self > other
+ if not self.is_comparable_with(other):
+ return False
+ return self.topology_type <= other.topology_type
def __hash__(self):
"""Each instance is identified by :attr:`topology_type`."""
--
2.54.0
^ permalink raw reply related
* [PATCH v5 08/10] common/mlx5: remove callbacks for MR registration
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Dariusz Sosnowski, Viacheslav Ovsiienko,
Bing Zhao, Ori Kam, Suanming Mou, Matan Azrad, Fan Zhang,
Ashish Gupta
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
The functions register/unregister for a Memory Region (MR)
were not called directly.
There are only 2 implementations for Linux and Windows,
no need of handling this difference with function pointers.
The callback pointers are replaced with direct calls
and link time decision based on the Operating System.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/common/mlx5/linux/mlx5_common_verbs.c | 26 +++----------
drivers/common/mlx5/mlx5_common.c | 6 +--
drivers/common/mlx5/mlx5_common_mr.c | 37 ++++++++-----------
drivers/common/mlx5/mlx5_common_mr.h | 26 +++----------
drivers/common/mlx5/windows/mlx5_common_os.c | 23 ++----------
drivers/compress/mlx5/mlx5_compress.c | 4 +-
drivers/crypto/mlx5/mlx5_crypto.h | 2 -
drivers/crypto/mlx5/mlx5_crypto_gcm.c | 6 +--
drivers/net/mlx5/mlx5.h | 3 +-
drivers/net/mlx5/mlx5_flow_aso.c | 21 +++++------
drivers/net/mlx5/mlx5_flow_hw.c | 11 ++----
drivers/net/mlx5/mlx5_flow_quota.c | 6 +--
drivers/net/mlx5/mlx5_hws_cnt.c | 19 ++++------
13 files changed, 61 insertions(+), 129 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index 6d44e1f566..5e23c5844d 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -106,10 +106,10 @@ mlx5_set_context_attr(struct rte_device *dev, struct ibv_context *ctx)
* @return
* 0 on successful registration, -1 otherwise
*/
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_common_verbs_reg_mr)
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_reg_mr)
int
-mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr)
+mlx5_os_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr)
{
struct ibv_mr *ibv_mr;
@@ -136,9 +136,9 @@ mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
* pmd_mr struct set with lkey, address, length and pointer to mr object
*
*/
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_common_verbs_dereg_mr)
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_dereg_mr)
void
-mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
+mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
{
if (pmd_mr && pmd_mr->obj != NULL) {
claim_zero(mlx5_glue->dereg_mr(pmd_mr->obj));
@@ -146,22 +146,6 @@ mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
}
}
-/**
- * Set the reg_mr and dereg_mr callbacks.
- *
- * @param[out] reg_mr_cb
- * Pointer to reg_mr func
- * @param[out] dereg_mr_cb
- * Pointer to dereg_mr func
- */
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_set_reg_mr_cb)
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
-{
- *reg_mr_cb = mlx5_common_verbs_reg_mr;
- *dereg_mr_cb = mlx5_common_verbs_dereg_mr;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
struct mlx5_pmd_mr *
mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index f71dbe4637..87de6d0ff0 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -1135,7 +1135,7 @@ mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
return -1;
}
mr = mlx5_create_mr_ext(dev->pd, (uintptr_t)addr, len,
- SOCKET_ID_ANY, dev->mr_scache.reg_mr_cb);
+ SOCKET_ID_ANY);
if (!mr) {
DRV_LOG(WARNING, "Device %s unable to DMA map", rte_dev->name);
rte_errno = EINVAL;
@@ -1165,7 +1165,7 @@ mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
ret = mlx5_mr_expand_cache(&dev->mr_scache, size,
rte_dev->numa_node);
if (ret < 0) {
- mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+ mlx5_mr_free(mr);
rte_errno = ret;
return -1;
}
@@ -1221,7 +1221,7 @@ mlx5_common_dev_dma_unmap(struct rte_device *rte_dev, void *addr,
}
LIST_REMOVE(mr, mr);
DRV_LOG(DEBUG, "MR(%p) is removed from list.", (void *)mr);
- mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+ mlx5_mr_free(mr);
mlx5_mr_rebuild_cache(&dev->mr_scache);
/*
* No explicit wmb is needed after updating dev_gen due to
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index 64ffc7f4ea..aa2d5e88a4 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -492,12 +492,12 @@ mlx5_mr_lookup_cache(struct mlx5_mr_share_cache *share_cache,
* Pointer to MR to free.
*/
void
-mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb)
+mlx5_mr_free(struct mlx5_mr *mr)
{
if (mr == NULL)
return;
DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
- dereg_mr_cb(&mr->pmd_mr);
+ mlx5_os_dereg_mr(&mr->pmd_mr);
rte_bitmap_free(mr->ms_bmp);
mlx5_free(mr);
}
@@ -545,7 +545,7 @@ mlx5_mr_garbage_collect(struct mlx5_mr_share_cache *share_cache)
struct mlx5_mr *mr = mr_next;
mr_next = LIST_NEXT(mr, mr);
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
}
}
@@ -821,7 +821,7 @@ mlx5_mr_create_primary(void *pd,
data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
data.end = data.start + msl->page_sz;
rte_mcfg_mem_read_unlock();
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
goto alloc_resources;
}
MLX5_ASSERT(data.msl == data_re.msl);
@@ -845,7 +845,7 @@ mlx5_mr_create_primary(void *pd,
* Must be unlocked before calling rte_free() because
* mlx5_mr_mem_event_free_cb() can be called inside.
*/
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
return entry->lkey;
}
/*
@@ -912,7 +912,7 @@ mlx5_mr_create_primary(void *pd,
* mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
* through mlx5_alloc_verbs_buf().
*/
- share_cache->reg_mr_cb(pd, (void *)data.start, len, &mr->pmd_mr);
+ mlx5_os_reg_mr(pd, (void *)data.start, len, &mr->pmd_mr);
if (mr->pmd_mr.obj == NULL) {
DRV_LOG(DEBUG, "Fail to create an MR for address (%p)",
(void *)addr);
@@ -948,7 +948,7 @@ mlx5_mr_create_primary(void *pd,
* calling rte_free() because mlx5_mr_mem_event_free_cb() can be called
* inside.
*/
- mlx5_mr_free(mr, share_cache->dereg_mr_cb);
+ mlx5_mr_free(mr);
return UINT32_MAX;
}
@@ -1139,9 +1139,6 @@ mlx5_mr_release_cache(struct mlx5_mr_share_cache *share_cache)
int
mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket)
{
- /* Set the reg_mr and dereg_mr callback functions */
- mlx5_os_set_reg_mr_cb(&share_cache->reg_mr_cb,
- &share_cache->dereg_mr_cb);
rte_rwlock_init(&share_cache->rwlock);
rte_rwlock_init(&share_cache->mprwlock);
/* Initialize B-tree and allocate memory for global MR cache table. */
@@ -1189,8 +1186,7 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
* Pointer to MR structure on success, NULL otherwise.
*/
struct mlx5_mr *
-mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
- mlx5_reg_mr_t reg_mr_cb)
+mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
{
struct mlx5_mr *mr = NULL;
@@ -1199,7 +1195,7 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
RTE_CACHE_LINE_SIZE, socket_id);
if (mr == NULL)
return NULL;
- reg_mr_cb(pd, (void *)addr, len, &mr->pmd_mr);
+ mlx5_os_reg_mr(pd, (void *)addr, len, &mr->pmd_mr);
if (mr->pmd_mr.obj == NULL) {
DRV_LOG(WARNING,
"Fail to create MR for address (%p)",
@@ -1624,14 +1620,13 @@ mlx5_mempool_reg_create(struct rte_mempool *mp, unsigned int mrs_n,
* Whether @p mpr owns its MRs exclusively, i.e. they are not shared.
*/
static void
-mlx5_mempool_reg_destroy(struct mlx5_mr_share_cache *share_cache,
- struct mlx5_mempool_reg *mpr, bool standalone)
+mlx5_mempool_reg_destroy(struct mlx5_mempool_reg *mpr, bool standalone)
{
if (standalone) {
unsigned int i;
for (i = 0; i < mpr->mrs_n; i++)
- share_cache->dereg_mr_cb(&mpr->mrs[i].pmd_mr);
+ mlx5_os_dereg_mr(&mpr->mrs[i].pmd_mr);
mlx5_free(mpr->mrs);
}
mlx5_free(mpr);
@@ -1748,7 +1743,7 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
const struct mlx5_range *range = &ranges[i];
size_t len = range->end - range->start;
- if (share_cache->reg_mr_cb(pd, (void *)range->start, len,
+ if (mlx5_os_reg_mr(pd, (void *)range->start, len,
&mr->pmd_mr) < 0) {
DRV_LOG(ERR,
"Failed to create an MR in PD %p for address range "
@@ -1763,7 +1758,7 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
mp->name);
}
if (i != ranges_n) {
- mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
+ mlx5_mempool_reg_destroy(new_mpr, true);
rte_errno = EINVAL;
goto exit;
}
@@ -1785,13 +1780,13 @@ mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
if (mpr != NULL) {
DRV_LOG(DEBUG, "Mempool %s is already registered for PD %p",
mp->name, pd);
- mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
+ mlx5_mempool_reg_destroy(new_mpr, true);
rte_errno = EEXIST;
goto exit;
} else if (old_mpr != NULL) {
DRV_LOG(DEBUG, "Mempool %s registration for PD %p updated for external memory",
mp->name, pd);
- mlx5_mempool_reg_destroy(share_cache, old_mpr, standalone);
+ mlx5_mempool_reg_destroy(old_mpr, standalone);
}
exit:
free(ranges);
@@ -1860,7 +1855,7 @@ mlx5_mr_mempool_unregister_primary(struct mlx5_mr_share_cache *share_cache,
rte_errno = ENOENT;
return -1;
}
- mlx5_mempool_reg_destroy(share_cache, mpr, standalone);
+ mlx5_mempool_reg_destroy(mpr, standalone);
return 0;
}
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index 00f3d832c3..5fb931a1b5 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -32,13 +32,6 @@ struct mlx5_pmd_mr {
struct mlx5_devx_obj *mkey; /* devx mkey object. */
};
-/**
- * mr operations typedef
- */
-typedef int (*mlx5_reg_mr_t)(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr);
-typedef void (*mlx5_dereg_mr_t)(struct mlx5_pmd_mr *pmd_mr);
-
/* Memory Region object. */
struct mlx5_mr {
LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
@@ -88,8 +81,6 @@ struct __rte_packed_begin mlx5_mr_share_cache {
struct mlx5_mr_list mr_list; /* Registered MR list. */
struct mlx5_mr_list mr_free_list; /* Freed MR list. */
struct mlx5_mempool_reg_list mempool_reg_list; /* Mempool database. */
- mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
- mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
} __rte_packed_end;
/* Multi-Packet RQ buffer header. */
@@ -233,9 +224,8 @@ struct mlx5_mr *
mlx5_mr_lookup_list(struct mlx5_mr_share_cache *share_cache,
struct mr_cache_entry *entry, uintptr_t addr);
struct mlx5_mr *
-mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
- mlx5_reg_mr_t reg_mr_cb);
-void mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
+mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id);
+void mlx5_mr_free(struct mlx5_mr *mr);
__rte_internal
uint32_t
mlx5_mr_create(struct mlx5_common_device *cdev,
@@ -246,19 +236,13 @@ __rte_internal
uint32_t
mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr);
-/* mlx5_common_verbs.c */
-
__rte_internal
int
-mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
- struct mlx5_pmd_mr *pmd_mr);
+mlx5_os_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr);
__rte_internal
void
-mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
-
-__rte_internal
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
__rte_internal
struct mlx5_pmd_mr *
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index 692517a9bf..bf1b654da3 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -377,7 +377,8 @@ mlx5_os_umem_dereg(void *pumem)
* @return
* 0 on successful registration, -1 otherwise
*/
-static int
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_reg_mr)
+int
mlx5_os_reg_mr(void *pd,
void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr)
{
@@ -425,7 +426,8 @@ mlx5_os_reg_mr(void *pd,
* @param[in] pmd_mr
* Pointer to PMD mr object
*/
-static void
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_dereg_mr)
+void
mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
{
if (!pmd_mr)
@@ -437,23 +439,6 @@ mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
memset(pmd_mr, 0, sizeof(*pmd_mr));
}
-/**
- * Set the reg_mr and dereg_mr callbacks.
- *
- * @param[out] reg_mr_cb
- * Pointer to reg_mr func
- * @param[out] dereg_mr_cb
- * Pointer to dereg_mr func
- *
- */
-RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_set_reg_mr_cb)
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
-{
- *reg_mr_cb = mlx5_os_reg_mr;
- *dereg_mr_cb = mlx5_os_dereg_mr;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
struct mlx5_pmd_mr *
mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index e5325c6150..1361dab630 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -117,7 +117,7 @@ mlx5_compress_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
if (qp->opaque_mr.obj != NULL) {
void *opaq = qp->opaque_mr.addr;
- mlx5_common_verbs_dereg_mr(&qp->opaque_mr);
+ mlx5_os_dereg_mr(&qp->opaque_mr);
rte_free(opaq);
}
mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
@@ -199,7 +199,7 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
qp->priv = priv;
qp->ops = (struct rte_comp_op **)RTE_ALIGN((uintptr_t)(qp + 1),
RTE_CACHE_LINE_SIZE);
- if (mlx5_common_verbs_reg_mr(priv->cdev->pd, opaq_buf, qp->entries_n *
+ if (mlx5_os_reg_mr(priv->cdev->pd, opaq_buf, qp->entries_n *
sizeof(union mlx5_gga_compress_opaque),
&qp->opaque_mr) != 0) {
rte_free(opaq_buf);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f9f127e9e6..93a2bb2c78 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -40,8 +40,6 @@ struct mlx5_crypto_priv {
TAILQ_ENTRY(mlx5_crypto_priv) next;
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
struct rte_cryptodev *crypto_dev;
- mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
- mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
struct mlx5_uar uar; /* User Access Region. */
uint32_t max_segs_num; /* Maximum supported data segs. */
uint32_t max_klm_num; /* Maximum supported klm. */
diff --git a/drivers/crypto/mlx5/mlx5_crypto_gcm.c b/drivers/crypto/mlx5/mlx5_crypto_gcm.c
index 89f32c7722..1a2600655a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto_gcm.c
+++ b/drivers/crypto/mlx5/mlx5_crypto_gcm.c
@@ -219,7 +219,6 @@ mlx5_crypto_gcm_mkey_klm_update(struct mlx5_crypto_priv *priv,
static int
mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
{
- struct mlx5_crypto_priv *priv = dev->data->dev_private;
struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
if (qp->umr_qp_obj.qp != NULL)
@@ -231,7 +230,7 @@ mlx5_crypto_gcm_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
if (qp->mr.obj != NULL) {
void *opaq = qp->mr.addr;
- priv->dereg_mr_cb(&qp->mr);
+ mlx5_os_dereg_mr(&qp->mr);
rte_free(opaq);
}
mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
@@ -363,7 +362,7 @@ mlx5_crypto_gcm_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
rte_errno = ENOMEM;
goto err;
}
- if (priv->reg_mr_cb(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) {
+ if (mlx5_os_reg_mr(priv->cdev->pd, mr_buf, mr_size, &qp->mr) != 0) {
rte_free(mr_buf);
DRV_LOG(ERR, "Failed to register opaque MR.");
rte_errno = ENOMEM;
@@ -1186,7 +1185,6 @@ mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
/* Override AES-GCM specified ops. */
dev_ops->sym_session_configure = mlx5_crypto_sym_gcm_session_configure;
- mlx5_os_set_reg_mr_cb(&priv->reg_mr_cb, &priv->dereg_mr_cb);
dev_ops->queue_pair_setup = mlx5_crypto_gcm_qp_setup;
dev_ops->queue_pair_release = mlx5_crypto_gcm_qp_release;
if (mlx5_crypto_is_ipsec_opt(priv)) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index bd6ef35b53..a4d5392e8f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -2706,8 +2706,7 @@ int mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh,
int mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
struct mlx5_aso_ct_pools_mng *ct_mng,
uint32_t nb_queues);
-int mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_aso_ct_pools_mng *ct_mng);
+int mlx5_aso_ct_queue_uninit(struct mlx5_aso_ct_pools_mng *ct_mng);
int
mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,
void *uar, uint16_t log_desc_n);
diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c
index 5e2a81ef9c..cd84ab1966 100644
--- a/drivers/net/mlx5/mlx5_flow_aso.c
+++ b/drivers/net/mlx5/mlx5_flow_aso.c
@@ -19,17 +19,15 @@
/**
* Free MR resources.
*
- * @param[in] cdev
- * Pointer to the mlx5 common device.
* @param[in] mr
* MR to free.
*/
static void
-mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr)
+mlx5_aso_dereg_mr(struct mlx5_pmd_mr *mr)
{
void *addr = mr->addr;
- cdev->mr_scache.dereg_mr_cb(mr);
+ mlx5_os_dereg_mr(mr);
mlx5_free(addr);
memset(mr, 0, sizeof(*mr));
}
@@ -59,7 +57,7 @@ mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length,
DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
return -1;
}
- ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr);
+ ret = mlx5_os_reg_mr(cdev->pd, mr->addr, length, mr);
if (ret) {
DRV_LOG(ERR, "Failed to create direct Mkey.");
mlx5_free(mr->addr);
@@ -362,7 +360,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq,
sh->tx_uar.obj,
MLX5_ASO_QUEUE_LOG_DESC)) {
- mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
return -1;
}
mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
@@ -399,14 +397,14 @@ mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
switch (aso_opc_mod) {
case ASO_OPC_MOD_FLOW_HIT:
- mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr);
+ mlx5_aso_dereg_mr(&sh->aso_age_mng->aso_sq.mr);
sq = &sh->aso_age_mng->aso_sq;
break;
case ASO_OPC_MOD_POLICER:
mlx5_aso_mtr_queue_uninit(sh, NULL, &sh->mtrmng->pools_mng);
break;
case ASO_OPC_MOD_CONNECTION_TRACKING:
- mlx5_aso_ct_queue_uninit(sh, sh->ct_mng);
+ mlx5_aso_ct_queue_uninit(sh->ct_mng);
break;
default:
DRV_LOG(ERR, "Unknown ASO operation mode");
@@ -1147,15 +1145,14 @@ __mlx5_aso_ct_get_pool(struct mlx5_dev_ctx_shared *sh,
}
int
-mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_aso_ct_pools_mng *ct_mng)
+mlx5_aso_ct_queue_uninit(struct mlx5_aso_ct_pools_mng *ct_mng)
{
uint32_t i;
/* 64B per object for query. */
for (i = 0; i < ct_mng->nb_sq; i++) {
if (ct_mng->aso_sqs[i].mr.addr)
- mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
+ mlx5_aso_dereg_mr(&ct_mng->aso_sqs[i].mr);
mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
}
return 0;
@@ -1197,7 +1194,7 @@ mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
error:
do {
if (ct_mng->aso_sqs[i].mr.addr)
- mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
+ mlx5_aso_dereg_mr(&ct_mng->aso_sqs[i].mr);
mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
} while (i--);
ct_mng->nb_sq = 0;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index b6bb9f12a6..7cc601d681 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -11086,12 +11086,9 @@ flow_hw_create_nic_ctrl_tables(struct rte_eth_dev *dev, struct rte_flow_error *e
}
static void
-flow_hw_ct_mng_destroy(struct rte_eth_dev *dev,
- struct mlx5_aso_ct_pools_mng *ct_mng)
+flow_hw_ct_mng_destroy(struct mlx5_aso_ct_pools_mng *ct_mng)
{
- struct mlx5_priv *priv = dev->data->dev_private;
-
- mlx5_aso_ct_queue_uninit(priv->sh, ct_mng);
+ mlx5_aso_ct_queue_uninit(ct_mng);
mlx5_free(ct_mng);
}
@@ -11230,7 +11227,7 @@ mlx5_flow_ct_init(struct rte_eth_dev *dev,
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ flow_hw_ct_mng_destroy(priv->ct_mng);
priv->ct_mng = NULL;
}
return ret;
@@ -11804,7 +11801,7 @@ __mlx5_flow_hw_resource_release(struct rte_eth_dev *dev, bool ctx_close)
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ flow_hw_ct_mng_destroy(priv->ct_mng);
priv->ct_mng = NULL;
}
mlx5_flow_quota_destroy(dev);
diff --git a/drivers/net/mlx5/mlx5_flow_quota.c b/drivers/net/mlx5/mlx5_flow_quota.c
index d94167d0b0..b661bd376e 100644
--- a/drivers/net/mlx5/mlx5_flow_quota.c
+++ b/drivers/net/mlx5/mlx5_flow_quota.c
@@ -412,12 +412,11 @@ mlx5_quota_alloc_sq(struct mlx5_priv *priv)
static void
mlx5_quota_destroy_read_buf(struct mlx5_priv *priv)
{
- struct mlx5_dev_ctx_shared *sh = priv->sh;
struct mlx5_quota_ctx *qctx = &priv->quota_ctx;
if (qctx->mr.lkey) {
void *addr = qctx->mr.addr;
- sh->cdev->mr_scache.dereg_mr_cb(&qctx->mr);
+ mlx5_os_dereg_mr(&qctx->mr);
mlx5_free(addr);
}
if (qctx->read_buf)
@@ -446,8 +445,7 @@ mlx5_quota_alloc_read_buf(struct mlx5_priv *priv)
DRV_LOG(DEBUG, "QUOTA: failed to allocate MTR ASO READ buffer [1]");
return -ENOMEM;
}
- ret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, buf,
- rd_buf_size, &qctx->mr);
+ ret = mlx5_os_reg_mr(sh->cdev->pd, buf, rd_buf_size, &qctx->mr);
if (ret) {
DRV_LOG(DEBUG, "QUOTA: failed to register MTR ASO READ MR");
return -errno;
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 1b6acb7a3b..d0c4ead71b 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -259,12 +259,11 @@ mlx5_hws_aging_check(struct mlx5_priv *priv, struct mlx5_hws_cnt_pool *cpool)
}
static void
-mlx5_hws_cnt_raw_data_free(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_hws_cnt_raw_data_mng *mng)
+mlx5_hws_cnt_raw_data_free(struct mlx5_hws_cnt_raw_data_mng *mng)
{
if (mng == NULL)
return;
- sh->cdev->mr_scache.dereg_mr_cb(&mng->mr);
+ mlx5_os_dereg_mr(&mng->mr);
mlx5_free(mng->raw);
mlx5_free(mng);
}
@@ -296,8 +295,7 @@ mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n,
NULL, "failed to allocate raw counters memory");
goto error;
}
- ret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, mng->raw, sz,
- &mng->mr);
+ ret = mlx5_os_reg_mr(sh->cdev->pd, mng->raw, sz, &mng->mr);
if (ret) {
rte_flow_error_set(error, errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -306,7 +304,7 @@ mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n,
}
return mng;
error:
- mlx5_hws_cnt_raw_data_free(sh, mng);
+ mlx5_hws_cnt_raw_data_free(mng);
return NULL;
}
@@ -639,8 +637,7 @@ mlx5_hws_cnt_pool_dcs_alloc(struct mlx5_dev_ctx_shared *sh,
}
static void
-mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,
- struct mlx5_hws_cnt_pool *cpool)
+mlx5_hws_cnt_pool_dcs_free(struct mlx5_hws_cnt_pool *cpool)
{
uint32_t idx;
@@ -649,7 +646,7 @@ mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,
for (idx = 0; idx < MLX5_HWS_CNT_DCS_NUM; idx++)
mlx5_devx_cmd_destroy(cpool->dcs_mng.dcs[idx].obj);
if (cpool->raw_mng) {
- mlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);
+ mlx5_hws_cnt_raw_data_free(cpool->raw_mng);
cpool->raw_mng = NULL;
}
}
@@ -842,8 +839,8 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
}
mlx5_hws_cnt_pool_action_destroy(cpool);
if (cpool->cfg.host_cpool == NULL) {
- mlx5_hws_cnt_pool_dcs_free(sh, cpool);
- mlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);
+ mlx5_hws_cnt_pool_dcs_free(cpool);
+ mlx5_hws_cnt_raw_data_free(cpool->raw_mng);
}
mlx5_free((void *)cpool->cfg.name);
mlx5_hws_cnt_pool_deinit(cpool);
--
2.54.0
^ permalink raw reply related
* [PATCH v5 07/10] net/mlx5: reindent previous changes
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Dariusz Sosnowski, Viacheslav Ovsiienko,
Bing Zhao, Ori Kam, Suanming Mou, Matan Azrad
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
Fix indent which was left untouched to help reviews.
This must be squashed before merging.
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/net/mlx5/mlx5_rx.c | 88 ++++++++++++++++-----------------
drivers/net/mlx5/mlx5_rxq.c | 32 ++++++------
drivers/net/mlx5/mlx5_trigger.c | 18 +++----
3 files changed, 68 insertions(+), 70 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index 6aeab8d423..a9a01d7c93 100644
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -1071,32 +1071,32 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
rte_prefetch0(cqe);
rte_prefetch0(wqe);
if (seg->pool) {
- /* Allocate the buf from the same pool. */
- rep = rte_mbuf_raw_alloc(seg->pool);
- if (unlikely(rep == NULL)) {
- ++rxq->stats.rx_nombuf;
- if (!pkt) {
- /*
- * no buffers before we even started,
- * bail out silently.
- */
+ /* Allocate the buf from the same pool. */
+ rep = rte_mbuf_raw_alloc(seg->pool);
+ if (unlikely(rep == NULL)) {
+ ++rxq->stats.rx_nombuf;
+ if (!pkt) {
+ /*
+ * no buffers before we even started,
+ * bail out silently.
+ */
+ break;
+ }
+ while (pkt != seg) {
+ MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
+ rep = NEXT(pkt);
+ NEXT(pkt) = NULL;
+ NB_SEGS(pkt) = 1;
+ rte_mbuf_raw_free(pkt);
+ pkt = rep;
+ }
+ rq_ci >>= sges_n;
+ ++rq_ci;
+ rq_ci <<= sges_n;
break;
}
- while (pkt != seg) {
- MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
- rep = NEXT(pkt);
- NEXT(pkt) = NULL;
- NB_SEGS(pkt) = 1;
- rte_mbuf_raw_free(pkt);
- pkt = rep;
- }
- rq_ci >>= sges_n;
- ++rq_ci;
- rq_ci <<= sges_n;
- break;
}
- }
- if (!pkt) {
+ if (!pkt) { /* new packet */
if (len == 0) { /* no CQE polled yet */
cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
@@ -1104,7 +1104,7 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
/* We drop packets with non-critical errors */
if (seg->pool)
- rte_mbuf_raw_free(rep);
+ rte_mbuf_raw_free(rep);
if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
rq_ci = rxq->rq_ci << sges_n;
break;
@@ -1118,7 +1118,7 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
}
if (len == 0) {
if (seg->pool)
- rte_mbuf_raw_free(rep);
+ rte_mbuf_raw_free(rep);
break;
}
MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
@@ -1143,34 +1143,34 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
}
}
if (seg->pool) { /* real segment: replenish WQE */
- tail = seg;
- DATA_LEN(rep) = DATA_LEN(seg);
- PKT_LEN(rep) = PKT_LEN(seg);
- SET_DATA_OFF(rep, DATA_OFF(seg));
- PORT(rep) = PORT(seg);
- (*rxq->elts)[idx] = rep;
- /*
- * Fill NIC descriptor with the new buffer. The lkey and size
- * of the buffers are already known, only the buffer address
- * changes.
- */
- wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
- /* If there's only one MR, no need to replace LKey in WQE. */
- if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
- wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+ tail = seg;
+ DATA_LEN(rep) = DATA_LEN(seg);
+ PKT_LEN(rep) = PKT_LEN(seg);
+ SET_DATA_OFF(rep, DATA_OFF(seg));
+ PORT(rep) = PORT(seg);
+ (*rxq->elts)[idx] = rep;
+ /*
+ * Fill NIC descriptor with the new buffer. The lkey and size
+ * of the buffers are already known, only the buffer address
+ * changes.
+ */
+ wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+ wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
}
- if (len > DATA_LEN(seg)) {
+ if (len > DATA_LEN(seg)) { /* more data: move to next segment */
if (seg->pool)
data_seg_len += DATA_LEN(seg);
len -= DATA_LEN(seg);
if (pkt)
- ++NB_SEGS(pkt);
+ ++NB_SEGS(pkt);
++rq_ci;
continue;
}
if (seg->pool) { /* last segment */
- DATA_LEN(seg) = len;
- data_seg_len += len;
+ DATA_LEN(seg) = len;
+ data_seg_len += len;
}
if (unlikely(!pkt)) { /* no real segment found, skip packet */
len = 0;
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 3fae189fa4..6ca29f7543 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -152,22 +152,22 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
struct rte_mbuf *buf;
if (seg->mp) {
- buf = rte_pktmbuf_alloc(seg->mp);
- if (buf == NULL) {
- if (rxq_ctrl->share_group == 0)
- DRV_LOG(ERR, "port %u queue %u empty mbuf pool",
- RXQ_PORT_ID(rxq_ctrl),
- rxq_ctrl->rxq.idx);
- else
- DRV_LOG(ERR, "share group %u queue %u empty mbuf pool",
- rxq_ctrl->share_group,
- rxq_ctrl->share_qid);
- rte_errno = ENOMEM;
- goto error;
- }
- /* Only vectored Rx routines rely on headroom size. */
- MLX5_ASSERT(!has_vec_support ||
- DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
+ buf = rte_pktmbuf_alloc(seg->mp);
+ if (buf == NULL) {
+ if (rxq_ctrl->share_group == 0)
+ DRV_LOG(ERR, "port %u queue %u empty mbuf pool",
+ RXQ_PORT_ID(rxq_ctrl),
+ rxq_ctrl->rxq.idx);
+ else
+ DRV_LOG(ERR, "share group %u queue %u empty mbuf pool",
+ rxq_ctrl->share_group,
+ rxq_ctrl->share_qid);
+ rte_errno = ENOMEM;
+ goto error;
+ }
+ /* Only vectored Rx routines rely on headroom size. */
+ MLX5_ASSERT(!has_vec_support ||
+ DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
} else {
buf = seg->null_mbuf;
}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 5b04d9a234..ac966c51b4 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -164,16 +164,14 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
seg = &rxq_ctrl->rxq.rxseg[s];
mp = seg->mp;
if (mp) { /* Regular segment */
- bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
- RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
- ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp,
- is_extmem);
- if (ret < 0 && rte_errno != EEXIST)
- goto error;
- ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
- mp);
- if (ret < 0)
- goto error;
+ bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
+ RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
+ ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp, is_extmem);
+ if (ret < 0 && rte_errno != EEXIST)
+ goto error;
+ ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl, mp);
+ if (ret < 0)
+ goto error;
} else { /* NULL segment used in selective Rx */
seg->null_mbuf = mlx5_alloc_null_mbuf(seg->length);
if (seg->null_mbuf == NULL) {
--
2.54.0
^ permalink raw reply related
* [PATCH v5 06/10] net/mlx5: support selective Rx
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Gregory Etelson, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Selective Rx may save some PCI bandwidth.
Implement selective Rx in the (quite slow) scalar SPRQ Rx path
mlx5_rx_burst() where the performance impact
of the added condition branches is acceptable.
Other Rx functions do not support this feature.
When using selective Rx, mlx5_rx_burst will be selected.
A null Memory Region (MR) is always allocated
at shared device context initialization.
The selective Rx capability is not advertised
if this special MR allocation fails.
For each Rx segment configured with a NULL mempool,
a "null mbuf" is created.
It is a fake mbuf allocated outside any mempool,
used as a placeholder in the Rx ring.
The null MR lkey is used in the WQE for these segments
so the NIC writes received data to a discard buffer.
The mbuf data room size is resolved from the first segment having a pool.
For null segments, the buffer length is from the last seen pool,
so that the WQE stride size remains consistent.
In mlx5_rx_burst, discarded segments are not chained
into the packet mbuf list, NB_SEGS is decremented accordingly,
and no replacement buffer is allocated.
A separate data_seg_len accumulator tracks the total length
of delivered segments only.
The packet length is adjusted to reflect only the data
actually delivered to the application.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
doc/guides/nics/features/mlx5.ini | 1 +
doc/guides/nics/mlx5.rst | 86 +++++++++++++++++++-------
doc/guides/rel_notes/release_26_07.rst | 4 ++
drivers/net/mlx5/mlx5.c | 7 +++
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_ethdev.c | 25 ++++++++
drivers/net/mlx5/mlx5_rx.c | 48 +++++++++++---
drivers/net/mlx5/mlx5_rx.h | 1 +
drivers/net/mlx5/mlx5_rxq.c | 45 ++++++++++++--
drivers/net/mlx5/mlx5_trigger.c | 52 +++++++++++++---
10 files changed, 229 insertions(+), 41 deletions(-)
diff --git a/doc/guides/nics/features/mlx5.ini b/doc/guides/nics/features/mlx5.ini
index 3b3eda28b8..ae8c83057b 100644
--- a/doc/guides/nics/features/mlx5.ini
+++ b/doc/guides/nics/features/mlx5.ini
@@ -16,6 +16,7 @@ Burst mode info = Y
Power mgmt address monitor = Y
MTU update = Y
Buffer split on Rx = Y
+Selective Rx = Y
Scattered Rx = Y
LRO = Y
TSO = Y
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 00bfb31370..afbf040e66 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -84,6 +84,9 @@ The Rx / Tx data path use different techniques to offer the best performance.
with :ref:`multi-packet Rx queues (MPRQ) <mlx5_mprq_params>`.
This feature is disabled by default.
+- Some PCI bandwidth is saved by receiving partial packets
+ with :ref:`selective Rx <mlx5_selective_rx>`.
+
More details about Rx implementations and their configurations are provided
in the chapter about :ref:`mlx5_rx_functions`.
@@ -879,6 +882,8 @@ MLX5 supports various methods to report statistics:
Basic port statistics can be queried using ``rte_eth_stats_get()``.
The received and sent statistics are through SW only
and counts the number of packets received or sent successfully by the PMD.
+In the case of :ref:`selective Rx <mlx5_selective_rx>`,
+the ``ibytes`` counter matches segments delivered, not the skipped ones.
The ``imissed`` counter is the amount of packets that could not be delivered
to SW because a queue was full.
Packets not received due to congestion in the bus or on the NIC
@@ -992,25 +997,26 @@ These configurations may also have an impact on the behavior:
.. table:: Rx burst functions
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- || Function Name || Parameters to Enable || Scatter|| Error Recovery || CQE || Large|| Shared |
- | | | | || comp|| MTU | RxQ |
- +===================+========================+=========+=================+======+=======+=========+
- | rx_burst | rx_vec_en=0 | Yes | Yes | Yes | Yes | No |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_vec | rx_vec_en=1 (default) | No | if CQE comp off | Yes | No | No |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_mprq || mprq_en=1 | No | Yes | Yes | Yes | No |
- | || RxQs >= rxqs_min_mprq | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst_mprq_vec || rx_vec_en=1 (default) | No | if CQE comp off | Yes | Yes | No |
- | || mprq_en=1 | | | | | |
- | || RxQs >= rxqs_min_mprq | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
- | rx_burst | at least one Rx queue | Yes | Yes | Yes | Yes | Yes |
- | (out of order) | on the device | | | | | |
- | | is shared | | | | | |
- +-------------------+------------------------+---------+-----------------+------+-------+---------+
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ || Function|| Parameters to Enable || Scatter|| Selec-|| Error || CQE || Large|| Shared|
+ || Name | | || tive || Recovery|| comp|| MTU || RxQ |
+ +==========+=======================+=========+========+==========+======+=======+========+
+ | rx_burst | rx_vec_en=0 | Yes | Yes | Yes | Yes | Yes | No |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _vec | rx_vec_en=1 (default) | No | No || if CQE | Yes | No | No |
+ | | | | || comp off| | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _mprq || mprq_en=1 | No | No | Yes | Yes | Yes | No |
+ | || RxQs >= rxqs_min_mprq| | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ | _mprq_vec|| rx_vec_en=1 (default)| No | No || if CQE | Yes | Yes | No |
+ | || mprq_en=1 | | || comp off| | | |
+ | || RxQs >= rxqs_min_mprq| | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
+ || _out_of || at least one Rx queue| Yes | No | Yes | Yes | Yes | Yes |
+ || _order || on the device | | | | | | |
+ | || is shared | | | | | | |
+ +----------+-----------------------+---------+--------+----------+------+-------+--------+
Rx/Tx Tuning
@@ -1105,13 +1111,14 @@ Rx interrupt X
:ref:`Rx threshold <mlx5_rx_threshold>` X X
:ref:`Rx drop delay <mlx5_drop>` X X
:ref:`Rx timestamp <mlx5_rx_timstp>` X X
+:ref:`buffer split <mlx5_buf_split>` X X
+:ref:`selective Rx <mlx5_selective_rx>` X
+:ref:`multi-segment <mlx5_multiseg>` X X
:ref:`Tx scheduling <mlx5_tx_sched>` X
:ref:`Tx rate limit <mlx5_rate_limit>` X
:ref:`Tx inline <mlx5_tx_inline>` X X
:ref:`Tx fast free <mlx5_tx_fast_free>` X X
:ref:`Tx affinity <mlx5_aggregated>` X
-:ref:`buffer split <mlx5_buf_split>` X X
-:ref:`multi-segment <mlx5_multiseg>` X X
promiscuous X X
multicast promiscuous X X
multiple MAC addresses X
@@ -2248,13 +2255,50 @@ OFED 5.1-2
DPDK 20.11
========= ==========
+Runtime configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+The offload flag ``RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT`` is required.
+
+When calling ``rte_eth_rx_queue_setup()``,
+the input ``rte_eth_rxconf::rx_seg`` defines the configuration of the segments,
+mainly offset and length.
+
Limitations
^^^^^^^^^^^
+#. Splitting per protocol header is not supported.
+
#. Buffer split offload is supported with regular Rx burst routine only,
no MPRQ feature or vectorized code can be engaged.
+.. _mlx5_selective_rx:
+
+Selective Rx
+~~~~~~~~~~~~
+
+Some PCI bandwidth can be saved
+by :ref:`skipping some parts of Rx data <nic_features_selective_rx>`.
+It is enabled when using :ref:`buffer split <mlx5_buf_split>`
+and configuring no mempool in some segments to discard.
+
+Runtime configuration
+^^^^^^^^^^^^^^^^^^^^^
+
+The offload flag ``RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT`` is required.
+
+When calling ``rte_eth_rx_queue_setup()``,
+the segment to discard (``rte_eth_rxconf::rx_seg::split``)
+is marked by the absence of mempool (``mp = NULL``).
+
+Limitations
+^^^^^^^^^^^
+
+#. Selective Rx is supported with regular Rx burst routine only,
+ no MPRQ feature or vectorized code can be engaged.
+
+
.. _mlx5_multiseg:
Multi-Segment Scatter/Gather
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index 90a7948c1d..3532792229 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -97,6 +97,10 @@ New Features
* Added support for transmitting LLDP packets based on mbuf packet type.
* Implemented AVX2 context descriptor transmit paths.
+* **Updated NVIDIA mlx5 ethernet driver.**
+
+ * Added support for selective Rx in scalar SPRQ Rx path.
+
* **Updated PCAP ethernet driver.**
* Added support for VLAN insertion and stripping.
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f190654756..61c26d1206 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1975,6 +1975,9 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
/* Init counter pool list header and lock. */
LIST_INIT(&sh->hws_cpool_list);
rte_spinlock_init(&sh->cpool_lock);
+ sh->null_mr = mlx5_os_alloc_null_mr(sh->cdev->dev, sh->cdev->pd);
+ if (!sh->null_mr)
+ DRV_LOG(DEBUG, "Fail to initialize NULL MR, selective Rx is disabled.");
exit:
pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
return sh;
@@ -2139,6 +2142,10 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL);
pthread_mutex_destroy(&sh->txpp.mutex);
mlx5_lwm_unset(sh);
+ if (sh->null_mr) {
+ mlx5_os_free_null_mr(sh->null_mr);
+ sh->null_mr = NULL;
+ }
mlx5_physical_device_destroy(sh->phdev);
mlx5_free(sh);
return;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 92a00cfaa8..bd6ef35b53 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1674,6 +1674,7 @@ struct mlx5_dev_ctx_shared {
rte_spinlock_t cpool_lock;
LIST_HEAD(hws_cpool_list, mlx5_hws_cnt_pool) hws_cpool_list; /* Count pool list. */
struct mlx5_dev_registers registers;
+ struct mlx5_pmd_mr *null_mr;
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index a29cdeeb50..7b7536fa1e 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -381,6 +381,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
info->rx_seg_capa.multi_pools = !priv->config.mprq.enabled;
info->rx_seg_capa.offset_allowed = !priv->config.mprq.enabled;
info->rx_seg_capa.offset_align_log2 = 0;
+ info->rx_seg_capa.selective_rx = !!priv->sh->null_mr;
info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
info->rx_queue_offload_capa);
info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
@@ -708,6 +709,25 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
return -rte_errno;
}
+static bool
+mlx5_selective_rx_enabled(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ for (uint32_t q = 0; q < priv->rxqs_n; ++q) {
+ struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, q);
+
+ if (rxq_ctrl == NULL || rxq_ctrl->is_hairpin)
+ continue;
+ for (uint16_t s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
+ if (rxq_ctrl->rxq.rxseg[s].mp == NULL)
+ return true;
+ }
+ }
+
+ return false;
+}
+
/**
* Configure the RX function to use.
*
@@ -723,6 +743,11 @@ mlx5_select_rx_function(struct rte_eth_dev *dev)
eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
MLX5_ASSERT(dev != NULL);
+ if (mlx5_selective_rx_enabled(dev)) {
+ DRV_LOG(DEBUG, "port %u forced to scalar SPRQ Rx (selective Rx configured)",
+ dev->data->port_id);
+ return rx_pkt_burst;
+ }
if (mlx5_shared_rq_enabled(dev)) {
rx_pkt_burst = mlx5_rx_burst_out_of_order;
DRV_LOG(DEBUG, "port %u forced to use SPRQ"
diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index 185bfd4fff..6aeab8d423 100644
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -486,7 +486,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
rxq->wqes)[i];
addr = rte_pktmbuf_mtod(buf, uintptr_t);
byte_count = DATA_LEN(buf);
- lkey = mlx5_rx_mb2mr(rxq, buf);
+ lkey = buf->pool ? mlx5_rx_mb2mr(rxq, buf) : rxq->sh->null_mr->lkey;
}
/* scat->addr must be able to store a pointer. */
MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
@@ -1044,11 +1044,14 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
const unsigned int sges_n = rxq->sges_n;
struct rte_mbuf *pkt = NULL;
struct rte_mbuf *seg = NULL;
+ struct rte_mbuf *tail = NULL;
volatile struct mlx5_cqe *cqe =
&(*rxq->cqes)[rxq->cq_ci & cqe_mask];
+ volatile struct mlx5_mini_cqe8 *mcqe = NULL;
unsigned int i = 0;
unsigned int rq_ci = rxq->rq_ci << sges_n;
int len = 0; /* keep its value across iterations. */
+ uint32_t data_seg_len = 0;
while (pkts_n) {
uint16_t skip_cnt;
@@ -1056,14 +1059,18 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
volatile struct mlx5_wqe_data_seg *wqe =
&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
struct rte_mbuf *rep = (*rxq->elts)[idx];
- volatile struct mlx5_mini_cqe8 *mcqe = NULL;
- if (pkt)
- NEXT(seg) = rep;
+ if (pkt) {
+ if (rep->pool)
+ NEXT(tail) = rep;
+ else
+ --NB_SEGS(pkt);
+ }
seg = rep;
rte_prefetch0(seg);
rte_prefetch0(cqe);
rte_prefetch0(wqe);
+ if (seg->pool) {
/* Allocate the buf from the same pool. */
rep = rte_mbuf_raw_alloc(seg->pool);
if (unlikely(rep == NULL)) {
@@ -1088,12 +1095,15 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
rq_ci <<= sges_n;
break;
}
+ }
if (!pkt) {
+ if (len == 0) { /* no CQE polled yet */
cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask,
&mcqe, &skip_cnt, false, NULL);
if (unlikely(len & MLX5_ERROR_CQE_MASK)) {
/* We drop packets with non-critical errors */
+ if (seg->pool)
rte_mbuf_raw_free(rep);
if (len == MLX5_CRITICAL_ERROR_CQE_RET) {
rq_ci = rxq->rq_ci << sges_n;
@@ -1107,17 +1117,21 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
continue;
}
if (len == 0) {
+ if (seg->pool)
rte_mbuf_raw_free(rep);
break;
}
- pkt = seg;
MLX5_ASSERT(len >= (int)(rxq->crc_present << 2));
+ if (rxq->crc_present)
+ len -= RTE_ETHER_CRC_LEN;
+ }
+ if (seg->pool) { /* first real segment */
+ pkt = seg;
+ tail = seg;
pkt->ol_flags &= RTE_MBUF_F_EXTERNAL;
if (rxq->cqe_comp_layout && mcqe)
cqe = &rxq->title_cqe;
rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
- if (rxq->crc_present)
- len -= RTE_ETHER_CRC_LEN;
PKT_LEN(pkt) = len;
if (cqe->lro_num_seg > 1) {
mlx5_lro_update_hdr
@@ -1127,6 +1141,9 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
pkt->tso_segsz = len / cqe->lro_num_seg;
}
}
+ }
+ if (seg->pool) { /* real segment: replenish WQE */
+ tail = seg;
DATA_LEN(rep) = DATA_LEN(seg);
PKT_LEN(rep) = PKT_LEN(seg);
SET_DATA_OFF(rep, DATA_OFF(seg));
@@ -1141,20 +1158,37 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
/* If there's only one MR, no need to replace LKey in WQE. */
if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+ }
if (len > DATA_LEN(seg)) {
+ if (seg->pool)
+ data_seg_len += DATA_LEN(seg);
len -= DATA_LEN(seg);
+ if (pkt)
++NB_SEGS(pkt);
++rq_ci;
continue;
}
+ if (seg->pool) { /* last segment */
DATA_LEN(seg) = len;
+ data_seg_len += len;
+ }
+ if (unlikely(!pkt)) { /* no real segment found, skip packet */
+ len = 0;
+ rq_ci >>= sges_n;
+ ++rq_ci;
+ rq_ci <<= sges_n;
+ continue;
+ }
+ PKT_LEN(pkt) = RTE_MIN(PKT_LEN(pkt), data_seg_len);
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment bytes counter. */
rxq->stats.ibytes += PKT_LEN(pkt);
#endif
+ data_seg_len = 0;
/* Return packet. */
*(pkts++) = pkt;
pkt = NULL;
+ len = 0;
--pkts_n;
++i;
/* Align consumer index to the next stride. */
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 01b563d981..cd48ee37ef 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -96,6 +96,7 @@ struct mlx5_eth_rxseg {
uint16_t length; /**< Segment data length, configures split point. */
uint16_t offset; /**< Data offset from beginning of mbuf data buffer. */
uint32_t reserved; /**< Reserved field. */
+ struct rte_mbuf *null_mbuf; /**< For selective Rx. */
};
/* RX queue descriptor. */
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 48d982a8c2..3fae189fa4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -151,6 +151,7 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
struct mlx5_eth_rxseg *seg = &rxq_ctrl->rxq.rxseg[i % sges_n];
struct rte_mbuf *buf;
+ if (seg->mp) {
buf = rte_pktmbuf_alloc(seg->mp);
if (buf == NULL) {
if (rxq_ctrl->share_group == 0)
@@ -167,6 +168,9 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
/* Only vectored Rx routines rely on headroom size. */
MLX5_ASSERT(!has_vec_support ||
DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM);
+ } else {
+ buf = seg->null_mbuf;
+ }
/* Buffer is supposed to be empty. */
MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0);
MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0);
@@ -324,10 +328,14 @@ rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
rxq->rq_pi = elts_ci;
}
for (i = 0; i != q_n; ++i) {
- if ((*rxq->elts)[i] != NULL)
+ if ((*rxq->elts)[i] != NULL && (*rxq->elts)[i]->pool != NULL)
rte_pktmbuf_free_seg((*rxq->elts)[i]);
(*rxq->elts)[i] = NULL;
}
+ for (i = 0; i < rxq->rxseg_n; i++) {
+ mlx5_free(rxq->rxseg[i].null_mbuf);
+ rxq->rxseg[i].null_mbuf = NULL;
+ }
}
/**
@@ -1815,7 +1823,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
int ret;
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *tmpl;
- unsigned int mb_len = rte_pktmbuf_data_room_size(rx_seg[0].mp);
+ struct rte_mempool *first_mp = NULL;
+ struct rte_mempool *last_mp = NULL;
+ unsigned int mb_len;
struct mlx5_port_config *config = &priv->config;
uint64_t offloads = conf->offloads |
dev->data->dev_conf.rxmode.offloads;
@@ -1827,7 +1837,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int non_scatter_min_mbuf_size = max_rx_pktlen +
RTE_PKTMBUF_HEADROOM;
unsigned int max_lro_size = 0;
- unsigned int first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
+ unsigned int first_mb_free_size;
uint32_t mprq_log_actual_stride_num = 0;
uint32_t mprq_log_actual_stride_size = 0;
bool rx_seg_en = n_seg != 1 || rx_seg[0].offset || rx_seg[0].length;
@@ -1845,6 +1855,21 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
const struct rte_eth_rxseg_split *qs_seg = rx_seg;
unsigned int tail_len;
+ /* Find first segment with a mempool. */
+ for (uint16_t seg = 0; seg < n_seg; seg++) {
+ if (rx_seg[seg].mp != NULL) {
+ first_mp = rx_seg[seg].mp;
+ break;
+ }
+ }
+ if (first_mp == NULL) {
+ DRV_LOG(ERR, "port %u Rx queue %u has no mempool", dev->data->port_id, idx);
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ mb_len = rte_pktmbuf_data_room_size(first_mp);
+ first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
+
if (mprq_en) {
/* Trim the number of descs needed. */
desc >>= mprq_log_actual_stride_num;
@@ -1884,13 +1909,20 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
do {
struct mlx5_eth_rxseg *hw_seg =
&tmpl->rxq.rxseg[tmpl->rxq.rxseg_n];
- uint32_t buf_len, offset, seg_len;
+ uint32_t buf_len = 0, offset, seg_len;
/*
* For the buffers beyond descriptions offset is zero,
* the first buffer contains head room.
*/
- buf_len = rte_pktmbuf_data_room_size(qs_seg->mp);
+ if (qs_seg->mp != NULL) {
+ last_mp = qs_seg->mp;
+ buf_len = rte_pktmbuf_data_room_size(qs_seg->mp);
+ } else if (last_mp != NULL) {
+ buf_len = rte_pktmbuf_data_room_size(last_mp);
+ } else {
+ buf_len = mb_len;
+ }
offset = (tmpl->rxq.rxseg_n >= n_seg ? 0 : qs_seg->offset) +
(tmpl->rxq.rxseg_n ? 0 : RTE_PKTMBUF_HEADROOM);
/*
@@ -2077,7 +2109,8 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save port ID. */
tmpl->rxq.port_id = dev->data->port_id;
tmpl->sh = priv->sh;
- tmpl->rxq.mp = rx_seg[0].mp;
+ tmpl->rxq.sh = priv->sh;
+ tmpl->rxq.mp = first_mp;
tmpl->rxq.elts_n = log2above(desc);
tmpl->rxq.rq_repl_thresh = MLX5_VPMD_RXQ_RPLNSH_THRESH(desc_n);
tmpl->rxq.elts = (struct rte_mbuf *(*)[])(tmpl + 1);
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index a070aaecfd..5b04d9a234 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -116,6 +116,27 @@ mlx5_txq_start(struct rte_eth_dev *dev)
return -rte_errno;
}
+static struct rte_mbuf *
+mlx5_alloc_null_mbuf(uint32_t data_len)
+{
+ size_t alloc_size = sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM +
+ rte_align32pow2(data_len);
+ struct rte_mbuf *m;
+
+ m = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, 0, SOCKET_ID_ANY);
+ if (m == NULL)
+ return NULL;
+ m->buf_addr = RTE_PTR_ADD(m, sizeof(*m));
+ m->buf_len = alloc_size - sizeof(*m);
+ rte_mbuf_iova_set(m, rte_mem_virt2iova(m->buf_addr));
+ m->data_off = RTE_PKTMBUF_HEADROOM;
+ m->refcnt = 1;
+ m->nb_segs = 1;
+ m->port = RTE_MBUF_PORT_INVALID;
+ m->pool = NULL;
+ return m;
+}
+
/**
* Register Rx queue mempools and fill the Rx queue cache.
* This function tolerates repeated mempool registration.
@@ -130,7 +151,8 @@ static int
mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
{
struct rte_mempool *mp;
- uint32_t s;
+ struct mlx5_eth_rxseg *seg;
+ uint16_t s;
int ret = 0;
mlx5_mr_flush_local_cache(&rxq_ctrl->rxq.mr_ctrl);
@@ -139,21 +161,37 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
return mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
rxq_ctrl->rxq.mprq_mp);
for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
- bool is_extmem;
-
- mp = rxq_ctrl->rxq.rxseg[s].mp;
- is_extmem = (rte_pktmbuf_priv_flags(mp) &
+ seg = &rxq_ctrl->rxq.rxseg[s];
+ mp = seg->mp;
+ if (mp) { /* Regular segment */
+ bool is_extmem = (rte_pktmbuf_priv_flags(mp) &
RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) != 0;
ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp,
is_extmem);
if (ret < 0 && rte_errno != EEXIST)
- return ret;
+ goto error;
ret = mlx5_mr_mempool_populate_cache(&rxq_ctrl->rxq.mr_ctrl,
mp);
if (ret < 0)
- return ret;
+ goto error;
+ } else { /* NULL segment used in selective Rx */
+ seg->null_mbuf = mlx5_alloc_null_mbuf(seg->length);
+ if (seg->null_mbuf == NULL) {
+ rte_errno = ENOMEM;
+ ret = -rte_errno;
+ goto error;
+ }
+ }
}
return 0;
+
+error:
+ while (s-- > 0) {
+ seg = &rxq_ctrl->rxq.rxseg[s];
+ mlx5_free(seg->null_mbuf);
+ seg->null_mbuf = NULL;
+ }
+ return ret;
}
/**
--
2.54.0
^ permalink raw reply related
* [PATCH v5 05/10] net/mlx5: fix Rx split segment counter type
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, stable, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
In the API, rx_nseg and max_nseg are uint16_t.
In mlx5, MLX5_MAX_RXQ_NSEG is 32.
So there is no reason to have rxseg_n as uint32_t.
Reduce the fields to uint16_t and move them to avoid struct holes.
Fixes: 9f209b59c8b0 ("net/mlx5: support Rx buffer split description")
Fixes: 572c9d4bda08 ("net/mlx5: fix shared Rx queue segment configuration match")
Cc: stable@dpdk.org
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/net/mlx5/mlx5_rx.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index dffab3955b..01b563d981 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -164,9 +164,9 @@ struct __rte_cache_aligned mlx5_rxq_data {
uint64_t flow_meta_mask;
int32_t flow_meta_offset;
uint32_t flow_meta_port_mask;
- uint32_t rxseg_n; /* Number of split segment descriptions. */
struct mlx5_eth_rxseg rxseg[MLX5_MAX_RXQ_NSEG];
/* Buffer split segment descriptions - sizes, offsets, pools. */
+ uint16_t rxseg_n; /* Number of split segment descriptions. */
uint16_t rq_win_cnt; /* Number of packets in the sliding window data. */
uint16_t rq_win_idx_mask; /* Sliding window index wrapping mask. */
uint16_t rq_win_idx; /* Index of the first element in sliding window. */
@@ -191,9 +191,9 @@ struct mlx5_rxq_ctrl {
unsigned int irq:1; /* Whether IRQ is enabled. */
uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
uint32_t wqn; /* WQ number. */
- uint32_t rxseg_n; /* Number of split segment descriptions. */
struct rte_eth_rxseg_split rxseg[MLX5_MAX_RXQ_NSEG];
/* Saved original buffer split segment configuration. */
+ uint16_t rxseg_n; /* Number of split segment descriptions. */
uint16_t dump_file_n; /* Number of dump files. */
};
--
2.54.0
^ permalink raw reply related
* [PATCH v5 04/10] common/mlx5: add null MR functions
From: Thomas Monjalon @ 2026-06-02 21:38 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Gregory Etelson, Dariusz Sosnowski,
Viacheslav Ovsiienko, Bing Zhao, Ori Kam, Suanming Mou,
Matan Azrad
In-Reply-To: <20260602214036.3606359-1-thomas@monjalon.net>
From: Gregory Etelson <getelson@nvidia.com>
Add functions to allocate and free a null Memory Region (MR)
using ibverbs on Linux.
There is no implementation for DevX on Windows.
Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
drivers/common/mlx5/linux/mlx5_common_verbs.c | 35 +++++++++++++++++++
drivers/common/mlx5/mlx5_common_mr.h | 9 +++++
drivers/common/mlx5/windows/mlx5_common_os.c | 16 +++++++++
3 files changed, 60 insertions(+)
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index 2322d9d033..6d44e1f566 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -161,3 +161,38 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
*reg_mr_cb = mlx5_common_verbs_reg_mr;
*dereg_mr_cb = mlx5_common_verbs_dereg_mr;
}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
+{
+ struct ibv_mr *ibv_mr;
+ struct mlx5_pmd_mr *null_mr;
+
+ null_mr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*null_mr), 0, dev->numa_node);
+ if (!null_mr)
+ return NULL;
+ ibv_mr = mlx5_glue->alloc_null_mr(pd);
+ if (!ibv_mr) {
+ mlx5_free(null_mr);
+ return NULL;
+ }
+ *null_mr = (struct mlx5_pmd_mr) {
+ .lkey = rte_cpu_to_be_32(ibv_mr->lkey),
+ .addr = ibv_mr->addr,
+ .len = ibv_mr->length,
+ .obj = (void *)ibv_mr,
+ };
+ return null_mr;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_free_null_mr)
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr)
+{
+ if (!null_mr)
+ return;
+ if (null_mr->obj)
+ claim_zero(mlx5_glue->dereg_mr(null_mr->obj));
+ mlx5_free(null_mr);
+}
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index cf7c685e9b..00f3d832c3 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -21,6 +21,8 @@
#include "mlx5_common_mp.h"
#include "mlx5_common_defs.h"
+struct rte_device;
+
/* mlx5 PMD MR struct. */
struct mlx5_pmd_mr {
uint32_t lkey;
@@ -258,6 +260,13 @@ __rte_internal
void
mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+__rte_internal
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd);
+__rte_internal
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr);
+
__rte_internal
int
mlx5_mr_mempool_register(struct mlx5_common_device *cdev,
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index 16fcc5f9fc..692517a9bf 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -454,6 +454,22 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
*dereg_mr_cb = mlx5_os_dereg_mr;
}
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_alloc_null_mr)
+struct mlx5_pmd_mr *
+mlx5_os_alloc_null_mr(struct rte_device *dev, void *pd)
+{
+ RTE_SET_USED(dev);
+ RTE_SET_USED(pd);
+ return NULL;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(mlx5_os_free_null_mr)
+void
+mlx5_os_free_null_mr(struct mlx5_pmd_mr *null_mr)
+{
+ RTE_SET_USED(null_mr);
+}
+
/*
* In Windows, no need to wrap the MR, no known issue for it in kernel.
* Use the regular function to create direct MR.
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox