* [PATCH v2 11/23] drivers: add support for VF representors
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add support for VF representors in sxe2 PMD. This allows the host
application (e.g., OVS-DPDK) to control and monitor virtual functions
through a dedicated ethdev on the PF (Physical Function) side.
Key changes include:
- Added representor enumeration and identification logic.
- Implemented representor-specific dev_ops (link update, stats, etc.).
- Configured back-channel communication between PF and VF for control
messages.
- Supported the "-a <DBDF>,representor=[0-N]" EAL parameter to
instantiate representor ports.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_common.c | 46 +
drivers/common/sxe2/sxe2_common.h | 2 +
drivers/net/sxe2/meson.build | 8 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 311 +++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 28 +
drivers/net/sxe2/sxe2_drv_cmd.h | 55 +-
drivers/net/sxe2/sxe2_ethdev.c | 201 ++-
drivers/net/sxe2/sxe2_ethdev.h | 11 +
drivers/net/sxe2/sxe2_ethdev_repr.c | 607 +++++++
drivers/net/sxe2/sxe2_ethdev_repr.h | 32 +
drivers/net/sxe2/sxe2_filter.c | 121 +-
drivers/net/sxe2/sxe2_filter.h | 2 +
drivers/net/sxe2/sxe2_flow.c | 1337 ++++++++++++++
drivers/net/sxe2/sxe2_flow.h | 29 +
drivers/net/sxe2/sxe2_flow_parse_action.c | 1182 +++++++++++++
drivers/net/sxe2/sxe2_flow_parse_action.h | 23 +
drivers/net/sxe2/sxe2_flow_parse_engine.c | 106 ++
drivers/net/sxe2/sxe2_flow_parse_engine.h | 13 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 1822 ++++++++++++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 40 +
drivers/net/sxe2/sxe2_irq.c | 54 +
drivers/net/sxe2/sxe2_irq.h | 4 +
drivers/net/sxe2/sxe2_queue.c | 6 +-
drivers/net/sxe2/sxe2_stats.c | 17 +-
drivers/net/sxe2/sxe2_switchdev.c | 332 ++++
drivers/net/sxe2/sxe2_switchdev.h | 33 +
drivers/net/sxe2/sxe2_txrx.c | 7 +
drivers/net/sxe2/sxe2_txrx_poll.c | 8 +
drivers/net/sxe2/sxe2_vsi.c | 146 ++
drivers/net/sxe2/sxe2_vsi.h | 12 +-
30 files changed, 6572 insertions(+), 23 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.c
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.h
create mode 100644 drivers/net/sxe2/sxe2_flow.c
create mode 100644 drivers/net/sxe2/sxe2_flow.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.h
create mode 100644 drivers/net/sxe2/sxe2_switchdev.c
create mode 100644 drivers/net/sxe2/sxe2_switchdev.h
diff --git a/drivers/common/sxe2/sxe2_common.c b/drivers/common/sxe2/sxe2_common.c
index f34427c569..a5d36998e1 100644
--- a/drivers/common/sxe2/sxe2_common.c
+++ b/drivers/common/sxe2/sxe2_common.c
@@ -169,6 +169,37 @@ static int32_t sxe2_parse_class_type(const char *key, const char *value, void *a
return ret;
}
+static int32_t sxe2_parse_driver(const char *key, const char *value, void *args)
+{
+ int32_t ret = -EINVAL;
+
+ if (value == NULL || args == NULL) {
+ ret = 0;
+ goto l_end;
+ }
+
+ if (strcmp(value, "sxe2") != 0) {
+ PMD_LOG_ERR(COM, "%s: \"%s\" is not a valid driver.",
+ key, value);
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_parse_representor(const char *key, const char *value, void *args)
+{
+ int32_t ret = 0;
+
+ if (value == NULL || args == NULL)
+ goto l_end;
+
+ PMD_LOG_INFO(COM, "representor arg %s: \"%s\".", key, value);
+
+l_end:
+ return ret;
+}
+
static int32_t sxe2_common_device_setup(struct sxe2_common_device *cdev)
{
struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(cdev->dev);
@@ -394,6 +425,21 @@ static int32_t sxe2_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused
goto l_free_args;
}
+ ret = sxe2_kvargs_process(kv_info_p, SXE2_DEVARGS_KEY_DRIVER,
+ sxe2_parse_driver, NULL);
+ if (ret < 0) {
+ PMD_LOG_ERR(COM, "Unsupported sxe2 driver name: %s",
+ rte_dev->devargs->args);
+ goto l_free_args;
+ }
+
+ ret = sxe2_kvargs_process(kv_info_p, SXE2_DEVARGS_KEY_REPR,
+ sxe2_parse_representor, NULL);
+ if (ret < 0) {
+ PMD_LOG_ERR(COM, "Unsupported sxe2 driver representor: %s",
+ rte_dev->devargs->args);
+ goto l_free_args;
+ }
}
cdev = sxe2_common_device_alloc(rte_dev, class_type);
diff --git a/drivers/common/sxe2/sxe2_common.h b/drivers/common/sxe2/sxe2_common.h
index 5fe218db99..23cffac581 100644
--- a/drivers/common/sxe2/sxe2_common.h
+++ b/drivers/common/sxe2/sxe2_common.h
@@ -18,6 +18,8 @@
((cdev)->config.cmd_fd)
#define SXE2_DEVARGS_KEY_CLASS "class"
+#define SXE2_DEVARGS_KEY_DRIVER "driver"
+#define SXE2_DEVARGS_KEY_REPR "representor"
struct sxe2_class_driver;
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 0658b2ee3a..61925ee80f 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -71,6 +71,12 @@ sources += files(
'sxe2_mp.c',
'sxe2_stats.c',
'sxe2_irq.c',
+ 'sxe2_switchdev.c',
+ 'sxe2_ethdev_repr.c',
+ 'sxe2_flow.c',
+ 'sxe2_flow_parse_action.c',
+ 'sxe2_flow_parse_pattern.c',
+ 'sxe2_flow_parse_engine.c',
)
-allow_internal_get_api = true
\ No newline at end of file
+allow_internal_get_api = true
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index d1f15084ed..6e2dd139a5 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -64,6 +64,23 @@ int32_t sxe2_drv_dev_caps_get(struct sxe2_adapter *adapter, struct sxe2_drv_dev_
return ret;
}
+int32_t sxe2_drv_switchdev_info_get(struct sxe2_adapter *adapter,
+ struct sxe2_switchdev_info *switchdev_info)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_DEV_GET_SWITCHDEV_INFO,
+ NULL, 0, switchdev_info,
+ sizeof(struct sxe2_switchdev_info));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "get switchdev info failed, ret=%d", ret);
+
+ return ret;
+}
+
int32_t sxe2_drv_dev_info_get(struct sxe2_adapter *adapter,
struct sxe2_drv_dev_info_resp *dev_info_resp)
{
@@ -167,7 +184,11 @@ static int32_t sxe2_rxq_ctxt_cfg_fill(struct sxe2_rx_queue *rxq,
req->q_cnt = rxq_cnt;
req->max_frame_size = dev_data->mtu + SXE2_ETH_OVERHEAD;
- ctxt->queue_id = rxq->queue_id;
+ if (adapter->is_dev_repr)
+ ctxt->queue_id = adapter->repr_priv_data->repr_q_id;
+ else
+ ctxt->queue_id = rxq->queue_id;
+
ctxt->depth = rxq->ring_depth;
ctxt->buf_len = RTE_ALIGN(rxq->rx_buf_len, SXE2_RXQ_CTXT_CFG_BUF_LEN_ALIGN);
ctxt->dma_addr = rxq->base_addr;
@@ -241,7 +262,10 @@ static void sxe2_txq_ctxt_cfg_fill(struct sxe2_tx_queue *txq,
ctxt = &req->cfg[q_idx];
ctxt->depth = txq[q_idx].ring_depth;
ctxt->dma_addr = txq[q_idx].base_addr;
- ctxt->queue_id = txq[q_idx].queue_id;
+ if (adapter->is_dev_repr)
+ ctxt->queue_id = adapter->repr_priv_data->repr_q_id;
+ else
+ ctxt->queue_id = txq[q_idx].queue_id;
ctxt->sched_mode = sxe2_sched_mode_get(adapter);
}
@@ -288,7 +312,10 @@ int32_t sxe2_drv_rxq_switch(struct sxe2_adapter *adapter, struct sxe2_rx_queue *
struct sxe2_drv_q_switch_req req;
req.vsi_id = rte_cpu_to_le_16(rxq->vsi->vsi_id);
- req.q_idx = rxq->queue_id;
+ if (adapter->is_dev_repr)
+ req.q_idx = adapter->repr_priv_data->repr_q_id;
+ else
+ req.q_idx = rxq->queue_id;
req.is_enable = (uint8_t)enable;
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RXQ_DISABLE,
@@ -310,7 +337,10 @@ int32_t sxe2_drv_txq_switch(struct sxe2_adapter *adapter, struct sxe2_tx_queue *
struct sxe2_drv_q_switch_req req;
req.vsi_id = rte_cpu_to_le_16(txq->vsi->vsi_id);
- req.q_idx = txq->queue_id;
+ if (adapter->is_dev_repr)
+ req.q_idx = adapter->repr_priv_data->repr_q_id;
+ else
+ req.q_idx = txq->queue_id;
req.is_enable = (uint8_t)enable;
req.sched_mode = sxe2_sched_mode_get(adapter);
@@ -326,6 +356,37 @@ int32_t sxe2_drv_txq_switch(struct sxe2_adapter *adapter, struct sxe2_tx_queue *
return ret;
}
+int32_t sxe2_drv_vsi_info_get(struct sxe2_adapter *adapter, struct sxe2_vsi *vsi)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_vsi_info_get_req vsi_info_get_req = {0};
+ struct sxe2_drv_vsi_info_get_resp vsi_info_get_resp = {0};
+
+ vsi_info_get_req.vsi_id = vsi->vsi_id;
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VSI_INFO_GET,
+ &vsi_info_get_req, sizeof(vsi_info_get_req),
+ &vsi_info_get_resp, sizeof(vsi_info_get_resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "switchdev cpvsi info get failed, ret=%d", ret);
+ goto l_end;
+ }
+
+ vsi->txqs.q_cnt = vsi_info_get_resp.used_queues.queues_cnt;
+ vsi->txqs.base_idx_in_func = vsi_info_get_resp.used_queues.base_idx_in_pf;
+
+ vsi->rxqs.q_cnt = vsi_info_get_resp.used_queues.queues_cnt;
+ vsi->rxqs.base_idx_in_func = vsi_info_get_resp.used_queues.base_idx_in_pf;
+
+ vsi->irqs.avail_cnt = vsi_info_get_resp.used_msix.msix_vectors_cnt;
+ vsi->irqs.base_idx_in_pf = vsi_info_get_resp.used_msix.base_idx_in_func;
+
+l_end:
+ return ret;
+}
+
int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
{
int32_t ret = 0;
@@ -614,6 +675,101 @@ int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set)
return ret;
}
+int32_t sxe2_drv_switchdev_uplink_config(struct sxe2_adapter *adapter, bool set)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_switchdev_uplink_info switchdev_uplink_info_req = {0};
+
+ switchdev_uplink_info_req.pf_id = adapter->pf_idx;
+ switchdev_uplink_info_req.is_set = set;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SWITCH_UPLINK,
+ &switchdev_uplink_info_req,
+ sizeof(switchdev_uplink_info_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "switchdev uplink info config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_switchdev_repr_vf_config(struct sxe2_adapter *adapter,
+ struct sxe2_switchdev_repr_info *repr_vf,
+ bool set)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_switchdev_repr_info switchdev_repr_info_req = {0};
+
+ switchdev_repr_info_req.pf_id = adapter->pf_idx;
+ switchdev_repr_info_req.is_set = set;
+ switchdev_repr_info_req.cp_vsi_id = repr_vf->cp_vsi_id;
+ switchdev_repr_info_req.repr_pf_id = repr_vf->repr_pf_id;
+ switchdev_repr_info_req.repr_vf_id = repr_vf->repr_vf_id;
+ switchdev_repr_info_req.repr_q_id = repr_vf->repr_q_id;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SWITCH_REPR,
+ &switchdev_repr_info_req,
+ sizeof(switchdev_repr_info_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "switchdev repr info config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_switchdev_mode_get(struct sxe2_adapter *adapter, bool *is_switchdev)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_switchdev_mode_info switchdev_mode_info_req = {0};
+ struct sxe2_switchdev_mode_info switchdev_mode_info_resp = {0};
+
+ switchdev_mode_info_req.pf_id = adapter->pf_idx;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SWITCH_MODE,
+ &switchdev_mode_info_req,
+ sizeof(switchdev_mode_info_req),
+ &switchdev_mode_info_resp,
+ sizeof(switchdev_mode_info_resp));
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "switchdev mode info get failed, ret=%d", ret);
+ else
+ *is_switchdev = (bool)switchdev_mode_info_resp.is_switchdev;
+
+ return ret;
+}
+
+int32_t sxe2_drv_switchdev_cpvsi_get(struct sxe2_adapter *adapter, uint16_t *cp_vsi_id)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_switchdev_cpvsi_info switchdev_cpvsi_resp = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SWITCH_CPVSI,
+ NULL, 0,
+ &switchdev_cpvsi_resp,
+ sizeof(switchdev_cpvsi_resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "switchdev cpvsi info get failed, ret=%d", ret);
+ else
+ *cp_vsi_id = switchdev_cpvsi_resp.cp_vsi_id;
+
+ return ret;
+}
+
int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add)
{
int32_t ret = 0;
@@ -1434,3 +1590,150 @@ int32_t sxe2_drv_mapping_stats_info_clear(struct rte_eth_dev *eth_dev)
return ret;
}
+
+int32_t sxe2_drv_flow_filter_add(struct sxe2_adapter *adapter, struct sxe2_flow *flow)
+{
+ struct sxe2_drv_flow_filter_req req = { 0 };
+ struct sxe2_drv_flow_filter_resp resp = { 0 };
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret = -1;
+
+ memcpy(&req.pattern_inner, &flow->pattern_inner, sizeof(req.pattern_inner));
+ memcpy(&req.pattern_outer, &flow->pattern_outer, sizeof(req.pattern_outer));
+ memcpy(&req.action, &flow->action, sizeof(req.action));
+ memcpy(&req.meta, &flow->meta, sizeof(req.meta));
+ req.engine_type = flow->engine_type;
+ req.flow_id = 0;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_FLOW_FILTER_ADD, &req,
+ sizeof(req), &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add flow filter, ret: %d.", ret);
+ flow->flow_id = resp.flow_id;
+ flow->create_err = ret;
+ return ret;
+}
+
+int32_t sxe2_drv_flow_filter_del(struct sxe2_adapter *adapter, struct sxe2_flow *flow)
+{
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_drv_flow_filter_req req = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret = -1;
+ memcpy(&req.pattern_inner, &flow->pattern_inner, sizeof(req.pattern_inner));
+ memcpy(&req.pattern_outer, &flow->pattern_outer, sizeof(req.pattern_outer));
+ memcpy(&req.action, &flow->action, sizeof(req.action));
+ memcpy(&req.meta, &flow->meta, sizeof(req.meta));
+ req.engine_type = flow->engine_type;
+ req.flow_id = flow->flow_id;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_FLOW_FILTER_DEL, &req,
+ sizeof(req), NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to delete flow filter, flow id: %u, ret: %d.",
+ flow->flow_id, ret);
+ return ret;
+}
+
+int32_t sxe2_drv_flow_fnav_get_stat_id(struct sxe2_adapter *adapter, uint32_t *stat_id)
+{
+ struct sxe2_drv_flow_fnav_get_stat_id_req req = { 0 };
+ struct sxe2_drv_flow_fnav_get_stat_id_resp resp = { 0 };
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret = -1;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_FLOW_FNAV_STAT_ALLOC,
+ &req, sizeof(req),
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to get fnav stat id, ret: %d.", ret);
+ goto l_end;
+ }
+ *stat_id = resp.stat_id;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_flow_fnav_free_stat(struct sxe2_adapter *adapter, uint32_t stat_id)
+{
+ struct sxe2_drv_flow_fnav_free_stat_id_req req = { 0 };
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret = -1;
+
+ req.stat_id = stat_id;
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_FLOW_FNAV_STAT_FREE,
+ &req, sizeof(req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to free fnav stat id, ret: %d.", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_flow_fnav_query_stat(struct sxe2_adapter *adapter,
+ struct sxe2_fnav_cid_mgr *mgr)
+{
+ struct sxe2_drv_flow_fnav_query_stat_req req = { 0 };
+ struct sxe2_drv_flow_fnav_query_stat_resp resp = { 0 };
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret = -1;
+
+ req.stat_id = mgr->stat_index;
+ req.stat_ctrl = mgr->count_type;
+ req.is_clear = 1;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_FLOW_FNAV_STAT_QUERY,
+ &req, sizeof(req),
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to query fnav stat, stat id: %u, ret: %d.",
+ req.stat_id, ret);
+ goto l_end;
+ }
+ mgr->hits += resp.stat_hits;
+ mgr->bytes += resp.stat_bytes;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_srcvsi_prune_config(struct sxe2_adapter *adapter,
+ uint16_t *vsi_list, uint16_t vsi_cnt, bool set)
+{
+ int32_t ret = 0;
+ uint16_t idx;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_srcvsi_ext_cfg_req srcvsi_list_prune_cfg_req = {0};
+
+ srcvsi_list_prune_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ srcvsi_list_prune_cfg_req.is_add = set;
+ srcvsi_list_prune_cfg_req.srcvsi_cnt = vsi_cnt;
+ for (idx = 0; idx < vsi_cnt; idx++)
+ srcvsi_list_prune_cfg_req.srcvsi_list[idx] = vsi_list[idx];
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VSI_SRCVSI_PRUNE,
+ &srcvsi_list_prune_cfg_req,
+ sizeof(srcvsi_list_prune_cfg_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "srcvsi prune config failed, ret=%d", ret);
+
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index c13653e8af..d505f93dc1 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -12,6 +12,9 @@
int32_t sxe2_drv_dev_caps_get(struct sxe2_adapter *adapter,
struct sxe2_drv_dev_caps_resp *dev_caps);
+int32_t sxe2_drv_switchdev_info_get(struct sxe2_adapter *adapter,
+ struct sxe2_switchdev_info *switchdev_info);
+
int32_t sxe2_drv_dev_info_get(struct sxe2_adapter *adapter,
struct sxe2_drv_dev_info_resp *dev_info_resp);
@@ -64,6 +67,8 @@ int32_t sxe2_drv_ipsec_txsa_delete(struct sxe2_adapter *adapter,
int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+int32_t sxe2_drv_vsi_info_get(struct sxe2_adapter *adapter, struct sxe2_vsi *vsi);
+
int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
int32_t sxe2_drv_get_mac_stats(struct sxe2_adapter *adapter);
@@ -91,6 +96,15 @@ int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set);
int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
+int32_t sxe2_drv_switchdev_uplink_config(struct sxe2_adapter *adapter, bool set);
+
+int32_t sxe2_drv_switchdev_repr_vf_config(struct sxe2_adapter *adapter,
+ struct sxe2_switchdev_repr_info *repr_vf, bool set);
+
+int32_t sxe2_drv_switchdev_cpvsi_get(struct sxe2_adapter *adapter, uint16_t *cp_vsi_id);
+
+int32_t sxe2_drv_switchdev_mode_get(struct sxe2_adapter *adapter, bool *is_switchdev);
+
int32_t sxe2_drv_mc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
int32_t sxe2_drv_vlan_config_query(struct sxe2_adapter *adapter);
@@ -122,4 +136,18 @@ int32_t sxe2_drv_rxq_bind_irq(struct sxe2_adapter *adapter, uint16_t rxq_idx, ui
int32_t sxe2_drv_rxq_unbind_irq(struct sxe2_adapter *adapter, uint16_t rxq_idx);
+int32_t sxe2_drv_flow_filter_add(struct sxe2_adapter *adapter, struct sxe2_flow *flow);
+
+int32_t sxe2_drv_flow_filter_del(struct sxe2_adapter *adapter, struct sxe2_flow *flow);
+
+int32_t sxe2_drv_flow_fnav_get_stat_id(struct sxe2_adapter *adapter, uint32_t *stat_id);
+
+int32_t sxe2_drv_flow_fnav_free_stat(struct sxe2_adapter *adapter, uint32_t stat_id);
+
+int32_t sxe2_drv_flow_fnav_query_stat(struct sxe2_adapter *adapter,
+ struct sxe2_fnav_cid_mgr *mgr);
+
+int32_t sxe2_drv_srcvsi_prune_config(struct sxe2_adapter *adapter,
+ uint16_t *vsi_list, uint16_t vsi_cnt, bool set);
+
#endif /* __SXE2_CMD_CHNL_H__ */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index a839d9dfe0..9d89039bee 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -108,7 +108,6 @@ enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_LEGACY,
SXE2_PHYS_PORT_NAME_TYPE_UPLINK,
SXE2_PHYS_PORT_NAME_TYPE_PFVF,
-
SXE2_PHYS_PORT_NAME_TYPE_UNKNOWN,
};
@@ -564,6 +563,60 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_irq_bind_req {
uint8_t rsv[2];
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_uplink_info {
+ uint8_t pf_id;
+ uint8_t is_set;
+ uint8_t rsv[2];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_repr_info {
+ uint8_t pf_id;
+ uint8_t is_set;
+ uint8_t rsv[2];
+ uint16_t cp_vsi_id;
+ uint16_t repr_pf_id;
+ uint16_t repr_vf_id;
+ uint16_t repr_q_id;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_filter_req {
+ uint32_t flow_id;
+ struct sxe2_flow_meta meta;
+ enum sxe2_flow_engine_type engine_type;
+ struct sxe2_flow_pattern pattern_outer;
+ struct sxe2_flow_pattern pattern_inner;
+ struct sxe2_flow_action action;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_filter_resp {
+ enum sxe2_flow_engine_type engine_type;
+ uint32_t flow_id;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_fnav_get_stat_id_req {
+ uint8_t need_update;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_fnav_get_stat_id_resp {
+ uint32_t stat_id;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_fnav_free_stat_id_req {
+ uint32_t stat_id;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_fnav_query_stat_req {
+ uint32_t stat_id;
+ uint32_t stat_ctrl;
+ uint32_t is_clear;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_fnav_query_stat_resp {
+ uint32_t stat_index;
+ uint64_t stat_hits;
+ uint64_t stat_bytes;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index f3fee74ddf..317101fb60 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -33,9 +33,13 @@
#include "sxe2_ptype.h"
#include "sxe2_common_log.h"
#include "sxe2_mp.h"
+#include "sxe2_flow.h"
#include "sxe2_stats.h"
#include "sxe2_host_regs.h"
+#include "sxe2_switchdev.h"
#include "sxe2_ioctl_chnl_func.h"
+#include "sxe2_ethdev_repr.h"
+#include "sxe2vf_regs.h"
#define SXE2_PCI_VENDOR_ID_1 0x1ff2
#define SXE2_PCI_DEVICE_ID_PF_1 0x10b1
@@ -83,6 +87,27 @@ static struct sxe2_pci_map_addr_info sxe2_net_map_addr_info_pf[SXE2_PCI_MAP_RES_
.reg_width = 10},
};
+static struct sxe2_pci_map_addr_info sxe2_net_map_addr_info_vf[SXE2_PCI_MAP_RES_MAX_COUNT] = {
+ [SXE2_PCI_MAP_RES_INVALID] = {.addr_base = 0,
+ .bar_idx = 0,
+ .reg_width = 0},
+ [SXE2_PCI_MAP_RES_DOORBELL_TX] = {.addr_base = SXE2VF_TXQ_TAIL(0),
+ .bar_idx = 0,
+ .reg_width = 4},
+ [SXE2_PCI_MAP_RES_DOORBELL_RX_TAIL] = {.addr_base = SXE2VF_RXQ_TAIL(0),
+ .bar_idx = 0,
+ .reg_width = 4},
+ [SXE2_PCI_MAP_RES_IRQ_DYN] = {.addr_base = SXE2VF_VF_DYN_CTL(0),
+ .bar_idx = 0,
+ .reg_width = 4},
+ [SXE2_PCI_MAP_RES_IRQ_ITR] = {.addr_base = SXE2VF_VF_INT_ITR(0, 0),
+ .bar_idx = 0,
+ .reg_width = 4},
+ [SXE2_PCI_MAP_RES_IRQ_MSIX] = {.addr_base = SXE2VF_BAR4_MSIX_CTL(0),
+ .bar_idx = 4,
+ .reg_width = 0x10},
+};
+
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev);
static int32_t sxe2_dev_start(struct rte_eth_dev *dev);
static int32_t sxe2_dev_stop(struct rte_eth_dev *dev);
@@ -137,6 +162,7 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.rss_hash_update = sxe2_dev_rss_hash_update,
.rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
+ .flow_ops_get = sxe2_flow_ops_get,
.tm_ops_get = sxe2_tm_ops_get,
.stats_get = sxe2_stats_info_get,
@@ -566,7 +592,7 @@ static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
return ret;
}
-static void sxe2_eth_uinit(struct rte_eth_dev *dev __rte_unused)
+void sxe2_eth_uinit(struct rte_eth_dev *dev __rte_unused)
{
sxe2_mac_addr_uinit(dev);
(void)sxe2_filter_uinit(dev);
@@ -607,6 +633,16 @@ static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
adapter->cap_flags |= SXE2_DEV_CAPS_OFFLOAD_FC_STATE;
}
+static void sxe2_sw_representor_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_representor_caps *repr_caps)
+{
+ adapter->repr_ctxt.nb_vf = repr_caps->cnt_repr_vf;
+ if (adapter->repr_ctxt.nb_vf > 0) {
+ memcpy(adapter->repr_ctxt.repr_vf_id, repr_caps->repr_vf_id,
+ adapter->repr_ctxt.nb_vf * sizeof(struct sxe2_drv_vsi_caps));
+ }
+}
+
static void sxe2_sw_sched_hw_cap_set(struct sxe2_adapter *adapter,
struct sxe2_txsch_caps *txsch_caps)
{
@@ -636,20 +672,47 @@ static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
sxe2_sw_vsi_ctx_hw_cap_set(adapter, &dev_caps.vsi_caps);
+ sxe2_sw_representor_ctx_hw_cap_set(adapter, &dev_caps.repr_caps);
+
sxe2_sw_sched_hw_cap_set(adapter, &dev_caps.txsch_caps);
l_end:
return ret;
}
+static int32_t sxe2_switchdev_info_get(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_switchdev_info switchdev_info = {0};
+
+ ret = sxe2_drv_switchdev_info_get(adapter, &switchdev_info);
+ if (ret)
+ goto l_end;
+ if (switchdev_info.master && switchdev_info.representor) {
+ PMD_LOG_ERR(INIT, "device could not be both master and representor");
+ ret = -ENODEV;
+ goto l_end;
+ }
+ adapter->switchdev_info = switchdev_info;
+l_end:
+ return ret;
+}
+
static int32_t sxe2_dev_caps_get(struct sxe2_adapter *adapter)
{
int32_t ret = -1;
ret = sxe2_func_caps_get(adapter);
- if (ret)
+ if (ret) {
PMD_LOG_ERR(INIT, "get function caps failed, ret=%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_switchdev_info_get(adapter);
+ if (ret)
+ PMD_LOG_ERR(INIT, "get switchdev info failed, ret=%d", ret);
+l_end:
return ret;
}
@@ -935,7 +998,10 @@ int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev)
bar_info[1].seg_info = seg_info;
map_ctxt->bar_info = bar_info;
- map_ctxt->addr_info = sxe2_net_map_addr_info_pf;
+ if (adapter->dev_type == SXE2_DEV_T_VF)
+ map_ctxt->addr_info = sxe2_net_map_addr_info_vf;
+ else
+ map_ctxt->addr_info = sxe2_net_map_addr_info_pf;
ret = sxe2_dev_pci_res_seg_map(adapter, SXE2_PCI_MAP_RES_DOORBELL_TX,
txq_cnt, txq_base);
@@ -1138,6 +1204,12 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_dev_info_err;
}
+ ret = sxe2_switchdev_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize switchdev mode, ret=[%d]", ret);
+ goto init_switchdev_err;
+ }
+
ret = sxe2_sw_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to initialize sw parameters, ret=[%d]", ret);
@@ -1168,6 +1240,12 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_rss_err;
}
+ ret = sxe2_flow_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init flow, ret=%d", ret);
+ goto init_flow_err;
+ }
+
ret = sxe2_sched_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to init sched, ret=%d", ret);
@@ -1191,15 +1269,19 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
init_xstats_err:
(void)sxe2_sched_uinit(dev);
init_sched_err:
+ (void)sxe2_flow_uninit(dev);
+init_flow_err:
init_rss_err:
sxe2_security_uinit(dev);
init_security_err:
+ sxe2_eth_uinit(dev);
+init_eth_err:
sxe2_intr_uninit(dev);
init_irq_err:
sxe2_sw_uninit(dev);
init_sw_err:
- sxe2_eth_uinit(dev);
-init_eth_err:
+ (void)sxe2_switchdev_uninit(dev);
+init_switchdev_err:
init_dev_info_err:
sxe2_vsi_uninit(dev);
init_vsi_err:
@@ -1214,6 +1296,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
sxe2_mp_uninit(dev);
goto l_end;
}
+ sxe2_repr_all_close(dev);
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
sxe2_mp_uninit(dev);
@@ -1222,6 +1305,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
sxe2_vsi_uninit(dev);
sxe2_security_uinit(dev);
sxe2_intr_uninit(dev);
+ (void)sxe2_switchdev_uninit(dev);
sxe2_sw_uninit(dev);
sxe2_eth_uinit(dev);
sxe2_dev_pci_map_uinit(dev);
@@ -1233,10 +1317,29 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
static int32_t sxe2_dev_uninit(struct rte_eth_dev *dev)
{
int32_t ret = 0;
+ int32_t i = 0;
+ struct sxe2_adapter *adapter = NULL;
+ struct rte_eth_dev *rep_dev = NULL;
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
goto l_end;
+ adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ for (i = 0; i < adapter->repr_ctxt.nb_repr_vf; i++) {
+ rep_dev = adapter->repr_ctxt.vf_rep_eth_dev[i];
+ if (rep_dev) {
+ ret = rep_dev->dev_ops->dev_close(rep_dev);
+ if (ret)
+ goto l_end;
+ if (rep_dev->intr_handle)
+ rte_intr_instance_free(rep_dev->intr_handle);
+ ret = rte_eth_dev_release_port(rep_dev);
+ if (ret)
+ goto l_end;
+ adapter->repr_ctxt.vf_rep_eth_dev[i] = NULL;
+ }
+ }
+
ret = sxe2_dev_close(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Sxe2 dev close failed, ret=%d", ret);
@@ -1270,6 +1373,65 @@ static int32_t sxe2_eth_pmd_remove(struct sxe2_common_device *cdev)
return ret;
}
+static uint16_t sxe2_switchdev_repr_id_encode_get(struct sxe2_switchdev_info *switchdev_info)
+{
+ enum rte_eth_representor_type type;
+ uint16_t repr = switchdev_info->vf_num;
+ uint32_t pf = switchdev_info->pf_num;
+
+ switch (switchdev_info->port_name_type) {
+ case SXE2_PHYS_PORT_NAME_TYPE_UPLINK:
+ if (!switchdev_info->representor)
+ return UINT16_MAX;
+ type = RTE_ETH_REPRESENTOR_PF;
+ pf = switchdev_info->mpesw_owner;
+ break;
+ case SXE2_PHYS_PORT_NAME_TYPE_PFVF:
+ default:
+ type = RTE_ETH_REPRESENTOR_VF;
+ break;
+ }
+
+ return SXE2_REPRESENTOR_ID(pf, type, repr);
+}
+
+static bool sxe2_switchdev_repr_match(struct sxe2_adapter *adapter,
+ struct rte_eth_devargs *req_eth_da)
+{
+ uint32_t port_idx = 0;
+ uint32_t repr_idx;
+ uint16_t kernel_repr_id = sxe2_switchdev_repr_id_encode_get(&adapter->switchdev_info);
+ uint16_t repr_id;
+
+ switch (req_eth_da->type) {
+ case RTE_ETH_REPRESENTOR_PF:
+ break;
+ case RTE_ETH_REPRESENTOR_VF:
+ if (adapter->switchdev_info.port_name_type !=
+ SXE2_PHYS_PORT_NAME_TYPE_PFVF) {
+ rte_errno = EBUSY;
+ return false;
+ }
+ break;
+ case RTE_ETH_REPRESENTOR_NONE:
+ rte_errno = EBUSY;
+ return false;
+ default:
+ rte_errno = ENOTSUP;
+ return false;
+ }
+
+ for (repr_idx = 0; repr_idx < req_eth_da->nb_representor_ports; ++repr_idx) {
+ repr_id = SXE2_REPRESENTOR_ID(req_eth_da->ports[port_idx],
+ req_eth_da->type,
+ req_eth_da->representor_ports[repr_idx]);
+ if (repr_id == kernel_repr_id)
+ return true;
+ }
+ rte_errno = EBUSY;
+ return false;
+}
+
static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
struct rte_eth_devargs *req_eth_da __rte_unused,
uint16_t owner_id __rte_unused,
@@ -1311,10 +1473,34 @@ static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
goto l_release_port;
}
+ if (req_eth_da->nb_representor_ports > 0) {
+ if (!adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Representor requested but Switchdev not enabled");
+ ret = -ENOTSUP;
+ goto l_dev_uinit;
+ }
+
+ if (!sxe2_switchdev_repr_match(adapter, req_eth_da)) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Representor parameters mismatch");
+ ret = -ENOTSUP;
+ goto l_dev_uinit;
+ }
+
+ ret = sxe2_switchdev_repr_devs_init(adapter, req_eth_da);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init representor, ret=%d", ret);
+ goto l_dev_uinit;
+ }
+ } else {
+ PMD_DEV_LOG_DEBUG(adapter, INIT, "No representors requested, skipping.");
+ }
+
rte_eth_dev_probing_finish(eth_dev);
PMD_DEV_LOG_DEBUG(adapter, INIT, "Sxe2 eth pmd probe successful!");
goto l_end;
+l_dev_uinit:
+ (void)sxe2_dev_uninit(eth_dev);
l_release_port:
(void)rte_eth_dev_release_port(eth_dev);
l_end:
@@ -1384,6 +1570,11 @@ static struct sxe2_class_driver sxe2_eth_pmd = {
.intr_rmv = 1,
};
+bool sxe2_ethdev_check(struct rte_eth_dev *dev)
+{
+ return !strcmp(dev->device->driver->name, "sxe2_pci");
+}
+
RTE_INIT(rte_sxe2_pmd_init)
{
sxe2_common_init();
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index 65ada44c12..6397a2e5c6 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -64,6 +64,9 @@ enum sxe2_fnav_tunnel_flag_type {
#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
#define lower_32_bits(n) ((uint32_t)((n) & 0xffffffff))
+#define SXE2_REPRESENTOR_ID(pf, type, repr) \
+ (((pf) << 14) + ((type) << 12) + ((repr) & 0xfff))
+
#define SXE2_I2C_EEPROM_DEV_ADDR 0xA0
#define SXE2_I2C_EEPROM_DEV_ADDR2 0xA2
#define SXE2_MODULE_TYPE_SFP 0x03
@@ -309,16 +312,20 @@ struct sxe2_adapter {
struct sxe2_vsi_context vsi_ctxt;
struct sxe2_filter_context filter_ctxt;
struct sxe2_rss_context rss_ctxt;
+ struct sxe2_flow_context flow_ctxt;
struct sxe2_link_context link_ctxt;
struct sxe2_ptp_context ptp_ctxt;
struct sxe2_sched_hw_cap sched_ctxt;
struct sxe2_tm_context tm_ctxt;
struct sxe2_devargs devargs;
struct sxe2_security_ctx security_ctx;
+ struct sxe2_repr_context repr_ctxt;
struct sxe2_switchdev_info switchdev_info;
bool rule_started;
bool flow_isolated;
+ bool flow_isolate_cfg;
uint16_t dev_port_id;
+ bool is_dev_repr;
uint64_t cap_flags;
enum sxe2_dev_type dev_type;
uint32_t ptype_tbl[SXE2_MAX_PTYPE_NUM];
@@ -340,6 +347,8 @@ void *sxe2_pci_map_addr_get(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type,
uint16_t idx_in_func);
+bool sxe2_ethdev_check(struct rte_eth_dev *dev);
+
uint32_t sxe2_sched_mode_get(struct sxe2_adapter *adapter);
struct sxe2_pci_map_bar_info *sxe2_dev_get_bar_info(struct sxe2_adapter *adapter,
@@ -366,6 +375,8 @@ int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev);
void sxe2_dev_pci_map_uinit(struct rte_eth_dev *dev);
+void sxe2_eth_uinit(struct rte_eth_dev *dev);
+
static inline bool
sxe2_dev_port_vlan_check(struct rte_eth_dev *dev)
{
diff --git a/drivers/net/sxe2/sxe2_ethdev_repr.c b/drivers/net/sxe2/sxe2_ethdev_repr.c
new file mode 100644
index 0000000000..a43991c379
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_ethdev_repr.c
@@ -0,0 +1,607 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include "sxe2_ethdev_repr.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_common.h"
+#include "sxe2_common_log.h"
+#include "sxe2_tx.h"
+#include "sxe2_rx.h"
+#include "sxe2_txrx.h"
+#include "sxe2_switchdev.h"
+#include "sxe2_ptype.h"
+#include "sxe2_mp.h"
+#include "sxe2_stats.h"
+#include "sxe2_flow.h"
+
+static struct sxe2_pci_map_addr_info sxe2_net_map_addr_info_repr[SXE2_PCI_MAP_RES_MAX_COUNT] = {
+ {0, 0, 0},
+ { SXE2_TXQ_LEGACY_DBLL(0), 0, 4},
+ { SXE2_RXQ_TAIL(0), 0, 4},
+ { SXE2_VF_DYN_CTL(0), 0, 4},
+ { SXE2_VF_INT_ITR(0, 0), 0, 4},
+ { SXE2_BAR4_MSIX_CTL(0), 4, 0x10},
+};
+
+static void sxe2_repr_dev_uinit(struct rte_eth_dev *dev);
+
+static int32_t sxe2_repr_promisc_enable(struct rte_eth_dev *dev __rte_unused)
+{
+ return 0;
+}
+static int32_t sxe2_repr_promisc_disable(struct rte_eth_dev *dev __rte_unused)
+{
+ return 0;
+}
+static int32_t sxe2_repr_allmulti_enable(struct rte_eth_dev *dev __rte_unused)
+{
+ return 0;
+}
+static int32_t sxe2_repr_allmulti_disable(struct rte_eth_dev *dev __rte_unused)
+{
+ return 0;
+}
+
+static int32_t sxe2_repr_dev_configure(struct rte_eth_dev *dev)
+{
+ dev->data->mtu = SXE2_FRAME_SIZE_MAX - SXE2_ETH_OVERHEAD;
+ return 0;
+}
+
+static int32_t sxe2_repr_dev_start(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ PMD_INIT_FUNC_TRACE();
+
+ ret = sxe2_queues_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init queues.");
+ goto l_end;
+ }
+
+ sxe2_rx_mode_func_set(dev);
+ sxe2_tx_mode_func_set(dev);
+
+ ret = sxe2_link_update_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_repr_rxq_intr_enable(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to enable rx queue intr");
+ goto l_end;
+ }
+
+ ret = sxe2_queues_start(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "enable queues failed");
+ goto l_start_queues_err;
+ }
+
+ dev->data->dev_started = 1;
+ adapter->started = 1;
+ goto l_end;
+l_start_queues_err:
+ (void)sxe2_rxq_intr_disable(dev);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_repr_dev_stop(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ PMD_INIT_FUNC_TRACE();
+
+ if (adapter->started == 0)
+ goto l_end;
+
+ sxe2_repr_rxq_intr_disable(dev);
+
+ sxe2_txqs_all_stop(dev);
+ sxe2_rxqs_all_stop(dev);
+
+ dev->data->dev_started = 0;
+ adapter->started = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_repr_dev_close(struct rte_eth_dev *dev)
+{
+ PMD_DEV_LOG_INFO(SXE2_DEV_PRIVATE_TO_ADAPTER(dev),
+ INIT, "repr close");
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ sxe2_mp_uninit(dev);
+ goto l_end;
+ }
+ (void)sxe2_repr_dev_stop(dev);
+ (void)sxe2_queues_release(dev);
+ sxe2_mp_uninit(dev);
+ sxe2_repr_dev_uinit(dev);
+l_end:
+ return 0;
+}
+
+static int32_t sxe2_repr_dev_infos_get(struct rte_eth_dev *dev,
+ struct rte_eth_dev_info *dev_info)
+{
+ dev_info->max_rx_queues = 1;
+ dev_info->max_tx_queues = 1;
+ dev_info->min_rx_bufsize = SXE2_MIN_BUF_SIZE;
+ dev_info->max_rx_pktlen = SXE2_FRAME_SIZE_MAX;
+ dev_info->max_lro_pkt_size = SXE2_FRAME_SIZE_MAX * SXE2_RX_LRO_DESC_MAX_NUM;
+ dev_info->max_mtu = dev_info->max_rx_pktlen - SXE2_ETH_OVERHEAD;
+ dev_info->min_mtu = RTE_ETHER_MIN_MTU;
+ dev_info->max_mac_addrs = SXE2_NUM_MACADDR_MAX;
+
+ dev_info->rx_offload_capa =
+ RTE_ETH_RX_OFFLOAD_KEEP_CRC |
+ RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
+ RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
+ RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
+ RTE_ETH_RX_OFFLOAD_SCTP_CKSUM |
+ RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM;
+ dev_info->tx_offload_capa =
+ RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
+ RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
+ RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
+ RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
+ RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
+ RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM;
+
+ dev_info->tx_queue_offload_capa = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
+
+ dev_info->default_rxconf = (struct rte_eth_rxconf) {
+ .rx_thresh = {
+ .pthresh = SXE2_DEFAULT_RX_PTHRESH,
+ .hthresh = SXE2_DEFAULT_RX_HTHRESH,
+ .wthresh = SXE2_DEFAULT_RX_WTHRESH,
+ },
+ .rx_free_thresh = SXE2_DEFAULT_RX_FREE_THRESH,
+ .rx_drop_en = 0,
+ .offloads = 0,
+ };
+
+ dev_info->default_txconf = (struct rte_eth_txconf) {
+ .tx_thresh = {
+ .pthresh = SXE2_DEFAULT_TX_PTHRESH,
+ .hthresh = SXE2_DEFAULT_TX_HTHRESH,
+ .wthresh = SXE2_DEFAULT_TX_WTHRESH,
+ },
+ .tx_free_thresh = SXE2_DEFAULT_TX_FREE_THRESH,
+ .tx_rs_thresh = SXE2_DEFAULT_TX_RSBIT_THRESH,
+ .offloads = 0,
+ };
+
+ dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
+ .nb_max = SXE2_MAX_RING_DESC,
+ .nb_min = SXE2_MIN_RING_DESC,
+ .nb_align = SXE2_ALIGN,
+ };
+
+ dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
+ .nb_max = SXE2_MAX_RING_DESC,
+ .nb_min = SXE2_MIN_RING_DESC,
+ .nb_align = SXE2_ALIGN,
+ .nb_mtu_seg_max = SXE2_TX_MTU_SEG_MAX,
+ .nb_seg_max = SXE2_MAX_RING_DESC,
+ };
+
+ dev_info->speed_capa = RTE_ETH_LINK_SPEED_10G | RTE_ETH_LINK_SPEED_25G |
+ RTE_ETH_LINK_SPEED_50G | RTE_ETH_LINK_SPEED_100G;
+
+ dev_info->nb_rx_queues = dev->data->nb_rx_queues;
+ dev_info->nb_tx_queues = dev->data->nb_tx_queues;
+
+ dev_info->default_rxportconf.burst_size = SXE2_RX_MAX_BURST;
+ dev_info->default_txportconf.burst_size = SXE2_TX_MAX_BURST;
+ dev_info->default_rxportconf.nb_queues = 1;
+ dev_info->default_txportconf.nb_queues = 1;
+ dev_info->default_rxportconf.ring_size = SXE2_RING_SIZE_MIN;
+ dev_info->default_txportconf.ring_size = SXE2_RING_SIZE_MIN;
+
+ dev_info->rx_seg_capa.offset_allowed = false;
+
+ dev_info->rx_seg_capa.offset_align_log2 = false;
+
+ return 0;
+}
+
+static const struct eth_dev_ops sxe2_switchdev_repr_dev_ops = {
+ .dev_configure = sxe2_repr_dev_configure,
+
+ .dev_start = sxe2_repr_dev_start,
+ .dev_stop = sxe2_repr_dev_stop,
+
+ .rx_queue_start = sxe2_rx_queue_start,
+ .rx_queue_stop = sxe2_rx_queue_stop,
+ .tx_queue_start = sxe2_tx_queue_start,
+ .tx_queue_stop = sxe2_tx_queue_stop,
+ .rx_queue_setup = sxe2_rx_queue_setup,
+ .rx_queue_release = sxe2_rx_queue_release,
+ .tx_queue_setup = sxe2_tx_queue_setup,
+ .tx_queue_release = sxe2_tx_queue_release,
+
+ .dev_close = sxe2_repr_dev_close,
+ .dev_infos_get = sxe2_repr_dev_infos_get,
+ .dev_supported_ptypes_get = sxe2_dev_supported_ptypes_get,
+ .link_update = sxe2_link_update,
+
+ .promiscuous_enable = sxe2_repr_promisc_enable,
+ .promiscuous_disable = sxe2_repr_promisc_disable,
+ .allmulticast_enable = sxe2_repr_allmulti_enable,
+ .allmulticast_disable = sxe2_repr_allmulti_disable,
+
+ .stats_get = sxe2_stats_info_get,
+ .stats_reset = sxe2_stats_info_reset,
+ .xstats_get = sxe2_xstats_info_get,
+ .xstats_get_names = sxe2_xstats_names_get,
+ .xstats_reset = sxe2_stats_info_reset,
+};
+
+void sxe2_repr_all_close(struct rte_eth_dev *dev)
+{
+ uint16_t vf_id;
+ struct rte_eth_dev *repr_eth_dev = NULL;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (adapter->repr_ctxt.nb_repr_vf) {
+ for (vf_id = 0; vf_id < adapter->repr_ctxt.nb_repr_vf; vf_id++) {
+ repr_eth_dev = adapter->repr_ctxt.vf_rep_eth_dev[vf_id];
+ if (!repr_eth_dev || repr_eth_dev->data->dev_started == 0)
+ continue;
+
+ (void)rte_eth_dev_stop(repr_eth_dev->data->port_id);
+ (void)rte_eth_dev_close(repr_eth_dev->data->port_id);
+ }
+ }
+}
+
+static void sxe2_repr_adapter_init(struct rte_eth_dev *dev_repr,
+ struct sxe2_adapter *parent_adapter,
+ uint16_t repr_id)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev_repr);
+
+ dev_repr->data->backer_port_id = parent_adapter->dev_port_id;
+ dev_repr->data->representor_id = repr_id;
+ dev_repr->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
+
+ adapter->is_dev_repr = true;
+ adapter->dev_port_id = dev_repr->data->port_id;
+ adapter->dev_type = parent_adapter->dev_type;
+ adapter->switchdev_info.is_switchdev = parent_adapter->switchdev_info.is_switchdev;
+ adapter->port_idx = parent_adapter->port_idx;
+ adapter->pf_idx = parent_adapter->pf_idx;
+ adapter->dev_info.pci = parent_adapter->dev_info.pci;
+ adapter->dev_info.fw = parent_adapter->dev_info.fw;
+}
+
+static int32_t sxe2_repr_eth_init(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+
+ ret = sxe2_filter_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize l2 filter, ret:%d", ret);
+ goto l_end;
+ }
+ ret = sxe2_link_update_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_mac_addr_init(dev);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to initialize mac address, ret:%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_repr_dev_pci_map_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *rep_adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_pci_map_context *map_ctxt = &rep_adapter->map_ctxt;
+ struct sxe2_pci_map_bar_info *bar_info = NULL;
+ struct sxe2_pci_map_segment_info *seg_info = NULL;
+ uint16_t txq_cnt = rep_adapter->q_ctxt.qp_cnt_assign;
+ uint16_t txq_base = rep_adapter->q_ctxt.base_idx_in_pf;
+ uint16_t rxq_cnt = rep_adapter->q_ctxt.qp_cnt_assign;
+ uint16_t rxq_base = rep_adapter->q_ctxt.base_idx_in_pf;
+ uint16_t irq_cnt = rep_adapter->irq_ctxt.max_cnt_hw;
+ uint16_t irq_base = rep_adapter->irq_ctxt.base_idx_in_func;
+ int32_t ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ rep_adapter->dev_info.dev_data = dev->data;
+
+ map_ctxt->bar_cnt = 2;
+
+ bar_info = rte_zmalloc("repr_bar_info",
+ sizeof(struct sxe2_pci_map_bar_info) * map_ctxt->bar_cnt, 0);
+ if (bar_info == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to alloc bar_info");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ bar_info[0].bar_idx = 0;
+ bar_info[0].map_cnt = SXE2_PCI_MAP_RES_MAX_COUNT;
+ seg_info = rte_zmalloc("repr_seg_info_bar0",
+ sizeof(struct sxe2_pci_map_segment_info) * bar_info[0].map_cnt, 0);
+ if (seg_info == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to alloc seg_info");
+ ret = -ENOMEM;
+ goto l_free_bar;
+ }
+
+ bar_info[0].seg_info = seg_info;
+
+ bar_info[1].bar_idx = 4;
+ bar_info[1].map_cnt = SXE2_PCI_MAP_RES_MAX_COUNT;
+ seg_info = rte_zmalloc("repr_seg_info_bar4",
+ sizeof(struct sxe2_pci_map_segment_info) * bar_info[1].map_cnt,
+ 0);
+ if (!seg_info) {
+ PMD_LOG_ERR(INIT, "Failed to alloc seg_info");
+ ret = -ENOMEM;
+ goto l_free_seg0;
+ }
+
+ bar_info[1].seg_info = seg_info;
+ map_ctxt->bar_info = bar_info;
+
+ map_ctxt->addr_info = sxe2_net_map_addr_info_repr;
+
+ ret = sxe2_dev_pci_res_seg_map(rep_adapter, SXE2_PCI_MAP_RES_DOORBELL_TX,
+ txq_cnt, txq_base);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to map txq doorbell addr, ret=%d", ret);
+ goto l_free_seg1;
+ }
+
+ ret = sxe2_dev_pci_res_seg_map(rep_adapter, SXE2_PCI_MAP_RES_DOORBELL_RX_TAIL,
+ rxq_cnt, rxq_base);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to map rxq tail doorbell addr, ret=%d", ret);
+ goto l_free_txq;
+ }
+
+ ret = sxe2_dev_pci_res_seg_map(rep_adapter, SXE2_PCI_MAP_RES_IRQ_DYN,
+ irq_cnt, irq_base);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to map irq dyn addr, ret=%d", ret);
+ goto l_free_rxq_tail;
+ }
+
+ ret = sxe2_dev_pci_res_seg_map(rep_adapter, SXE2_PCI_MAP_RES_IRQ_ITR,
+ irq_cnt, irq_base);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to map irq itr addr, ret=%d", ret);
+ goto l_free_irq_dyn;
+ }
+
+ ret = sxe2_dev_pci_res_seg_map(rep_adapter, SXE2_PCI_MAP_RES_IRQ_MSIX,
+ irq_cnt, irq_base);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to map irq msix addr, ret=%d", ret);
+ goto l_free_irq_itr;
+ }
+ goto l_end;
+
+l_free_irq_itr:
+ (void)sxe2_dev_pci_seg_unmap(rep_adapter, SXE2_PCI_MAP_RES_IRQ_ITR);
+l_free_irq_dyn:
+ (void)sxe2_dev_pci_seg_unmap(rep_adapter, SXE2_PCI_MAP_RES_IRQ_DYN);
+l_free_rxq_tail:
+ (void)sxe2_dev_pci_seg_unmap(rep_adapter, SXE2_PCI_MAP_RES_DOORBELL_RX_TAIL);
+l_free_txq:
+ (void)sxe2_dev_pci_seg_unmap(rep_adapter, SXE2_PCI_MAP_RES_DOORBELL_TX);
+l_free_seg1:
+ if (bar_info[1].seg_info) {
+ rte_free(bar_info[1].seg_info);
+ bar_info[1].seg_info = NULL;
+ }
+l_free_seg0:
+ if (bar_info[0].seg_info) {
+ rte_free(bar_info[0].seg_info);
+ bar_info[0].seg_info = NULL;
+ }
+l_free_bar:
+ if (bar_info) {
+ rte_free(bar_info);
+ bar_info = NULL;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_repr_dev_init(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter,
+ uint16_t repr_id)
+{
+ int32_t ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ sxe2_set_common_function(dev);
+
+ sxe2_repr_adapter_init(dev, parent_adapter, repr_id);
+
+ dev->dev_ops = &sxe2_switchdev_repr_dev_ops;
+
+ ret = sxe2_vsi_repr_main_vsi_create(dev, parent_adapter, repr_id);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to create representor main vsi, ret=[%d]", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_switchdev_repr_private_data_init(dev, parent_adapter, repr_id);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to fill representor private data, ret=[%d]", ret);
+ goto l_init_priv_data_err;
+ }
+
+ ret = sxe2_repr_dev_pci_map_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to pci addr map, ret=[%d]", ret);
+ goto l_init_pci_error;
+ }
+
+ ret = sxe2_switchdev_dev_info_init(dev, parent_adapter);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to get device info, ret=[%d]", ret);
+ goto l_init_dev_info_err;
+ }
+
+ ret = sxe2_flow_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init flow, ret=%d", ret);
+ goto l_init_flow_err;
+ }
+
+ ret = sxe2_repr_eth_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init device, status = %d", ret);
+ goto l_init_eth_err;
+ }
+
+ ret = sxe2_sw_irq_ctxt_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize sw parameters, ret=[%d]", ret);
+ goto l_init_sw_err;
+ }
+
+ goto l_end;
+
+l_init_sw_err:
+ sxe2_eth_uinit(dev);
+l_init_eth_err:
+ (void)sxe2_flow_uninit(dev);
+l_init_flow_err:
+l_init_dev_info_err:
+ sxe2_dev_pci_map_uinit(dev);
+l_init_pci_error:
+ (void)sxe2_switchdev_uninit(dev);
+l_init_priv_data_err:
+ sxe2_vsi_repr_main_vsi_destroy(dev);
+l_end:
+ return ret;
+}
+
+static void sxe2_repr_dev_uinit(struct rte_eth_dev *dev)
+{
+ sxe2_eth_uinit(dev);
+ (void)sxe2_flow_uninit(dev);
+ sxe2_dev_pci_map_uinit(dev);
+ (void)sxe2_switchdev_uninit(dev);
+ sxe2_vsi_repr_main_vsi_destroy(dev);
+}
+
+int32_t sxe2_switchdev_repr_devs_init(struct sxe2_adapter *adapter,
+ struct rte_eth_devargs *req_eth_da)
+{
+ struct rte_eth_dev *eth_dev = NULL;
+ int32_t ret;
+ uint16_t repr_idx = 0, tmp_repr_idx = 0;
+ char name[RTE_ETH_NAME_MAX_LEN];
+
+ if (req_eth_da->nb_representor_ports == 0) {
+ ret = 0;
+ goto l_end;
+ }
+
+ if (req_eth_da->nb_representor_ports > adapter->repr_ctxt.nb_vf) {
+ PMD_LOG_ERR(INIT, "Failed to create repr vsi, nb_representor_ports=%d, nb_vf=%d",
+ req_eth_da->nb_representor_ports, adapter->repr_ctxt.nb_vf);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_other_vsi_create(adapter, req_eth_da->nb_representor_ports);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to create representor vsi, ret=%d", ret);
+ goto l_release_port;
+ }
+
+ adapter->repr_ctxt.vf_rep_eth_dev = rte_zmalloc("sxe2_repr_ethdev",
+ req_eth_da->nb_representor_ports * sizeof(struct rte_eth_dev *), 0);
+ if (adapter->repr_ctxt.vf_rep_eth_dev == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to malloc representor eth dev.");
+ ret = -ENOMEM;
+ goto l_release_port;
+ }
+
+ for (repr_idx = 0; repr_idx < req_eth_da->nb_representor_ports; ++repr_idx) {
+ snprintf(name, sizeof(name), "sxe2_representor_c%dpf%d%s%u",
+ adapter->pf_idx, adapter->pf_idx,
+ "vf",
+ req_eth_da->representor_ports[repr_idx]);
+
+ eth_dev = rte_eth_dev_allocate(name);
+ if (!eth_dev) {
+ ret = -ENOMEM;
+ goto l_release_port;
+ }
+ eth_dev->data->dev_private = rte_zmalloc_socket(name,
+ sizeof(struct sxe2_adapter),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+
+ if (!eth_dev->data->dev_private) {
+ rte_eth_dev_release_port(eth_dev);
+ ret = -ENOMEM;
+ goto l_release_port;
+ }
+
+ eth_dev->device = rte_eth_devices[adapter->dev_info.dev_data->port_id].device;
+
+ ret = sxe2_repr_dev_init(eth_dev, adapter,
+ req_eth_da->representor_ports[repr_idx]);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Sxe2 dev init failed, ret=%d", ret);
+ rte_eth_dev_release_port(eth_dev);
+ goto l_release_port;
+ }
+
+ eth_dev->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
+ if (eth_dev->intr_handle == NULL) {
+ PMD_LOG_ERR(INIT, "Sxe2 dev init representor intr_handle failed");
+ ret = -ENOMEM;
+ sxe2_repr_dev_uinit(eth_dev);
+ rte_eth_dev_release_port(eth_dev);
+ goto l_release_port;
+ }
+ adapter->repr_ctxt.vf_rep_eth_dev[repr_idx] = eth_dev;
+ rte_eth_dev_probing_finish(eth_dev);
+ }
+ adapter->repr_ctxt.nb_repr_vf = req_eth_da->nb_representor_ports;
+ goto l_end;
+
+l_release_port:
+ for (tmp_repr_idx = 0; tmp_repr_idx < repr_idx; ++tmp_repr_idx) {
+ struct rte_eth_dev *rep_dev = adapter->repr_ctxt.vf_rep_eth_dev[tmp_repr_idx];
+ if (rep_dev) {
+ sxe2_repr_dev_uinit(rep_dev);
+ if (rep_dev->intr_handle)
+ rte_intr_instance_free(rep_dev->intr_handle);
+ rte_eth_dev_release_port(rep_dev);
+ adapter->repr_ctxt.vf_rep_eth_dev[tmp_repr_idx] = NULL;
+ }
+ }
+
+ rte_free(adapter->repr_ctxt.vf_rep_eth_dev);
+ adapter->repr_ctxt.vf_rep_eth_dev = NULL;
+
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_ethdev_repr.h b/drivers/net/sxe2/sxe2_ethdev_repr.h
new file mode 100644
index 0000000000..71a666337f
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_ethdev_repr.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SX2_ETHDEV_REPR_H__
+#define __SX2_ETHDEV_REPR_H__
+#include <rte_compat.h>
+#include <rte_kvargs.h>
+#include <rte_time.h>
+#include <ethdev_driver.h>
+#include <ethdev_pci.h>
+#include <rte_tm_driver.h>
+#include <rte_io.h>
+#include <rte_ethdev.h>
+#include <rte_alarm.h>
+#include <rte_dev_info.h>
+
+#include "sxe2_vsi.h"
+#include "sxe2_irq.h"
+#include "sxe2_queue.h"
+struct sxe2_adapter;
+
+void sxe2_repr_all_close(struct rte_eth_dev *dev);
+
+int32_t sxe2_repr_dev_init(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter,
+ uint16_t repr_id);
+
+int32_t sxe2_switchdev_repr_devs_init(struct sxe2_adapter *adapter,
+ struct rte_eth_devargs *req_eth_da);
+
+#endif /* __SX2_ETHDEV_REPR_H__ */
diff --git a/drivers/net/sxe2/sxe2_filter.c b/drivers/net/sxe2/sxe2_filter.c
index cfeeb7a6c3..696242ed0c 100644
--- a/drivers/net/sxe2/sxe2_filter.c
+++ b/drivers/net/sxe2/sxe2_filter.c
@@ -9,6 +9,7 @@
#include "sxe2_common_log.h"
#include "sxe2_ethdev.h"
#include "sxe2_cmd_chnl.h"
+#include "sxe2_switchdev.h"
static struct sxe2_mac_filter *sxe2_uc_filter_find(struct sxe2_adapter *adapter,
struct rte_ether_addr *macaddr)
@@ -700,16 +701,96 @@ static int32_t sxe2_all_filter_hw_set(struct sxe2_adapter *adapter)
return ret;
}
+static int32_t sxe2_uplink_hw_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (adapter->filter_ctxt.hw_uplink_config) {
+ if (adapter->dev_type == SXE2_DEV_T_PF) {
+ ret = sxe2_uplink_clear(adapter);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to clear uplink, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_uplink_config = false;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_uplink_hw_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->filter_ctxt.hw_uplink_config) {
+ if (adapter->dev_type == SXE2_DEV_T_PF) {
+ ret = sxe2_uplink_set(adapter);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set uplink, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_uplink_config = true;
+ ret = 0;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_repr_hw_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (adapter->filter_ctxt.hw_repr_config) {
+ if (adapter->dev_type == SXE2_DEV_T_PF ||
+ adapter->dev_type == SXE2_DEV_T_PF_BOND) {
+ ret = sxe2_repr_clear(adapter);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to clear repr, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_repr_config = false;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_repr_hw_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->filter_ctxt.hw_repr_config) {
+ if (adapter->dev_type == SXE2_DEV_T_PF ||
+ adapter->dev_type == SXE2_DEV_T_PF_BOND) {
+ ret = sxe2_repr_set(adapter);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set repr, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_repr_config = true;
+ ret = 0;
+ }
+ }
+l_end:
+ return ret;
+}
+
int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter)
{
int32_t ret = 0;
- if (!adapter->flow_isolated && !adapter->switchdev_info.is_switchdev &&
- adapter->rule_started) {
+ if (!adapter->flow_isolated &&
+ !adapter->switchdev_info.is_switchdev &&
+ adapter->rule_started)
adapter->filter_ctxt.cur_l2_config = true;
- } else {
+ else
adapter->filter_ctxt.cur_l2_config = false;
- }
if (adapter->filter_ctxt.cur_l2_config !=
adapter->filter_ctxt.hw_l2_config) {
@@ -726,6 +807,38 @@ int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter)
return ret;
}
+int32_t sxe2_switchdev_rule_update(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated &&
+ adapter->switchdev_info.is_switchdev) {
+ adapter->filter_ctxt.cur_uplink_config = true;
+ adapter->filter_ctxt.cur_repr_config = true;
+ } else {
+ adapter->filter_ctxt.cur_uplink_config = false;
+ adapter->filter_ctxt.cur_repr_config = false;
+ }
+
+ if (adapter->filter_ctxt.cur_uplink_config !=
+ adapter->filter_ctxt.hw_uplink_config) {
+ if (adapter->filter_ctxt.cur_uplink_config)
+ ret = sxe2_uplink_hw_set(adapter);
+ else
+ ret = sxe2_uplink_hw_clear(adapter);
+ }
+
+ if (adapter->filter_ctxt.cur_repr_config !=
+ adapter->filter_ctxt.hw_repr_config) {
+ if (adapter->filter_ctxt.cur_repr_config)
+ ret = sxe2_repr_hw_set(adapter);
+ else
+ ret = sxe2_repr_hw_clear(adapter);
+ }
+
+ return ret;
+}
+
int32_t sxe2_filter_rule_stop(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
diff --git a/drivers/net/sxe2/sxe2_filter.h b/drivers/net/sxe2/sxe2_filter.h
index 6262e8c845..b2538ed22f 100644
--- a/drivers/net/sxe2/sxe2_filter.h
+++ b/drivers/net/sxe2/sxe2_filter.h
@@ -89,6 +89,8 @@ int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter);
int32_t sxe2_filter_rule_stop(struct rte_eth_dev *dev);
+int32_t sxe2_switchdev_rule_update(struct sxe2_adapter *adapter);
+
int32_t sxe2_filter_rule_start(struct rte_eth_dev *dev);
int32_t sxe2_filter_init(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_flow.c b/drivers/net/sxe2/sxe2_flow.c
new file mode 100644
index 0000000000..6999cb0725
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow.c
@@ -0,0 +1,1337 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <sys/queue.h>
+#include <unistd.h>
+#include "sxe2_ethdev.h"
+#include "sxe2_flow.h"
+#include "sxe2_flow_parse_pattern.h"
+#include "sxe2_flow_parse_action.h"
+#include "sxe2_flow_parse_engine.h"
+#include "sxe2_cmd_chnl.h"
+#include "sxe2_flow_public.h"
+#include "sxe2_common_log.h"
+
+static int32_t sxe2_check_para(const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ if (!pattern) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
+ NULL, "NULL pattern.");
+ ret = -rte_errno;
+ goto l_end;
+ }
+
+ if (!actions) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
+ NULL, "NULL action.");
+ ret = -rte_errno;
+ goto l_end;
+ }
+
+ if (!attr) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR,
+ NULL, "NULL attribute.");
+ ret = -rte_errno;
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_valid_attr(const struct rte_flow_attr *attr, struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+
+ if (!attr->ingress) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+ attr, "Only support ingress.");
+ ret = -rte_errno;
+ goto l_end;
+ }
+
+ if (attr->egress) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ attr, "Not support egress.");
+ ret = -rte_errno;
+ goto l_end;
+ }
+
+ if (attr->group >= 4) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+ attr, "Not support group >= 4.");
+ ret = -rte_errno;
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_check_hdr_duplicate(struct sxe2_flow_item *item_new,
+ struct sxe2_flow_item *item_exist)
+{
+ int32_t ret = 0;
+ uint16_t i = 0;
+ uint16_t size = sizeof(struct sxe2_flow_item);
+ union sxe2_flow_item_raw item_raw_new;
+ union sxe2_flow_item_raw item_raw_exist;
+ rte_memcpy(&item_raw_new.item, item_new, size);
+ rte_memcpy(&item_raw_exist.item, item_exist, size);
+
+ for (i = 0; i < size; i++) {
+ if (item_raw_new.raw[i] != item_raw_exist.raw[i])
+ goto l_end;
+ }
+ ret = -EEXIST;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_check_flow_duplicate(struct sxe2_flow *flow_new,
+ struct sxe2_flow *flow_exist)
+{
+ int32_t ret = 0;
+ int32_t ret_mask1 = 0;
+ int32_t ret_mask2 = 0;
+ int32_t ret_spec1 = 0;
+ int32_t ret_spec2 = 0;
+
+ if (flow_new->engine_type != flow_exist->engine_type)
+ goto l_end;
+ if (flow_new->meta.flow_type != flow_exist->meta.flow_type)
+ goto l_end;
+ if (!sxe2_bitmap_equal(flow_new->flow_type, flow_exist->flow_type,
+ SXE2_EXPANSION_MAX))
+ goto l_end;
+ if (flow_new->meta.flow_prio != flow_exist->meta.flow_prio)
+ goto l_end;
+
+ ret_mask1 = sxe2_flow_check_hdr_duplicate(&flow_new->pattern_outer.item_mask,
+ &flow_exist->pattern_outer.item_mask);
+ ret_mask2 = sxe2_flow_check_hdr_duplicate(&flow_new->pattern_inner.item_mask,
+ &flow_exist->pattern_inner.item_mask);
+
+ ret_spec1 = sxe2_flow_check_hdr_duplicate(&flow_new->pattern_outer.item_spec,
+ &flow_exist->pattern_outer.item_spec);
+ ret_spec2 = sxe2_flow_check_hdr_duplicate(&flow_new->pattern_inner.item_spec,
+ &flow_exist->pattern_inner.item_spec);
+
+ if (flow_new->engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ if (ret_mask1 == 0 || ret_mask2 == 0) {
+ ret = -EEXIST;
+ goto l_end;
+ }
+
+ if (ret_spec1 == -EEXIST && ret_spec2 == -EEXIST) {
+ ret = -EEXIST;
+ goto l_end;
+ }
+ } else {
+ if (ret_mask1 == -EEXIST && ret_mask2 == -EEXIST &&
+ ret_spec1 == -EEXIST && ret_spec2 == -EEXIST) {
+ ret = -EEXIST;
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_check_flow_list_duplicate(struct rte_eth_dev *dev,
+ struct rte_flow *flow_list)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_flow *sxe2_flow_new = NULL;
+ struct rte_flow *rte_flow_exist = NULL;
+ struct sxe2_flow *sxe2_flow_exist = NULL;
+ TAILQ_FOREACH(sxe2_flow_new, &flow_list->sxe2_flow_list, next) {
+ TAILQ_FOREACH(rte_flow_exist, &adapter->flow_ctxt.rte_flow_list, next) {
+ TAILQ_FOREACH(sxe2_flow_exist, &rte_flow_exist->sxe2_flow_list, next) {
+ ret = sxe2_flow_check_flow_duplicate(sxe2_flow_new,
+ sxe2_flow_exist);
+ if (ret != 0)
+ goto l_end;
+ }
+ }
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_check_function(struct rte_eth_dev *dev,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow *flow = TAILQ_FIRST(sxe2_flow_list);
+ int32_t ret = 0;
+
+ uint16_t flow_dst_vsi = UINT16_MAX;
+
+ if (adapter->dev_type == SXE2_DEV_T_VF) {
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_TO_VSI, flow->action.act_types)) {
+ flow_dst_vsi = flow->action.vsi.vsi_index;
+
+ if (adapter->vsi_ctxt.dpdk_vsi_id != flow_dst_vsi &&
+ adapter->vsi_ctxt.kernel_vsi_id != flow_dst_vsi) {
+ PMD_LOG_ERR(DRV, "Failed to redirect other function");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to redirect other function");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+ }
+
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_TO_VSI_LIST, flow->action.act_types)) {
+ PMD_LOG_ERR(DRV,
+ "Failed to redirect multiple driver or function");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to redirect multiple driver or function");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (!adapter->flow_isolated &&
+ flow->engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ PMD_LOG_ERR(DRV,
+ "Failed to switch engine rules in a non-flow-isolated state");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to switch engine rules in a non-flow-isolated state");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (adapter->switchdev_info.is_switchdev &&
+ flow->engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ PMD_LOG_ERR(DRV,
+ "Failed to switch engine rules in a switchdev mode state");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to switch engine rules in a switchdev mode state");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+ }
+
+ if (adapter->is_dev_repr) {
+ if (flow->engine_type != SXE2_FLOW_ENGINE_SWITCH) {
+ PMD_LOG_ERR(DRV,
+ "Failed to config non switch engine rules in representor dev");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to config non switch engine rules in representor dev");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_QUEUE, flow->action.act_types) ||
+ sxe2_test_bit(SXE2_FLOW_ACTION_Q_REGION, flow->action.act_types)) {
+ PMD_LOG_ERR(DRV,
+ "Failed to config queue rules in representor dev");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to config queue rules in representor dev");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+ }
+
+ if (adapter->switchdev_info.is_switchdev &&
+ adapter->dev_type == SXE2_DEV_T_PF &&
+ !adapter->is_dev_repr &&
+ !adapter->flow_isolated) {
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_TO_VSI, flow->action.act_types)) {
+ if (flow->action.vsi.vsi_index == adapter->vsi_ctxt.dpdk_vsi_id) {
+ PMD_LOG_ERR(DRV,
+ "Failed to config rx fwd rule to current uplink dev");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to config rx fwd rule to current uplink dev");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_meta_proc(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow *flow = TAILQ_FIRST(sxe2_flow_list);
+ int32_t ret = 0;
+
+ if (attr->priority >= 1) {
+ if (flow->engine_type != SXE2_FLOW_ENGINE_SWITCH) {
+ PMD_LOG_ERR(DRV, "Only support priority 0.");
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+ attr, "Only support priority 0.");
+ ret = -rte_errno;
+ goto l_end;
+ } else if (!adapter->switchdev_info.is_switchdev) {
+ PMD_LOG_ERR(DRV, "Legacy mode only support priority 0.");
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+ attr, "Legacy mode only priority 0.");
+ ret = -rte_errno;
+ goto l_end;
+ } else {
+ flow->meta.flow_prio = attr->priority;
+ }
+ }
+
+ flow->meta.flow_src_vsi = adapter->vsi_ctxt.dpdk_vsi_id;
+
+ if (adapter->is_dev_repr && adapter->repr_priv_data &&
+ adapter->repr_priv_data->parent_adapter) {
+ flow->meta.flow_rule_vsi =
+ adapter->repr_priv_data->parent_adapter->vsi_ctxt.dpdk_vsi_id;
+ } else {
+ flow->meta.flow_rule_vsi = adapter->vsi_ctxt.dpdk_vsi_id;
+ }
+
+ if (flow->engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ flow->meta.switch_pattern_dup_allow =
+ adapter->devargs.flow_dup_pattern_mode;
+
+ flow->meta.switch_src_direct = SXE2_FLOW_SW_DIRECT_RX;
+
+ if (adapter->switchdev_info.is_switchdev && adapter->is_dev_repr) {
+ flow->meta.switch_src_direct = SXE2_FLOW_SW_DIRECT_TX;
+ flow->meta.flow_src_vsi = adapter->repr_priv_data->repr_vf_vsi_id;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_src_split_proc(struct rte_eth_dev *dev,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow *flow = TAILQ_FIRST(sxe2_flow_list);
+ int32_t ret = 0;
+
+ int32_t idx = 0;
+ uint8_t flow_cnt = 0;
+ uint8_t flow_create_cnt = 0;
+ uint8_t flow_bond_num = 1;
+ uint16_t flow_src_vsi[SXE2_MAX_DRV_TYPE_CNT][SXE2_MAX_BOND_MEMBER_CNT];
+ uint16_t flow_dst_vsi = UINT16_MAX;
+ struct sxe2_flow *flow_new = NULL;
+
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_TO_VSI, flow->action.act_types))
+ flow_dst_vsi = flow->action.vsi.vsi_index;
+
+ for (idx = 0; idx < SXE2_MAX_BOND_MEMBER_CNT; idx++) {
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx] = UINT16_MAX;
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] = UINT16_MAX;
+ }
+
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][0] = adapter->vsi_ctxt.dpdk_vsi_id;
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][0] = adapter->vsi_ctxt.kernel_vsi_id;
+ if (flow->engine_type == SXE2_FLOW_ENGINE_FNAV ||
+ flow->engine_type == SXE2_FLOW_ENGINE_ACL) {
+ if (!adapter->devargs.func_flow_direct_en &&
+ adapter->dev_type != SXE2_DEV_T_PF_BOND) {
+ if (adapter->flow_isolated) {
+ for (idx = 0; idx < flow_bond_num; idx++) {
+ if (flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] !=
+ UINT16_MAX)
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx] =
+ UINT16_MAX;
+ }
+ } else {
+ for (idx = 0; idx < flow_bond_num; idx++)
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] = UINT16_MAX;
+ }
+ }
+
+ for (idx = 0; idx < flow_bond_num; idx++) {
+ if (flow_dst_vsi == flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx])
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] = UINT16_MAX;
+ if (flow_dst_vsi == flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx])
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx] = UINT16_MAX;
+ }
+ } else {
+ for (idx = 0; idx < flow_bond_num; idx++)
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] = UINT16_MAX;
+ }
+
+ if (adapter->switchdev_info.is_switchdev && adapter->is_dev_repr) {
+ flow_bond_num = 1;
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][0] =
+ adapter->repr_priv_data->repr_vf_u_vsi_id;
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][0] =
+ adapter->repr_priv_data->repr_vf_k_vsi_id;
+ }
+
+ for (idx = 0; idx < flow_bond_num; idx++) {
+ if (flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] != UINT16_MAX)
+ flow_cnt++;
+ if (flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx] != UINT16_MAX)
+ flow_cnt++;
+ }
+
+ if (flow_cnt == 0) {
+ PMD_LOG_ERR(DRV, "Failed to redirect same device.");
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to redirect same device");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (idx = 0; idx < flow_bond_num; idx++) {
+ if (flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx] != UINT16_MAX) {
+ if (flow_create_cnt == 0) {
+ flow->meta.flow_src_vsi =
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx];
+ flow_create_cnt++;
+ } else {
+ flow_new = rte_zmalloc("sxe2_flow", sizeof(*flow), 0);
+ if (!flow_new) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to alloc memory for flow rule");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ rte_memcpy(flow_new, flow, sizeof(struct sxe2_flow));
+ TAILQ_INSERT_TAIL(sxe2_flow_list, flow_new, next);
+ flow_new->meta.flow_src_vsi =
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_DPDK][idx];
+ flow_create_cnt++;
+ }
+ }
+ if (flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx] != UINT16_MAX) {
+ if (flow_create_cnt == 0) {
+ flow->meta.flow_src_vsi =
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx];
+ flow_create_cnt++;
+ } else {
+ flow_new = rte_zmalloc("sxe2_flow", sizeof(*flow), 0);
+ if (!flow_new) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to alloc memory for flow rule");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ rte_memcpy(flow_new, flow, sizeof(struct sxe2_flow));
+ TAILQ_INSERT_TAIL(sxe2_flow_list, flow_new, next);
+ flow_new->meta.flow_src_vsi =
+ flow_src_vsi[SXE2_MAX_DRV_TYPE_KERNEL][idx];
+ flow_create_cnt++;
+ }
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_adjust_action(struct rte_eth_dev *dev __rte_unused,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error __rte_unused)
+{
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow *flow = NULL;
+ int32_t ret = 0;
+ int32_t dest_num = 0;
+ int32_t pass_num = 0;
+ int32_t mark_num = 0;
+ int32_t count_num = 0;
+ int32_t drop_num = 0;
+
+ TAILQ_FOREACH(flow, sxe2_flow_list, next) {
+ if (flow->engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ dest_num = sxe2_test_bit(SXE2_FLOW_ACTION_Q_REGION,
+ flow->action.act_types) +
+ sxe2_test_bit(SXE2_FLOW_ACTION_QUEUE,
+ flow->action.act_types);
+ pass_num = sxe2_test_bit(SXE2_FLOW_ACTION_PASSTHRU,
+ flow->action.act_types);
+ mark_num = sxe2_test_bit(SXE2_FLOW_ACTION_MARK,
+ flow->action.act_types);
+ count_num = sxe2_test_bit(SXE2_FLOW_ACTION_COUNT,
+ flow->action.act_types);
+ drop_num = sxe2_test_bit(SXE2_FLOW_ACTION_DROP,
+ flow->action.act_types);
+
+ if (dest_num) {
+ sxe2_clear_bit(SXE2_FLOW_ACTION_PASSTHRU,
+ flow->action.act_types);
+ pass_num = 0;
+ sxe2_clear_bit(SXE2_FLOW_ACTION_TO_VSI,
+ flow->action.act_types);
+ }
+
+ if (pass_num)
+ flow->action.passthru.vsi_index = flow->meta.flow_src_vsi;
+
+ if (mark_num) {
+ if (dest_num == 0) {
+ flow->action.q_region.q_index = 0;
+ flow->action.q_region.region = 7;
+ flow->action.q_region.vsi_index = flow->meta.flow_src_vsi;
+ sxe2_set_bit(SXE2_FLOW_ACTION_Q_REGION,
+ flow->action.act_types);
+ dest_num++;
+ }
+ sxe2_clear_bit(SXE2_FLOW_ACTION_PASSTHRU,
+ flow->action.act_types);
+ pass_num = 0;
+ }
+ if (count_num) {
+ if (dest_num == 0 && drop_num == 0) {
+ if (pass_num == 0) {
+ sxe2_set_bit(SXE2_FLOW_ACTION_PASSTHRU,
+ flow->action.act_types);
+ flow->action.passthru.vsi_index =
+ flow->meta.flow_src_vsi;
+ pass_num++;
+ }
+ }
+ }
+ PMD_LOG_DEBUG(DRV, "dest_num: %d, pass_num: %d, mark_num: %d, count_num: "
+ "%d, drop_num: %d", dest_num, pass_num, mark_num, count_num,
+ drop_num);
+ PMD_LOG_DEBUG(DRV, "src_vsi: %d", flow->meta.flow_src_vsi);
+ }
+ }
+
+ return ret;
+}
+
+static int32_t sxe2_flow_check_item_empty(uint8_t *item, uint16_t size)
+{
+ uint16_t i = 0;
+
+ for (i = 0; i < size; i++) {
+ if (item[i] != 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int32_t sxe2_flowlist_add_proto_type(struct rte_eth_dev *dev __rte_unused,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow *flow = TAILQ_FIRST(sxe2_flow_list);
+ struct sxe2_flow_pattern *pattern = &flow->pattern_outer;
+ int32_t ret = 0;
+
+ if (flow->engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV4, flow->flow_type) &&
+ sxe2_flow_check_item_empty((uint8_t *)&pattern->item_mask.ipv4,
+ sizeof(pattern->item_mask.ipv4)) == 0) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_TYPE, pattern->map_spec);
+ pattern->item_spec.eth.ether_type =
+ rte_cpu_to_be_16(SXE2_FLOW_ETH_TYPE_IPV4);
+ pattern->item_mask.eth.ether_type = 0xffff;
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV6, flow->flow_type) &&
+ sxe2_flow_check_item_empty((uint8_t *)&pattern->item_mask.ipv6,
+ sizeof(pattern->item_mask.ipv6)) == 0) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_TYPE, pattern->map_spec);
+ pattern->item_spec.eth.ether_type =
+ rte_cpu_to_be_16(SXE2_FLOW_ETH_TYPE_IPV6);
+ pattern->item_mask.eth.ether_type = 0xffff;
+ }
+
+ if (flow->meta.tunnel_type == SXE2_FLOW_TUNNEL_TYPE_NONE) {
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_UDP, flow->flow_type) &&
+ sxe2_flow_check_item_empty((uint8_t *)&pattern->item_mask.udp,
+ sizeof(pattern->item_mask.udp)) == 0) {
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV4,
+ flow->flow_type)) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_PROT, pattern->map_spec);
+ pattern->item_spec.ipv4.protocol =
+ SXE2_FLOW_IP_PROTOCOL_UDP;
+ pattern->item_mask.ipv4.protocol = 0xff;
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV6,
+ flow->flow_type)) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, ENOENT,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "UDP after IPv6 must has pattern item.");
+ PMD_LOG_ERR(DRV,
+ "UDP after IPv6 must has pattern item.");
+ goto l_end;
+ }
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_TCP, flow->flow_type) &&
+ sxe2_flow_check_item_empty((uint8_t *)&pattern->item_mask.tcp,
+ sizeof(pattern->item_mask.tcp)) == 0) {
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV4,
+ flow->flow_type)) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_PROT,
+ pattern->map_spec);
+ pattern->item_spec.ipv4.protocol =
+ SXE2_FLOW_IP_PROTOCOL_TCP;
+ pattern->item_mask.ipv4.protocol = 0xff;
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV6,
+ flow->flow_type)) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, ENOENT,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "TCP after IPv6 must has pattern item.");
+ PMD_LOG_ERR(DRV,
+ "TCP after IPv6 must has pattern item.");
+ goto l_end;
+ }
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_SCTP,
+ flow->flow_type)) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, ENOENT,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "SWITCH not support SCTP.");
+ PMD_LOG_ERR(DRV, "SWITCH not support SCTP.");
+ goto l_end;
+ }
+ }
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_tunnel_split_proc(struct rte_eth_dev *dev __rte_unused,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow_list_t tunnel_flow_list;
+ struct sxe2_flow *sxe2_flow_exist = NULL;
+ struct sxe2_flow *sxe2_flow_new = NULL;
+ struct sxe2_flow_pattern *pattern = NULL;
+ int32_t ret = 0;
+
+ TAILQ_INIT(&tunnel_flow_list);
+
+ TAILQ_FOREACH(sxe2_flow_exist, sxe2_flow_list, next) {
+ if (sxe2_flow_exist->engine_type != SXE2_FLOW_ENGINE_SWITCH)
+ continue;
+ if (sxe2_test_bit(SXE2_FLOW_FLD_ID_IPV4_PROT,
+ sxe2_flow_exist->pattern_outer.map_spec)) {
+ pattern = &sxe2_flow_exist->pattern_outer;
+ if ((pattern->item_spec.ipv4.protocol &
+ pattern->item_mask.ipv4.protocol) ==
+ (SXE2_FLOW_IP_PROTOCOL_GRE &
+ pattern->item_mask.ipv4.protocol)) {
+ sxe2_flow_new = rte_zmalloc("sxe2_flow",
+ sizeof(struct sxe2_flow), 0);
+ if (!sxe2_flow_new) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to alloc memory for flow rule");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ rte_memcpy(sxe2_flow_new, sxe2_flow_exist,
+ sizeof(struct sxe2_flow));
+ pattern = &sxe2_flow_new->pattern_outer;
+ sxe2_flow_new->meta.tunnel_type =
+ SXE2_FLOW_TUNNEL_TYPE_GRE;
+ sxe2_set_bit(SXE2_FLOW_HDR_GRE, pattern->hdrs);
+ pattern->item_spec.ipv4.protocol =
+ SXE2_FLOW_IP_PROTOCOL_GRE;
+ pattern->item_mask.ipv4.protocol = 0xff;
+ TAILQ_INSERT_TAIL(&tunnel_flow_list, sxe2_flow_new, next);
+ }
+ }
+ }
+ TAILQ_FOREACH(sxe2_flow_exist, &tunnel_flow_list, next)
+ TAILQ_INSERT_TAIL(sxe2_flow_list, sxe2_flow_exist, next);
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_post_proc(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+
+ ret = sxe2_flowlist_add_proto_type(dev, flow_list, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_check_function(dev, flow_list, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_meta_proc(dev, attr, flow_list, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_src_split_proc(dev, flow_list, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_adjust_action(dev, flow_list, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_tunnel_split_proc(dev, flow_list, error);
+ if (ret)
+ goto l_end;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_validate_with_flow(struct rte_eth_dev *dev,
+ struct rte_flow *flow_list,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_flow *flow = NULL;
+
+ ret = sxe2_check_para(attr, pattern, actions, error);
+ if (ret != 0)
+ goto l_end;
+
+ ret = sxe2_flow_valid_attr(attr, error);
+ if (ret != 0)
+ goto l_end;
+
+ flow = rte_zmalloc("sxe2_flow", sizeof(*flow), 0);
+ if (!flow) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to alloc memory for flow rule");
+ PMD_LOG_ERR(DRV, "Failed to alloc memory for flow rule.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ TAILQ_INSERT_TAIL(&flow_list->sxe2_flow_list, flow, next);
+ flow->create_err = -1;
+
+ ret = sxe2_flow_parse_pattern(dev, pattern, error, flow);
+ if (ret != 0)
+ goto l_end;
+
+ ret = sxe2_flow_parse_engine(dev, attr, actions, error, flow);
+ if (ret != 0)
+ goto l_end;
+
+ ret = sxe2_flow_parse_action(dev, actions, error, flow);
+ if (ret != 0)
+ goto l_end;
+
+ ret = sxe2_flow_post_proc(dev, attr, flow_list, error);
+ if (ret != 0)
+ goto l_end;
+
+ ret = sxe2_flow_check_flow_list_duplicate(dev, flow_list);
+ if (ret != 0) {
+ rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_ITEM,
+ NULL, "Duplicate flow.");
+ PMD_LOG_ERR(DRV, "Duplicate flow.");
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static const char *sxe2_flow_convert_ret_to_flow_msg(int32_t ret)
+{
+ const char *msg = NULL;
+ if (ret > 0)
+ ret = -ret;
+ switch (ret) {
+ case -ENOMEM:
+ msg = "no memory";
+ break;
+ case -ENOTSUP:
+ msg = "not support";
+ break;
+ case -EEXIST:
+ msg = "rule already exist";
+ break;
+ case -ETIMEDOUT:
+ msg = "timeout";
+ break;
+ case -EINVAL:
+ msg = "invalid parameter";
+ break;
+ case -ENOSPC:
+ msg = "no space";
+ break;
+ case -ENOENT:
+ msg = "no such rule";
+ break;
+ default:
+ msg = "unknown error";
+ break;
+ }
+ return msg;
+}
+
+static int32_t sxe2_flow_rte_list_free(struct sxe2_adapter *adapter,
+ struct rte_flow **flow_ptr,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ int32_t ret1 = 0;
+ struct rte_flow *flow = *flow_ptr;
+ struct rte_flow *flow_temp = NULL;
+ struct sxe2_flow *hw_flow = NULL;
+ struct sxe2_flow *hw_flow_temp = NULL;
+ struct sxe2_fnav_cid_mgr *mgr = NULL;
+ rte_spinlock_lock(&adapter->flow_ctxt.flow_list_lock);
+ TAILQ_FOREACH(flow_temp, &adapter->flow_ctxt.rte_flow_list, next) {
+ if (flow_temp == flow)
+ TAILQ_REMOVE(&adapter->flow_ctxt.rte_flow_list, flow, next);
+ }
+
+ TAILQ_FOREACH_SAFE(hw_flow, &flow->sxe2_flow_list, next, hw_flow_temp) {
+ if (hw_flow->create_err == 0) {
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_COUNT, hw_flow->action.act_types)) {
+ ret = sxe2_flow_query_mgr(adapter, hw_flow, &mgr, error);
+ if (ret) {
+ PMD_LOG_ERR(DRV,
+ "Failed to query flow count, flow id: %u, ret: %d.",
+ hw_flow->flow_id, ret);
+ rte_flow_error_set(error, EIO,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to query flow count");
+ ret1 = ret;
+ }
+ }
+
+ ret = sxe2_drv_flow_filter_del(adapter, hw_flow);
+ if (ret) {
+ PMD_LOG_ERR(DRV,
+ "Failed to delete flow filter, ret: %d:%s",
+ ret, sxe2_flow_convert_ret_to_flow_msg(ret));
+ rte_flow_error_set(error, EIO,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Failed to delete flow filter");
+ ret1 = ret;
+ }
+
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_COUNT, hw_flow->action.act_types)) {
+ ret = sxe2_flow_free_mgr(adapter, hw_flow,
+ &mgr, error);
+ if (ret)
+ ret1 = ret;
+ }
+ }
+
+ TAILQ_REMOVE(&flow->sxe2_flow_list, hw_flow, next);
+ rte_free(hw_flow);
+ }
+ rte_free(flow);
+ *flow_ptr = NULL;
+ rte_spinlock_unlock(&adapter->flow_ctxt.flow_list_lock);
+
+ return ret1;
+}
+
+static int32_t sxe2_flow_validate(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct rte_flow *flow_list = NULL;
+ struct sxe2_flow *hw_flow = NULL;
+ struct sxe2_flow *hw_flow_temp = NULL;
+ flow_list = rte_zmalloc("rte_flow_va", sizeof(*flow_list), 0);
+ if (!flow_list) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to alloc memory for flow rule");
+ PMD_LOG_ERR(DRV, "Failed to alloc memory for flow rule.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ TAILQ_INIT(&flow_list->sxe2_flow_list);
+
+ ret = sxe2_flow_validate_with_flow(dev, flow_list, attr, pattern, actions, error);
+ if (ret != 0)
+ goto l_free;
+l_free:
+
+ TAILQ_FOREACH_SAFE(hw_flow, &flow_list->sxe2_flow_list, next, hw_flow_temp) {
+ TAILQ_REMOVE(&flow_list->sxe2_flow_list, hw_flow, next);
+ rte_free(hw_flow);
+ }
+ rte_free(flow_list);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_isolate(struct rte_eth_dev *dev,
+ int32_t enable,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (dev->data->dev_started) {
+ rte_flow_error_set(error, EBUSY,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "port must be stopped first");
+ ret = -EBUSY;
+ goto l_end;
+ }
+
+ if (adapter->is_dev_repr) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "representor dev cannot change isolated mode ");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (enable == adapter->flow_isolated)
+ goto l_end;
+
+ if (adapter->dev_type == SXE2_DEV_T_VF &&
+ adapter->switchdev_info.is_switchdev) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "isolated mode cannot be change when port in switch dev mode");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ rte_spinlock_lock(&adapter->flow_ctxt.flow_list_lock);
+ if (!TAILQ_EMPTY(&adapter->flow_ctxt.rte_flow_list))
+ PMD_DEV_LOG_WARN(adapter, DRV,
+ "The configured flow item may not take effect.");
+ rte_spinlock_unlock(&adapter->flow_ctxt.flow_list_lock);
+
+ adapter->flow_isolated = !!enable;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ ret = sxe2_switchdev_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update switchdev rule");
+
+l_end:
+ if (ret == 0)
+ adapter->flow_isolate_cfg = !!enable;
+ return ret;
+}
+
+static struct rte_flow *sxe2_flow_create(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item pattern[],
+ const struct rte_flow_action action[],
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_flow *flow_list = NULL;
+ struct sxe2_flow *flow = NULL;
+
+ flow_list = rte_zmalloc("sxe2_flow_create", sizeof(*flow_list), 0);
+ if (!flow_list) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to alloc memory for flow rule");
+ PMD_LOG_ERR(DRV, "Failed to alloc memory for flow rule.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ TAILQ_INIT(&flow_list->sxe2_flow_list);
+
+ ret = sxe2_flow_validate_with_flow(dev, flow_list, attr, pattern, action, error);
+ if (ret != 0)
+ goto l_free_flow;
+
+ TAILQ_FOREACH(flow, &flow_list->sxe2_flow_list, next) {
+ ret = sxe2_fnav_get_filter_cid(adapter, flow);
+ if (ret != 0) {
+ PMD_LOG_ERR(DRV, "fnav get stats id failed, ret:%d", ret);
+ rte_flow_error_set(error, EIO,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to add fnav rule:alloc cid failed.");
+ goto l_free_flow;
+ }
+ ret = sxe2_drv_flow_filter_add(adapter, flow);
+ if (ret) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, NULL,
+ "Failed to add flow filter to hw.");
+ PMD_LOG_ERR(DRV, "Failed to add flow filter to hw.ret:%d:%s",
+ ret, sxe2_flow_convert_ret_to_flow_msg(ret));
+ goto l_free_flow;
+ }
+ }
+
+ TAILQ_INSERT_TAIL(&adapter->flow_ctxt.rte_flow_list, flow_list, next);
+ goto l_end;
+l_free_flow:
+ (void)sxe2_flow_rte_list_free(adapter, &flow_list, error);
+l_end:
+ return flow_list;
+}
+
+static int32_t sxe2_flow_destroy(struct rte_eth_dev *dev,
+ struct rte_flow *flow,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ ret = sxe2_flow_rte_list_free(adapter, &flow, error);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Failed to destroy flow.ret:%d.", ret);
+ return ret;
+}
+
+static int32_t sxe2_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_flow *flow = NULL;
+ struct rte_flow *tmp_flow = NULL;
+ struct rte_flow_list_t *flow_list = &adapter->flow_ctxt.rte_flow_list;
+ TAILQ_FOREACH_SAFE(flow, flow_list, next, tmp_flow) {
+ ret = sxe2_flow_destroy(dev, flow, error);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to flush flows.ret:%d.", ret);
+
+ if (ret != -EAGAIN)
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_fnav_get_filter_cid(struct sxe2_adapter *adapter, struct sxe2_flow *flow)
+{
+ int32_t ret = 0;
+ struct sxe2_fnav_cid_mgr_list_t *cid_mgr_list =
+ &adapter->flow_ctxt.hw_res.fnav_cid_mgr_list;
+ uint32_t stat_index;
+ uint32_t user_id;
+ uint32_t driver_id;
+ struct sxe2_fnav_cid_mgr *temp = NULL;
+ struct sxe2_fnav_cid_mgr *mgr = NULL;
+
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_COUNT, flow->action.act_types)) {
+ user_id = flow->action.count.user_id;
+ driver_id = flow->action.count.driver_id;
+
+ TAILQ_FOREACH(temp, cid_mgr_list, next) {
+ if (temp->user_id == user_id &&
+ temp->driver_id == driver_id) {
+ mgr = temp;
+ break;
+ }
+ }
+ if (mgr == NULL) {
+ mgr = rte_zmalloc("sxe2_fnav_cid_mgr",
+ sizeof(struct sxe2_fnav_cid_mgr), 0);
+ if (!mgr) {
+ PMD_LOG_ERR(DRV,
+ "Failed to alloc sxe2vf_fnav_cid_mgr memory.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ ret = sxe2_drv_flow_fnav_get_stat_id(adapter, &stat_index);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to alloc fw count id.");
+ rte_free(mgr);
+ goto l_end;
+ }
+
+ TAILQ_INSERT_TAIL(cid_mgr_list, mgr, next);
+ mgr->user_id = user_id;
+ mgr->driver_id = driver_id;
+ mgr->stat_index = stat_index;
+ mgr->count_type = adapter->flow_ctxt.hw_res.count_type;
+ }
+ flow->action.count.stat_index = mgr->stat_index;
+ flow->action.count.stat_ctrl = mgr->count_type;
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_flow_free_mgr(struct sxe2_adapter *adapter,
+ struct sxe2_flow *flow,
+ struct sxe2_fnav_cid_mgr **mgr_ptr,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_fnav_cid_mgr_list_t *cid_mgr_list =
+ &adapter->flow_ctxt.hw_res.fnav_cid_mgr_list;
+ struct sxe2_fnav_cid_mgr *mgr = *mgr_ptr;
+ uint32_t user_id = flow->action.count.user_id;
+ if (user_id == 0) {
+ TAILQ_REMOVE(cid_mgr_list, mgr, next);
+ ret = sxe2_drv_flow_fnav_free_stat(adapter, mgr->stat_index);
+ if (ret) {
+ rte_flow_error_set(error, EIO,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to free flow count.");
+ PMD_LOG_ERR(DRV,
+ "Failed to free flow count, flow id: %u, ret: %d.",
+ flow->flow_id, ret);
+ }
+ rte_free(mgr);
+ *mgr_ptr = NULL;
+ }
+ return ret;
+}
+
+int32_t sxe2_flow_query_mgr(struct sxe2_adapter *adapter,
+ struct sxe2_flow *flow,
+ struct sxe2_fnav_cid_mgr **mgr_ptr,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_fnav_cid_mgr_list_t *cid_mgr_list =
+ &adapter->flow_ctxt.hw_res.fnav_cid_mgr_list;
+ struct sxe2_fnav_cid_mgr *temp = NULL;
+ struct sxe2_fnav_cid_mgr *mgr = NULL;
+ uint32_t user_id = flow->action.count.user_id;
+ uint32_t driver_id = flow->action.count.driver_id;
+
+ TAILQ_FOREACH(temp, cid_mgr_list, next) {
+ if (temp->user_id == user_id &&
+ temp->driver_id == driver_id) {
+ mgr = temp;
+ break;
+ }
+ }
+ if (!mgr) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "fnav flow query count invalid user_id or driver_id.");
+ PMD_LOG_ERR(DRV,
+ "fnav flow query count invalid user_id or driver_id.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ ret = sxe2_drv_flow_fnav_query_stat(adapter, mgr);
+ if (ret) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, NULL,
+ "Failed to query flow count.");
+ PMD_LOG_ERR(DRV,
+ "Failed to query flow count, flow id: %u, ret: %d.",
+ flow->flow_id, ret);
+ goto l_end;
+ }
+ *mgr_ptr = mgr;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_query_count(struct sxe2_adapter *adapter,
+ struct sxe2_flow *flow,
+ struct rte_flow_query_count *count,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_fnav_cid_mgr *mgr = NULL;
+ switch (flow->action.count.stat_ctrl) {
+ case SXE2_FNAV_STAT_ENA_NONE:
+ count->hits_set = 0;
+ count->bytes_set = 0;
+ break;
+ case SXE2_FNAV_STAT_ENA_PKTS:
+ count->hits_set = 1;
+ count->bytes_set = 0;
+ break;
+ case SXE2_FNAV_STAT_ENA_BYTES:
+ count->hits_set = 0;
+ count->bytes_set = 1;
+ break;
+ case SXE2_FNAV_STAT_ENA_ALL:
+ count->hits_set = 1;
+ count->bytes_set = 1;
+ break;
+ default:
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (!sxe2_test_bit(SXE2_FLOW_ACTION_COUNT, flow->action.act_types)) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "this flow don't have count action.");
+ PMD_LOG_ERR(DRV, "this flow don't have count action.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ ret = sxe2_flow_query_mgr(adapter, flow, &mgr, error);
+ if (ret) {
+ PMD_LOG_ERR(DRV,
+ "Failed to query flow count, flow id: %u, ret: %d.",
+ flow->flow_id, ret);
+ goto l_end;
+ }
+ count->hits = mgr->hits;
+ count->bytes = mgr->bytes;
+ if (count->reset) {
+ mgr->hits = 0;
+ mgr->bytes = 0;
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_query(struct rte_eth_dev *dev,
+ struct rte_flow *flow_list,
+ const struct rte_flow_action *actions,
+ void *data,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct rte_flow_query_count *count = data;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_flow *flow = NULL;
+
+ if (!flow_list) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
+ NULL, "Invalid flow");
+ PMD_LOG_ERR(DRV, "Invalid flow to query flow.");
+ goto l_end;
+ }
+
+ rte_spinlock_lock(&adapter->flow_ctxt.flow_list_lock);
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_VOID:
+ break;
+ case RTE_FLOW_ACTION_TYPE_COUNT:
+ flow = TAILQ_FIRST(&flow_list->sxe2_flow_list);
+ ret = sxe2_flow_query_count(adapter, flow, count, error);
+ if (ret) {
+ PMD_LOG_ERR(DRV,
+ "Failed to query flow count, flow id: %u, ret: %d.",
+ flow->flow_id, ret);
+ goto l_end_unlock;
+ }
+ break;
+ default:
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "action not supported");
+ PMD_LOG_ERR(DRV,
+ "Failed to query flow action type:%d.",
+ actions->type);
+ ret = -ENOTSUP;
+ goto l_end_unlock;
+ }
+ }
+
+l_end_unlock:
+ rte_spinlock_unlock(&adapter->flow_ctxt.flow_list_lock);
+
+l_end:
+ return ret;
+}
+
+const struct rte_flow_ops sxe2_flow_ops = {
+ .validate = sxe2_flow_validate,
+ .create = sxe2_flow_create,
+ .destroy = sxe2_flow_destroy,
+ .flush = sxe2_flow_flush,
+ .query = sxe2_flow_query,
+ .isolate = sxe2_flow_isolate,
+};
+
+int32_t sxe2_flow_ops_get(struct rte_eth_dev *dev, const struct rte_flow_ops **ops)
+{
+ int32_t ret = 0;
+
+ if (dev == NULL) {
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ *ops = &sxe2_flow_ops;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_flow_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ TAILQ_INIT(&adapter->flow_ctxt.rte_flow_list);
+ TAILQ_INIT(&adapter->flow_ctxt.hw_res.fnav_cid_mgr_list);
+ if (adapter->devargs.fnav_stat_type)
+ adapter->flow_ctxt.hw_res.count_type =
+ adapter->devargs.fnav_stat_type;
+ else
+ adapter->flow_ctxt.hw_res.count_type = SXE2_FNAV_STAT_ENA_ALL;
+
+ adapter->flow_ctxt.fnav_inited = 1;
+ rte_spinlock_init(&adapter->flow_ctxt.flow_list_lock);
+ return ret;
+}
+
+int32_t sxe2_flow_uninit(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_flow_error error;
+ struct sxe2_fnav_cid_mgr *mgr = NULL;
+ struct sxe2_fnav_cid_mgr *temp = NULL;
+ struct sxe2_fnav_cid_mgr_list_t *cid_mgr_list =
+ &adapter->flow_ctxt.hw_res.fnav_cid_mgr_list;
+
+ ret = sxe2_flow_flush(dev, &error);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Failed to flush flow, ret: %d.", ret);
+
+ TAILQ_FOREACH_SAFE(mgr, cid_mgr_list, next, temp) {
+ TAILQ_REMOVE(cid_mgr_list, mgr, next);
+ ret = sxe2_drv_flow_fnav_free_stat(adapter, mgr->stat_index);
+ if (ret)
+ PMD_LOG_ERR(DRV,
+ "Failed to free fnav stat id, ret: %d.", ret);
+ rte_free(mgr);
+ }
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_flow.h b/drivers/net/sxe2/sxe2_flow.h
new file mode 100644
index 0000000000..9970fddcf0
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FLOW_H__
+#define __SXE2_FLOW_H__
+#include <rte_flow_driver.h>
+#include "sxe2_osal.h"
+#include "sxe2_common.h"
+
+
+int32_t sxe2_flow_ops_get(struct rte_eth_dev *dev, const struct rte_flow_ops **ops);
+
+int32_t sxe2_flow_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_flow_uninit(struct rte_eth_dev *dev);
+
+int32_t sxe2_fnav_get_filter_cid(struct sxe2_adapter *adapter, struct sxe2_flow *flow);
+
+int32_t sxe2_flow_free_mgr(struct sxe2_adapter *adapter,
+ struct sxe2_flow *flow,
+ struct sxe2_fnav_cid_mgr **mgr_ptr,
+ struct rte_flow_error *error);
+
+int32_t sxe2_flow_query_mgr(struct sxe2_adapter *adapter,
+ struct sxe2_flow *flow,
+ struct sxe2_fnav_cid_mgr **mgr_ptr,
+ struct rte_flow_error *error);
+#endif /* __SXE2_FLOW_H__ */
diff --git a/drivers/net/sxe2/sxe2_flow_parse_action.c b/drivers/net/sxe2/sxe2_flow_parse_action.c
new file mode 100644
index 0000000000..a9559e2d7e
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_parse_action.c
@@ -0,0 +1,1182 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include "sxe2_flow_parse_action.h"
+#include "sxe2_common_log.h"
+#include "sxe2_flow_public.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_vsi.h"
+
+
+static int32_t sxe2_flow_check_rss_action_attr(const struct rte_flow_action_rss *rss,
+ struct rte_flow_error *error)
+{
+ int32_t ret = ENOTSUP;
+ switch (rss->func) {
+ case RTE_ETH_HASH_FUNCTION_DEFAULT:
+ case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
+ case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ:
+ case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR:
+ break;
+ default:
+ PMD_LOG_ERR(DRV, "RSS hash function[%d] not support.", rss->func);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (rss->level > 2)
+ rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "RSS level is could not be greater than 2");
+ if (rss->key_len)
+ rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "a nonzero RSS key_len is not supported");
+ if (rss->queue_num)
+ rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "a non-NULL RSS queue is not supported");
+ ret = 0;
+l_end:
+ return ret;
+}
+
+
+static int32_t sxe2_flow_set_rss_action_func(enum rte_eth_hash_function rss_func,
+ uint64_t rss_type,
+ struct sxe2_flow *flow,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ if (rss_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
+ if (flow->has_hdr) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, -EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to cfg Simple XOR hash with not empty pattern");
+ PMD_LOG_ERR(DRV, "Failed to cfg Simple XOR hash with not empty pattern.");
+ goto l_end;
+ }
+ } else {
+ if (!flow->has_hdr) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, -EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to cfg Simple hash with empty pattern");
+ PMD_LOG_ERR(DRV, "Failed to cfg Simple hash with empty pattern.");
+ goto l_end;
+ }
+ }
+
+ if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
+ if (rss_type & (RTE_ETH_RSS_L3_SRC_ONLY |
+ RTE_ETH_RSS_L3_DST_ONLY |
+ RTE_ETH_RSS_L4_SRC_ONLY |
+ RTE_ETH_RSS_L4_DST_ONLY)) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, -EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to cfg symm func rss_type l3/l4 only.");
+ PMD_LOG_ERR(DRV, "Failed to cfg symm func rss_type l3/l4 only.");
+ goto l_end;
+ }
+
+ if (!(rss_type & (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_IPV6 |
+ RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_FRAG_IPV6 |
+ RTE_ETH_RSS_NONFRAG_IPV4_UDP |
+ RTE_ETH_RSS_NONFRAG_IPV6_UDP |
+ RTE_ETH_RSS_NONFRAG_IPV4_TCP |
+ RTE_ETH_RSS_NONFRAG_IPV6_TCP |
+ RTE_ETH_RSS_NONFRAG_IPV4_SCTP |
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP))) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, -EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to cfg symm func unsupported rss_type.");
+ PMD_LOG_ERR(DRV, "Failed to cfg symm func unsupported rss_type.");
+ goto l_end;
+ }
+ flow->action.rss.func = SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ;
+ }
+ if (rss_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
+ flow->action.rss.func = SXE2_RSS_HASH_FUNC_XOR;
+ if (rss_func == RTE_ETH_HASH_FUNCTION_DEFAULT)
+ flow->action.rss.func = SXE2_RSS_HASH_FUNC_TOEPLITZ;
+ if (rss_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+ flow->action.rss.func = SXE2_RSS_HASH_FUNC_TOEPLITZ;
+l_end:
+ return ret;
+}
+
+
+static uint64_t sxe2_hash_invalid_comb[] = {
+ RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_UDP,
+ RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_TCP,
+ RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_SCTP,
+ RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_UDP,
+ RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_TCP,
+ RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_SCTP,
+ RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
+ RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
+ RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE64,
+};
+
+struct sxe2_rss_attr_type {
+ uint64_t attr;
+ uint64_t type;
+};
+
+static struct sxe2_rss_attr_type sxe2_rss_attr_valid_type[] = {
+ {RTE_ETH_RSS_L2_SRC_ONLY | RTE_ETH_RSS_L2_DST_ONLY, RTE_ETH_RSS_ETH},
+ {RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY, SXE2_VALID_RSS_L3},
+ {RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY, SXE2_VALID_RSS_L4},
+
+ {RTE_ETH_RSS_L3_PRE32, SXE2_VALID_RSS_IPV6},
+ {RTE_ETH_RSS_L3_PRE48, SXE2_VALID_RSS_IPV6},
+ {RTE_ETH_RSS_L3_PRE64, SXE2_VALID_RSS_IPV6},
+ {SXE2_INVALID_RSS_ATTR, 0}
+};
+
+
+static void sxe2_flow_action_pre(struct sxe2_flow *flow)
+{
+ flow->action.vsi.vsi_index = UINT16_MAX;
+ flow->action.vsi_list.vsi_cnt = 0;
+ sxe2_bitmap_zero(flow->action.vsi_list.vsi_list_map, SXE2_VSI_MAX);
+}
+
+static void sxe2_flow_action_post(struct sxe2_flow *flow, uint8_t action_num[])
+{
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_TO_VSI, flow->action.act_types))
+ action_num[SXE2_FLOW_ACTION_TO_VSI] = 1;
+
+ if (sxe2_test_bit(SXE2_FLOW_ACTION_TO_VSI_LIST, flow->action.act_types))
+ action_num[SXE2_FLOW_ACTION_TO_VSI_LIST] = 1;
+}
+
+
+static void sxe2_flow_action_vsi_merge(struct sxe2_flow *flow, uint16_t add_vsi_id)
+{
+ if (flow->action.vsi_list.vsi_cnt == 0) {
+ if (flow->action.vsi.vsi_index == UINT16_MAX) {
+ flow->action.vsi.vsi_index = add_vsi_id;
+ sxe2_set_bit(SXE2_FLOW_ACTION_TO_VSI, flow->action.act_types);
+ goto l_end;
+ }
+
+ if (flow->action.vsi.vsi_index == add_vsi_id)
+ goto l_end;
+
+ sxe2_set_bit(flow->action.vsi.vsi_index, flow->action.vsi_list.vsi_list_map);
+ sxe2_set_bit(add_vsi_id, flow->action.vsi_list.vsi_list_map);
+ flow->action.vsi_list.vsi_cnt = 2;
+ flow->action.vsi.vsi_index = UINT16_MAX;
+ sxe2_clear_bit(SXE2_FLOW_ACTION_TO_VSI, flow->action.act_types);
+ sxe2_set_bit(SXE2_FLOW_ACTION_TO_VSI_LIST, flow->action.act_types);
+ }
+
+ if (sxe2_test_bit(add_vsi_id, flow->action.vsi_list.vsi_list_map))
+ goto l_end;
+
+ sxe2_set_bit(add_vsi_id, flow->action.vsi_list.vsi_list_map);
+ flow->action.vsi_list.vsi_cnt++;
+
+l_end:
+ return;
+}
+
+
+static int32_t sxe2_flow_vsi_get_ethdev(struct rte_eth_dev *dev,
+ uint16_t dev_port_id, uint16_t *vsi_index)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_eth_dev *dst_dev;
+ struct sxe2_adapter *dst_adapter;
+ int32_t ret = 0;
+
+ dst_dev = &rte_eth_devices[dev_port_id];
+ if (!dst_dev->data) {
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (!sxe2_ethdev_check(dst_dev)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "dst dev is not sxe2 ethdev.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ dst_adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dst_dev);
+ if (!dst_adapter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "dst dev adapter is null.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (adapter->dev_info.pci.serial_number != dst_adapter->dev_info.pci.serial_number) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "dst dev sn is miss match current dev.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (adapter->dev_type != SXE2_DEV_T_PF_BOND) {
+ if (adapter->pf_idx != dst_adapter->pf_idx) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "dst dev pf id is miss match current dev.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+
+ if (dst_adapter->is_dev_repr) {
+ if (dst_adapter->repr_priv_data == NULL) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "dst dev repr data is null.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ *vsi_index = dst_adapter->repr_priv_data->repr_vf_vsi_id;
+ } else {
+ *vsi_index = dst_adapter->vsi_ctxt.dpdk_vsi_id;
+ }
+
+l_end:
+ return ret;
+}
+static int32_t sxe2_flow_check_rss_action_type_with_pattern(struct sxe2_flow_pattern *pattern,
+ uint64_t rss_type)
+{
+ uint64_t rss_type_allow = pattern->rss_type_allow;
+ int32_t ret = -EINVAL;
+
+ if ((rss_type & rss_type_allow) != rss_type)
+ goto l_end;
+
+ if (sxe2_test_bit(SXE2_FLOW_HDR_VLAN, pattern->hdrs) &&
+ !sxe2_test_bit(SXE2_FLOW_HDR_QINQ, pattern->hdrs)) {
+ if ((rss_type & RTE_ETH_RSS_C_VLAN) != 0 &&
+ (rss_type & RTE_ETH_RSS_S_VLAN) == 0)
+ goto l_end;
+ }
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_check_rss_action_type_valid(uint64_t rss_type)
+{
+ struct sxe2_rss_attr_type *attr_type;
+ uint32_t i;
+ int32_t ret = -EINVAL;
+
+ for (i = 0; i < RTE_DIM(sxe2_hash_invalid_comb); i++) {
+ if (rte_popcount64(rss_type & sxe2_hash_invalid_comb[i]) > 1) {
+ PMD_LOG_ERR(DRV, "Error rss_type invalid comb[%d].", i);
+ goto l_end;
+ }
+ }
+
+ for (i = 0; i < RTE_DIM(sxe2_rss_attr_valid_type); i++) {
+ attr_type = &sxe2_rss_attr_valid_type[i];
+ if ((attr_type->attr & rss_type) &&
+ !(attr_type->type & rss_type)) {
+ PMD_LOG_ERR(DRV, "Rss_type valid_comb[%d] check error.", i);
+ goto l_end;
+ }
+ }
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+
+static void sxe2_flow_set_rss_action_type_l234(BITMAP_TYPE *hdr,
+ BITMAP_TYPE *fld,
+ uint64_t rss_type)
+{
+ if (rss_type & RTE_ETH_RSS_ETH) {
+ if (rss_type & RTE_ETH_RSS_L2_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_SA, fld);
+ } else if (rss_type & RTE_ETH_RSS_L2_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_DA, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_SA, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_DA, fld);
+ }
+ }
+ if (rss_type & RTE_ETH_RSS_L2_PAYLOAD) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_TYPE, fld);
+ sxe2_set_bit(SXE2_FLOW_HDR_ETH_NON_IP, hdr);
+ }
+
+ if (sxe2_test_bit(SXE2_FLOW_HDR_VLAN, hdr)) {
+ if (rss_type & RTE_ETH_RSS_S_VLAN)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_S_TCI, fld);
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_QINQ, hdr)) {
+ if (rss_type & RTE_ETH_RSS_C_VLAN)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_C_TCI, fld);
+ }
+
+ if (rss_type & (RTE_ETH_RSS_IPV4 |
+ RTE_ETH_RSS_NONFRAG_IPV4_OTHER |
+ RTE_ETH_RSS_NONFRAG_IPV4_UDP |
+ RTE_ETH_RSS_NONFRAG_IPV4_TCP |
+ RTE_ETH_RSS_NONFRAG_IPV4_SCTP |
+ RTE_ETH_RSS_FRAG_IPV4 |
+ RTE_ETH_RSS_IPV4_CHKSUM)) {
+ if (rss_type & RTE_ETH_RSS_L3_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_SA, fld);
+ } else if (rss_type & RTE_ETH_RSS_L3_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_DA, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_SA, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_DA, fld);
+ }
+
+ if (rss_type & RTE_ETH_RSS_NONFRAG_IPV4_OTHER)
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_OTHER, hdr);
+ if (rss_type & RTE_ETH_RSS_FRAG_IPV4) {
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_FRAG, hdr);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_ID, fld);
+ }
+
+ if (rss_type & RTE_ETH_RSS_IPV4_CHKSUM)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_CHKSUM, fld);
+ }
+
+ if (rss_type & (RTE_ETH_RSS_IPV6 |
+ RTE_ETH_RSS_FRAG_IPV6 |
+ RTE_ETH_RSS_NONFRAG_IPV6_OTHER |
+ RTE_ETH_RSS_NONFRAG_IPV6_UDP |
+ RTE_ETH_RSS_NONFRAG_IPV6_TCP |
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP)) {
+ if (rss_type & RTE_ETH_RSS_L3_PRE32) {
+ if (rss_type & RTE_ETH_RSS_L3_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE32_SA, fld);
+ } else if (rss_type & RTE_ETH_RSS_L3_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE32_DA, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE32_SA, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE32_DA, fld);
+ }
+ } else if (rss_type & RTE_ETH_RSS_L3_PRE48) {
+ if (rss_type & RTE_ETH_RSS_L3_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE48_SA, fld);
+ } else if (rss_type & RTE_ETH_RSS_L3_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE48_DA, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE48_SA, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE48_DA, fld);
+ }
+ } else if (rss_type & RTE_ETH_RSS_L3_PRE64) {
+ if (rss_type & RTE_ETH_RSS_L3_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE64_SA, fld);
+ } else if (rss_type & RTE_ETH_RSS_L3_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE64_DA, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE64_SA, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PRE64_DA, fld);
+ }
+ } else {
+ if (rss_type & RTE_ETH_RSS_L3_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_SA, fld);
+ } else if (rss_type & RTE_ETH_RSS_L3_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_DA, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_SA, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_DA, fld);
+ }
+ }
+ if (rss_type & RTE_ETH_RSS_FRAG_IPV6) {
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_FRAG, hdr);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_ID, fld);
+ }
+ if (rss_type & RTE_ETH_RSS_NONFRAG_IPV6_OTHER)
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_OTHER, hdr);
+ }
+
+ if (rss_type & (RTE_ETH_RSS_NONFRAG_IPV4_TCP |
+ RTE_ETH_RSS_NONFRAG_IPV6_TCP)) {
+ if (rss_type & RTE_ETH_RSS_L4_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_SRC_PORT, fld);
+ } else if (rss_type & RTE_ETH_RSS_L4_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_DST_PORT, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_SRC_PORT, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_DST_PORT, fld);
+ }
+ if (rss_type & RTE_ETH_RSS_L4_CHKSUM)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_CHKSUM, fld);
+ }
+
+ if (rss_type & (RTE_ETH_RSS_NONFRAG_IPV4_UDP |
+ RTE_ETH_RSS_NONFRAG_IPV6_UDP)) {
+ if (rss_type & RTE_ETH_RSS_L4_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_SRC_PORT, fld);
+ } else if (rss_type & RTE_ETH_RSS_L4_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_DST_PORT, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_SRC_PORT, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_DST_PORT, fld);
+ }
+ if (rss_type & RTE_ETH_RSS_L4_CHKSUM)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_CHKSUM, fld);
+ }
+
+ if (rss_type & (RTE_ETH_RSS_NONFRAG_IPV4_SCTP |
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP)) {
+ if (rss_type & RTE_ETH_RSS_L4_SRC_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_SRC_PORT, fld);
+ } else if (rss_type & RTE_ETH_RSS_L4_DST_ONLY) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_DST_PORT, fld);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_SRC_PORT, fld);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_DST_PORT, fld);
+ }
+ if (rss_type & RTE_ETH_RSS_L4_CHKSUM)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_CHKSUM, fld);
+ }
+}
+
+
+static int32_t sxe2_flow_set_rss_action_hdr_type(struct sxe2_flow *flow,
+ bool is_inner)
+{
+ struct sxe2_flow_action_rss *rss = &flow->action.rss;
+ BITMAP_TYPE *hdr = rss->hdr_out;
+ int32_t ret = 0;
+
+ rss->hdr_type = SXE2_RSS_ANY_HEADERS;
+ if (!is_inner) {
+ rss->hdr_type = SXE2_RSS_OUTER_HEADERS;
+ goto l_end;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV4, hdr)) {
+ if (sxe2_test_bit(SXE2_FLOW_HDR_UDP, hdr)) {
+ if (sxe2_test_bit(SXE2_FLOW_HDR_GRE, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GRE;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_GENEVE, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GENEVE;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_VXLAN, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_VXLAN;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_GTPU, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GTPU;
+ }
+ } else {
+ if (sxe2_test_bit(SXE2_FLOW_HDR_GRE, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_GRE;
+ } else {
+ rss->hdr_type = SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4;
+ }
+ }
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_IPV6, hdr)) {
+ if (sxe2_test_bit(SXE2_FLOW_HDR_UDP, hdr)) {
+ if (sxe2_test_bit(SXE2_FLOW_HDR_GRE, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GRE;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_GENEVE, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GENEVE;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_VXLAN, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_VXLAN;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_GTPU, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GTPU;
+ }
+ } else {
+ if (sxe2_test_bit(SXE2_FLOW_HDR_GRE, hdr)) {
+ rss->hdr_type =
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_GRE;
+ } else {
+ rss->hdr_type = SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6;
+ }
+ }
+ }
+
+l_end:
+ if (rss->hdr_type == SXE2_RSS_ANY_HEADERS) {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "Unsupported rss hdr type.");
+ }
+ return ret;
+}
+
+static int32_t sxe2_flow_set_rss_action_level(uint32_t level,
+ struct sxe2_flow *flow, struct rte_flow_error *error)
+{
+ bool is_inner = false;
+ struct sxe2_flow_action_rss *rss = &flow->action.rss;
+ int32_t ret = 0;
+
+ if (flow->meta.tunnel_type != SXE2_FLOW_TUNNEL_TYPE_NONE) {
+ if (level == 0 || level == 2)
+ is_inner = true;
+ else if (level == 1)
+ is_inner = false;
+ } else {
+ if (level == 2) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "RSS hash level 2 is not allowed no tunnel flow.");
+ PMD_LOG_ERR(DRV, "RSS hash level 2 is not allowed no tunnel flow.");
+ goto l_end;
+ }
+ is_inner = false;
+ }
+ rss->is_inner = is_inner;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_set_rss_action_type(uint64_t rss_type,
+ struct sxe2_flow *flow, struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_flow_pattern *pattern = NULL;
+ struct sxe2_flow_action_rss *rss = &flow->action.rss;
+ BITMAP_TYPE *hdr;
+ BITMAP_TYPE *fld;
+ bool is_inner = rss->is_inner;
+
+ ret = sxe2_flow_check_rss_action_type_valid(rss_type);
+ if (ret) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "RSS hash type has invalid combination.");
+ PMD_LOG_ERR(DRV, "RSS hash type has invalid combination.");
+ goto l_end;
+ }
+
+ pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ ret = sxe2_flow_check_rss_action_type_with_pattern(pattern,
+ rss_type);
+ if (ret) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "RSS hash type is not allowed by pattern.");
+ PMD_LOG_ERR(DRV, "RSS hash type is not allowed by pattern.");
+ goto l_end;
+ }
+
+ sxe2_bitmap_copy(rss->hdr_out, flow->pattern_outer.hdrs,
+ SXE2_FLOW_HDR_MAX);
+ sxe2_bitmap_copy(rss->hdr_in, flow->pattern_inner.hdrs,
+ SXE2_FLOW_HDR_MAX);
+ hdr = is_inner ? rss->hdr_in : rss->hdr_out;
+ fld = rss->fld;
+ sxe2_flow_set_rss_action_type_l234(hdr, fld, rss_type);
+
+ ret = sxe2_flow_set_rss_action_hdr_type(flow, is_inner);
+ if (ret) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Unsupported rss hdr type.");
+ PMD_LOG_ERR(DRV, "Unsupported rss hdr type.");
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_action_rss(const struct rte_flow_action *action,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ const struct rte_flow_action_rss *rss = action->conf;
+ int32_t ret = 0;
+ uint64_t rss_type = rss->types;
+ enum rte_eth_hash_function rss_func = rss->func;
+ uint32_t level = rss->level;
+
+ rss_type = rte_eth_rss_hf_refine(rss_type);
+
+ ret = sxe2_flow_check_rss_action_attr(rss, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_set_rss_action_func(rss_func, rss_type, flow, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_set_rss_action_level(level, flow, error);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_flow_set_rss_action_type(rss_type, flow, error);
+ if (ret)
+ goto l_end;
+ sxe2_set_bit(SXE2_FLOW_ACTION_RSS, flow->action.act_types);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_action_qregion(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error, struct sxe2_flow *flow)
+{
+ int32_t ret = 0;
+ uint8_t i = 0;
+ const struct rte_flow_action_rss *rss = action->conf;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (rss->types != 0 || rss->key_len != 0) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Queue region not support rss types or key.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (rss->queue_num <= 1) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Queue region size can't be 0 or 1.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < rss->queue_num - 1; i++) {
+ if (rss->queue[i + 1] != rss->queue[i] + 1) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Queue index for queue region is not continuous.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ if (rss->queue[rss->queue_num - 1] >= adapter->dev_info.dev_data->nb_rx_queues) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Queue index for queue region is out of range.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (!(rte_is_power_of_2(rss->queue_num))) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Queue region size must be power of 2.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ flow->action.q_region.vsi_index = adapter->vsi_ctxt.dpdk_vsi_id;
+ flow->action.q_region.q_index = rss->queue[0];
+ flow->action.q_region.region = (uint8_t)rte_log2_u32(rss->queue_num);
+ sxe2_set_bit(SXE2_FLOW_ACTION_Q_REGION, flow->action.act_types);
+l_end:
+ return ret;
+}
+
+
+static int32_t sxe2_flow_parse_action_queue(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ int32_t ret = 0;
+ const struct rte_flow_action_queue *queue = action->conf;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (queue->index >= adapter->dev_info.dev_data->nb_rx_queues) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Invalid queue index.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ flow->action.queue.vsi_index = adapter->vsi_ctxt.dpdk_vsi_id;
+ flow->action.queue.q_index = queue->index;
+ sxe2_set_bit(SXE2_FLOW_ACTION_QUEUE, flow->action.act_types);
+l_end:
+ return ret;
+}
+
+
+static int32_t sxe2_flow_parse_action_represented_port(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ const struct rte_flow_action_ethdev *action_ethdev_conf;
+ const struct rte_eth_dev *dst_repr_dev;
+ uint16_t dst_repr_vsi_id;
+ uint16_t dst_backer_port_id;
+ uint16_t src_backer_port_id;
+ int32_t ret = 0;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Represented port action only support in switchdev mode.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (adapter->dev_type == SXE2_DEV_T_VF) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to cfg vf dev type.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ action_ethdev_conf = action->conf;
+ if (!rte_eth_dev_is_valid_port(action_ethdev_conf->port_id)) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Invalid port for represented port action.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ dst_repr_dev = &rte_eth_devices[action_ethdev_conf->port_id];
+ if (!dst_repr_dev->data) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Invalid port for represented port action.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ dst_backer_port_id = dst_repr_dev->data->backer_port_id;
+ if (adapter->is_dev_repr)
+ src_backer_port_id = dev->data->backer_port_id;
+ else
+ src_backer_port_id = adapter->dev_port_id;
+
+ if (src_backer_port_id != dst_backer_port_id) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Represented port action only support to cfg port in same device.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_flow_vsi_get_ethdev(dev, action_ethdev_conf->port_id, &dst_repr_vsi_id);
+ if (ret != 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Port representor action port dev invalid.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ sxe2_flow_action_vsi_merge(flow, dst_repr_vsi_id);
+l_end:
+ return ret;
+}
+
+
+static int32_t sxe2_flow_parse_action_port_representor(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ const struct rte_flow_action_ethdev *action_ethdev_conf;
+ uint16_t dst_vsi_id;
+ int32_t ret = 0;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Port representor action only support in switchdev mode.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (adapter->dev_type == SXE2_DEV_T_VF) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Cfg rule dev type is vf.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (!adapter->is_dev_repr) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Cfg rule dev type is not repr.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ action_ethdev_conf = action->conf;
+ if (!action_ethdev_conf || !rte_eth_dev_is_valid_port(action_ethdev_conf->port_id)) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Invalid port for port representor action.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (dev->data->backer_port_id != action_ethdev_conf->port_id) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Invalid port for port representor.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_flow_vsi_get_ethdev(dev, action_ethdev_conf->port_id, &dst_vsi_id);
+ if (ret != 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Port representor action port dev invalid.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ sxe2_flow_action_vsi_merge(flow, dst_vsi_id);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_flow_parse_action_port_id(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ const struct rte_flow_action_port_id *action_port_id_conf;
+ uint16_t dst_port_id;
+ uint16_t dst_vsi_id;
+ int32_t ret = 0;
+
+ action_port_id_conf = (const struct rte_flow_action_port_id *)action->conf;
+ dst_port_id = action_port_id_conf->original ?
+ adapter->dev_port_id : action_port_id_conf->id;
+
+ if (!rte_eth_dev_is_valid_port(dst_port_id)) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "Invalid port id.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_flow_vsi_get_ethdev(dev, dst_port_id, &dst_vsi_id);
+ if (ret != 0) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Failed to cfg port dev invalid.");
+ goto l_end;
+ }
+
+ sxe2_flow_action_vsi_merge(flow, dst_vsi_id);
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_action_send_to_kernel(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action, struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ if (adapter->vsi_ctxt.kernel_vsi_id == UINT16_MAX) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Failed to cfg send to kernel action without kernel vsi.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ sxe2_flow_action_vsi_merge(flow, adapter->vsi_ctxt.kernel_vsi_id);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flow_check_actions(struct rte_eth_dev *dev __rte_unused, struct sxe2_flow *flow,
+ uint8_t action_num[], struct rte_flow_error *error)
+{
+ enum sxe2_flow_engine_type engine_type = flow->engine_type;
+ int32_t ret = 0;
+ int32_t dest_num = action_num[SXE2_FLOW_ACTION_Q_REGION] +
+ action_num[SXE2_FLOW_ACTION_QUEUE];
+ int32_t vsi_num = action_num[SXE2_FLOW_ACTION_TO_VSI];
+ int32_t vsi_list_num = action_num[SXE2_FLOW_ACTION_TO_VSI_LIST];
+ int32_t pass_num = action_num[SXE2_FLOW_ACTION_PASSTHRU];
+ int32_t drop_num = action_num[SXE2_FLOW_ACTION_DROP];
+ int32_t mark_num = action_num[SXE2_FLOW_ACTION_MARK];
+ int32_t count_num = action_num[SXE2_FLOW_ACTION_COUNT];
+ int32_t rss_num = action_num[SXE2_FLOW_ACTION_RSS];
+ int32_t fwd_num = dest_num + vsi_num + vsi_list_num;
+ int32_t total_num = dest_num + vsi_num + vsi_list_num + pass_num +
+ drop_num + mark_num + count_num + rss_num;
+
+ if (pass_num > 1 || drop_num > 1 || mark_num > 1 ||
+ count_num > 1 || rss_num > 1 || dest_num > 1 ||
+ vsi_num > 1 || vsi_list_num > 1) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "ecah action can only be used once.");
+ PMD_LOG_ERR(DRV, "ecah action can only be used once.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (vsi_list_num && engine_type != SXE2_FLOW_ENGINE_SWITCH) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "VSI_LIST action is only supported for switch engine.");
+ PMD_LOG_ERR(DRV, "VSI_LIST action is only supported for switch engine.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (drop_num) {
+ if (total_num > drop_num + count_num) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Drop action can't be used with other actions unless count.");
+ PMD_LOG_ERR(DRV,
+ "Drop action can't be used with other actions unless count.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+
+ if (fwd_num > 1) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Only supports one type of forwarding action.");
+ PMD_LOG_ERR(DRV, "Only supports one type of forwarding action.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (vsi_list_num) {
+ if (total_num > vsi_list_num) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "VSI_LIST action can't be used with other actions.");
+ PMD_LOG_ERR(DRV,
+ "VSI_LIST action can't be used with other actions.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+
+ if (engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ if (vsi_num) {
+ flow->action.q_region.q_index = 0;
+ flow->action.q_region.region = 7;
+ flow->action.q_region.vsi_index = flow->action.vsi.vsi_index;
+ sxe2_set_bit(SXE2_FLOW_ACTION_Q_REGION, flow->action.act_types);
+ dest_num++;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+
+int32_t sxe2_flow_parse_action(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ int32_t ret = 0;
+ const struct rte_flow_action *action;
+ const struct rte_flow_action_count *act_count;
+ const struct rte_flow_action_mark *act_mark;
+ uint8_t action_num[SXE2_FLOW_ACTION_MAX] = {0};
+ enum sxe2_flow_engine_type engine_type = flow->engine_type;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ sxe2_flow_action_pre(flow);
+
+ for (action = actions; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
+ switch (action->type) {
+ case RTE_FLOW_ACTION_TYPE_VOID:
+ break;
+ case RTE_FLOW_ACTION_TYPE_PASSTHRU:
+ if (engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ sxe2_set_bit(SXE2_FLOW_ACTION_PASSTHRU, flow->action.act_types);
+ action_num[SXE2_FLOW_ACTION_PASSTHRU]++;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Passthru action is not supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "Passthru action is not supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_DROP:
+ if (engine_type == SXE2_FLOW_ENGINE_ACL ||
+ engine_type == SXE2_FLOW_ENGINE_SWITCH ||
+ engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ sxe2_set_bit(SXE2_FLOW_ACTION_DROP, flow->action.act_types);
+ action_num[SXE2_FLOW_ACTION_DROP]++;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Drop action is not supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "Drop action is not supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_MARK:
+ if (engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ sxe2_set_bit(SXE2_FLOW_ACTION_MARK, flow->action.act_types);
+ act_mark = action->conf;
+ flow->action.mark.mark_id = act_mark->id;
+ action_num[SXE2_FLOW_ACTION_MARK]++;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Mark action is not supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "Mark action is not supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_COUNT:
+ if (engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ sxe2_set_bit(SXE2_FLOW_ACTION_COUNT, flow->action.act_types);
+ act_count = action->conf;
+ flow->action.count.user_id = act_count->id;
+ flow->action.count.driver_id = 0;
+ if (flow->action.count.user_id == 0)
+ flow->action.count.driver_id =
+ ++adapter->flow_ctxt.hw_res.global_index;
+ action_num[SXE2_FLOW_ACTION_COUNT]++;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Count action is not supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "Count action is not supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_RSS:
+ if (engine_type == SXE2_FLOW_ENGINE_RSS) {
+ ret = sxe2_flow_parse_action_rss(action, error, flow);
+ if (ret != 0)
+ goto l_end;
+ action_num[SXE2_FLOW_ACTION_RSS]++;
+ } else if (engine_type == SXE2_FLOW_ENGINE_ACL ||
+ engine_type == SXE2_FLOW_ENGINE_SWITCH ||
+ engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ ret = sxe2_flow_parse_action_qregion(dev, action, error, flow);
+ if (ret != 0)
+ goto l_end;
+ action_num[SXE2_FLOW_ACTION_Q_REGION]++;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "RSS action is only supported for RSS flow.");
+ PMD_LOG_ERR(DRV,
+ "RSS action is only supported for RSS flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_QUEUE:
+ if (engine_type == SXE2_FLOW_ENGINE_ACL ||
+ engine_type == SXE2_FLOW_ENGINE_SWITCH ||
+ engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ ret = sxe2_flow_parse_action_queue(dev, action, error, flow);
+ if (ret != 0)
+ goto l_end;
+ action_num[SXE2_FLOW_ACTION_QUEUE]++;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "Queue action is not supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "Queue action is not supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
+ if (engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ ret = sxe2_flow_parse_action_represented_port(dev,
+ action, error, flow);
+ if (ret != 0)
+ goto l_end;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "REPRESENTED PORT action is only supported for SWITCH flow.");
+ PMD_LOG_ERR(DRV,
+ "REPRESENTED PORT action is only supported for SWITCH flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
+ if (engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ ret = sxe2_flow_parse_action_port_representor(dev,
+ action, error, flow);
+ if (ret != 0)
+ goto l_end;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "PORT REPRESENTOR action is only supported for SWITCH flow.");
+ PMD_LOG_ERR(DRV,
+ "PORT REPRESENTOR action is only supported for SWITCH flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_PORT_ID:
+ if (engine_type == SXE2_FLOW_ENGINE_SWITCH ||
+ engine_type == SXE2_FLOW_ENGINE_ACL ||
+ engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ ret = sxe2_flow_parse_action_port_id(dev, action,
+ error, flow);
+ if (ret != 0)
+ goto l_end;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "PORT ID action is only supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "PORT ID action is only supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
+ if (engine_type == SXE2_FLOW_ENGINE_ACL ||
+ engine_type == SXE2_FLOW_ENGINE_FNAV ||
+ engine_type == SXE2_FLOW_ENGINE_SWITCH) {
+ ret = sxe2_flow_parse_action_send_to_kernel(dev,
+ action, error, flow);
+ if (ret != 0)
+ goto l_end;
+ } else {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, action,
+ "SEND TO KERNEL action is only supported for this flow.");
+ PMD_LOG_ERR(DRV,
+ "SEND TO KERNEL action is only supported for this flow.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ break;
+ default:
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, actions,
+ "Invalid action.");
+ PMD_LOG_ERR(DRV, "Invalid action type:%d", actions->type);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+
+ sxe2_flow_action_post(flow, action_num);
+
+ ret = sxe2_flow_check_actions(dev, flow, action_num, error);
+ if (ret != 0)
+ goto l_end;
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_flow_parse_action.h b/drivers/net/sxe2/sxe2_flow_parse_action.h
new file mode 100644
index 0000000000..479d10a522
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_parse_action.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_FLOW_PARSE_ACTION_H_
+#define SXE2_FLOW_PARSE_ACTION_H_
+#include <rte_flow_driver.h>
+
+#include "sxe2_osal.h"
+#include "sxe2_flow_define.h"
+
+
+int32_t sxe2_flow_parse_action(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow);
+
+int32_t sxe2_flow_parse_action_port_id(struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow);
+
+#endif /* SXE2_FLOW_PARSE_ACTION_H_ */
diff --git a/drivers/net/sxe2/sxe2_flow_parse_engine.c b/drivers/net/sxe2/sxe2_flow_parse_engine.c
new file mode 100644
index 0000000000..09de1b94c4
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_parse_engine.c
@@ -0,0 +1,106 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include "sxe2_flow_parse_engine.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_flow_public.h"
+#include "sxe2_flow_parse_action.h"
+#include "sxe2_common_log.h"
+
+static int32_t sxe2_flow_parse_engine_chk(struct sxe2_flow *flow,
+ struct rte_flow_error *error)
+{
+ int32_t ret = 0;
+
+ if (flow->engine_type == SXE2_FLOW_ENGINE_FNAV) {
+ if (flow->has_mask) {
+ ret = -EINVAL;
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM_MASK, NULL,
+ "FNAV flow doesn't support mask");
+ PMD_LOG_ERR(DRV, "FNAV flow doesn't support mask");
+ goto l_end;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_FLD_ID_S_VID,
+ flow->pattern_outer.map_spec) &&
+ sxe2_test_bit(SXE2_FLOW_FLD_ID_C_VID,
+ flow->pattern_outer.map_spec)) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, NULL,
+ "Can't set double vid,please use tci.");
+ PMD_LOG_ERR(DRV,
+ "Can't set double vid,please use tci.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_flow_parse_engine(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ const struct rte_flow_action *action;
+
+ if (flow->has_mask == 0 && flow->has_spec == 0) {
+ flow->engine_type = SXE2_FLOW_ENGINE_RSS;
+ goto l_end;
+ }
+
+ if (attr->group == 1) {
+ flow->engine_type = SXE2_FLOW_ENGINE_SWITCH;
+ goto l_end;
+ }
+ if (attr->group == 2) {
+ flow->engine_type = SXE2_FLOW_ENGINE_ACL;
+ goto l_end;
+ }
+ if (attr->group == 3) {
+ flow->engine_type = SXE2_FLOW_ENGINE_FNAV;
+ goto l_end;
+ }
+
+ if (adapter->is_dev_repr) {
+ flow->engine_type = SXE2_FLOW_ENGINE_SWITCH;
+ goto l_end;
+ }
+
+ if (adapter->switchdev_info.is_switchdev &&
+ adapter->dev_type == SXE2_DEV_T_VF) {
+ flow->engine_type = SXE2_FLOW_ENGINE_FNAV;
+ goto l_end;
+ }
+
+ for (action = actions; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
+ switch (action->type) {
+ case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
+ case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
+ case RTE_FLOW_ACTION_TYPE_PORT_ID:
+ flow->engine_type = SXE2_FLOW_ENGINE_SWITCH;
+ goto l_end;
+ default:
+ break;
+ }
+ }
+
+ if (adapter->switchdev_info.is_switchdev) {
+ flow->engine_type = SXE2_FLOW_ENGINE_FNAV;
+ goto l_end;
+ }
+
+ if (adapter->flow_isolated)
+ flow->engine_type = SXE2_FLOW_ENGINE_SWITCH;
+ else
+ flow->engine_type = SXE2_FLOW_ENGINE_FNAV;
+
+l_end:
+ ret = sxe2_flow_parse_engine_chk(flow, error);
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_flow_parse_engine.h b/drivers/net/sxe2/sxe2_flow_parse_engine.h
new file mode 100644
index 0000000000..1485beecb4
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_parse_engine.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_FLOW_PARSE_ENGINE_H_
+#define SXE2_FLOW_PARSE_ENGINE_H_
+#include "sxe2_osal.h"
+#include "sxe2_flow_define.h"
+
+int32_t sxe2_flow_parse_engine(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
+ const struct rte_flow_action actions[], struct rte_flow_error *error,
+ struct sxe2_flow *flow);
+#endif /* SXE2_FLOW_PARSE_ENGINE_H_ */
diff --git a/drivers/net/sxe2/sxe2_flow_parse_pattern.c b/drivers/net/sxe2/sxe2_flow_parse_pattern.c
new file mode 100644
index 0000000000..189abb1a33
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_parse_pattern.c
@@ -0,0 +1,1822 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include "sxe2_flow_parse_pattern.h"
+#include "rte_common.h"
+#include "rte_flow.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+#include "sxe2_flow_define.h"
+
+const struct sxe2_flow_expand_node sxe2_support_expansion[SXE2_EXPANSION_MAX] = {
+ [SXE2_EXPANSION_OUTER_ETH] = {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "eth",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_VLAN,
+ SXE2_EXPANSION_OUTER_IPV4,
+ SXE2_EXPANSION_OUTER_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_OUTER_VLAN] = {
+ .type = RTE_FLOW_ITEM_TYPE_VLAN,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "vlan",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_QINQ,
+ SXE2_EXPANSION_OUTER_IPV4,
+ SXE2_EXPANSION_OUTER_IPV6,
+ SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_QINQ] = {
+ .type = RTE_FLOW_ITEM_TYPE_VLAN,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "vlan",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_IPV4,
+ SXE2_EXPANSION_OUTER_IPV6,
+ SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_IPV4] = {
+ .type = RTE_FLOW_ITEM_TYPE_IPV4,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_IPIP,
+ .is_tunnel = false,
+ .name = "ipv4",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_UDP,
+ SXE2_EXPANSION_OUTER_TCP,
+ SXE2_EXPANSION_OUTER_SCTP,
+ SXE2_EXPANSION_GRE,
+ SXE2_EXPANSION_NVGRE,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_IPV6] = {
+ .type = RTE_FLOW_ITEM_TYPE_IPV6,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_IPIP,
+ .is_tunnel = false,
+ .name = "ipv6",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_IPV6_FRAG_EXT,
+ SXE2_EXPANSION_OUTER_UDP,
+ SXE2_EXPANSION_OUTER_TCP,
+ SXE2_EXPANSION_OUTER_SCTP,
+ SXE2_EXPANSION_GRE,
+ SXE2_EXPANSION_NVGRE,
+ SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_IPV6_FRAG_EXT] = {
+ .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "ipv6_frag_ext",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_UDP] = {
+ .type = RTE_FLOW_ITEM_TYPE_UDP,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "udp",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_VXLAN,
+ SXE2_EXPANSION_VXLAN_GPE,
+ SXE2_EXPANSION_GENEVE,
+ SXE2_EXPANSION_GTPU,
+ SXE2_EXPANSION_GRE,
+ SXE2_EXPANSION_NVGRE,
+ SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_TCP] = {
+ .type = RTE_FLOW_ITEM_TYPE_TCP,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "tcp",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_SCTP] = {
+ .type = RTE_FLOW_ITEM_TYPE_SCTP,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "sctp",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_OUTER_END),
+ },
+ [SXE2_EXPANSION_OUTER_END] = {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE,
+ .is_tunnel = false,
+ .name = "end",
+ .next = SXE2_FLOW_EXPAND_NEXT(0),
+ },
+ [SXE2_EXPANSION_VXLAN] = {
+ .type = RTE_FLOW_ITEM_TYPE_VXLAN,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_VXLAN,
+ .is_tunnel = true,
+ .name = "vxlan",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_VXLAN_GPE] = {
+ .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_VXLAN,
+ .is_tunnel = true,
+ .name = "vxlan_gpe",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_GRE] = {
+ .type = RTE_FLOW_ITEM_TYPE_GRE,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_GRE,
+ .is_tunnel = true,
+ .name = "gre",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_NVGRE] = {
+ .type = RTE_FLOW_ITEM_TYPE_NVGRE,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_GRE,
+ .is_tunnel = true,
+ .name = "nvgre",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_GENEVE] = {
+ .type = RTE_FLOW_ITEM_TYPE_GENEVE,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_GENEVE,
+ .is_tunnel = true,
+ .name = "geneve",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_GTPU] = {
+ .type = RTE_FLOW_ITEM_TYPE_GTPU,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_GTPU,
+ .is_tunnel = true,
+ .name = "gtpu",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_ETH] = {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "eth",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_VLAN,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_VLAN] = {
+ .type = RTE_FLOW_ITEM_TYPE_VLAN,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "vlan",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_IPV4] = {
+ .type = RTE_FLOW_ITEM_TYPE_IPV4,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "ipv4",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_UDP,
+ SXE2_EXPANSION_TCP,
+ SXE2_EXPANSION_SCTP,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_IPV6] = {
+ .type = RTE_FLOW_ITEM_TYPE_IPV6,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "ipv6",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_UDP,
+ SXE2_EXPANSION_TCP,
+ SXE2_EXPANSION_SCTP,
+ SXE2_EXPANSION_IPV6_FRAG_EXT,
+ SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_UDP] = {
+ .type = RTE_FLOW_ITEM_TYPE_UDP,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "udp",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_TCP] = {
+ .type = RTE_FLOW_ITEM_TYPE_TCP,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "tcp",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_SCTP] = {
+ .type = RTE_FLOW_ITEM_TYPE_SCTP,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "sctp",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_IPV6_FRAG_EXT] = {
+ .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "ipv6_frag_ext",
+ .next = SXE2_FLOW_EXPAND_NEXT(SXE2_EXPANSION_END),
+ },
+ [SXE2_EXPANSION_END] = {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ .tunnel_type = SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ .is_tunnel = true,
+ .name = "end",
+ .next = SXE2_FLOW_EXPAND_NEXT(0),
+ }
+};
+
+const char *sxe2_flow_type_name[SXE2_FLOW_TYPE_MAX] = {
+ [SXE2_FLOW_MAC_PAY] = "SXE2_FLOW_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_FRAG_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_UDP_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_TCP_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_SCTP_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_FRAG_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_UDP_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_TCP_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_SCTP_PAY] =
+ "SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_FRAG_PAY] =
+ "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_SCTP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_FRAG_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_FRAG_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_UDP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_UDP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_TCP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_TCP_PAY",
+ [SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_SCTP_PAY] = "SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_SCTP_PAY",
+};
+#define SXE2_FLOW_TYPE_NAME_MAX_LEN 128
+
+static int32_t sxe2_flow_get_flow_type(struct sxe2_flow *flow)
+{
+ int32_t ret = -EINVAL;
+ uint16_t i = 0;
+ uint16_t len = 0;
+ char flow_type_name[SXE2_FLOW_TYPE_NAME_MAX_LEN] = {0};
+ len = snprintf(flow_type_name, sizeof(flow_type_name), "SXE2_FLOW_");
+ i += len;
+ if (sxe2_test_bit(SXE2_FLOW_HDR_ETH, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "MAC_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV4, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "IPV4_");
+ i += len;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_IPV6, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "IPV6_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV_FRAG, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "FRAG_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_UDP, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "UDP_");
+ i += len;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_TCP, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "TCP_");
+ i += len;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_SCTP, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "SCTP_");
+ i += len;
+ }
+
+ if (sxe2_test_bit(SXE2_FLOW_HDR_VXLAN, flow->pattern_outer.hdrs) ||
+ sxe2_test_bit(SXE2_FLOW_HDR_GENEVE, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "VXGEN_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_GRE, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "GRE_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_GTPU, flow->pattern_outer.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "GTPU_");
+ i += len;
+ }
+
+ if (sxe2_test_bit(SXE2_FLOW_HDR_ETH, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "MAC_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_VLAN, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "VLAN_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV4, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "IPV4_");
+ i += len;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_IPV6, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "IPV6_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV_FRAG, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "FRAG_");
+ i += len;
+ }
+ if (sxe2_test_bit(SXE2_FLOW_HDR_UDP, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "UDP_");
+ i += len;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_TCP, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "TCP_");
+ i += len;
+ } else if (sxe2_test_bit(SXE2_FLOW_HDR_SCTP, flow->pattern_inner.hdrs)) {
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name) - i,
+ "SCTP_");
+ i += len;
+ }
+
+ len = snprintf(flow_type_name + i, sizeof(flow_type_name), "PAY");
+ i += len;
+
+ for (i = 0; i < SXE2_FLOW_TYPE_MAX; i++) {
+ if (sxe2_flow_type_name[i] == NULL)
+ continue;
+ if (strcmp(flow_type_name, sxe2_flow_type_name[i]) == 0) {
+ flow->meta.flow_type = i;
+ ret = 0;
+ break;
+ }
+ }
+ if (ret != 0)
+ PMD_LOG_ERR(DRV,
+ "Unsupported flow type. %s is not supported.", flow_type_name);
+ return ret;
+}
+
+static int32_t sxe2_flow_is_expandable_item(const struct rte_flow_item *item)
+{
+ int32_t ret = -EINVAL;
+ switch (item->type) {
+ case RTE_FLOW_ITEM_TYPE_ETH:
+ case RTE_FLOW_ITEM_TYPE_VLAN:
+ case RTE_FLOW_ITEM_TYPE_IPV4:
+ case RTE_FLOW_ITEM_TYPE_IPV6:
+ case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
+ case RTE_FLOW_ITEM_TYPE_UDP:
+ case RTE_FLOW_ITEM_TYPE_TCP:
+ case RTE_FLOW_ITEM_TYPE_SCTP:
+ case RTE_FLOW_ITEM_TYPE_VXLAN:
+ case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+ case RTE_FLOW_ITEM_TYPE_GRE:
+ case RTE_FLOW_ITEM_TYPE_NVGRE:
+ case RTE_FLOW_ITEM_TYPE_GENEVE:
+ case RTE_FLOW_ITEM_TYPE_GTPU:
+ case RTE_FLOW_ITEM_TYPE_VOID:
+ case RTE_FLOW_ITEM_TYPE_END:
+ ret = 0;
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
+
+static int32_t sxe2_flow_valid_next_expansion(enum sxe2_expansion *current,
+ const struct rte_flow_item *item,
+ enum sxe2_flow_tunnel_type *tunnel_type, BITMAP_TYPE *flow_type)
+{
+ int32_t ret = -EINVAL;
+ const struct rte_flow_item *next;
+ const enum sxe2_expansion *next_expansion = current;
+ const struct sxe2_flow_expand_node *node;
+ uint8_t len = 0;
+ char typelist[512] = {0};
+ char error[1024] = {0};
+ enum sxe2_flow_tunnel_type tunnel_type_now = SXE2_FLOW_TUNNEL_TYPE_NONE;
+ enum sxe2_flow_tunnel_type tunnel_type_next = SXE2_FLOW_TUNNEL_TYPE_NONE;
+ uint8_t is_tunnel_now = 0;
+ uint8_t is_tunnel_next = 0;
+
+ if (item->type != sxe2_support_expansion[*current].type) {
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ next = item;
+ do {
+ next++;
+ if (next->type == RTE_FLOW_ITEM_TYPE_VOID)
+ continue;
+ break;
+ } while (1);
+
+ node = &sxe2_support_expansion[*current];
+ next_expansion = node->next;
+ while (*next_expansion != 0) {
+ len = strlen(typelist);
+ snprintf(typelist + len, sizeof(typelist) - len,
+ "%s|", sxe2_support_expansion[*next_expansion].name);
+ if (sxe2_support_expansion[*next_expansion].type == next->type) {
+ ret = 0;
+ break;
+ }
+ next_expansion++;
+ }
+ if (ret != 0) {
+ snprintf(error, sizeof(error),
+ "The next item of %s only can be one of [%s].",
+ sxe2_support_expansion[*current].name, typelist);
+ PMD_LOG_ERR(INIT, "Invalid pattern sequence. %s", error);
+ goto l_end;
+ }
+ tunnel_type_now = sxe2_support_expansion[*current].tunnel_type;
+ tunnel_type_next = sxe2_support_expansion[*next_expansion].tunnel_type;
+ is_tunnel_now = sxe2_support_expansion[*current].is_tunnel;
+ is_tunnel_next = sxe2_support_expansion[*next_expansion].is_tunnel;
+
+
+ if (!is_tunnel_now && is_tunnel_next) {
+ if (tunnel_type_next == SXE2_FLOW_TUNNEL_TYPE_PARENT)
+ *tunnel_type = tunnel_type_now;
+ else
+ *tunnel_type = tunnel_type_next;
+ }
+
+l_end:
+ sxe2_set_bit(*current, flow_type);
+ *current = *next_expansion;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_eth(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next,
+ bool is_inner)
+{
+ const struct rte_flow_item_eth *eth_spec;
+ const struct rte_flow_item_eth *eth_mask;
+ const struct rte_ether_addr *dst_addr_mask;
+ const struct rte_ether_addr *src_addr_mask;
+ const struct rte_ether_addr *dst_addr_spec;
+ const struct rte_ether_addr *src_addr_spec;
+ rte_be16_t type_mask;
+ rte_be16_t type_spec;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ uint16_t ether_type;
+ eth_spec = item->spec;
+ eth_mask = item->mask;
+
+ if (eth_spec == NULL && eth_mask == NULL)
+ goto l_end;
+
+ dst_addr_mask = ð_mask->hdr.dst_addr;
+ src_addr_mask = ð_mask->hdr.src_addr;
+ dst_addr_spec = ð_spec->hdr.dst_addr;
+ src_addr_spec = ð_spec->hdr.src_addr;
+ type_mask = eth_mask->hdr.ether_type;
+ type_spec = eth_spec->hdr.ether_type;
+
+ if (eth_mask->has_vlan) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported eth mask has_vlan.");
+ PMD_LOG_ERR(DRV, "Unsupported eth mask has_vlan");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (!rte_is_zero_ether_addr(dst_addr_mask)) {
+ if (!rte_is_broadcast_ether_addr(dst_addr_mask))
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_DA, pattern->map_mask);
+
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_DA, pattern->map_spec);
+ rte_memcpy(pattern->item_spec.eth.dst_addr, dst_addr_spec,
+ RTE_ETHER_ADDR_LEN);
+ rte_memcpy(pattern->item_mask.eth.dst_addr, dst_addr_mask,
+ RTE_ETHER_ADDR_LEN);
+ }
+ if (!rte_is_zero_ether_addr(src_addr_mask)) {
+ if (!rte_is_broadcast_ether_addr(src_addr_mask))
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_SA, pattern->map_mask);
+
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_SA, pattern->map_spec);
+ rte_memcpy(pattern->item_spec.eth.src_addr, src_addr_spec,
+ RTE_ETHER_ADDR_LEN);
+ rte_memcpy(pattern->item_mask.eth.src_addr, src_addr_mask,
+ RTE_ETHER_ADDR_LEN);
+ }
+ if (type_mask != 0) {
+ if (type_mask != UINT16_MAX) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported eth ether_type mask");
+ PMD_LOG_ERR(DRV, "unsupported eth ether_type mask[0x%x].",
+ type_mask);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (next != SXE2_EXPANSION_OUTER_END && next != SXE2_EXPANSION_END) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported eth ether_type match with next item.");
+ PMD_LOG_ERR(DRV, "unsupported eth ether_type match with next item.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ ether_type = rte_be_to_cpu_16(type_spec);
+ if (ether_type == RTE_ETHER_TYPE_IPV4 ||
+ ether_type == RTE_ETHER_TYPE_IPV6) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Ether_type unsupported ipv4/ipv6(0x0800/0x86DD).");
+ PMD_LOG_ERR(DRV, "Ether_type unsupported ipv4/ipv6(0x0800/0x86DD).");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (ether_type == RTE_ETHER_TYPE_VLAN || ether_type == RTE_ETHER_TYPE_QINQ ||
+ ether_type == RTE_ETHER_TYPE_QINQ1) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Ether_type unsupported vlan(0x8100/0x88a8/0x9100).");
+ PMD_LOG_ERR(DRV, "Ether_type unsupported vlan(0x8100/0x88a8/0x9100).");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (ether_type <= SXE2_FLOW_ETH_TYPE_MIN) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Ether_type need max 1500.");
+ PMD_LOG_ERR(DRV, "Ether_type need max 1500.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_ETH_TYPE, pattern->map_spec);
+ pattern->item_spec.eth.ether_type = type_spec;
+ pattern->item_mask.eth.ether_type = type_mask;
+ }
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_ETH, pattern->hdrs);
+ pattern->rss_type_allow |= RTE_ETH_RSS_ETH;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L2_DST_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L2_SRC_ONLY;
+ if (next == SXE2_EXPANSION_OUTER_END || next == SXE2_EXPANSION_END)
+ pattern->rss_type_allow |= RTE_ETH_RSS_L2_PAYLOAD;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_vlan(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_vlan *vlan_spec;
+ const struct rte_flow_item_vlan *vlan_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ rte_be16_t vlan_tci_mask;
+ rte_be16_t vlan_tci_spec;
+ rte_be16_t eth_proto_mask;
+ rte_be16_t eth_proto_spec;
+ int32_t ret = 0;
+ vlan_spec = item->spec;
+ vlan_mask = item->mask;
+ bool is_qinq = false;
+
+ if (sxe2_test_bit(SXE2_FLOW_HDR_VLAN, pattern->hdrs))
+ is_qinq = true;
+
+ if (vlan_spec == NULL && vlan_mask == NULL)
+ goto l_end;
+
+ vlan_tci_mask = vlan_mask->hdr.vlan_tci;
+ vlan_tci_spec = vlan_spec->hdr.vlan_tci;
+ eth_proto_mask = vlan_mask->hdr.eth_proto;
+ eth_proto_spec = vlan_spec->hdr.eth_proto;
+
+ if (vlan_mask->has_more_vlan || vlan_mask->reserved) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported vlan mask has_qinq.");
+ PMD_LOG_ERR(DRV, "Unsupported vlan mask has_qinq");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (vlan_tci_mask) {
+ if (vlan_tci_spec == 0) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "vlan id can't be 0.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+
+ if (eth_proto_mask) {
+ if (eth_proto_mask != UINT16_MAX) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported vlan ether_type mask");
+ PMD_LOG_ERR(DRV, "unsupported vlan ether_type mask.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (eth_proto_spec != RTE_BE16(0x8100) &&
+ eth_proto_spec != RTE_BE16(0x88a8) &&
+ eth_proto_spec != RTE_BE16(0x9100)) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported vlan ether_type, only support 0x8100, 0x88a8 and 0x9100.");
+ PMD_LOG_ERR(DRV, "Unsupported vlan ether_type, only support 0x8100, 0x88a8 and 0x9100.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ if (!is_qinq) {
+ if (vlan_tci_mask) {
+ if (vlan_tci_mask == RTE_BE16(0x0fff)) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_S_VID, pattern->map_spec);
+ } else if (vlan_tci_mask == UINT16_MAX) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_S_TCI, pattern->map_spec);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_S_TCI, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_S_TCI, pattern->map_spec);
+ }
+ }
+ if (eth_proto_mask)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_S_TPID, pattern->map_spec);
+ } else {
+ if (vlan_tci_mask) {
+ if (vlan_tci_mask == RTE_BE16(0x0fff)) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_C_VID, pattern->map_spec);
+ } else if (vlan_tci_mask == UINT16_MAX) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_C_TCI, pattern->map_spec);
+ } else {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_C_TCI, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_C_TCI, pattern->map_spec);
+ }
+ }
+ if (eth_proto_mask)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_C_TPID, pattern->map_spec);
+ }
+ if (is_qinq) {
+ pattern->item_spec.qinq.type = eth_proto_spec;
+ pattern->item_mask.qinq.type = eth_proto_mask;
+ pattern->item_spec.qinq.vlan = vlan_tci_spec;
+ pattern->item_mask.qinq.vlan = vlan_tci_mask;
+ } else {
+ pattern->item_spec.vlan.type = eth_proto_spec;
+ pattern->item_mask.vlan.type = eth_proto_mask;
+ pattern->item_spec.vlan.vlan = vlan_tci_spec;
+ pattern->item_mask.vlan.vlan = vlan_tci_mask;
+ }
+l_end:
+ pattern->rss_type_allow |= RTE_ETH_RSS_S_VLAN;
+ pattern->rss_type_allow |= RTE_ETH_RSS_C_VLAN;
+ if (!is_qinq)
+ sxe2_set_bit(SXE2_FLOW_HDR_VLAN, pattern->hdrs);
+ else
+ sxe2_set_bit(SXE2_FLOW_HDR_QINQ, pattern->hdrs);
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_ipv4(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next,
+ bool is_inner)
+{
+ const struct rte_flow_item_ipv4 *ipv4_spec;
+ const struct rte_flow_item_ipv4 *ipv4_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ ipv4_spec = item->spec;
+ ipv4_mask = item->mask;
+
+ if (ipv4_mask == NULL && ipv4_spec == NULL)
+ goto l_end;
+
+ if (ipv4_mask->hdr.version_ihl || ipv4_mask->hdr.total_length ||
+ ipv4_mask->hdr.hdr_checksum || ipv4_mask->hdr.packet_id) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some IPv4 mask.");
+ PMD_LOG_ERR(DRV, "Unsupported some IPv4 mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (ipv4_mask->hdr.src_addr) {
+ if (ipv4_mask->hdr.src_addr != UINT32_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_SA, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_SA, pattern->map_spec);
+ pattern->item_spec.ipv4.saddr = ipv4_spec->hdr.src_addr;
+ pattern->item_mask.ipv4.saddr = ipv4_mask->hdr.src_addr;
+ }
+ if (ipv4_mask->hdr.dst_addr) {
+ if (ipv4_mask->hdr.dst_addr != UINT32_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_DA, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_DA, pattern->map_spec);
+ pattern->item_spec.ipv4.daddr = ipv4_spec->hdr.dst_addr;
+ pattern->item_mask.ipv4.daddr = ipv4_mask->hdr.dst_addr;
+ }
+
+ if (ipv4_mask->hdr.next_proto_id) {
+ if (next != SXE2_EXPANSION_OUTER_END && next != SXE2_EXPANSION_END) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "IPv4 proto id must be the last partten.");
+ PMD_LOG_ERR(DRV, "IPv4 proto id must be the last partten.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (ipv4_mask->hdr.next_proto_id != UINT8_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_PROT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_PROT, pattern->map_spec);
+ pattern->item_spec.ipv4.protocol = ipv4_spec->hdr.next_proto_id;
+ pattern->item_mask.ipv4.protocol = ipv4_mask->hdr.next_proto_id;
+ }
+ if (ipv4_mask->hdr.time_to_live) {
+ if (ipv4_mask->hdr.time_to_live != UINT8_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_TTL, pattern->map_mask);
+ if (ipv4_spec->hdr.time_to_live == 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "ipv4 ttl must be not 0.");
+ PMD_LOG_ERR(DRV, "ipv4 ttl must be not 0.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_TTL, pattern->map_spec);
+ pattern->item_spec.ipv4.ttl = ipv4_spec->hdr.time_to_live;
+ pattern->item_mask.ipv4.ttl = ipv4_mask->hdr.time_to_live;
+ }
+ if (ipv4_mask->hdr.type_of_service) {
+ if (ipv4_mask->hdr.type_of_service != UINT8_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_TOS, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_TOS, pattern->map_spec);
+ pattern->item_spec.ipv4.tos = ipv4_spec->hdr.type_of_service;
+ pattern->item_mask.ipv4.tos = ipv4_mask->hdr.type_of_service;
+ }
+ if (ipv4_mask->hdr.fragment_offset) {
+ if (ipv4_spec->hdr.fragment_offset == rte_cpu_to_be_16(RTE_IPV4_HDR_MF_FLAG) &&
+ ipv4_mask->hdr.fragment_offset == rte_cpu_to_be_16(RTE_IPV4_HDR_MF_FLAG)) {
+ if (next != SXE2_EXPANSION_OUTER_END && next != SXE2_EXPANSION_END) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "IPv4 frag offset must be the last partten.");
+ PMD_LOG_ERR(DRV, "IPv4 frag offset must be the last partten.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_FRAG, pattern->hdrs);
+ } else {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported ipv4 fragment_offset cfg.");
+ PMD_LOG_ERR(DRV, "Unsupported ipv4 fragment_offset cfg.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV4, pattern->hdrs);
+ pattern->rss_type_allow |= RTE_ETH_RSS_IPV4;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_SRC_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_DST_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_IPV4_CHKSUM;
+ if (next == SXE2_EXPANSION_OUTER_END || next == SXE2_EXPANSION_END) {
+ pattern->rss_type_allow |= RTE_ETH_RSS_FRAG_IPV4;
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV4_OTHER;
+ }
+
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_ipv6(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_ipv6 *ipv6_spec;
+ const struct rte_flow_item_ipv6 *ipv6_mask;
+ uint32_t vtc_flow_mask;
+ uint32_t tc_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ ipv6_spec = item->spec;
+ ipv6_mask = item->mask;
+ uint8_t ipv6_addr_mask[16] = {
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
+ };
+ uint8_t ipv6_addr_empty[16] = { 0 };
+
+ if (ipv6_mask == NULL && ipv6_spec == NULL)
+ goto l_end;
+
+ if (ipv6_mask->hdr.payload_len || ipv6_mask->has_hop_ext ||
+ ipv6_mask->has_route_ext || ipv6_mask->has_frag_ext ||
+ ipv6_mask->has_auth_ext || ipv6_mask->has_esp_ext ||
+ ipv6_mask->has_dest_ext || ipv6_mask->has_mobil_ext ||
+ ipv6_mask->has_hip_ext || ipv6_mask->has_shim6_ext) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some IPv6 mask");
+ PMD_LOG_ERR(DRV, "Unsupported some IPv6 mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (memcmp(&ipv6_mask->hdr.src_addr, ipv6_addr_empty,
+ sizeof(ipv6_addr_empty)) != 0) {
+ if (memcmp(&ipv6_mask->hdr.src_addr, ipv6_addr_mask,
+ sizeof(ipv6_addr_mask)) != 0)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_SA, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_SA, pattern->map_spec);
+ rte_memcpy(&pattern->item_spec.ipv6.saddr, &ipv6_spec->hdr.src_addr,
+ sizeof(ipv6_spec->hdr.src_addr));
+ rte_memcpy(&pattern->item_mask.ipv6.saddr, &ipv6_mask->hdr.src_addr,
+ sizeof(ipv6_mask->hdr.src_addr));
+ }
+ if (memcmp(&ipv6_mask->hdr.dst_addr, ipv6_addr_empty,
+ sizeof(ipv6_addr_empty)) != 0) {
+ if (memcmp(&ipv6_mask->hdr.dst_addr, ipv6_addr_mask,
+ sizeof(ipv6_addr_mask)) != 0)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_DA, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_DA, pattern->map_spec);
+ rte_memcpy(&pattern->item_spec.ipv6.daddr, &ipv6_spec->hdr.dst_addr,
+ sizeof(ipv6_spec->hdr.dst_addr));
+ rte_memcpy(&pattern->item_mask.ipv6.daddr, &ipv6_mask->hdr.dst_addr,
+ sizeof(ipv6_mask->hdr.dst_addr));
+ }
+ if (ipv6_mask->hdr.vtc_flow) {
+ vtc_flow_mask = rte_be_to_cpu_32(ipv6_mask->hdr.vtc_flow);
+ tc_mask = vtc_flow_mask & (SXE2_IPV6_TC_MASK << SXE2_IPV6_TC_SHIFT);
+ if (tc_mask != vtc_flow_mask) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "ipv6 vtc_flow only support TC mask.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (tc_mask != (SXE2_IPV6_TC_MASK << SXE2_IPV6_TC_SHIFT))
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_DSCP, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_DSCP, pattern->map_spec);
+ pattern->item_spec.ipv6.pri_ver_flow = ipv6_spec->hdr.vtc_flow;
+ pattern->item_mask.ipv6.pri_ver_flow = ipv6_mask->hdr.vtc_flow;
+ }
+ if (ipv6_mask->hdr.proto) {
+ if (next != SXE2_EXPANSION_OUTER_END && next != SXE2_EXPANSION_END) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "IPv6 proto id must be the last partten.");
+ PMD_LOG_ERR(DRV, "IPv6 proto id must be the last partten.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (ipv6_mask->hdr.proto != UINT8_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PROT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PROT, pattern->map_spec);
+ pattern->item_spec.ipv6.nexthdr = ipv6_spec->hdr.proto;
+ pattern->item_mask.ipv6.nexthdr = ipv6_mask->hdr.proto;
+ }
+ if (ipv6_mask->hdr.hop_limits) {
+ if (ipv6_mask->hdr.hop_limits != UINT8_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_TTL, pattern->map_mask);
+
+ if (ipv6_spec->hdr.hop_limits == 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "ipv6 hop must be not 0.");
+ PMD_LOG_ERR(DRV, "ipv6 hop must be not 0.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_TTL, pattern->map_spec);
+ pattern->item_spec.ipv6.hop_limit = ipv6_spec->hdr.hop_limits;
+ pattern->item_mask.ipv6.hop_limit = ipv6_mask->hdr.hop_limits;
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV6, pattern->hdrs);
+ pattern->rss_type_allow |= RTE_ETH_RSS_IPV6;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_SRC_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_DST_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_PRE32;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_PRE48;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L3_PRE64;
+ if (next == SXE2_EXPANSION_OUTER_END || next == SXE2_EXPANSION_END)
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV6_OTHER;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_ipv6_frag_ext(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_spec;
+ const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ ipv6_frag_spec = item->spec;
+ ipv6_frag_mask = item->mask;
+
+ if (ipv6_frag_mask == NULL && ipv6_frag_spec == NULL)
+ goto l_end;
+
+ if (ipv6_frag_mask->hdr.reserved || ipv6_frag_mask->hdr.frag_data ||
+ ipv6_frag_mask->hdr.id || ipv6_frag_mask->hdr.next_header) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some IPv6 frag ext mask");
+ PMD_LOG_ERR(DRV, "Unsupported some IPv6 frag ext mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_FRAG, pattern->hdrs);
+ pattern->rss_type_allow |= RTE_ETH_RSS_FRAG_IPV6;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_tcp(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_tcp *tcp_spec;
+ const struct rte_flow_item_tcp *tcp_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ tcp_spec = item->spec;
+ tcp_mask = item->mask;
+
+ if (tcp_mask == NULL && tcp_spec == NULL)
+ goto l_end;
+
+ if (tcp_mask->hdr.sent_seq || tcp_mask->hdr.recv_ack ||
+ tcp_mask->hdr.data_off || tcp_mask->hdr.tcp_flags ||
+ tcp_mask->hdr.rx_win || tcp_mask->hdr.cksum ||
+ tcp_mask->hdr.tcp_urp) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some TCP mask");
+ PMD_LOG_ERR(DRV, "Unsupported some TCP mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (tcp_mask->hdr.src_port) {
+ if (tcp_mask->hdr.src_port != UINT16_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_SRC_PORT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_SRC_PORT, pattern->map_spec);
+ pattern->item_spec.tcp.source = tcp_spec->hdr.src_port;
+ pattern->item_mask.tcp.source = tcp_mask->hdr.src_port;
+ }
+ if (tcp_mask->hdr.dst_port) {
+ if (tcp_mask->hdr.dst_port != UINT16_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_DST_PORT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_TCP_DST_PORT, pattern->map_spec);
+ pattern->item_spec.tcp.dest = tcp_spec->hdr.dst_port;
+ pattern->item_mask.tcp.dest = tcp_mask->hdr.dst_port;
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_TCP, pattern->hdrs);
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV4, pattern->hdrs))
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
+ else if (sxe2_test_bit(SXE2_FLOW_HDR_IPV6, pattern->hdrs))
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
+
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_SRC_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_DST_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_CHKSUM;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_udp(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_udp *udp_spec;
+ const struct rte_flow_item_udp *udp_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ udp_spec = item->spec;
+ udp_mask = item->mask;
+
+ if (udp_mask == NULL && udp_spec == NULL)
+ goto l_end;
+
+ if (udp_mask->hdr.dgram_len || udp_mask->hdr.dgram_cksum) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some UDP mask");
+ PMD_LOG_ERR(DRV, "Unsupported some UDP mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (udp_mask->hdr.src_port) {
+ if (udp_mask->hdr.src_port != UINT16_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_SRC_PORT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_SRC_PORT, pattern->map_spec);
+ pattern->item_spec.udp.source = udp_spec->hdr.src_port;
+ pattern->item_mask.udp.source = udp_mask->hdr.src_port;
+ }
+ if (udp_mask->hdr.dst_port) {
+ if (udp_mask->hdr.dst_port != UINT16_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_DST_PORT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_DST_PORT, pattern->map_spec);
+ pattern->item_spec.udp.dest = udp_spec->hdr.dst_port;
+ pattern->item_mask.udp.dest = udp_mask->hdr.dst_port;
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_UDP, pattern->hdrs);
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV4, pattern->hdrs))
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
+ else if (sxe2_test_bit(SXE2_FLOW_HDR_IPV6, pattern->hdrs))
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
+
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_SRC_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_DST_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_CHKSUM;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_sctp(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_sctp *sctp_spec;
+ const struct rte_flow_item_sctp *sctp_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ sctp_spec = item->spec;
+ sctp_mask = item->mask;
+
+ if (sctp_mask == NULL && sctp_spec == NULL)
+ goto l_end;
+
+ if (sctp_mask->hdr.cksum || sctp_mask->hdr.tag) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some SCTP mask");
+ PMD_LOG_ERR(DRV, "Unsupported some SCTP mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (sctp_mask->hdr.src_port) {
+ if (sctp_mask->hdr.src_port != UINT16_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_SRC_PORT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_SRC_PORT, pattern->map_spec);
+ pattern->item_spec.sctp.src_port = sctp_spec->hdr.src_port;
+ pattern->item_mask.sctp.src_port = sctp_mask->hdr.src_port;
+ }
+ if (sctp_mask->hdr.dst_port) {
+ if (sctp_mask->hdr.dst_port != UINT16_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_DST_PORT, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_SCTP_DST_PORT, pattern->map_spec);
+ pattern->item_spec.sctp.dst_port = sctp_spec->hdr.dst_port;
+ pattern->item_mask.sctp.dst_port = sctp_mask->hdr.dst_port;
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_SCTP, pattern->hdrs);
+ if (sxe2_test_bit(SXE2_FLOW_HDR_IPV4, pattern->hdrs))
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV4_SCTP;
+ else if (sxe2_test_bit(SXE2_FLOW_HDR_IPV6, pattern->hdrs))
+ pattern->rss_type_allow |= RTE_ETH_RSS_NONFRAG_IPV6_SCTP;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_SRC_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_DST_ONLY;
+ pattern->rss_type_allow |= RTE_ETH_RSS_L4_CHKSUM;
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_geneve(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_geneve *geneve_spec;
+ const struct rte_flow_item_geneve *geneve_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ geneve_spec = item->spec;
+ geneve_mask = item->mask;
+
+ if (!(geneve_spec && geneve_mask))
+ goto l_end;
+
+ if (geneve_mask->protocol || geneve_mask->ver_opt_len_o_c_rsvd0 || geneve_mask->rsvd1) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some Geneve mask");
+ PMD_LOG_ERR(DRV, "Unsupported some Geneve mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (geneve_mask->vni[0] || geneve_mask->vni[1] || geneve_mask->vni[2]) {
+ if (strcmp((const char *)geneve_mask->vni, "\xFF\xFF\xFF") != 0)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_GENEVE_VNI, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_GENEVE_VNI, pattern->map_spec);
+ pattern->item_spec.geneve.vni = (geneve_spec->vni[2] << 16) |
+ (geneve_spec->vni[1] << 8) | geneve_spec->vni[0];
+ pattern->item_mask.geneve.vni = (geneve_mask->vni[2] << 16) |
+ (geneve_mask->vni[1] << 8) | geneve_mask->vni[0];
+ }
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_GENEVE, pattern->hdrs);
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_gtpu(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_gtp *gtpu_spec;
+ const struct rte_flow_item_gtp *gtpu_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ gtpu_spec = item->spec;
+ gtpu_mask = item->mask;
+
+ if (gtpu_mask == NULL && gtpu_spec == NULL)
+ goto l_end;
+
+ if (gtpu_mask->v_pt_rsv_flags || gtpu_mask->msg_type || gtpu_mask->msg_len) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some GTPU mask");
+ PMD_LOG_ERR(DRV, "Unsupported some GTPU mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (gtpu_mask->teid) {
+ if (gtpu_mask->teid != UINT32_MAX)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_GTPU_TEID, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_GTPU_TEID, pattern->map_spec);
+ pattern->item_spec.gtpu.teid = gtpu_spec->teid;
+ pattern->item_mask.gtpu.teid = gtpu_mask->teid;
+ }
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_GTPU, pattern->hdrs);
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_gre(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_gre *gre_spec;
+ const struct rte_flow_item_gre *gre_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ gre_spec = item->spec;
+ gre_mask = item->mask;
+
+ if (gre_mask == NULL && gre_spec == NULL)
+ goto l_end;
+
+ if (gre_mask->c_rsvd0_ver || gre_mask->protocol) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some GRE mask");
+ PMD_LOG_ERR(DRV, "Unsupported some GRE mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_GRE, pattern->hdrs);
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_nvgre(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_nvgre *nvgre_spec;
+ const struct rte_flow_item_nvgre *nvgre_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ nvgre_spec = item->spec;
+ nvgre_mask = item->mask;
+
+ if (nvgre_mask == NULL && nvgre_spec == NULL)
+ goto l_end;
+
+ if (nvgre_mask->c_k_s_rsvd0_ver || nvgre_mask->protocol || nvgre_mask->flow_id) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some NVGRE mask");
+ PMD_LOG_ERR(DRV, "Unsupported some NVGRE mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (nvgre_mask->tni[0] || nvgre_mask->tni[1] || nvgre_mask->tni[2]) {
+ if (strcmp((const char *)nvgre_mask->tni, "\xFF\xFF\xFF") != 0)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_NVGRE_TNI, pattern->map_mask);
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_NVGRE_TNI, pattern->map_spec);
+ pattern->item_spec.nvgre.tni =
+ (nvgre_spec->tni[2] << 16) |
+ (nvgre_spec->tni[1] << 8) |
+ (nvgre_spec->tni[0]);
+ pattern->item_mask.nvgre.tni =
+ (nvgre_mask->tni[2] << 16) |
+ (nvgre_mask->tni[1] << 8) |
+ (nvgre_mask->tni[0]);
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_GRE, pattern->hdrs);
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_vxlan(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_vxlan *vxlan_spec;
+ const struct rte_flow_item_vxlan *vxlan_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ vxlan_spec = item->spec;
+ vxlan_mask = item->mask;
+
+ if (vxlan_mask == NULL && vxlan_spec == NULL)
+ goto l_end;
+
+ if (vxlan_mask->flags ||
+ vxlan_mask->rsvd1 ||
+ vxlan_mask->rsvd0[0] ||
+ vxlan_mask->rsvd0[1] ||
+ vxlan_mask->rsvd0[2]) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some VXLAN mask");
+ PMD_LOG_ERR(DRV, "Unsupported some VXLAN mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (vxlan_mask->vni[0] || vxlan_mask->vni[1] || vxlan_mask->vni[2]) {
+ if (strcmp((const char *)vxlan_mask->vni, "\xFF\xFF\xFF") != 0)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_VXLAN_VNI, pattern->map_mask);
+
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_VXLAN_VNI, pattern->map_spec);
+ pattern->item_spec.vxlan.vni =
+ (vxlan_spec->vni[2] << 16) |
+ (vxlan_spec->vni[1] << 8) |
+ (vxlan_spec->vni[0]);
+ pattern->item_mask.vxlan.vni =
+ (vxlan_mask->vni[2] << 16) |
+ (vxlan_mask->vni[1] << 8) |
+ (vxlan_mask->vni[0]);
+ }
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_VXLAN, pattern->hdrs);
+ return ret;
+}
+
+static int32_t sxe2_flow_parse_pattern_vxlan_gpe(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next __rte_unused,
+ bool is_inner)
+{
+ const struct rte_flow_item_vxlan_gpe *vxlan_gpe_spec;
+ const struct rte_flow_item_vxlan_gpe *vxlan_gpe_mask;
+ struct sxe2_flow_pattern *pattern = is_inner ? &flow->pattern_inner : &flow->pattern_outer;
+ int32_t ret = 0;
+ vxlan_gpe_spec = item->spec;
+ vxlan_gpe_mask = item->mask;
+
+ if (vxlan_gpe_mask == NULL && vxlan_gpe_spec == NULL)
+ goto l_end;
+
+ if (vxlan_gpe_mask->flags ||
+ vxlan_gpe_mask->protocol ||
+ vxlan_gpe_mask->rsvd1 ||
+ vxlan_gpe_mask->rsvd0[0] ||
+ vxlan_gpe_mask->rsvd0[1]) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Unsupported some VXLAN-GPE mask");
+ PMD_LOG_ERR(DRV, "Unsupported some VXLAN-GPE mask");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if (vxlan_gpe_mask->vni[0] || vxlan_gpe_mask->vni[1] || vxlan_gpe_mask->vni[2]) {
+ if (strcmp((const char *)vxlan_gpe_mask->vni, "\xFF\xFF\xFF") != 0)
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_VXLAN_VNI, pattern->map_mask);
+
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_VXLAN_VNI, pattern->map_spec);
+ pattern->item_spec.vxlan.vni =
+ (vxlan_gpe_spec->vni[2] << 16) |
+ (vxlan_gpe_spec->vni[1] << 8) |
+ (vxlan_gpe_spec->vni[0]);
+ pattern->item_mask.vxlan.vni =
+ (vxlan_gpe_mask->vni[2] << 16) |
+ (vxlan_gpe_mask->vni[1] << 8) |
+ (vxlan_gpe_mask->vni[0]);
+ }
+
+l_end:
+ sxe2_set_bit(SXE2_FLOW_HDR_VXLAN, pattern->hdrs);
+ return ret;
+}
+
+struct sxe2_flow_parse_pattern_ops sxe2_flow_parse_pattern_list[] = {
+ [SXE2_EXPANSION_OUTER_ETH] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_eth,
+ },
+ [SXE2_EXPANSION_OUTER_VLAN] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_vlan,
+ },
+ [SXE2_EXPANSION_OUTER_QINQ] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_vlan,
+ },
+ [SXE2_EXPANSION_OUTER_IPV4] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_ipv4,
+ },
+ [SXE2_EXPANSION_OUTER_IPV6] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_ipv6,
+ },
+ [SXE2_EXPANSION_OUTER_IPV6_FRAG_EXT] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_ipv6_frag_ext,
+ },
+ [SXE2_EXPANSION_OUTER_TCP] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_tcp,
+ },
+ [SXE2_EXPANSION_OUTER_UDP] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_udp,
+ },
+ [SXE2_EXPANSION_OUTER_SCTP] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_sctp,
+ },
+ [SXE2_EXPANSION_GENEVE] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_geneve,
+ },
+ [SXE2_EXPANSION_GTPU] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_gtpu,
+ },
+ [SXE2_EXPANSION_GRE] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_gre,
+ },
+ [SXE2_EXPANSION_NVGRE] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_nvgre,
+ },
+ [SXE2_EXPANSION_VXLAN] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_vxlan,
+ },
+ [SXE2_EXPANSION_VXLAN_GPE] = {
+ .is_inner = false,
+ .func = sxe2_flow_parse_pattern_vxlan_gpe,
+ },
+ [SXE2_EXPANSION_ETH] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_eth,
+ },
+ [SXE2_EXPANSION_VLAN] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_vlan,
+ },
+ [SXE2_EXPANSION_IPV4] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_ipv4,
+ },
+ [SXE2_EXPANSION_IPV6] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_ipv6,
+ },
+ [SXE2_EXPANSION_IPV6_FRAG_EXT] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_ipv6_frag_ext,
+ },
+ [SXE2_EXPANSION_TCP] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_tcp,
+ },
+ [SXE2_EXPANSION_UDP] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_udp,
+ },
+ [SXE2_EXPANSION_SCTP] = {
+ .is_inner = true,
+ .func = sxe2_flow_parse_pattern_sctp,
+ },
+
+};
+
+int32_t sxe2_flow_parse_pattern(struct rte_eth_dev *dev __rte_unused,
+ const struct rte_flow_item patterns[],
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow)
+{
+ int32_t ret = 0;
+ const struct rte_flow_item *item = patterns;
+ enum sxe2_expansion now = SXE2_EXPANSION_OUTER_ETH;
+ enum sxe2_expansion next = SXE2_EXPANSION_OUTER_ETH;
+ enum sxe2_flow_tunnel_type tunnel_type = SXE2_FLOW_TUNNEL_TYPE_NONE;
+ DECLARE_BITMAP(flow_type, SXE2_EXPANSION_MAX);
+ sxe2_bitmap_zero(flow_type, SXE2_EXPANSION_MAX);
+ sxe2_flow_parse_pattern_func_t func;
+ bool is_inner = false;
+
+ for (item = patterns; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+ if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
+ continue;
+
+ if (item->last) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, item,
+ "FANV not support range pattern.");
+ PMD_LOG_ERR(DRV, "flow not supported range pattern.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ ret = sxe2_flow_is_expandable_item(item);
+ if (ret != 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, item,
+ "Unsupported item type.");
+ PMD_LOG_ERR(DRV, "Unsupported item type: %d", item->type);
+ goto l_end;
+ }
+ next = now;
+ ret = sxe2_flow_valid_next_expansion(&next, item,
+ &tunnel_type, flow_type);
+ if (ret != 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, item,
+ "Invalid pattern sequence for rule.");
+ PMD_LOG_ERR(DRV, "Invalid pattern sequence for rule.");
+ goto l_end;
+ }
+
+ if ((item->spec != NULL && item->mask == NULL) ||
+ (item->spec == NULL && item->mask != NULL)) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, item,
+ "Invalid pattern spec miss macth mask for rule.");
+ PMD_LOG_ERR(DRV, "Invalid pattern spec miss macth mask for rule.");
+ goto l_end;
+ }
+
+ func = sxe2_flow_parse_pattern_list[now].func;
+ is_inner = sxe2_flow_parse_pattern_list[now].is_inner;
+ ret = func(item, error, flow, next, is_inner);
+ if (ret != 0)
+ goto l_end;
+ now = next;
+ }
+ if (sxe2_bitmap_weight(flow->pattern_outer.hdrs, SXE2_FLOW_HDR_MAX) > 0)
+ flow->has_hdr = 1;
+
+ if (sxe2_bitmap_weight(flow->pattern_inner.map_mask, SXE2_FLOW_FLD_ID_MAX) > 0 ||
+ sxe2_bitmap_weight(flow->pattern_outer.map_mask, SXE2_FLOW_FLD_ID_MAX) > 0)
+ flow->has_mask = 1;
+
+ if (sxe2_bitmap_weight(flow->pattern_inner.map_spec, SXE2_FLOW_FLD_ID_MAX) > 0 ||
+ sxe2_bitmap_weight(flow->pattern_outer.map_spec, SXE2_FLOW_FLD_ID_MAX) > 0)
+ flow->has_spec = 1;
+
+ flow->meta.tunnel_type = tunnel_type;
+ if (flow->has_hdr) {
+ ret = sxe2_flow_get_flow_type(flow);
+ if (ret != 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ NULL, "Unsupported flow type.");
+ goto l_end;
+ }
+ }
+ sxe2_bitmap_copy(flow->flow_type, flow_type, SXE2_EXPANSION_MAX);
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_flow_parse_pattern.h b/drivers/net/sxe2/sxe2_flow_parse_pattern.h
new file mode 100644
index 0000000000..69d83a6ea6
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_parse_pattern.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_FLOW_PARSE_PATTERN_H_
+#define SXE2_FLOW_PARSE_PATTERN_H_
+#include <rte_flow_driver.h>
+#include "sxe2_osal.h"
+#include "sxe2_flow_define.h"
+
+#define SXE2_FLOW_EXPAND_NEXT(...) \
+ ((const enum sxe2_expansion []){ \
+ __VA_ARGS__, 0, \
+ })
+
+struct sxe2_flow_expand_node {
+ const enum rte_flow_item_type type;
+ const enum sxe2_flow_tunnel_type tunnel_type;
+ const uint8_t is_tunnel;
+ const enum sxe2_expansion *const next;
+ const char *const name;
+};
+
+typedef int32_t (*sxe2_flow_parse_pattern_func_t)(const struct rte_flow_item *item,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ enum sxe2_expansion next,
+ bool is_inner);
+
+struct sxe2_flow_parse_pattern_ops {
+ bool is_inner;
+ sxe2_flow_parse_pattern_func_t func;
+};
+
+int32_t sxe2_flow_parse_pattern(struct rte_eth_dev *dev,
+ const struct rte_flow_item patterns[],
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow);
+
+#endif /* SXE2_FLOW_PARSE_PATTERN_H_ */
diff --git a/drivers/net/sxe2/sxe2_irq.c b/drivers/net/sxe2/sxe2_irq.c
index 13619500ea..3634eb99b6 100644
--- a/drivers/net/sxe2/sxe2_irq.c
+++ b/drivers/net/sxe2/sxe2_irq.c
@@ -20,6 +20,7 @@
#include "sxe2vf_regs.h"
#include "sxe2_host_regs.h"
#include "sxe2_cmd_chnl.h"
+#include "sxe2_switchdev.h"
#define SXE2_INT_EVENT_OICR_ALL (SXE2_PF_INT_OICR_SWINT | \
SXE2_PF_INT_OICR_LAN_TX_ERR | \
@@ -58,6 +59,14 @@ static void sxe2_event_irq_common_handler(struct sxe2_adapter *adapter, uint64_t
RTE_ETH_EVENT_INTR_LSC,
NULL);
}
+ if (oicr & RTE_BIT32(SXE2_COM_SW_MODE_SWITCHDEV)) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "event notify switchdev");
+ (void)sxe2_switchdev_notify_callback(adapter, true);
+ }
+ if (oicr & RTE_BIT32(SXE2_COM_SW_MODE_LEGACY)) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "event notify legacy");
+ (void)sxe2_switchdev_notify_callback(adapter, false);
+ }
}
static uint32_t sxe2_event_intr_handle(void *param __rte_unused)
@@ -881,6 +890,42 @@ int32_t sxe2_rxq_intr_enable(struct rte_eth_dev *dev)
return ret;
}
+int32_t sxe2_repr_rxq_intr_enable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint16_t rxq_cnt = dev->data->nb_rx_queues;
+ int32_t ret = 0;
+ uint16_t qid = adapter->repr_priv_data->repr_q_id;
+ uint32_t val;
+
+ if (!rxq_cnt)
+ goto l_end;
+
+ sxe2_pci_hw_irq_disable(adapter, qid);
+ sxe2_pci_hw_int_itr_set(adapter, qid, SXE2_ITR_INTERVAL_NORMAL);
+ ret = sxe2_drv_rxq_bind_irq(adapter, qid, qid);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "RXQ[%u] bind IRQ[%u] failed.",
+ qid, qid);
+ goto l_end;
+ }
+ sxe2_pci_hw_irq_enable(adapter, qid);
+
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, qid);
+ if ((val & SXE2VF_DYN_CTL_INTENABLE) == 0)
+ goto l_end;
+
+ sxe2_pci_hw_msix_disable(adapter, qid);
+ sxe2_pci_hw_irq_trigger(adapter, qid);
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, qid);
+ sxe2_pci_hw_irq_clear_pba(adapter, qid);
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, qid);
+ sxe2_pci_hw_msix_enable(adapter, qid);
+
+l_end:
+ return ret;
+}
+
void sxe2_rxq_intr_disable(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter =
@@ -901,6 +946,15 @@ void sxe2_rxq_intr_disable(struct rte_eth_dev *dev)
return;
}
+void sxe2_repr_rxq_intr_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint16_t qid = adapter->repr_priv_data->repr_q_id;
+
+ sxe2_pci_hw_irq_disable(adapter, qid);
+ (void)sxe2_drv_rxq_unbind_irq(adapter, qid);
+}
+
int32_t sxe2_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct sxe2_adapter *adapter =
diff --git a/drivers/net/sxe2/sxe2_irq.h b/drivers/net/sxe2/sxe2_irq.h
index c898c16f84..e35278610b 100644
--- a/drivers/net/sxe2/sxe2_irq.h
+++ b/drivers/net/sxe2/sxe2_irq.h
@@ -60,8 +60,12 @@ void sxe2_sw_irq_ctx_hw_cap_set(struct sxe2_adapter *adapter,
int32_t sxe2_rxq_intr_enable(struct rte_eth_dev *dev);
+int32_t sxe2_repr_rxq_intr_enable(struct rte_eth_dev *dev);
+
void sxe2_rxq_intr_disable(struct rte_eth_dev *dev);
+void sxe2_repr_rxq_intr_disable(struct rte_eth_dev *dev);
+
int32_t sxe2_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
int32_t sxe2_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
diff --git a/drivers/net/sxe2/sxe2_queue.c b/drivers/net/sxe2/sxe2_queue.c
index 220cab6fce..afb2681b72 100644
--- a/drivers/net/sxe2/sxe2_queue.c
+++ b/drivers/net/sxe2/sxe2_queue.c
@@ -24,7 +24,11 @@ int32_t sxe2_queues_init(struct rte_eth_dev *dev)
struct sxe2_rx_queue *rxq;
uint16_t nb_rxq;
- frame_size = dev->data->mtu + SXE2_ETH_OVERHEAD;
+ if (adapter->is_dev_repr)
+ frame_size = SXE2_FRAME_SIZE_MAX - SXE2_ETH_OVERHEAD;
+ else
+ frame_size = dev->data->mtu + SXE2_ETH_OVERHEAD;
+
for (nb_rxq = 0; nb_rxq < dev->data->nb_rx_queues; nb_rxq++) {
rxq = dev->data->rx_queues[nb_rxq];
if (!rxq)
diff --git a/drivers/net/sxe2/sxe2_stats.c b/drivers/net/sxe2/sxe2_stats.c
index 7ea2815fa3..2f8bb432c4 100644
--- a/drivers/net/sxe2/sxe2_stats.c
+++ b/drivers/net/sxe2/sxe2_stats.c
@@ -154,7 +154,12 @@ static int32_t sxe2_vsi_hw_stats_get_update(struct sxe2_adapter *adapter)
ret = sxe2_drv_get_vsi_stats(adapter);
if (ret) {
- PMD_LOG_ERR(DRV, "get vsi stats failed, ret:%d.", ret);
+ if (adapter->is_dev_repr) {
+ PMD_LOG_WARN(DRV, "get repr vsi stats failed, ret:%d.", ret);
+ ret = 0;
+ } else {
+ PMD_LOG_ERR(DRV, "get vsi stats failed, ret:%d.", ret);
+ }
goto l_end;
}
@@ -231,7 +236,7 @@ static void sxe2_stats_update(struct sxe2_adapter *adapter)
stats->rx_sw_drop_bytes = sw_stats->rx_sw_drop_bytes +
sw_stats_prev->rx_sw_drop_bytes;
- if (adapter->dev_type != SXE2_DEV_T_VF) {
+ if (adapter->dev_type != SXE2_DEV_T_VF && !adapter->is_dev_repr) {
stats->rx_out_of_buffer = hw_stats->rx_out_of_buffer;
stats->rx_qblock_drop = hw_stats->rx_qblock_drop;
stats->tx_frame_good = hw_stats->tx_frame_good;
@@ -364,7 +369,7 @@ int32_t sxe2_xstats_info_get(struct rte_eth_dev *dev,
if (rte_eal_process_type() == RTE_PROC_SECONDARY)
return sxe2_mp_req_get_xstats(dev, xstats, usr_cnt);
- if (adapter->dev_type == SXE2_DEV_T_VF)
+ if (adapter->dev_type == SXE2_DEV_T_VF || adapter->is_dev_repr)
xstats_cnt = SXE2_XSTAT_CNT_VF;
else
xstats_cnt = SXE2_XSTAT_CNT_PF;
@@ -387,7 +392,7 @@ int32_t sxe2_xstats_info_get(struct rte_eth_dev *dev,
goto end;
}
- if (adapter->dev_type == SXE2_DEV_T_VF) {
+ if (adapter->dev_type == SXE2_DEV_T_VF || adapter->is_dev_repr) {
sxe2_stats_update(adapter);
for (i = 0; i < xstats_cnt; i++) {
(void)sxe2_xstat_vf_offset_get(i, &offset);
@@ -431,7 +436,7 @@ int32_t sxe2_xstats_names_get(__rte_unused struct rte_eth_dev *dev,
int32_t ret = -1;
uint32_t xstats_cnt = 0;
- if (adapter->dev_type == SXE2_DEV_T_VF) {
+ if (adapter->dev_type == SXE2_DEV_T_VF || adapter->is_dev_repr) {
field = sxe2_xstats_field_vf;
xstats_cnt = SXE2_XSTAT_CNT_VF;
} else {
@@ -476,7 +481,7 @@ int32_t sxe2_stats_hw_reset(struct rte_eth_dev *dev)
PMD_LOG_ERR(DRV, "reset vsi stats failed, ret:%d.", ret);
goto l_end;
}
- if (adapter->dev_type != SXE2_DEV_T_VF) {
+ if (adapter->dev_type != SXE2_DEV_T_VF && !adapter->is_dev_repr) {
ret = sxe2_drv_mac_stats_reset(adapter);
if (ret) {
PMD_LOG_ERR(DRV, "reset mac stats failed, ret:%d.", ret);
diff --git a/drivers/net/sxe2/sxe2_switchdev.c b/drivers/net/sxe2/sxe2_switchdev.c
new file mode 100644
index 0000000000..44703cfb5c
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_switchdev.c
@@ -0,0 +1,332 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_os.h>
+#include <rte_tailq.h>
+#include "sxe2_osal.h"
+#include "sxe2_mac.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_switchdev.h"
+#include "sxe2_cmd_chnl.h"
+#include "sxe2_host_regs.h"
+
+int32_t sxe2_uplink_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "Current mode is not switchdev");
+ goto l_end;
+ }
+
+ ret = sxe2_drv_switchdev_uplink_config(adapter, false);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_uplink_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "Current mode is not switchdev");
+ goto l_end;
+ }
+
+ ret = sxe2_drv_switchdev_uplink_config(adapter, true);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_repr_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ uint16_t repr_id = 0;
+ struct rte_eth_dev *repr_dev;
+ struct sxe2_adapter *repr_adapter;
+ struct sxe2_switchdev_repr_info repr_vf;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "Current mode is not switchdev");
+ goto l_end;
+ }
+
+ for (repr_id = 0; repr_id < adapter->repr_ctxt.nb_repr_vf; repr_id++) {
+ repr_dev = adapter->repr_ctxt.vf_rep_eth_dev[repr_id];
+ if (!repr_dev)
+ continue;
+ repr_adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(repr_dev);
+ if (repr_adapter &&
+ repr_adapter->repr_priv_data &&
+ repr_adapter->repr_priv_data->cp_vsi) {
+ memset(&repr_vf, 0, sizeof(struct sxe2_switchdev_repr_info));
+
+ repr_vf.repr_pf_id = repr_adapter->repr_priv_data->repr_pf_id;
+ repr_vf.repr_vf_id = repr_adapter->repr_priv_data->repr_vf_id;
+ repr_vf.cp_vsi_id = repr_adapter->repr_priv_data->cp_vsi->vsi_id;
+ repr_vf.repr_q_id = repr_adapter->repr_priv_data->repr_q_id;
+ ret = sxe2_drv_switchdev_repr_vf_config(adapter, &repr_vf,
+ false);
+ }
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_repr_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ uint16_t repr_id = 0;
+ struct rte_eth_dev *repr_dev;
+ struct sxe2_adapter *repr_adapter;
+ struct sxe2_switchdev_repr_info repr_vf;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "Current mode is not switchdev");
+ goto l_end;
+ }
+
+ for (repr_id = 0; repr_id < adapter->repr_ctxt.nb_repr_vf; repr_id++) {
+ repr_dev = adapter->repr_ctxt.vf_rep_eth_dev[repr_id];
+ if (!repr_dev)
+ continue;
+ repr_adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(repr_dev);
+ if (repr_adapter &&
+ repr_adapter->repr_priv_data &&
+ repr_adapter->repr_priv_data->cp_vsi) {
+ memset(&repr_vf, 0, sizeof(struct sxe2_switchdev_repr_info));
+
+ repr_vf.repr_pf_id = repr_adapter->repr_priv_data->repr_pf_id;
+ repr_vf.repr_vf_id = repr_adapter->repr_priv_data->repr_vf_id;
+ repr_vf.cp_vsi_id = repr_adapter->repr_priv_data->cp_vsi->vsi_id;
+ repr_vf.repr_q_id = repr_adapter->repr_priv_data->repr_q_id;
+ ret = sxe2_drv_switchdev_repr_vf_config(adapter, &repr_vf, true);
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+void sxe2_free_repr_info(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (adapter->repr_ctxt.vf_rep_eth_dev) {
+ rte_free(adapter->repr_ctxt.vf_rep_eth_dev);
+ adapter->repr_ctxt.vf_rep_eth_dev = NULL;
+ }
+
+ adapter->repr_ctxt.nb_repr_vf = 0;
+}
+
+static int32_t sxe2_switchdev_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "Current mode is not switchdev");
+ goto l_end;
+ }
+
+ adapter->switchdev_info.is_switchdev = false;
+
+ if (!adapter->flow_isolate_cfg && adapter->flow_isolated)
+ adapter->flow_isolated = false;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ ret = sxe2_switchdev_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update switchdev rule");
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_switchdev_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "Current mode switch dev");
+ goto l_end;
+ }
+
+ adapter->switchdev_info.is_switchdev = true;
+
+ if (adapter->flow_isolate_cfg && !adapter->flow_isolated)
+ adapter->flow_isolated = true;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ ret = sxe2_switchdev_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update switchdev rule");
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_switchdev_notify_callback(struct sxe2_adapter *adapter, bool set)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[adapter->dev_info.dev_data->port_id];
+ int32_t ret = 0;
+ bool cur_switchdev_set = false;
+
+ if (adapter->repr_ctxt.nb_repr_vf) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "switch dev notify remove dev");
+ rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL);
+ goto l_end;
+ }
+
+ ret = sxe2_drv_switchdev_mode_get(adapter, &cur_switchdev_set);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to get switchdev mode");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (set != cur_switchdev_set) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "current switchdev mode miss macth");
+ goto l_end;
+ }
+
+ if (set) {
+ ret = sxe2_switchdev_set(adapter);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set switchdev");
+ goto l_end;
+ }
+ } else {
+ ret = sxe2_switchdev_clear(adapter);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to clear switchdev");
+ goto l_end;
+ }
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_switchdev_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ PMD_DEV_LOG_INFO(adapter, INIT, "switchdev init");
+
+ if (adapter->switchdev_info.is_switchdev)
+ adapter->flow_isolated = true;
+
+ adapter->repr_priv_data = NULL;
+ adapter->repr_ctxt.nb_repr_vf = 0;
+
+ return 0;
+}
+
+int32_t sxe2_switchdev_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ PMD_DEV_LOG_INFO(adapter, INIT, "switchdev uinit");
+
+ if (adapter->repr_priv_data) {
+ rte_free(adapter->repr_priv_data);
+ adapter->repr_priv_data = NULL;
+ }
+
+ return 0;
+}
+
+int32_t sxe2_switchdev_dev_info_init(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_dev_info *dev_info = &adapter->dev_info;
+ struct sxe2_dev_info *parent_dev_info = &parent_adapter->dev_info;
+ struct sxe2_drv_dev_info_resp dev_info_resp = {0};
+ struct sxe2_drv_dev_fw_info_resp dev_fw_info_resp = {0};
+ int32_t ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ dev_info->pci = parent_dev_info->pci;
+ dev_info->pci.max_vfs = 0;
+
+ ret = sxe2_drv_dev_info_get(adapter, &dev_info_resp);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to get device info, ret=[%d]", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_drv_dev_fw_info_get(adapter, &dev_fw_info_resp);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to get device fw info, ret=[%d]", ret);
+ goto l_end;
+ }
+ dev_info->fw.build_id = dev_fw_info_resp.build_id;
+ dev_info->fw.fix_version_id = dev_fw_info_resp.fix_version_id;
+ dev_info->fw.sub_version_id = dev_fw_info_resp.sub_version_id;
+ dev_info->fw.main_version_id = dev_fw_info_resp.main_version_id;
+
+ rte_ether_addr_copy((struct rte_ether_addr *)dev_info_resp.mac_addr,
+ (struct rte_ether_addr *)dev_info->mac.perm_addr);
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_switchdev_repr_private_data_init(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter, uint16_t repr_id)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_repr_private_data *repr_priv_data = adapter->repr_priv_data;
+ int32_t ret = 0;
+
+ if (repr_priv_data != NULL)
+ goto l_end;
+
+ repr_priv_data = rte_zmalloc("sxe2_repr_priv_data",
+ sizeof(struct sxe2_repr_private_data), 0);
+ if (repr_priv_data == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to malloc representor private data.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ repr_priv_data->parent_adapter = parent_adapter;
+ repr_priv_data->repr_id = repr_id;
+ repr_priv_data->cp_vsi =
+ TAILQ_FIRST(&parent_adapter->vsi_ctxt.other_vsi_list);
+ if (repr_priv_data->cp_vsi == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to get cp vsi.");
+ ret = -EINVAL;
+ goto l_free;
+ }
+ repr_priv_data->repr_q_id = repr_id;
+ repr_priv_data->repr_pf_id = parent_adapter->pf_idx;
+ repr_priv_data->repr_vf_id = repr_id;
+ repr_priv_data->repr_vf_k_vsi_id =
+ parent_adapter->repr_ctxt.repr_vf_id[repr_id].kernel_vsi_id;
+ repr_priv_data->repr_vf_u_vsi_id =
+ parent_adapter->repr_ctxt.repr_vf_id[repr_id].dpdk_vsi_id;
+
+ repr_priv_data->repr_vf_vsi_id =
+ parent_adapter->repr_ctxt.repr_vf_id[repr_id].kernel_vsi_id !=
+ SXE2_INVALID_VSI_ID ?
+ parent_adapter->repr_ctxt.repr_vf_id[repr_id].kernel_vsi_id :
+ parent_adapter->repr_ctxt.repr_vf_id[repr_id].dpdk_vsi_id;
+
+ adapter->repr_priv_data = repr_priv_data;
+ goto l_end;
+l_free:
+ rte_free(repr_priv_data);
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_switchdev.h b/drivers/net/sxe2/sxe2_switchdev.h
new file mode 100644
index 0000000000..0a74454424
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_switchdev.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_SWITCHDEV_H__
+#define __SXE2_SWITCHDEV_H__
+#include <ethdev_driver.h>
+
+struct sxe2_adapter;
+
+int32_t sxe2_uplink_clear(struct sxe2_adapter *adapter);
+
+int32_t sxe2_uplink_set(struct sxe2_adapter *adapter);
+
+int32_t sxe2_repr_clear(struct sxe2_adapter *adapter);
+
+int32_t sxe2_repr_set(struct sxe2_adapter *adapter);
+
+int32_t sxe2_switchdev_notify_callback(struct sxe2_adapter *adapter, bool set);
+
+int32_t sxe2_switchdev_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_switchdev_uninit(struct rte_eth_dev *dev);
+
+void sxe2_free_repr_info(struct rte_eth_dev *dev);
+
+int32_t sxe2_switchdev_dev_info_init(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter);
+
+int32_t sxe2_switchdev_repr_private_data_init(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter, uint16_t repr_id);
+
+#endif /* __SXE2_SWITCHDEV_H__ */
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 2eb8365457..a2cea954d5 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -152,6 +152,13 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
uint32_t batch_flags = 0;
PMD_INIT_FUNC_TRACE();
+
+ if (adapter->is_dev_repr) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts;
+ tx_mode_flags = 0;
+ return;
+ }
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
tx_mode_flags = 0;
ret = sxe2_tx_vec_support_check(dev, &vec_flags);
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index 8b6e585c36..f3c4fa0d91 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -454,6 +454,14 @@ uint16_t sxe2_tx_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkt
desc_l2tag2 = tx_pkt->vlan_tci_outer;
desc_type_cmd_tso_mss |= SXE2_TX_CTXT_DESC_CMD_IL2TAG2_MASK;
}
+ if (unlikely(vsi->vsi_type == SXE2_VSI_T_DPDK_ESW)) {
+ desc_type_cmd_tso_mss |=
+ (SXE2_TX_CTXT_DESC_CMD_SWTCH_VSI <<
+ SXE2_TX_CTXT_DESC_CMD_SHIFT);
+ desc_type_cmd_tso_mss |=
+ ((vsi->adapter->repr_priv_data->repr_vf_vsi_id & 0x3FFULL)
+ << SXE2_TX_CTXT_DESC_VSI_SHIFT);
+ }
ctxt_desc->tunneling_params =
rte_cpu_to_le_32(desc_tunneling_params);
diff --git a/drivers/net/sxe2/sxe2_vsi.c b/drivers/net/sxe2/sxe2_vsi.c
index baaa20c02e..d29480b931 100644
--- a/drivers/net/sxe2/sxe2_vsi.c
+++ b/drivers/net/sxe2/sxe2_vsi.c
@@ -98,9 +98,15 @@ static struct sxe2_vsi *sxe2_vsi_node_create(struct sxe2_adapter *adapter,
static void sxe2_vsi_node_free(struct sxe2_vsi *vsi)
{
+ struct sxe2_adapter *adapter;
+
if (!vsi)
return;
+ adapter = vsi->adapter;
+ if (vsi->vsi_type == SXE2_VSI_T_ESW)
+ TAILQ_REMOVE(&adapter->vsi_ctxt.other_vsi_list, vsi, next);
+
rte_free(vsi);
vsi = NULL;
}
@@ -174,10 +180,54 @@ static int32_t sxe2_main_vsi_create(struct sxe2_adapter *adapter)
return ret;
}
+int32_t sxe2_other_vsi_create(struct sxe2_adapter *adapter, uint16_t cnt_vf)
+{
+ int32_t ret = 0;
+ struct sxe2_vsi *other_vsi = NULL;
+ uint16_t vsi_id = SXE2_INVALID_VSI_ID;
+
+ PMD_INIT_FUNC_TRACE();
+
+ other_vsi = sxe2_vsi_node_create(adapter, SXE2_INVALID_VSI_ID, SXE2_VSI_T_DPDK_ESW);
+ if (other_vsi == NULL) {
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ ret = sxe2_drv_switchdev_cpvsi_get(adapter, &vsi_id);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to query vsi from fw, ret=%d", ret);
+ goto l_free_vsi;
+ }
+
+ other_vsi->vsi_id = vsi_id;
+ other_vsi->vsi_type = SXE2_VSI_T_DPDK_ESW;
+
+ ret = sxe2_drv_vsi_info_get(adapter, other_vsi);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to query vsi info from fw, ret=%d", ret);
+ goto l_free_vsi;
+ }
+
+ other_vsi->txqs.q_cnt = RTE_MIN(cnt_vf, other_vsi->txqs.q_cnt);
+ other_vsi->rxqs.q_cnt = RTE_MIN(cnt_vf, other_vsi->rxqs.q_cnt);
+ other_vsi->irqs.avail_cnt = RTE_MIN(cnt_vf, other_vsi->irqs.avail_cnt);
+
+ TAILQ_INSERT_TAIL(&adapter->vsi_ctxt.other_vsi_list, other_vsi, next);
+ goto l_end;
+
+l_free_vsi:
+ sxe2_vsi_node_free(other_vsi);
+l_end:
+ return ret;
+}
+
int32_t sxe2_vsi_init(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
int32_t ret = 0;
+ uint16_t srcvsi_list[SXE2_SRCVSI_PRUNE_MAX_NUM];
+ uint16_t srcvsi_cnt;
PMD_INIT_FUNC_TRACE();
@@ -187,6 +237,18 @@ int32_t sxe2_vsi_init(struct rte_eth_dev *dev)
goto l_end;
}
+ if (adapter->vsi_ctxt.kernel_vsi_id != SXE2_INVALID_VSI_ID) {
+ srcvsi_list[0] = adapter->vsi_ctxt.kernel_vsi_id;
+ srcvsi_list[1] = adapter->vsi_ctxt.dpdk_vsi_id;
+ srcvsi_cnt = 2;
+ ret = sxe2_drv_srcvsi_prune_config(adapter, srcvsi_list, srcvsi_cnt, true);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to set src vsi to fw, ret=%d", ret);
+ goto l_end;
+ }
+ PMD_LOG_DEBUG(DRV, "Successfully set src vsi");
+ }
+
l_end:
return ret;
}
@@ -194,21 +256,105 @@ int32_t sxe2_vsi_init(struct rte_eth_dev *dev)
void sxe2_vsi_uninit(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vsi *var, *tvar;
int32_t ret;
+ uint16_t srcvsi_list[SXE2_SRCVSI_PRUNE_MAX_NUM];
+ uint16_t srcvsi_cnt;
if (adapter->vsi_ctxt.main_vsi == NULL) {
PMD_LOG_INFO(DRV, "vsi is not created, no need to destroy.");
goto l_end;
}
+ if (adapter->vsi_ctxt.kernel_vsi_id != SXE2_INVALID_VSI_ID) {
+ srcvsi_list[0] = adapter->vsi_ctxt.kernel_vsi_id;
+ srcvsi_list[1] = adapter->vsi_ctxt.dpdk_vsi_id;
+ srcvsi_cnt = 2;
+ ret = sxe2_drv_srcvsi_prune_config(adapter, srcvsi_list,
+ srcvsi_cnt, false);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to clear src vsi to fw, ret=%d", ret);
+ if (ret == -EPERM)
+ goto l_free;
+ goto l_end;
+ }
+ PMD_LOG_DEBUG(DRV, "Successfully clear src vsi");
+ }
+
+l_free:
ret = sxe2_vsi_destroy(adapter, adapter->vsi_ctxt.main_vsi);
if (ret) {
PMD_LOG_ERR(DRV, "Failed to del vsi from fw, ret=%d", ret);
goto l_end;
}
+ RTE_TAILQ_FOREACH_SAFE(var, &adapter->vsi_ctxt.other_vsi_list, next, tvar) {
+ ret = sxe2_vsi_destroy(adapter, var);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to del vsi from fw, ret=%d", ret);
+ break;
+ }
+ }
PMD_LOG_DEBUG(DRV, "vsi destroyed.");
l_end:
return;
}
+
+int32_t sxe2_vsi_repr_main_vsi_create(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter,
+ uint16_t repr_id)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vsi *vsi = NULL;
+ struct sxe2_vsi *pos;
+ int32_t ret = 0;
+
+ TAILQ_FOREACH(pos, &parent_adapter->vsi_ctxt.other_vsi_list, next) {
+ if (pos->vsi_type == SXE2_VSI_T_DPDK_ESW) {
+ vsi = pos;
+ break;
+ }
+ }
+
+ if (vsi == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to get dpdk vsi.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ TAILQ_INIT(&adapter->vsi_ctxt.other_vsi_list);
+
+ adapter->vsi_ctxt.vsi_type = SXE2_VSI_T_DPDK_ESW;
+ adapter->vsi_ctxt.kernel_vsi_id = SXE2_INVALID_VSI_ID;
+
+ adapter->cdev = parent_adapter->cdev;
+
+ adapter->q_ctxt.base_idx_in_pf = vsi->txqs.base_idx_in_func +
+ RTE_MIN(vsi->txqs.q_cnt, repr_id);
+ adapter->irq_ctxt.base_idx_in_func = vsi->irqs.base_idx_in_pf +
+ RTE_MIN(vsi->irqs.avail_cnt, repr_id);
+ adapter->q_ctxt.qp_cnt_assign = RTE_MIN(vsi->txqs.q_cnt, 1);
+ adapter->irq_ctxt.max_cnt_hw = RTE_MIN(vsi->irqs.avail_cnt, 1);
+
+ adapter->vsi_ctxt.main_vsi =
+ sxe2_vsi_node_create(adapter, vsi->vsi_id, SXE2_VSI_T_DPDK_ESW);
+ if (adapter->vsi_ctxt.main_vsi == NULL) {
+ ret = -ENOMEM;
+ PMD_LOG_ERR(DRV, "Failed to create vsi struct, ret=%d", ret);
+ goto l_end;
+ }
+ adapter->vsi_ctxt.dpdk_vsi_id = adapter->vsi_ctxt.main_vsi->vsi_id;
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+void sxe2_vsi_repr_main_vsi_destroy(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ sxe2_vsi_node_free(adapter->vsi_ctxt.main_vsi);
+}
diff --git a/drivers/net/sxe2/sxe2_vsi.h b/drivers/net/sxe2/sxe2_vsi.h
index e712f738f1..176f34b4f8 100644
--- a/drivers/net/sxe2/sxe2_vsi.h
+++ b/drivers/net/sxe2/sxe2_vsi.h
@@ -192,13 +192,23 @@ struct sxe2_vsi_context {
uint16_t bond_member_dpdk_vsi_id[SXE2_MAX_BOND_MEMBER_CNT];
struct sxe2_vsi *main_vsi;
+
+ struct sxe2_vsi_list_head other_vsi_list;
};
void sxe2_sw_vsi_ctx_hw_cap_set(struct sxe2_adapter *adapter,
- struct sxe2_drv_vsi_caps *vsi_caps);
+ struct sxe2_drv_vsi_caps *vsi_caps);
+
+int32_t sxe2_other_vsi_create(struct sxe2_adapter *adapter, uint16_t cnt_vf);
int32_t sxe2_vsi_init(struct rte_eth_dev *dev);
void sxe2_vsi_uninit(struct rte_eth_dev *dev);
+int32_t sxe2_vsi_repr_main_vsi_create(struct rte_eth_dev *dev,
+ struct sxe2_adapter *parent_adapter,
+ uint16_t repr_id);
+
+void sxe2_vsi_repr_main_vsi_destroy(struct rte_eth_dev *dev);
+
#endif /* __SXE2_VSI_H__ */
--
2.47.3
^ permalink raw reply related
* [PATCH v2 17/23] net/sxe2: implement private dump info
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch implements the 'eth_dev_priv_dump' ops for the sxe2 PMD.
This interface allows applications to dump driver-specific internal
state and configuration information to a file stream.
The output includes:
- capabilities.
- device base info.
- device args info.
- device filter info.
- reprenstor info.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_dump.c | 289 ++++++++++++++++++++++++++++
drivers/net/sxe2/sxe2_dump.h | 12 ++
drivers/net/sxe2/sxe2_ethdev.c | 3 +
drivers/net/sxe2/sxe2_ethdev_repr.c | 3 +
5 files changed, 308 insertions(+)
create mode 100644 drivers/net/sxe2/sxe2_dump.c
create mode 100644 drivers/net/sxe2/sxe2_dump.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 61925ee80f..631730bf3e 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -77,6 +77,7 @@ sources += files(
'sxe2_flow_parse_action.c',
'sxe2_flow_parse_pattern.c',
'sxe2_flow_parse_engine.c',
+ 'sxe2_dump.c',
)
allow_internal_get_api = true
diff --git a/drivers/net/sxe2/sxe2_dump.c b/drivers/net/sxe2/sxe2_dump.c
new file mode 100644
index 0000000000..9898456aea
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_dump.c
@@ -0,0 +1,289 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_malloc.h>
+#include <arpa/inet.h>
+
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_dump.h"
+#include "sxe2_stats.h"
+
+static void
+sxe2_dump_dev_feature_capability(FILE *file, struct rte_eth_dev *dev)
+{
+ uint32_t i;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ const struct {
+ uint32_t cap_bit;
+ const char *name;
+ } caps_name[] = {
+ {SXE2_DEV_CAPS_OFFLOAD_L2, "L2"},
+ {SXE2_DEV_CAPS_OFFLOAD_VLAN, "VLAN"},
+ {SXE2_DEV_CAPS_OFFLOAD_IPSEC, "IPSEC"},
+ {SXE2_DEV_CAPS_OFFLOAD_RSS, "RSS"},
+ {SXE2_DEV_CAPS_OFFLOAD_FNAV, "FNAV"},
+ {SXE2_DEV_CAPS_OFFLOAD_TM, "TM"},
+ {SXE2_DEV_CAPS_OFFLOAD_PTP, "PTP"},
+ };
+ if (adapter->is_dev_repr)
+ goto l_end;
+
+ fprintf(file, " - Dev Capability:\n");
+ for (i = 0; i < RTE_DIM(caps_name); i++) {
+ fprintf(file, "\t -- support %s: %s\n", caps_name[i].name,
+ (adapter->cap_flags & caps_name[i].cap_bit) ? "Yes" :
+ "No");
+ }
+l_end:
+ return;
+}
+
+static void
+sxe2_dump_device_basic_info(FILE *file, struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ fprintf(file,
+ " - Device Base Info:\n"
+ "\t -- name: %s\n"
+ "\t -- pf_idx: %u port_idx: %u\n"
+ "\t -- tx_mode_flags: 0x%x rx_mode_flags: 0x%x\n"
+ "\t -- flow_isolate_cfg: 0x%x flow_isolated: 0x%x\n"
+ "\t -- dev_type: 0x%x is_switchdev: 0x%x\n"
+ "\t -- is_dev_repr: 0x%x dev_port_id: 0x%x\n"
+ "\t -- dev_flags: 0x%x\n"
+ "\t -- intr_conf lsc: %u rxq: %u rmv: %u\n",
+ dev->data->name,
+ adapter->pf_idx, adapter->port_idx,
+ adapter->tx_mode_flags, adapter->rx_mode_flags,
+ adapter->flow_isolate_cfg, adapter->flow_isolated,
+ adapter->dev_type, adapter->switchdev_info.is_switchdev,
+ adapter->is_dev_repr, adapter->dev_port_id,
+ dev->data->dev_flags,
+ dev->data->dev_conf.intr_conf.lsc,
+ dev->data->dev_conf.intr_conf.rxq,
+ dev->data->dev_conf.intr_conf.rmv);
+}
+
+static void
+sxe2_dump_dev_args_info(FILE *file, struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (adapter->is_dev_repr)
+ goto l_end;
+
+ fprintf(file,
+ " - Device Args Info:\n"
+ "\t -- sw-stats-en: %s\n"
+ "\t -- high-performance-mode: %s\n"
+ "\t -- flow-duplicate-pattern: %u\n"
+ "\t -- fnav-stat-type: %u\n"
+ "\t -- sched_layer_mode: %u\n"
+ "\t -- rx_low_latency: %s\n"
+ "\t -- function-flow-direct: %s\n",
+ adapter->devargs.sw_stats_en ? "On" : "Off",
+ adapter->devargs.high_performance_mode ? "On" : "Off",
+ adapter->devargs.flow_dup_pattern_mode,
+ adapter->devargs.fnav_stat_type,
+ adapter->devargs.sched_layer_mode,
+ adapter->devargs.rx_low_latency ? "On" : "Off",
+ adapter->devargs.func_flow_direct_en ? "On" : "Off");
+l_end:
+ return;
+}
+
+static void sxe2_dump_filter_info(FILE *file, struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_mac_filter *mac_entry;
+ struct sxe2_mac_filter *next_mac_entry;
+ struct sxe2_vlan_filter *vlan_entry;
+ struct sxe2_vlan_filter *next_vlan_entry;
+
+ if (adapter->is_dev_repr)
+ goto l_end;
+
+ fprintf(file,
+ " - Device Filter Info:\n"
+ "\t -- cur_promisc:0x%x hw_promisc:0x%x\n"
+ "\t -- unicast_num: %u multicast_num: %u\n"
+ "\t -- vlan_num: %u filter_on: %u hw_filter_on: %u\n"
+ "\t -- vlan max_cnt: %u cnt: %u\n"
+ "\t -- tpid: 0x%x vid: 0x%x\n"
+ "\t -- vlan_outer_insert: 0x%x vlan_outer_strip: 0x%x\n"
+ "\t -- vlan_inner_insert: 0x%x vlan_inner_strip: 0x%x\n",
+ adapter->filter_ctxt.cur_promisc_flags,
+ adapter->filter_ctxt.hw_promisc_flags,
+ adapter->filter_ctxt.uc_num,
+ adapter->filter_ctxt.mc_num,
+ adapter->filter_ctxt.vlan_num,
+ adapter->filter_ctxt.vlan_info.filter_on,
+ adapter->filter_ctxt.vlan_info.hw_filter_on,
+ adapter->filter_ctxt.vlan_info.max_cnt,
+ adapter->filter_ctxt.vlan_info.cnt,
+ adapter->filter_ctxt.vlan_info.tpid,
+ adapter->filter_ctxt.vlan_info.vid,
+ adapter->filter_ctxt.vlan_info.outer_insert,
+ adapter->filter_ctxt.vlan_info.outer_strip,
+ adapter->filter_ctxt.vlan_info.inner_insert,
+ adapter->filter_ctxt.vlan_info.inner_strip);
+
+ if (adapter->filter_ctxt.uc_num > 0) {
+ fprintf(file,
+ "\t -- Unicast entry:\n");
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.uc_list, next,
+ next_mac_entry) {
+ fprintf(file,
+ "\t -- addr: %02x:%02x:%02x:%02x:%02x:%02x hw status:%u "
+ "default:%u\n",
+ mac_entry->mac_addr.addr_bytes[0],
+ mac_entry->mac_addr.addr_bytes[1],
+ mac_entry->mac_addr.addr_bytes[2],
+ mac_entry->mac_addr.addr_bytes[3],
+ mac_entry->mac_addr.addr_bytes[4],
+ mac_entry->mac_addr.addr_bytes[5],
+ mac_entry->hw_config,
+ mac_entry->default_config);
+ }
+ }
+
+ if (adapter->filter_ctxt.mc_num > 0) {
+ fprintf(file,
+ "\t -- Multicast entry:\n");
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.mc_list,
+ next, next_mac_entry) {
+ fprintf(file,
+ "\t -- addr: %02x:%02x:%02x:%02x:%02x:%02x "
+ "hw status:%u default:%u\n",
+ mac_entry->mac_addr.addr_bytes[0],
+ mac_entry->mac_addr.addr_bytes[1],
+ mac_entry->mac_addr.addr_bytes[2],
+ mac_entry->mac_addr.addr_bytes[3],
+ mac_entry->mac_addr.addr_bytes[4],
+ mac_entry->mac_addr.addr_bytes[5],
+ mac_entry->hw_config,
+ mac_entry->default_config);
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_num > 0) {
+ fprintf(file,
+ "\t -- Vlan entry:\n");
+ RTE_TAILQ_FOREACH_SAFE(vlan_entry, &adapter->filter_ctxt.vlan_list,
+ next, next_vlan_entry) {
+ fprintf(file,
+ "\t -- vlan tpid:0x%04x vid:0x%04x prio:%d "
+ "hw status:%u default:%u\n",
+ vlan_entry->vlan_info.tpid,
+ vlan_entry->vlan_info.vid,
+ vlan_entry->vlan_info.prio,
+ vlan_entry->hw_config,
+ vlan_entry->default_config);
+ }
+ }
+l_end:
+ return;
+}
+
+static const char *sxe2_vsi_id_str(uint16_t vsi_id, char *buf, size_t len)
+{
+ if (vsi_id == SXE2_INVALID_VSI_ID)
+ return "NA";
+
+ snprintf(buf, len, "%u", vsi_id);
+ return buf;
+}
+
+static void
+sxe2_dump_switchdev_info(FILE *file, struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint32_t idx;
+ char k_vsi_buf[16];
+ char u_vsi_buf[16];
+
+ if (adapter->is_dev_repr && adapter->repr_priv_data) {
+ fprintf(file,
+ " - Reprenstor Info:\n"
+ "\t -- repr_id: %u\n"
+ "\t -- repr_q_id: %u\n"
+ "\t -- repr_pf_id: %u\n"
+ "\t -- repr_vf_id: %u\n"
+ "\t -- repr_vf_vsi_id: %u\n"
+ "\t -- repr_vf_k_vsi_id: %s\n"
+ "\t -- repr_vf_u_vsi_id: %s\n",
+ adapter->repr_priv_data->repr_id,
+ adapter->repr_priv_data->repr_q_id,
+ adapter->repr_priv_data->repr_pf_id,
+ adapter->repr_priv_data->repr_vf_id,
+ adapter->repr_priv_data->repr_vf_vsi_id,
+ sxe2_vsi_id_str(adapter->repr_priv_data->repr_vf_k_vsi_id,
+ k_vsi_buf, sizeof(k_vsi_buf)),
+ sxe2_vsi_id_str(adapter->repr_priv_data->repr_vf_u_vsi_id,
+ u_vsi_buf, sizeof(u_vsi_buf)));
+ goto l_end;
+ }
+ if (adapter->switchdev_info.is_switchdev) {
+ fprintf(file,
+ " - Switchdev Info:\n"
+ "\t -- master:0x%x\n"
+ "\t -- representor: 0x%x\n"
+ "\t -- port_name_type: 0x%x\n"
+ "\t -- nb_vf: %u nb_repr_vf: %u\n",
+ adapter->switchdev_info.master,
+ adapter->switchdev_info.representor,
+ adapter->switchdev_info.port_name_type,
+ adapter->repr_ctxt.nb_vf,
+ adapter->repr_ctxt.nb_repr_vf);
+ if (adapter->repr_ctxt.nb_vf > 0) {
+ fprintf(file,
+ "\t -- vf entry:\n");
+ for (idx = 0; idx < adapter->repr_ctxt.nb_vf; idx++) {
+ fprintf(file,
+ "\t -- func_id:%u vsi_type:%u kernel_vsi_id:%u dpdk_vsi_id:%u\n",
+ adapter->repr_ctxt.repr_vf_id[idx].func_id,
+ adapter->repr_ctxt.repr_vf_id[idx].vsi_type,
+ adapter->repr_ctxt.repr_vf_id[idx].kernel_vsi_id,
+ adapter->repr_ctxt.repr_vf_id[idx].dpdk_vsi_id);
+ }
+ }
+ }
+
+l_end:
+ return;
+}
+
+int32_t sxe2_eth_dev_priv_dump(struct rte_eth_dev *dev, FILE *file)
+{
+ char *buf = NULL;
+ size_t size = 0;
+ FILE *str;
+ int32_t ret = -1;
+
+ str = open_memstream(&buf, &size);
+ if (!str) {
+ PMD_LOG_ERR(DRV, "fopen fail.");
+ goto l_end;
+ }
+
+ sxe2_dump_dev_feature_capability(str, dev);
+ sxe2_dump_device_basic_info(str, dev);
+ sxe2_dump_dev_args_info(str, dev);
+ sxe2_dump_filter_info(str, dev);
+ sxe2_dump_switchdev_info(str, dev);
+
+ (void)fflush(str);
+
+ (void)fwrite(buf, 1, size, file);
+ (void)fflush(file);
+
+ ret = 0;
+
+ (void)fclose(str);
+ free(buf);
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_dump.h b/drivers/net/sxe2/sxe2_dump.h
new file mode 100644
index 0000000000..05d6db9b3d
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_dump.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_DUMP_H__
+#define __SXE2_DUMP_H__
+
+#include <ethdev_driver.h>
+
+int32_t sxe2_eth_dev_priv_dump(struct rte_eth_dev *dev, FILE *file);
+
+#endif /* __SXE2_DUMP_H__ */
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index edcedbab45..73a92d99f8 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -38,6 +38,7 @@
#include "sxe2_host_regs.h"
#include "sxe2_switchdev.h"
#include "sxe2_ioctl_chnl_func.h"
+#include "sxe2_dump.h"
#include "sxe2_ethdev_repr.h"
#include "sxe2vf_regs.h"
#include "sxe2_switchdev.h"
@@ -194,6 +195,8 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.get_module_info = sxe2_get_module_info,
.get_module_eeprom = sxe2_get_module_eeprom,
+
+ .eth_dev_priv_dump = sxe2_eth_dev_priv_dump,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
diff --git a/drivers/net/sxe2/sxe2_ethdev_repr.c b/drivers/net/sxe2/sxe2_ethdev_repr.c
index a43991c379..faac1b2701 100644
--- a/drivers/net/sxe2/sxe2_ethdev_repr.c
+++ b/drivers/net/sxe2/sxe2_ethdev_repr.c
@@ -12,6 +12,7 @@
#include "sxe2_switchdev.h"
#include "sxe2_ptype.h"
#include "sxe2_mp.h"
+#include "sxe2_dump.h"
#include "sxe2_stats.h"
#include "sxe2_flow.h"
@@ -237,6 +238,8 @@ static const struct eth_dev_ops sxe2_switchdev_repr_dev_ops = {
.allmulticast_enable = sxe2_repr_allmulti_enable,
.allmulticast_disable = sxe2_repr_allmulti_disable,
+ .eth_dev_priv_dump = sxe2_eth_dev_priv_dump,
+
.stats_get = sxe2_stats_info_get,
.stats_reset = sxe2_stats_info_reset,
.xstats_get = sxe2_xstats_info_get,
--
2.47.3
^ permalink raw reply related
* [PATCH v2 18/23] net/sxe2: add mbuf validation in Tx debug mode
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Introduce the `sxe2_txrx_check_mbuf` helper function to validate outgoing
mbufs when `RTE_ETHDEV_DEBUG_TX` is enabled. This helps developers catch
malformed mbufs (e.g., invalid segment lengths, bad offload flags, or
unaligned buffers) before passing them to the hardware rings, avoiding
potential hardware hangs or silent packet drops.
The validation is fully wrapped inside `RTE_ETHDEV_DEBUG_TX` conditional
compilation blocks to ensure zero performance overhead in standard
production builds.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 2 +
drivers/net/sxe2/sxe2_txrx.c | 8 +-
drivers/net/sxe2/sxe2_txrx_check_mbuf.c | 595 ++++++++++++++++++++++++
drivers/net/sxe2/sxe2_txrx_check_mbuf.h | 38 ++
4 files changed, 641 insertions(+), 2 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 631730bf3e..5a02b1c3d3 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -78,6 +78,8 @@ sources += files(
'sxe2_flow_parse_pattern.c',
'sxe2_flow_parse_engine.c',
'sxe2_dump.c',
+ 'sxe2_txrx_check_mbuf.c',
+
)
allow_internal_get_api = true
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index a2cea954d5..55e477db1d 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -13,6 +13,7 @@
#include "sxe2_txrx_common.h"
#include "sxe2_txrx_vec.h"
#include "sxe2_txrx_poll.h"
+#include "sxe2_txrx_check_mbuf.h"
#include "sxe2_ethdev.h"
#include "sxe2_common_log.h"
#include "sxe2_osal.h"
@@ -121,13 +122,11 @@ uint16_t sxe2_tx_pkts_prepare(void *tx_queue,
rte_errno = -EINVAL;
goto l_end;
}
-#ifdef RTE_ETHDEV_DEBUG_TX
ret = rte_validate_tx_offload(mbuf);
if (ret != 0) {
rte_errno = -ret;
goto l_end;
}
-#endif
ret = rte_net_intel_cksum_prepare(mbuf);
if (ret != 0) {
rte_errno = -ret;
@@ -138,6 +137,11 @@ uint16_t sxe2_tx_pkts_prepare(void *tx_queue,
rte_errno = -ret;
goto l_end;
}
+ ret = sxe2_txrx_check_mbuf(mbuf);
+ if (ret != 0) {
+ rte_errno = -ret;
+ goto l_end;
+ }
}
l_end:
return i;
diff --git a/drivers/net/sxe2/sxe2_txrx_check_mbuf.c b/drivers/net/sxe2/sxe2_txrx_check_mbuf.c
new file mode 100644
index 0000000000..5a12b42120
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_check_mbuf.c
@@ -0,0 +1,595 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_common.h>
+#include <rte_net.h>
+#include <rte_vect.h>
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+#include <ethdev_driver.h>
+#include <rte_geneve.h>
+
+#include "sxe2_txrx_check_mbuf.h"
+#include "sxe2_common_log.h"
+
+#define TX_IPPROTO_IPIP 4
+#define TX_IPPROTO_GRE 47
+#define GRE_CHECKSUM_PRESENT 0x8000
+#define GRE_KEY_PRESENT 0x2000
+#define GRE_SEQUENCE_PRESENT 0x1000
+#define GRE_EXT_LEN 4
+#define GRE_SUPPORTED_FIELDS (GRE_CHECKSUM_PRESENT | GRE_KEY_PRESENT | GRE_SEQUENCE_PRESENT)
+
+
+static uint16_t vxlan_gpe_udp_port = RTE_VXLAN_GPE_DEFAULT_PORT;
+static uint16_t geneve_udp_port = RTE_GENEVE_DEFAULT_PORT;
+
+static inline int32_t check_mbuf_len(struct offload_info *info, struct rte_mbuf *m)
+{
+ int32_t ret = 0;
+ if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
+ if (info->outer_l2_len != m->outer_l2_len) {
+ PMD_LOG_ERR(TX, "outer_l2_len error in mbuf. Original "
+ "length:%hu calculated length:%u", m->outer_l2_len,
+ info->outer_l2_len);
+ ret = -1;
+ goto end;
+ }
+ if (info->outer_l3_len != m->outer_l3_len) {
+ PMD_LOG_ERR(TX, "outer_l3_len error in mbuf. Original "
+ "length:%hu calculated length:%u", m->outer_l3_len,
+ info->outer_l3_len);
+ ret = -1;
+ goto end;
+ }
+ }
+
+ if (info->l2_len != m->l2_len) {
+ PMD_LOG_ERR(TX, "l2_len error in mbuf. Original "
+ "length:%hu calculated length:%u", m->l2_len, info->l2_len);
+ ret = -1;
+ goto end;
+ }
+ if (info->l3_len != m->l3_len) {
+ PMD_LOG_ERR(TX, "l3_len error in mbuf. Original "
+ "length:%hu calculated length:%u", m->l3_len, info->l3_len);
+ ret = -1;
+ goto end;
+ }
+ if (info->l4_len != m->l4_len) {
+ PMD_LOG_ERR(TX, "l4_len error in mbuf. Original "
+ "length:%hu calculated length:%u", m->l4_len, info->l4_len);
+ ret = -1;
+ goto end;
+ }
+ ret = 0;
+
+end:
+ return ret;
+}
+
+static inline int32_t check_ether_type(struct offload_info *info, struct rte_mbuf *m)
+{
+ int32_t ret = 0;
+
+ if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
+ if (info->outer_ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
+ if (!(m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)) {
+ PMD_LOG_ERR(TX, "Outer ethernet type is ipv4, "
+ "tx offload missing `RTE_MBUF_F_TX_OUTER_IPV4` flag");
+ ret = -1;
+ goto end;
+ }
+ if (m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) {
+ PMD_LOG_ERR(TX, "Outer ethernet type is ipv4, tx "
+ "offload contains wrong `RTE_MBUF_F_TX_OUTER_IPV6` flag");
+ ret = -1;
+ goto end;
+ }
+ } else if (info->outer_ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
+ if (!(m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)) {
+ PMD_LOG_ERR(TX, "Outer ethernet type is ipv6, "
+ "tx offload missing `RTE_MBUF_F_TX_OUTER_IPV6` flag");
+ ret = -1;
+ goto end;
+ }
+ if (m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) {
+ PMD_LOG_ERR(TX, "Outer ethernet type is ipv6, tx "
+ "offload contains wrong `RTE_MBUF_F_TX_OUTER_IPV4` flag");
+ ret = -1;
+ goto end;
+ }
+ }
+ }
+
+ if (info->ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
+ if (!(m->ol_flags & RTE_MBUF_F_TX_IPV4)) {
+ PMD_LOG_ERR(TX, "Ethernet type is ipv4, tx offload "
+ "missing `RTE_MBUF_F_TX_IPV4` flag.");
+ ret = -1;
+ goto end;
+ }
+ if (m->ol_flags & RTE_MBUF_F_TX_IPV6) {
+ PMD_LOG_ERR(TX, "Ethernet type is ipv4, tx "
+ "offload contains wrong `RTE_MBUF_F_TX_IPV6` flag");
+ ret = -1;
+ goto end;
+ }
+ } else if (info->ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
+ if (!(m->ol_flags & RTE_MBUF_F_TX_IPV6)) {
+ PMD_LOG_ERR(TX, "Ethernet type is ipv6, tx offload "
+ "missing `RTE_MBUF_F_TX_IPV6` flag.");
+ ret = -1;
+ goto end;
+ }
+ if (m->ol_flags & RTE_MBUF_F_TX_IPV4) {
+ PMD_LOG_ERR(TX, "Ethernet type is ipv6, tx offload "
+ "contains wrong `RTE_MBUF_F_TX_IPV4` flag");
+ ret = -1;
+ goto end;
+ }
+ }
+ ret = 0;
+
+end:
+ return ret;
+}
+
+static inline void parse_ipv4(struct rte_ipv4_hdr *ipv4_hdr, struct offload_info *info)
+{
+ struct rte_tcp_hdr *tcp_hdr;
+
+ info->l3_len = rte_ipv4_hdr_len(ipv4_hdr);
+ info->l4_proto = ipv4_hdr->next_proto_id;
+
+ if (info->l4_proto == IPPROTO_TCP) {
+ tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + info->l3_len);
+ info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
+ } else if (info->l4_proto == IPPROTO_UDP) {
+ info->l4_len = sizeof(struct rte_udp_hdr);
+ } else {
+ info->l4_len = 0;
+ }
+}
+
+static inline void parse_ipv6(struct rte_ipv6_hdr *ipv6_hdr, struct offload_info *info)
+{
+ struct rte_tcp_hdr *tcp_hdr;
+
+ info->l3_len = sizeof(struct rte_ipv6_hdr);
+ info->l4_proto = ipv6_hdr->proto;
+
+ if (info->l4_proto == IPPROTO_TCP) {
+ tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + info->l3_len);
+ info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
+ } else if (info->l4_proto == IPPROTO_UDP) {
+ info->l4_len = sizeof(struct rte_udp_hdr);
+ } else {
+ info->l4_len = 0;
+ }
+}
+
+static inline void parse_ethernet(struct rte_ether_hdr *eth_hdr, struct offload_info *info)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_vlan_hdr *vlan_hdr;
+
+ info->l2_len = sizeof(struct rte_ether_hdr);
+ info->ethertype = eth_hdr->ether_type;
+
+ while (info->ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN) ||
+ info->ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_QINQ)) {
+ vlan_hdr = (struct rte_vlan_hdr *)
+ ((char *)eth_hdr + info->l2_len);
+ info->l2_len += sizeof(struct rte_vlan_hdr);
+ info->ethertype = vlan_hdr->eth_proto;
+ }
+
+ switch (info->ethertype) {
+ case RTE_STATIC_BSWAP16(RTE_ETHER_TYPE_IPV4):
+ ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + info->l2_len);
+ parse_ipv4(ipv4_hdr, info);
+ break;
+ case RTE_STATIC_BSWAP16(RTE_ETHER_TYPE_IPV6):
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + info->l2_len);
+ parse_ipv6(ipv6_hdr, info);
+ break;
+ default:
+ info->l4_len = 0;
+ info->l3_len = 0;
+ info->l4_proto = 0;
+ break;
+ }
+}
+
+static inline void update_tunnel_outer(struct offload_info *info)
+{
+ info->is_tunnel = 1;
+ info->outer_ethertype = info->ethertype;
+ info->outer_l2_len = info->l2_len;
+ info->outer_l3_len = info->l3_len;
+ info->outer_l4_proto = info->l4_proto;
+}
+
+static inline void parse_gtp(struct rte_udp_hdr *udp_hdr, struct offload_info *info)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_gtp_hdr *gtp_hdr;
+ uint8_t gtp_len = sizeof(*gtp_hdr);
+ uint8_t ip_ver;
+
+ if (udp_hdr->dst_port != rte_cpu_to_be_16(RTE_GTPC_UDP_PORT) &&
+ udp_hdr->src_port != rte_cpu_to_be_16(RTE_GTPC_UDP_PORT) &&
+ udp_hdr->dst_port != rte_cpu_to_be_16(RTE_GTPU_UDP_PORT))
+ goto end;
+
+ update_tunnel_outer(info);
+ info->l2_len = 0;
+
+ gtp_hdr = (struct rte_gtp_hdr *)((char *)udp_hdr + sizeof(*udp_hdr));
+
+ if (gtp_hdr->msg_type == 0xff) {
+ ip_ver = *(uint8_t *)((char *)udp_hdr + sizeof(*udp_hdr) + sizeof(*gtp_hdr));
+ ip_ver = (ip_ver) & 0xf0;
+
+ if (ip_ver == RTE_GTP_TYPE_IPV4) {
+ ipv4_hdr = (struct rte_ipv4_hdr *)((char *)gtp_hdr + gtp_len);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
+ parse_ipv4(ipv4_hdr, info);
+ } else if (ip_ver == RTE_GTP_TYPE_IPV6) {
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)gtp_hdr + gtp_len);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
+ parse_ipv6(ipv6_hdr, info);
+ }
+ } else {
+ info->ethertype = 0;
+ info->l4_len = 0;
+ info->l3_len = 0;
+ info->l4_proto = 0;
+ }
+
+ info->l2_len += RTE_ETHER_GTP_HLEN;
+
+end:
+ return;
+}
+
+static inline void parse_vxlan(struct rte_udp_hdr *udp_hdr, struct offload_info *info)
+{
+ struct rte_ether_hdr *eth_hdr;
+
+ if (udp_hdr->dst_port != rte_cpu_to_be_16(RTE_VXLAN_DEFAULT_PORT))
+ goto end;
+
+ update_tunnel_outer(info);
+
+ eth_hdr = (struct rte_ether_hdr *)((char *)udp_hdr +
+ sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr));
+
+ parse_ethernet(eth_hdr, info);
+ info->l2_len += RTE_ETHER_VXLAN_HLEN;
+
+end:
+ return;
+}
+
+static inline void parse_vxlan_gpe(struct rte_udp_hdr *udp_hdr, struct offload_info *info)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_vxlan_gpe_hdr *vxlan_gpe_hdr;
+ uint8_t vxlan_gpe_len = sizeof(*vxlan_gpe_hdr);
+
+ if (udp_hdr->dst_port != rte_cpu_to_be_16(vxlan_gpe_udp_port))
+ goto end;
+
+ vxlan_gpe_hdr = (struct rte_vxlan_gpe_hdr *)((char *)udp_hdr + sizeof(struct rte_udp_hdr));
+
+ if (!vxlan_gpe_hdr->proto || vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_IPV4) {
+ update_tunnel_outer(info);
+
+ ipv4_hdr = (struct rte_ipv4_hdr *)((char *)vxlan_gpe_hdr + vxlan_gpe_len);
+
+ parse_ipv4(ipv4_hdr, info);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
+ info->l2_len = 0;
+
+ } else if (vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_IPV6) {
+ update_tunnel_outer(info);
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)vxlan_gpe_hdr + vxlan_gpe_len);
+
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
+ parse_ipv6(ipv6_hdr, info);
+ info->l2_len = 0;
+
+ } else if (vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_ETH) {
+ update_tunnel_outer(info);
+
+ eth_hdr = (struct rte_ether_hdr *)((char *)vxlan_gpe_hdr + vxlan_gpe_len);
+
+ parse_ethernet(eth_hdr, info);
+ } else {
+ goto end;
+ }
+
+ info->l2_len += RTE_ETHER_VXLAN_GPE_HLEN;
+
+end:
+ return;
+}
+
+static inline void parse_geneve(struct rte_udp_hdr *udp_hdr, struct offload_info *info)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_geneve_hdr *geneve_hdr;
+ uint16_t geneve_len;
+
+ if (udp_hdr->dst_port != rte_cpu_to_be_16(geneve_udp_port))
+ goto end;
+
+ geneve_hdr = (struct rte_geneve_hdr *)((char *)udp_hdr + sizeof(struct rte_udp_hdr));
+ geneve_len = sizeof(struct rte_geneve_hdr) + geneve_hdr->opt_len * 4;
+ if (!geneve_hdr->proto || geneve_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
+ update_tunnel_outer(info);
+ ipv4_hdr = (struct rte_ipv4_hdr *)((char *)geneve_hdr + geneve_len);
+ parse_ipv4(ipv4_hdr, info);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
+ info->l2_len = 0;
+ } else if (geneve_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
+ update_tunnel_outer(info);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)geneve_hdr + geneve_len);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
+ parse_ipv6(ipv6_hdr, info);
+ info->l2_len = 0;
+
+ } else if (geneve_hdr->proto == rte_cpu_to_be_16(RTE_GENEVE_TYPE_ETH)) {
+ update_tunnel_outer(info);
+ eth_hdr = (struct rte_ether_hdr *)((char *)geneve_hdr + geneve_len);
+ parse_ethernet(eth_hdr, info);
+ } else {
+ goto end;
+ }
+
+ info->l2_len += (sizeof(struct rte_udp_hdr) + sizeof(struct rte_geneve_hdr) +
+ ((struct rte_geneve_hdr *)geneve_hdr)->opt_len * 4);
+
+end:
+ return;
+}
+
+static inline void parse_gre(struct simple_gre_hdr *gre_hdr, struct offload_info *info)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ uint8_t gre_len = 0;
+
+ gre_len += sizeof(struct simple_gre_hdr);
+
+ if (gre_hdr->flags & rte_cpu_to_be_16(GRE_KEY_PRESENT))
+ gre_len += GRE_EXT_LEN;
+ if (gre_hdr->flags & rte_cpu_to_be_16(GRE_SEQUENCE_PRESENT))
+ gre_len += GRE_EXT_LEN;
+ if (gre_hdr->flags & rte_cpu_to_be_16(GRE_CHECKSUM_PRESENT))
+ gre_len += GRE_EXT_LEN;
+
+ if (gre_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
+ update_tunnel_outer(info);
+
+ ipv4_hdr = (struct rte_ipv4_hdr *)((char *)gre_hdr + gre_len);
+
+ parse_ipv4(ipv4_hdr, info);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
+ info->l2_len = 0;
+
+ } else if (gre_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
+ update_tunnel_outer(info);
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)gre_hdr + gre_len);
+
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
+ parse_ipv6(ipv6_hdr, info);
+ info->l2_len = 0;
+
+ } else if (gre_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_TEB)) {
+ update_tunnel_outer(info);
+
+ eth_hdr = (struct rte_ether_hdr *)((char *)gre_hdr + gre_len);
+
+ parse_ethernet(eth_hdr, info);
+ } else {
+ goto end;
+ }
+
+ info->l2_len += gre_len;
+
+end:
+ return;
+}
+
+static inline void parse_encap_ip(void *encap_ip, struct offload_info *info)
+{
+ struct rte_ipv4_hdr *ipv4_hdr = encap_ip;
+ struct rte_ipv6_hdr *ipv6_hdr = encap_ip;
+ uint8_t ip_version;
+
+ ip_version = ((ipv4_hdr->version_ihl & 0xf0) >> 4);
+
+ if (ip_version != 4 && ip_version != 6)
+ goto end;
+
+ info->is_tunnel = 1;
+ info->outer_ethertype = info->ethertype;
+ info->outer_l2_len = info->l2_len;
+ info->outer_l3_len = info->l3_len;
+
+ if (ip_version == 4) {
+ parse_ipv4(ipv4_hdr, info);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
+ } else {
+ parse_ipv6(ipv6_hdr, info);
+ info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
+ }
+ info->l2_len = 0;
+
+end:
+ return;
+}
+
+__rte_unused int32_t sxe2_txrx_check_mbuf(struct rte_mbuf *m)
+{
+ int32_t ret = 0;
+ struct rte_ether_hdr *eth_hdr;
+ void *l3_hdr = NULL;
+ struct offload_info info = {0};
+ uint64_t ol_flags = m->ol_flags;
+ uint64_t tunnel_type = ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK;
+
+ eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
+ parse_ethernet(eth_hdr, &info);
+ l3_hdr = (char *)eth_hdr + info.l2_len;
+ if (info.l4_proto == IPPROTO_UDP) {
+ struct rte_udp_hdr *udp_hdr;
+
+ udp_hdr = (struct rte_udp_hdr *)((char *)l3_hdr + info.l3_len);
+ if ((info.l2_len + info.l3_len + sizeof(struct rte_udp_hdr)) > m->data_len) {
+ PMD_LOG_ERR(TX, "UDP header exceeds mbuf data length");
+ ret = -1;
+ goto end;
+ }
+ parse_gtp(udp_hdr, &info);
+ if (info.is_tunnel) {
+ if (!tunnel_type) {
+ PMD_LOG_ERR(TX, "gtp tunnel packet missing tx "
+ "offload missing `RTE_MBUF_F_TX_TUNNEL_GTP` flag");
+ ret = -1;
+ goto end;
+ }
+ if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_GTP) {
+ PMD_LOG_ERR(TX, "gtp tunnel packet, tx offload has wrong "
+ "`%s` flag correct is `RTE_MBUF_F_TX_TUNNEL_GTP` flag",
+ rte_get_tx_ol_flag_name(tunnel_type));
+ ret = -1;
+ goto end;
+ }
+ goto check_len;
+ }
+ parse_vxlan_gpe(udp_hdr, &info);
+ if (info.is_tunnel) {
+ if (!tunnel_type) {
+ PMD_LOG_ERR(TX, "vxlan gpe tunnel packet missing tx "
+ "offload missing `RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE` flag");
+ ret = -1;
+ goto end;
+ }
+ if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE) {
+ PMD_LOG_ERR(TX, "vxlan gpe tunnel packet, tx offload has "
+ "wrong `%s` flag correct is `RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE` flag",
+ rte_get_tx_ol_flag_name(tunnel_type));
+ ret = -1;
+ goto end;
+ }
+ goto check_len;
+ }
+ parse_vxlan(udp_hdr, &info);
+ if (info.is_tunnel) {
+ if (!tunnel_type) {
+ PMD_LOG_ERR(TX, "vxlan tunnel packet missing tx "
+ "offload missing `RTE_MBUF_F_TX_TUNNEL_VXLAN` flag");
+ ret = -1;
+ goto end;
+ }
+ if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_VXLAN) {
+ PMD_LOG_ERR(TX, "vxlan tunnel packet, tx offload has "
+ "wrong `%s` flag correct is `RTE_MBUF_F_TX_TUNNEL_VXLAN` flag",
+ rte_get_tx_ol_flag_name(tunnel_type));
+ ret = -1;
+ goto end;
+ }
+ goto check_len;
+ }
+ parse_geneve(udp_hdr, &info);
+ if (info.is_tunnel) {
+ if (!tunnel_type) {
+ PMD_LOG_ERR(TX, "geneve tunnel packet missing tx "
+ "offload missing `RTE_MBUF_F_TX_TUNNEL_GENEVE` flag");
+ ret = -1;
+ goto end;
+ }
+ if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_GENEVE) {
+ PMD_LOG_ERR(TX, "geneve tunnel packet, tx offload has "
+ "wrong `%s` flag correct is `RTE_MBUF_F_TX_TUNNEL_GENEVE` flag",
+ rte_get_tx_ol_flag_name(tunnel_type));
+ ret = -1;
+ goto end;
+ }
+ goto check_len;
+ }
+
+ if (unlikely(RTE_ETH_IS_TUNNEL_PKT(m->packet_type) != 0)) {
+ PMD_LOG_ERR(TX, "Unknown tunnel packet UDP dst port:%hu",
+ udp_hdr->dst_port);
+ ret = -1;
+ goto end;
+ }
+ } else if (info.l4_proto == TX_IPPROTO_GRE) {
+ struct simple_gre_hdr *gre_hdr;
+
+ gre_hdr = (struct simple_gre_hdr *)((char *)l3_hdr + info.l3_len);
+ parse_gre(gre_hdr, &info);
+ if (info.is_tunnel) {
+ if (!tunnel_type) {
+ PMD_LOG_ERR(TX, "gre tunnel packet missing tx "
+ "offload missing `RTE_MBUF_F_TX_TUNNEL_GRE` flag.");
+ ret = -1;
+ goto end;
+ }
+ if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_GRE) {
+ PMD_LOG_ERR(TX, "gre tunnel packet, tx offload has "
+ "wrong `%s` flag, correct is `RTE_MBUF_F_TX_TUNNEL_GRE` flag",
+ rte_get_tx_ol_flag_name(tunnel_type));
+ ret = -1;
+ goto end;
+ }
+ goto check_len;
+ }
+ } else if (info.l4_proto == TX_IPPROTO_IPIP) {
+ void *encap_ip_hdr;
+
+ encap_ip_hdr = (char *)l3_hdr + info.l3_len;
+ parse_encap_ip(encap_ip_hdr, &info);
+ if (info.is_tunnel) {
+ if (!tunnel_type) {
+ PMD_LOG_ERR(TX, "Ipip tunnel packet missing tx "
+ "offload missing `RTE_MBUF_F_TX_TUNNEL_IPIP` flag");
+ ret = -1;
+ goto end;
+ }
+ if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_IPIP) {
+ PMD_LOG_ERR(TX, "Ipip tunnel packet, tx offload has "
+ "wrong `%s` flag, correct is `RTE_MBUF_F_TX_TUNNEL_IPIP` flag",
+ rte_get_tx_ol_flag_name(tunnel_type));
+ ret = -1;
+ goto end;
+ }
+ goto check_len;
+ }
+ }
+
+check_len:
+ if (check_mbuf_len(&info, m) != 0) {
+ ret = -1;
+ goto end;
+ }
+ ret = check_ether_type(&info, m);
+
+end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_txrx_check_mbuf.h b/drivers/net/sxe2/sxe2_txrx_check_mbuf.h
new file mode 100644
index 0000000000..98197f85d9
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_check_mbuf.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_TXRX_CHECK_MBUF_H__
+#define __SXE2_TXRX_CHECK_MBUF_H__
+
+#include <rte_common.h>
+#include <rte_net.h>
+#include <rte_vect.h>
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+#include <ethdev_driver.h>
+
+struct offload_info {
+ uint16_t ethertype;
+ uint8_t gso_enable;
+ uint16_t l2_len;
+ uint16_t l3_len;
+ uint16_t l4_len;
+ uint8_t l4_proto;
+ uint8_t is_tunnel;
+ uint16_t outer_ethertype;
+ uint16_t outer_l2_len;
+ uint16_t outer_l3_len;
+ uint8_t outer_l4_proto;
+ uint16_t tso_segsz;
+ uint16_t tunnel_tso_segsz;
+ uint32_t pkt_len;
+};
+
+struct simple_gre_hdr {
+ uint16_t flags;
+ uint16_t proto;
+};
+
+__rte_unused int32_t sxe2_txrx_check_mbuf(struct rte_mbuf *m);
+#endif /* __SXE2_TXRX_CHECK_MBUF_H__ */
--
2.47.3
^ permalink raw reply related
* [PATCH v2 15/23] common/sxe2: add shared SFP module definitions
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch adds a new shared header file 'sxe2_msg.h' which
contains definitions for SFP/SFP+ modules. This file is shared across
Firmware, Kernel driver, and DPDK PMD to ensure consistent protocol
handling.
The header includes:
- SFP EEPROM memory map offsets.
- Module type encoding definitions.
By using this shared header, the PMD can correctly identify module
capabilities and report diagnostic information in a way that is
consistent with the underlying firmware logic.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_msg.h | 118 +++++++++++++++++++++++++++++++++
1 file changed, 118 insertions(+)
create mode 100644 drivers/common/sxe2/sxe2_msg.h
diff --git a/drivers/common/sxe2/sxe2_msg.h b/drivers/common/sxe2/sxe2_msg.h
new file mode 100644
index 0000000000..f08944f7c9
--- /dev/null
+++ b/drivers/common/sxe2/sxe2_msg.h
@@ -0,0 +1,118 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_MSG_H__
+#define __SXE2_MSG_H__
+
+enum sfp_type_identifier {
+ SXE2_SFP_TYPE_UNKNOWN = 0x00,
+ SXE2_SFP_TYPE_SFP = 0x03,
+
+ SXE2_SFP_TYPE_QSFP_PLUS = 0x0D,
+ SXE2_SFP_TYPE_QSFP28 = 0x11,
+
+ SXE2_SFP_TYPE_MAX = 0xFF,
+};
+
+#ifndef SFP_DEFINE
+#define SFP_DEFINE
+
+#define SXE2_SFP_EEP_WR 0x1
+#define SXE2_SFP_EEP_QSFP 0x1
+
+enum sfp_bus_addr {
+ SXE2_SFP_EEP_I2C_ADDR0 = 0xA0,
+ SXE2_SFP_EEP_I2C_ADDR1 = 0xA2,
+ SXE2_SFP_EEP_I2C_ADDR_NR = 0xFFFF,
+};
+
+struct sxe2_sfp_req {
+ uint8_t is_wr;
+ uint8_t is_qsfp;
+ uint16_t bus_addr;
+ uint16_t page_cnt;
+ uint16_t offset;
+ uint16_t data_len;
+ uint16_t rvd;
+ uint8_t data[];
+};
+
+struct sxe2_sfp_resp {
+ uint8_t is_wr;
+ uint8_t is_qsfp;
+ uint16_t data_len;
+ uint8_t data[];
+};
+
+enum sfp_page_cnt {
+ SXE2_SFP_EEP_PAGE_CNT0 = 0,
+ SXE2_SFP_EEP_PAGE_CNT1,
+ SXE2_SFP_EEP_PAGE_CNT2,
+ SXE2_SFP_EEP_PAGE_CNT3,
+ SXE2_SFP_EEP_PAGE_CNT20 = 20,
+ SXE2_SFP_EEP_PAGE_CNT21 = 21,
+
+ SXE2_SFP_EEP_PAGE_CNT_NR = 0xFFFF,
+};
+
+#define SXE2_SFP_E2P_I2C_7BIT_ADDR0 (SXE2_SFP_EEP_I2C_ADDR0 >> 1)
+#define SXE2_SFP_E2P_I2C_7BIT_ADDR1 (SXE2_SFP_EEP_I2C_ADDR1 >> 1)
+
+#define SXE2_QSFP_PAGE_OFST_START 128
+#define SXE2_SFP_EEP_OFST_MAX 255
+#define SXE2_SFP_EEP_LEN_MAX 256
+#endif
+
+#ifndef FW_STATE_DEFINE
+#define FW_STATE_DEFINE
+
+#define SXE2_FW_STATUS_MAIN_SHIF (16)
+#define SXE2_FW_STATUS_MAIN_MASK (0xFF0000)
+#define SXE2_FW_STATUS_SUB_MASK (0xFFFF)
+
+enum Sxe2FwStateMain {
+ SXE2_FW_STATE_MAIN_UNDEFINED = 0x00,
+ SXE2_FW_STATE_MAIN_INIT = 0x10000,
+ SXE2_FW_STATE_MAIN_RUN = 0x20000,
+ SXE2_FW_STATE_MAIN_ABNOMAL = 0x30000,
+};
+
+enum Sxe2FwState {
+ SXE2_FW_START_STATE_UNDEFINED = SXE2_FW_STATE_MAIN_UNDEFINED,
+ SXE2_FW_START_STATE_INIT_BASE = (SXE2_FW_STATE_MAIN_INIT + 0x1),
+ SXE2_FW_START_STATE_SCAN_DEVICE = (SXE2_FW_STATE_MAIN_INIT + 0x20),
+ SXE2_FW_START_STATE_FINISHED = (SXE2_FW_STATE_MAIN_RUN + 0x0),
+ SXE2_FW_START_STATE_UPGRADE = (SXE2_FW_STATE_MAIN_RUN + 0x1),
+ SXE2_FW_START_STATE_SYNC = (SXE2_FW_STATE_MAIN_RUN + 0x2),
+ SXE2_FW_RUNNING_STATE_ABNOMAL = (SXE2_FW_STATE_MAIN_ABNOMAL + 0x1),
+ SXE2_FW_RUNNING_STATE_ABNOMAL_CORE1 = (SXE2_FW_STATE_MAIN_ABNOMAL + 0x2),
+ SXE2_FW_RUNNING_STATE_ABNOMAL_HEART = (SXE2_FW_STATE_MAIN_ABNOMAL + 0x3),
+ SXE2_FW_START_STATE_MASK = (SXE2_FW_STATUS_MAIN_MASK | SXE2_FW_STATUS_SUB_MASK),
+};
+#endif
+
+#ifndef LED_DEFINE
+#define LED_DEFINE
+
+enum sxe2_led_mode {
+ SXE2_IDENTIFY_LED_BLINK_ON = 0,
+ SXE2_IDENTIFY_LED_BLINK_OFF,
+ SXE2_IDENTIFY_LED_ON,
+ SXE2_IDENTIFY_LED_OFF,
+ SXE2_IDENTIFY_LED_RESET,
+};
+
+
+typedef struct sxe2_led_ctrl {
+ uint32_t mode;
+ uint32_t duration;
+} sxe2_led_ctrl_s;
+
+typedef struct sxe2_led_ctrl_resp {
+ uint32_t ack;
+} sxe2_led_ctrl_resp_s;
+#endif
+
+#endif /* __SXE2_MSG_H__ */
--
2.47.3
^ permalink raw reply related
* [PATCH v2 14/23] net/sxe2: implement get monitor address
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch implements the 'get_monitor_addr' ethdev ops in the sxe2
PMD. This interface allows the Ethernet device to provide the
address of the next expected Rx descriptor to the power management
library.
The implementation calculates the address of the next Rx descriptor
based on the receive queue's current hardware ring position and
descriptor size. Applications can then use this address with
rte_power_monitor() to put the CPU into a low-power state until
new packets are written to the descriptor by the hardware.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/sxe2_ethdev.c | 2 ++
drivers/net/sxe2/sxe2_rx.c | 21 +++++++++++++++++++++
drivers/net/sxe2/sxe2_rx.h | 2 ++
3 files changed, 25 insertions(+)
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 215ae772b5..45e245740b 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -184,6 +184,8 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.queue_stats_mapping_set = sxe2_queue_stats_mapping_set,
.fw_version_get = sxe2_fw_version_string_get,
+
+ .get_monitor_addr = sxe2_get_monitor_addr,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
diff --git a/drivers/net/sxe2/sxe2_rx.c b/drivers/net/sxe2/sxe2_rx.c
index 007192c7d8..9055363ee3 100644
--- a/drivers/net/sxe2/sxe2_rx.c
+++ b/drivers/net/sxe2/sxe2_rx.c
@@ -557,3 +557,24 @@ void __rte_cold sxe2_rxqs_all_stop(struct rte_eth_dev *dev)
}
}
}
+
+static int32_t sxe2_monitor_callback(const uint64_t value,
+ const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
+{
+ const uint64_t dd_state = rte_cpu_to_le_64(1 << SXE2_RX_DESC_STATUS_DD_SHIFT);
+ return (value & dd_state) == dd_state ? -1 : 0;
+}
+
+int32_t sxe2_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
+{
+ volatile union sxe2_rx_desc *rxdp;
+ struct sxe2_rx_queue *rxq = (struct sxe2_rx_queue *)rx_queue;
+
+ rxdp = &rxq->desc_ring[rxq->processing_idx];
+
+ pmc->addr = &rxdp->wb.status_err_ptype_len;
+ pmc->fn = sxe2_monitor_callback;
+ pmc->size = sizeof(uint16_t);
+
+ return 0;
+}
diff --git a/drivers/net/sxe2/sxe2_rx.h b/drivers/net/sxe2/sxe2_rx.h
index 1c53f7f559..05b991e233 100644
--- a/drivers/net/sxe2/sxe2_rx.h
+++ b/drivers/net/sxe2/sxe2_rx.h
@@ -29,4 +29,6 @@ int32_t __rte_cold sxe2_rxqs_all_start(struct rte_eth_dev *dev);
void __rte_cold sxe2_rxqs_all_stop(struct rte_eth_dev *dev);
+int32_t sxe2_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc);
+
#endif /* __SXE2_RX_H__ */
--
2.47.3
^ permalink raw reply related
* [PATCH v2 13/23] net/sxe2: support firmware version reading
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch implements the logic to retrieve the firmware version and
Build ID from the hardware during device initialization.
The version is exposed to applications through the dev_info_get API.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/sxe2_ethdev.c | 35 +++++++++++++++++++++++++++++++++-
1 file changed, 34 insertions(+), 1 deletion(-)
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 14c8f6c16d..215ae772b5 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -120,7 +120,8 @@ static int32_t sxe2_udp_tunnel_port_add(struct rte_eth_dev *dev,
struct rte_eth_udp_tunnel *tunnel_udp);
static int32_t sxe2_udp_tunnel_port_del(struct rte_eth_dev *dev,
struct rte_eth_udp_tunnel *tunnel_udp);
-
+static int32_t sxe2_fw_version_string_get(struct rte_eth_dev *dev,
+ char *fw_version, size_t fw_size);
static const struct eth_dev_ops sxe2_eth_dev_ops = {
.dev_configure = sxe2_dev_configure,
@@ -181,6 +182,8 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.xstats_reset = sxe2_stats_info_reset,
.queue_stats_mapping_set = sxe2_queue_stats_mapping_set,
+
+ .fw_version_get = sxe2_fw_version_string_get,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -1575,6 +1578,36 @@ static int32_t sxe2_eth_pmd_remove(struct sxe2_common_device *cdev)
return ret;
}
+static int32_t sxe2_fw_version_string_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_fw_info *fw_info = &adapter->dev_info.fw;
+ int32_t ret_len;
+ int32_t ret;
+
+ ret_len = snprintf(fw_version, fw_size,
+ "%u.%u.%u.%u",
+ fw_info->main_version_id,
+ fw_info->sub_version_id,
+ fw_info->fix_version_id,
+ fw_info->build_id);
+
+ if (ret_len < 0) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret_len += 1;
+ if (fw_size < (size_t)ret_len)
+ ret = -EINVAL;
+ else
+ ret = 0;
+
+out:
+ return ret;
+}
+
static uint16_t sxe2_switchdev_repr_id_encode_get(struct sxe2_switchdev_info *switchdev_info)
{
enum rte_eth_representor_type type;
--
2.47.3
^ permalink raw reply related
* [PATCH v2 10/23] net/sxe2: add NEON vec Rx/Tx burst functions
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
- Implement sxe2_recv_pkts_vec_neon for bulk packet receiving.
- Implement sxe2_xmit_pkts_vec_neon for bulk packet transmission.
- Added logic to select the vectorized path based on runtime config
and CPU flags (RTE_ARCH_ARM64).
Vectorized path improves throughput for small packets by processing
multiple descriptors simultaneously using SIMD instructions.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 2 +
drivers/net/sxe2/sxe2_txrx.c | 36 ++
drivers/net/sxe2/sxe2_txrx_vec.h | 16 +-
drivers/net/sxe2/sxe2_txrx_vec_common.h | 1 -
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 707 ++++++++++++++++++++++++
5 files changed, 759 insertions(+), 3 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index c73e13bbad..0658b2ee3a 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -48,6 +48,8 @@ if arch_subdir == 'x86'
include_directories: includes,
c_args: [cflags, '-mavx2'])
objs += sxe2_avx2_lib.extract_objects('sxe2_txrx_vec_avx2.c')
+elif arch_subdir == 'arm'
+ sources += files('sxe2_txrx_vec_neon.c')
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index dcfaf7278d..2eb8365457 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -176,6 +176,10 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if ((0 == (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK)))
tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#elif defined(RTE_ARCH_ARM64)
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) == 1) {
+ tx_mode_flags |= (vec_flags | SXE2_TX_MODE_VEC_NEON);
+ }
#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
@@ -228,6 +232,13 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
}
}
+#elif defined(RTE_ARCH_ARM64)
+ if (adapter->tx_mode_flags & SXE2_TX_MODE_VEC_NEON) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon_simple;
+ }
} else {
#endif
if (tx_mode_flags & SXE2_TX_MODE_SIMPLE_BATCH) {
@@ -263,6 +274,12 @@ static const struct {
{ sxe2_tx_pkts_vec_sse_simple,
"Vector SSE Simple" },
#endif
+#ifdef RTE_ARCH_ARM64
+ { sxe2_tx_pkts_vec_neon,
+ "Vector NEON" },
+ { sxe2_tx_pkts_vec_neon_simple,
+ "Vector NEON Simple" },
+#endif
};
int32_t sxe2_tx_burst_mode_get(struct rte_eth_dev *dev,
@@ -366,6 +383,11 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+
+#elif defined(RTE_ARCH_ARM64)
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) == 1) {
+ rx_mode_flags |= (vec_flags | SXE2_RX_MODE_VEC_NEON);
+ }
#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
@@ -397,6 +419,14 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
}
return;
}
+#elif defined(RTE_ARCH_ARM64)
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_neon_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_neon;
+ goto l_end;
+ }
#endif
if (sxe2_rx_offload_en_check(dev, RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT))
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_split;
@@ -426,6 +456,12 @@ static const struct {
{ sxe2_rx_pkts_scattered_vec_sse_offload,
"Vector SSE Scattered" },
#endif
+#ifdef RTE_ARCH_ARM64
+ { sxe2_rx_pkts_scattered_vec_neon,
+ "Vector NEON Scattered" },
+ { sxe2_rx_pkts_scattered_vec_neon_offload,
+ "Offload Vector NEON Scattered" },
+#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index d7a0ce6ca5..02b1743e3e 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -12,19 +12,23 @@
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
#define SXE2_RX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
+#define SXE2_RX_MODE_VEC_NEON RTE_BIT32(5)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
- SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512)
+ SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512 | \
+ SXE2_RX_MODE_VEC_NEON)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
#define SXE2_TX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
+#define SXE2_TX_MODE_VEC_NEON RTE_BIT32(5)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
- SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512)
+ SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512 | \
+ SXE2_TX_MODE_VEC_NEON)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -75,6 +79,14 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+#elif defined(RTE_ARCH_ARM64)
+uint16_t sxe2_rx_pkts_scattered_vec_neon(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_neon_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_neon_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_common.h b/drivers/net/sxe2/sxe2_txrx_vec_common.h
index 138b748f4a..8fce2bb7cc 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_common.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec_common.h
@@ -4,7 +4,6 @@
#ifndef __SXE2_TXRX_VEC_COMMON_H__
#define __SXE2_TXRX_VEC_COMMON_H__
-#include <rte_atomic.h>
#ifdef PCLINT
#include "avx_stub.h"
#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_neon.c b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
new file mode 100644
index 0000000000..e50a0b21bf
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
@@ -0,0 +1,707 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifdef RTE_ARCH_ARM64
+#include <arm_neon.h>
+#include <rte_vect.h>
+
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_common_log.h"
+
+#define PKTLEN_SHIFT 10
+#define SXE2_UINT16_BIT (CHAR_BIT * sizeof(uint16_t))
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_neon(volatile union sxe2_tx_data_desc *desc,
+ struct rte_mbuf *pkt, uint64_t desc_cmd, bool with_offloads)
+{
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ uint64x2_t data_desc = { rte_pktmbuf_iova(pkt), desc_qw1 };
+
+ vst1q_u64((uint64_t *)desc, data_desc);
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+ uint16_t i;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_TX_DEBUG("Tx pkts neon batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+
+ for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ }
+
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+ }
+
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+
+ for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ }
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, txq->next_use);
+
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_neon_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num = sxe2_tx_pkts_vec_neon_batch(txq,
+ tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ PMD_LOG_TX_DEBUG("Tx pkts neon: port_id=%u, queue_id=%u, "
+ "nb_pkts=%u, tx_done_num=%u with_offloads=%u",
+ txq->port_id, txq->idx_in_pf, nb_pkts, tx_done_num, with_offloads);
+
+ SXE2_TX_STATS_CNT(txq, tx_pkts_num, tx_done_num);
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_neon_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_neon_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_neon_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static __rte_always_inline void
+sxe2_rx_desc_ptype_fill_neon(uint16x8_t staterr, struct rte_mbuf **__rte_restrict rx_pkts,
+ const uint32_t *__rte_restrict ptype_tbl)
+{
+ uint16x8_t ptype_mask = {
+ 0, 0x3FFULL,
+ 0, 0x3FFULL,
+ 0, 0x3FFULL,
+ 0, 0x3FFULL,
+ };
+ uint16x8_t ptype_all;
+
+ ptype_all = vandq_u16(staterr, ptype_mask);
+
+ rx_pkts[3]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 3)];
+ rx_pkts[2]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 7)];
+ rx_pkts[1]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 1)];
+ rx_pkts[0]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 5)];
+}
+
+static __rte_always_inline uint32x4_t
+sxe2_rx_desc_fnav_flags_neon(uint64x2_t descs_arr[4])
+{
+ uint32x4_t descs_tmp1, descs_tmp2;
+ uint32x4_t descs_fnav_vld;
+ uint32x4_t v_zeros, v_ffff, v_u32_one;
+ uint32x4_t m_flags;
+
+ const uint32x4_t fdir_flags = vdupq_n_u32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs_arr[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs_arr[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs_arr[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs_arr[3]);
+
+ descs_tmp1 = vzip1q_u32(d1, d
+
+static __rte_always_inline void
+sxe2_rx_desc_offloads_para_fill_neon(struct sxe2_rx_queue *rxq,
+ volatile union sxe2_rx_desc *desc,
+ uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
+{
+ uint32x4_t desc_lo, desc_hi, flags, tmp_flags;
+ const uint64x2_t mbuf_init = {rxq->mbuf_init_value, 0};
+ uint64x2_t rearm0, rearm1, rearm2, rearm3;
+
+ const uint32x4_t desc_msk = {
+ 0x00001C04, 0x00001C04, 0x00001C04, 0x00001C04};
+
+ const uint32x4_t rss_msk = {
+ 0x20000000, 0x20000000, 0x20000000, 0x20000000};
+
+ const uint32x4_t vlan_msk = {
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED
+ };
+ const uint8x16_t vlan_flags = {
+ 0, 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0
+ };
+
+ const uint8x16_t rss_flags = {
+ 0, 0, 0, 0,
+ RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0
+ };
+
+ const uint32x4_t cksum_mask = {
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ };
+
+ const uint8x16_t cksum_flags = {
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0
+ };
+
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
+ uint64x2_t f64, t64;
+
+ flags = vzip2q_u32(d1, d0);
+ tmp_flags = vzip2q_u32(d3, d2);
+ f64 = vreinterpretq_u64_u32(flags);
+ t64 = vreinterpretq_u64_u32(tmp_flags);
+ desc_lo = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(f64),
+ vget_low_u64(t64)));
+ desc_hi = vreinterpretq_u32_u64(vcombine_u64(vget_high_u64(f64),
+ vget_high_u64(t64)));
+ }
+
+ desc_lo = vandq_u32(desc_lo, desc_msk);
+ desc_hi = vandq_u32(desc_hi, rss_msk);
+
+ tmp_flags = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags,
+ vreinterpretq_u8_u32(desc_lo)));
+ flags = vandq_u32(tmp_flags, vlan_msk);
+
+ desc_lo = vshrq_n_u32(desc_lo, 10);
+ tmp_flags = vreinterpretq_u32_u8(vqtbl1q_u8(cksum_flags,
+ vreinterpretq_u8_u32(desc_lo)));
+ tmp_flags = vshlq_n_u32(tmp_flags, 1);
+ tmp_flags = vandq_u32(tmp_flags, cksum_mask);
+ flags = vorrq_u32(flags, tmp_flags);
+
+ desc_hi = vshrq_n_u32(desc_hi, 27);
+ tmp_flags = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags,
+ vreinterpretq_u8_u32(desc_hi)));
+ flags = vorrq_u32(flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ uint32x4_t tmp_fnav_flags = sxe2_rx_desc_fnav_flags_neon(descs);
+ flags = vorrq_u32(flags, tmp_fnav_flags);
+
+ rx_pkts[0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ }
+#endif
+
+ rearm0 = vsetq_lane_u64(vgetq_lane_u32(flags, 0), mbuf_init, 1);
+ rearm1 = vsetq_lane_u64(vgetq_lane_u32(flags, 1), mbuf_init, 1);
+ rearm2 = vsetq_lane_u64(vgetq_lane_u32(flags, 2), mbuf_init, 1);
+ rearm3 = vsetq_lane_u64(vgetq_lane_u32(flags, 3), mbuf_init, 1);
+
+ vst1q_u64((uint64_t *)&rx_pkts[0]->rearm_data, rearm0);
+ vst1q_u64((uint64_t *)&rx_pkts[1]->rearm_data, rearm1);
+ vst1q_u64((uint64_t *)&rx_pkts[2]->rearm_data, rearm2);
+ vst1q_u64((uint64_t *)&rx_pkts[3]->rearm_data, rearm3);
+}
+
+static inline void sxe2_rx_queue_rearm_neon(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ uint64x2_t dma_addr0, dma_addr1;
+ uint64x2_t zero = vdupq_n_u64(0);
+ uint64x2_t virt_addr0, virt_addr1;
+ uint64x2_t hdr_room = vdupq_n_u64(RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer,
+ SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ PMD_LOG_RX_INFO("Rx mbuf vec alloc failed port_id=%u "
+ "queue_id=%u", rxq->port_id,
+ rxq->idx_in_pf);
+
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_NEON; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ vst1q_u64((uint64_t *)&desc[i].read, zero);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+#if RTE_IOVA_IN_MBUF
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = vld1q_u64((uint64_t *)&mbuf0->buf_addr);
+ virt_addr1 = vld1q_u64((uint64_t *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+ dma_addr0 = vdupq_n_u64((uint64_t)vget_high_u64(virt_addr0));
+ dma_addr1 = vdupq_n_u64((uint64_t)vget_high_u64(virt_addr1));
+#else
+ dma_addr0 = vdupq_n_u64((uint64_t)vget_low_u64(virt_addr0));
+ dma_addr1 = vdupq_n_u64((uint64_t)vget_low_u64(virt_addr1));
+#endif
+ dma_addr0 = vaddq_u64(dma_addr0, hdr_room);
+ dma_addr1 = vaddq_u64(dma_addr1, hdr_room);
+
+ vst1q_u64((uint64_t *)&desc++->read, dma_addr0);
+ vst1q_u64((uint64_t *)&desc++->read, dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ?
+ (rxq->ring_depth - 1) : (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags, uint8_t *umbcast_flags,
+ bool do_offload)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ uint32_t i;
+ uint16_t done_num = 0;
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+
+ uint8x16_t rvp_shuf_mask = {
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 12, 13, 0xFF, 0xFF,
+ 12, 13,
+ 2, 3,
+ 4, 5, 6, 7
+ };
+
+ uint16x8_t crc_adjust = {
+ 0, 0,
+ rxq->crc_len,
+ 0, rxq->crc_len,
+ 0, 0, 0
+ };
+
+ desc = &rxq->desc_ring[rxq->processing_idx];
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_NEON);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_neon(rxq);
+
+ if ((rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_DD_MASK) == 0) {
+ goto l_end;
+ }
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_NEON,
+ desc += SXE2_RX_NUM_PER_LOOP_NEON) {
+ uint64x2_t descs[SXE2_RX_NUM_PER_LOOP_NEON];
+ uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+ uint64x2_t mbp1, mbp2;
+ uint16x8_t staterr;
+ uint16x8_t tmp;
+ uint16_t bit_num;
+
+ descs[3] = vld1q_u64((uint64_t *)(desc + 3));
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+ descs[2] = vld1q_u64((uint64_t *)(desc + 2));
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+ descs[1] = vld1q_u64((uint64_t *)(desc + 1));
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+ descs[0] = vld1q_u64((uint64_t *)(desc));
+
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+
+ descs[3] = vld1q_lane_u64((uint64_t *)(desc + 3), descs[3], 0);
+ descs[2] = vld1q_lane_u64((uint64_t *)(desc + 2), descs[2], 0);
+ descs[1] = vld1q_lane_u64((uint64_t *)(desc + 1), descs[1], 0);
+ descs[0] = vld1q_lane_u64((uint64_t *)(desc), descs[0], 0);
+
+ mbp1 = vld1q_u64((uint64_t *)&buffer[i]);
+ mbp2 = vld1q_u64((uint64_t *)&buffer[i + 2]);
+
+ vst1q_u64((uint64_t *)&rx_pkts[i], mbp1);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 2], mbp2);
+
+ if (split_rxe_flags) {
+ rte_mbuf_prefetch_part2(rx_pkts[i]);
+ rte_mbuf_prefetch_part2(rx_pkts[i + 1]);
+ rte_mbuf_prefetch_part2(rx_pkts[i + 2]);
+ rte_mbuf_prefetch_part2(rx_pkts[i + 3]);
+ }
+
+ pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), rvp_shuf_mask);
+ pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), rvp_shuf_mask);
+ pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), rvp_shuf_mask);
+ pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), rvp_shuf_mask);
+
+ if (do_offload) {
+ sxe2_rx_desc_offloads_para_fill_neon(rxq, desc, descs, &rx_pkts[i]);
+ } else {
+ const uint64x2_t mbuf_init = {
+ rxq->mbuf_init_value,
+ 0,
+ };
+
+ vst1q_u64((uint64_t *)&rx_pkts[i]->rearm_data, mbuf_init);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 1]->rearm_data, mbuf_init);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 2]->rearm_data, mbuf_init);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 3]->rearm_data, mbuf_init);
+ }
+
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
+ pkt_mb4 = vreinterpretq_u8_u16(tmp);
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
+ pkt_mb3 = vreinterpretq_u8_u16(tmp);
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
+ pkt_mb2 = vreinterpretq_u8_u16(tmp);
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
+ pkt_mb1 = vreinterpretq_u8_u16(tmp);
+
+ vst1q_u8((void *)&rx_pkts[i + 3]->rx_descriptor_fields1,
+ pkt_mb4);
+ vst1q_u8((void *)&rx_pkts[i + 2]->rx_descriptor_fields1,
+ pkt_mb3);
+ vst1q_u8((void *)&rx_pkts[i + 1]->rx_descriptor_fields1,
+ pkt_mb2);
+ vst1q_u8((void *)&rx_pkts[i]->rx_descriptor_fields1,
+ pkt_mb1);
+
+ if (likely(i + SXE2_RX_NUM_PER_LOOP_NEON < nb_pkts))
+ rte_prefetch_non_temporal(desc + SXE2_RX_NUM_PER_LOOP_NEON);
+
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
+ uint32x4_t sterr_tmp1 = vzip2q_u32(d1, d0);
+ uint32x4_t sterr_tmp2 = vzip2q_u32(d3, d2);
+ uint32x4_t sterr_u32 = vzip1q_u32(sterr_tmp1, sterr_tmp2);
+
+ staterr = vreinterpretq_u16_u32(sterr_u32);
+ }
+
+ sxe2_rx_desc_ptype_fill_neon(staterr, &rx_pkts[i], ptype_tbl);
+
+ if (umbcast_flags != NULL) {
+ uint32x4_t umbcast_mask = {
+ SXE2_RX_DESC_STATUS_UMBCAST_MASK, SXE2_RX_DESC_STATUS_UMBCAST_MASK,
+ SXE2_RX_DESC_STATUS_UMBCAST_MASK, SXE2_RX_DESC_STATUS_UMBCAST_MASK,
+ };
+
+ uint8x16_t umbcast_shuf_mask = {
+ 0x0B, 0x03, 0x0F, 0x07,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ };
+ uint8x16_t umbcast_bits =
+ vreinterpretq_u8_u32(vandq_u32(vreinterpretq_u32_u16(staterr),
+ umbcast_mask));
+
+ umbcast_bits = vqtbl1q_u8(umbcast_bits, umbcast_shuf_mask);
+ vst1q_lane_u32((uint32_t *)umbcast_flags,
+ vreinterpretq_u32_u8(umbcast_bits), 0);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_NEON;
+ }
+
+ if (split_rxe_flags) {
+ uint8x16_t eop_shuf_mask = {
+ 0x08, 0x00, 0x0C, 0x04,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF};
+ uint8x16_t eop_bits;
+ uint32x4_t rxe_mask = {
+ 0x2080, 0x2080, 0x2080, 0x2080
+ };
+ uint32x4_t rxe_bits;
+ uint32x4_t eop_mask;
+
+ eop_mask = vshlq_n_u32(vdupq_n_u32(1), SXE2_RX_DESC_STATUS_EOP_SHIFT);
+ eop_bits = vandq_u8(vmvnq_u8(vreinterpretq_u8_u16(staterr)),
+ vreinterpretq_u8_u32(eop_mask));
+
+ rxe_bits = vandq_u32(vreinterpretq_u32_u16(staterr), rxe_mask);
+ rxe_bits = vshrq_n_u32(rxe_bits, 7);
+
+ eop_bits = vorrq_u8(eop_bits, vreinterpretq_u8_u32(rxe_bits));
+
+ eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
+
+ vst1q_lane_u32((uint32_t *)split_rxe_flags,
+ vreinterpretq_u32_u8(eop_bits), 0);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_NEON;
+
+#ifdef RTE_IOVA_IN_MBUF
+ rx_pkts[i]->next = NULL;
+ rx_pkts[i + 1]->next = NULL;
+ rx_pkts[i + 2]->next = NULL;
+ rx_pkts[i + 3]->next = NULL;
+#endif
+ }
+
+ {
+ uint32x4_t dd_mask = vdupq_n_u32(1);
+ uint32x4_t sterr_dd = vandq_u32(vreinterpretq_u32_u16(staterr), dd_mask);
+ uint16x4_t packed_lo = vmovn_u32(sterr_dd);
+ uint64_t dd64 = vget_lane_u64(vreinterpret_u64_u16(packed_lo), 0);
+
+ bit_num = (uint16_t)rte_popcount64(dd64);
+ }
+ done_num += bit_num;
+ if (likely(bit_num != SXE2_RX_NUM_PER_LOOP_NEON))
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ rxq->realloc_num += done_num;
+
+l_end:
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_neon(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_neon((struct sxe2_rx_queue *)rxq,
+ rx_pkts, nb_pkts, split_rxe_flags, umbcast_flags,
+ do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_neon((struct sxe2_rx_queue *)rxq,
+ rx_pkts, nb_pkts, split_rxe_flags, NULL,
+ do_offload);
+ }
+
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ split_flags64[0] == 0 && split_flags64[1] == 0 &&
+ split_flags64[2] == 0 && split_flags64[3] == 0) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0) {
+ rx_pkt_done_num++;
+ }
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+ return rx_pkt_done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_neon_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM_VEC) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM_VEC,
+ true);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM_VEC)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ nb_pkts,
+ true);
+l_end:
+ SXE2_RX_STATS_CNT(rx_queue, rx_pkts_num, done_num);
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_neon(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM_VEC) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM_VEC,
+ false);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM_VEC)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ nb_pkts,
+ false);
+l_end:
+ SXE2_RX_STATS_CNT(rx_queue, rx_pkts_num, done_num);
+ return done_num;
+}
+#endif
--
2.47.3
^ permalink raw reply related
* [PATCH v2 12/23] net/sxe2: add support for custom UDP tunnel ports
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch enables the configuration of custom UDP port numbers for
tunneling protocols in the SXE2 PMD.
The change includes:
- Adding a new entry in the tunnel port lookup table.
- Updating the hardware profile to recognize
the custom UDP port as a tunnel type.
- Enabling inner header parsing for packets arriving on these ports.
This allows the Switch module to correctly apply recipes based on
inner packet fields (e.g., inner MAC or IP).
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/sxe2_cmd_chnl.c | 96 ++++++++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 17 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 16 ++
drivers/net/sxe2/sxe2_ethdev.c | 206 ++++++++++++++++++++-
drivers/net/sxe2/sxe2_ethdev.h | 12 ++
drivers/net/sxe2/sxe2_flow.c | 54 ++++++
drivers/net/sxe2/sxe2_flow.h | 3 +-
drivers/net/sxe2/sxe2_flow_define.h | 1 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 113 +++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 6 +
drivers/net/sxe2/sxe2_txrx_poll.c | 46 ++++-
11 files changed, 566 insertions(+), 4 deletions(-)
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index 6e2dd139a5..926eaee062 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -1455,6 +1455,102 @@ int32_t sxe2_drv_ipsec_txsa_delete(struct sxe2_adapter *adapter,
return ret;
}
+int32_t sxe2_drv_udp_tunnel_add(struct sxe2_adapter *adapter,
+ enum sxe2_udp_tunnel_protocol tunnel_proto,
+ uint16_t udp_port)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_udp_tunnel_req req = {};
+ struct sxe2_drv_cmd_params cmd = {};
+ int32_t ret = -1;
+
+ req.type = tunnel_proto;
+ req.port = udp_port;
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_UDPTUNNEL_ADD,
+ &req, sizeof(req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Failed to add udp proto %d port %d, ret=%d",
+ tunnel_proto, udp_port, ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_udp_tunnel_del(struct sxe2_adapter *adapter,
+ enum sxe2_udp_tunnel_protocol tunnel_proto,
+ uint16_t udp_port)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_udp_tunnel_req req = {};
+ struct sxe2_drv_cmd_params cmd = {};
+ int32_t ret = -1;
+
+ req.type = tunnel_proto;
+ req.port = udp_port;
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_UDPTUNNEL_DEL,
+ &req, sizeof(req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Failed to del udp proto %d port %d, ret=%d",
+ tunnel_proto, udp_port, ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_get_udp_tunnel_port(struct sxe2_adapter *adapter,
+ enum sxe2_flow_udp_tunnel_protocol proto,
+ uint16_t *port)
+{
+ int32_t ret = 0;
+ static const uint16_t flow_proto_to_udp_tunnel_proto[SXE2_FLOW_UDP_TUNNEL_MAX] = {
+ [SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN] = SXE2_UDP_TUNNEL_PROTOCOL_VXLAN,
+ [SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN_GPE] = SXE2_UDP_TUNNEL_PROTOCOL_VXLAN_GPE,
+ [SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GENEVE] = SXE2_UDP_TUNNEL_PROTOCOL_GENEVE,
+ [SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GTP_U] = SXE2_UDP_TUNNEL_PROTOCOL_GTP_U,
+ [SXE2_FLOW_UDP_TUNNEL_PROTOCOL_NVGRE] = SXE2_UDP_TUNNEL_PROTOCOL_NVGRE,
+ };
+ struct sxe2_udp_tunnel_cfg tunnel_config = {};
+
+ tunnel_config.protocol = flow_proto_to_udp_tunnel_proto[proto];
+ ret = sxe2_drv_udp_tunnel_get(adapter, &tunnel_config);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to get udp tunnel port, ret=%d", ret);
+ goto l_end;
+ }
+
+ *port = tunnel_config.fw_port;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_udp_tunnel_get(struct sxe2_adapter *adapter,
+ struct sxe2_udp_tunnel_cfg *tunnel_config)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_udp_tunnel_req req = {};
+ struct sxe2_drv_udp_tunnel_resp resp = {};
+ struct sxe2_drv_cmd_params cmd = {};
+ int32_t ret = -1;
+
+ req.type = tunnel_config->protocol;
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_UDPTUNNEL_GET,
+ &req, sizeof(req),
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Failed to get udp proto %d port, ret=%d", req.type, ret);
+
+ tunnel_config->fw_port = resp.port;
+ tunnel_config->fw_status = resp.enable;
+ tunnel_config->fw_dst_en = resp.dst;
+ tunnel_config->fw_src_en = resp.src;
+ tunnel_config->fw_used = resp.fw_used;
+
+ return ret;
+}
+
int32_t sxe2_drv_queue_info_get_update(struct sxe2_adapter *adapter, struct eth_queue_stats *qstats)
{
struct sxe2_drv_cmd_params param = {0};
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index d505f93dc1..43f28c8304 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -67,6 +67,23 @@ int32_t sxe2_drv_ipsec_txsa_delete(struct sxe2_adapter *adapter,
int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+int32_t sxe2_drv_udp_tunnel_add(struct sxe2_adapter *adapter,
+ enum sxe2_udp_tunnel_protocol tunnel_proto,
+ uint16_t udp_port);
+
+int32_t sxe2_drv_udp_tunnel_del(struct sxe2_adapter *adapter,
+ enum sxe2_udp_tunnel_protocol tunnel_proto,
+ uint16_t udp_port);
+
+int32_t sxe2_drv_udp_tunnel_get(struct sxe2_adapter *adapter,
+ struct sxe2_udp_tunnel_cfg *tunnel_config);
+
+int32_t sxe2_drv_get_udp_tunnel_port(struct sxe2_adapter *adapter,
+ enum sxe2_flow_udp_tunnel_protocol proto,
+ uint16_t *port);
+
+int32_t sxe2_drv_vsi_info_get(struct sxe2_adapter *adapter, struct sxe2_vsi *vsi);
+
int32_t sxe2_drv_vsi_info_get(struct sxe2_adapter *adapter, struct sxe2_vsi *vsi);
int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index 9d89039bee..12c7a02a71 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -617,6 +617,22 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_flow_fnav_query_stat_resp {
uint64_t stat_bytes;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_udp_tunnel_req {
+ uint8_t type;
+ uint8_t rsv;
+ uint16_t port;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_udp_tunnel_resp {
+ uint8_t type;
+ uint8_t enable;
+ uint8_t dst;
+ uint8_t src;
+ uint16_t port;
+ uint8_t fw_used;
+ uint8_t rsv;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 317101fb60..14c8f6c16d 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -40,10 +40,11 @@
#include "sxe2_ioctl_chnl_func.h"
#include "sxe2_ethdev_repr.h"
#include "sxe2vf_regs.h"
+#include "sxe2_switchdev.h"
#define SXE2_PCI_VENDOR_ID_1 0x1ff2
#define SXE2_PCI_DEVICE_ID_PF_1 0x10b1
-#define SXE2_PCI_DEVICE_ID_VF_1 0x10b2
+#define SXE2_PCI_DEVICE_ID_VF_1 0x10b
#define SXE2_PCI_VENDOR_ID_2 0x1d94
#define SXE2_PCI_DEVICE_ID_PF_2 0x1260
@@ -115,6 +116,11 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev);
static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info);
static const uint32_t *sxe2_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev
__rte_unused, size_t *no_of_elements __rte_unused);
+static int32_t sxe2_udp_tunnel_port_add(struct rte_eth_dev *dev,
+ struct rte_eth_udp_tunnel *tunnel_udp);
+static int32_t sxe2_udp_tunnel_port_del(struct rte_eth_dev *dev,
+ struct rte_eth_udp_tunnel *tunnel_udp);
+
static const struct eth_dev_ops sxe2_eth_dev_ops = {
.dev_configure = sxe2_dev_configure,
@@ -162,6 +168,9 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.rss_hash_update = sxe2_dev_rss_hash_update,
.rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
+ .udp_tunnel_port_add = sxe2_udp_tunnel_port_add,
+ .udp_tunnel_port_del = sxe2_udp_tunnel_port_del,
+
.flow_ops_get = sxe2_flow_ops_get,
.tm_ops_get = sxe2_tm_ops_get,
@@ -226,6 +235,12 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
PMD_INIT_FUNC_TRACE();
+ ret = sxe2_flow_init_udp_tunnel_port(dev);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to init udp tunnel port, ret: %d.", ret);
+ goto l_end;
+ }
+
ret = sxe2_queues_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to init queues.");
@@ -271,6 +286,188 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
return ret;
}
+static enum sxe2_udp_tunnel_protocol
+sxe2_udp_tunnel_type_rte_to_sxe2(enum rte_eth_tunnel_type rte_type)
+{
+ static enum sxe2_udp_tunnel_protocol sxe2_udp_proto_map[RTE_ETH_TUNNEL_TYPE_MAX] = {
+ [RTE_ETH_TUNNEL_TYPE_NONE] = SXE2_UDP_TUNNEL_MAX,
+ [RTE_ETH_TUNNEL_TYPE_VXLAN] = SXE2_UDP_TUNNEL_PROTOCOL_VXLAN,
+ [RTE_ETH_TUNNEL_TYPE_GENEVE] = SXE2_UDP_TUNNEL_PROTOCOL_GENEVE,
+ [RTE_ETH_TUNNEL_TYPE_TEREDO] = SXE2_UDP_TUNNEL_PROTOCOL_TEREDO,
+ [RTE_ETH_TUNNEL_TYPE_NVGRE] = SXE2_UDP_TUNNEL_PROTOCOL_NVGRE,
+ [RTE_ETH_TUNNEL_TYPE_IP_IN_GRE] = SXE2_UDP_TUNNEL_MAX,
+ [RTE_ETH_L2_TUNNEL_TYPE_E_TAG] = SXE2_UDP_TUNNEL_MAX,
+ [RTE_ETH_TUNNEL_TYPE_VXLAN_GPE] = SXE2_UDP_TUNNEL_PROTOCOL_VXLAN_GPE,
+ [RTE_ETH_TUNNEL_TYPE_ECPRI] = SXE2_UDP_TUNNEL_PROTOCOL_ECPRI
+ };
+
+ if (rte_type >= RTE_ETH_TUNNEL_TYPE_MAX) {
+ PMD_LOG_ERR(DRV, "Invalid rte_eth_tunnel_type %d!", rte_type);
+ rte_type = RTE_ETH_TUNNEL_TYPE_NONE;
+ }
+
+ return sxe2_udp_proto_map[rte_type];
+}
+
+int32_t sxe2_udp_tunnel_port_add_common(struct sxe2_adapter *ad,
+ enum sxe2_udp_tunnel_protocol tunnel_proto,
+ uint16_t udp_port)
+{
+ struct sxe2_udp_tunnel_cfg *tunnel_config;
+ int32_t ret = -1;
+
+ rte_spinlock_lock(&ad->udp_tunnel_ctx.lock);
+
+ tunnel_config = &ad->udp_tunnel_ctx.tunnel_conf[tunnel_proto];
+
+ if (tunnel_config->dev_status == SXE2_UDP_TUNNEL_ENABLE) {
+ if (udp_port == tunnel_config->dev_port &&
+ tunnel_config->dev_ref_cnt < 0xFFFFU) {
+ tunnel_config->dev_ref_cnt++;
+ ret = 0;
+ goto l_unlock_end;
+ } else {
+ PMD_LOG_ERR(DRV, "Adding multiple ports to the same protocol "
+ "is not supported!");
+ ret = -EINVAL;
+ goto l_unlock_end;
+ }
+ } else {
+ ret = sxe2_drv_udp_tunnel_add(ad, tunnel_proto, udp_port);
+ if (ret != 0)
+ goto l_unlock_end;
+
+ tunnel_config->protocol = tunnel_proto;
+ tunnel_config->dev_port = udp_port;
+ tunnel_config->dev_status = SXE2_UDP_TUNNEL_ENABLE;
+ tunnel_config->dev_ref_cnt++;
+ }
+
+l_unlock_end:
+ rte_spinlock_unlock(&ad->udp_tunnel_ctx.lock);
+ return ret;
+}
+
+int32_t sxe2_udp_tunnel_port_del_common(struct sxe2_adapter *ad,
+ enum sxe2_udp_tunnel_protocol tunnel_proto,
+ uint16_t udp_port)
+{
+ struct sxe2_udp_tunnel_cfg *tunnel_config;
+ int32_t ret = -1;
+
+ rte_spinlock_lock(&ad->udp_tunnel_ctx.lock);
+ tunnel_config = &ad->udp_tunnel_ctx.tunnel_conf[tunnel_proto];
+
+ if (tunnel_config->dev_status == SXE2_UDP_TUNNEL_ENABLE &&
+ udp_port == tunnel_config->dev_port) {
+ if (tunnel_config->dev_ref_cnt > 1) {
+ tunnel_config->dev_ref_cnt--;
+ ret = 0;
+ goto l_unlock_end;
+ } else {
+ ret = sxe2_drv_udp_tunnel_del(ad, tunnel_proto, udp_port);
+ if (ret != 0)
+ goto l_unlock_end;
+
+ tunnel_config->dev_status = SXE2_UDP_TUNNEL_DISABLE;
+ tunnel_config->dev_ref_cnt = 0;
+ }
+ goto l_unlock_end;
+ }
+
+ ret = -EINVAL;
+
+l_unlock_end:
+ rte_spinlock_unlock(&ad->udp_tunnel_ctx.lock);
+ return ret;
+}
+
+int32_t sxe2_udp_tunnel_port_get_common(struct sxe2_adapter *ad,
+ struct sxe2_udp_tunnel_cfg *tunnel_config)
+{
+ return sxe2_drv_udp_tunnel_get(ad, tunnel_config);
+}
+
+static int32_t sxe2_udp_tunnel_port_clear(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *ad = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_udp_tunnel_cfg *tunnel_config;
+ int32_t ret = 0;
+ uint16_t tunnel_proto = 0;
+
+ rte_spinlock_lock(&ad->udp_tunnel_ctx.lock);
+
+ for (tunnel_proto = 0; tunnel_proto < SXE2_UDP_TUNNEL_MAX; tunnel_proto++) {
+ tunnel_config = &ad->udp_tunnel_ctx.tunnel_conf[tunnel_proto];
+ if (tunnel_config->dev_status == SXE2_UDP_TUNNEL_ENABLE) {
+ ret = sxe2_drv_udp_tunnel_del(ad, tunnel_config->protocol,
+ tunnel_config->dev_port);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Failed to delete udp tunnel port %d, proto %d",
+ tunnel_config->dev_port, tunnel_config->protocol);
+ goto l_unlock_end;
+ }
+
+ tunnel_config->dev_status = SXE2_UDP_TUNNEL_DISABLE;
+ tunnel_config->dev_ref_cnt = 0;
+ }
+ }
+l_unlock_end:
+ rte_spinlock_unlock(&ad->udp_tunnel_ctx.lock);
+ return ret;
+}
+
+static int32_t sxe2_udp_tunnel_port_add(struct rte_eth_dev *dev,
+ struct rte_eth_udp_tunnel *tunnel_udp)
+{
+ int32_t ret = 0;
+ enum sxe2_udp_tunnel_protocol tunnel_proto;
+ struct sxe2_adapter *ad = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (tunnel_udp->udp_port == 0) {
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ tunnel_proto = sxe2_udp_tunnel_type_rte_to_sxe2(tunnel_udp->prot_type);
+ if (tunnel_proto >= SXE2_UDP_TUNNEL_MAX) {
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_udp_tunnel_port_add_common(ad, tunnel_proto, tunnel_udp->udp_port);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Add tunnel port failed, ret = %d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_udp_tunnel_port_del(struct rte_eth_dev *dev,
+ struct rte_eth_udp_tunnel *tunnel_udp)
+{
+ int32_t ret = 0;
+ enum sxe2_udp_tunnel_protocol tunnel_proto;
+ struct sxe2_adapter *ad = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ tunnel_proto = sxe2_udp_tunnel_type_rte_to_sxe2(tunnel_udp->prot_type);
+ if (tunnel_proto >= SXE2_UDP_TUNNEL_MAX) {
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_udp_tunnel_port_del_common(ad, tunnel_proto, tunnel_udp->udp_port);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Delete tunnel port failed, ret = %d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
@@ -1300,15 +1497,20 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
sxe2_mp_uninit(dev);
- (void)sxe2_rss_disable(dev);
(void)sxe2_sched_uinit(dev);
+ (void)sxe2_rss_disable(dev);
+ (void)sxe2_flow_uninit(dev);
+ (void)sxe2_udp_tunnel_port_clear(dev);
sxe2_vsi_uninit(dev);
sxe2_security_uinit(dev);
sxe2_intr_uninit(dev);
(void)sxe2_switchdev_uninit(dev);
sxe2_sw_uninit(dev);
+ (void)sxe2_switchdev_uninit(dev);
+ sxe2_dev_pci_map_uinit(dev);
sxe2_eth_uinit(dev);
sxe2_dev_pci_map_uinit(dev);
+ sxe2_free_repr_info(dev);
l_end:
return 0;
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index 6397a2e5c6..d6c6a152e7 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -318,6 +318,7 @@ struct sxe2_adapter {
struct sxe2_sched_hw_cap sched_ctxt;
struct sxe2_tm_context tm_ctxt;
struct sxe2_devargs devargs;
+ struct sxe2_udp_tunnel_ctx udp_tunnel_ctx;
struct sxe2_security_ctx security_ctx;
struct sxe2_repr_context repr_ctxt;
struct sxe2_switchdev_info switchdev_info;
@@ -373,6 +374,17 @@ void sxe2_dev_pci_seg_unmap(struct sxe2_adapter *adapter, uint32_t res_type);
int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev);
+void sxe2_dev_pci_seg_unmap(struct sxe2_adapter *adapter, uint32_t res_type);
+
+int32_t sxe2_udp_tunnel_port_get_common(struct sxe2_adapter *ad,
+ struct sxe2_udp_tunnel_cfg *tunnel_config);
+
+int32_t sxe2_udp_tunnel_port_del_common(struct sxe2_adapter *ad,
+ enum sxe2_udp_tunnel_protocol tunnel_proto, uint16_t udp_port);
+
+int32_t sxe2_udp_tunnel_port_add_common(struct sxe2_adapter *ad,
+ enum sxe2_udp_tunnel_protocol tunnel_proto, uint16_t udp_port);
+
void sxe2_dev_pci_map_uinit(struct rte_eth_dev *dev);
void sxe2_eth_uinit(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_flow.c b/drivers/net/sxe2/sxe2_flow.c
index 6999cb0725..63cfc36968 100644
--- a/drivers/net/sxe2/sxe2_flow.c
+++ b/drivers/net/sxe2/sxe2_flow.c
@@ -523,6 +523,51 @@ static int32_t sxe2_flow_adjust_action(struct rte_eth_dev *dev __rte_unused,
return ret;
}
+int32_t sxe2_flow_init_udp_tunnel_port(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ uint16_t i = 0;
+ uint16_t *flow_udp_tunnel_port = NULL;
+
+ memset(adapter->flow_ctxt.tunnel_port_list, 0,
+ sizeof(adapter->flow_ctxt.tunnel_port_list));
+
+ flow_udp_tunnel_port = adapter->flow_ctxt.tunnel_port_list;
+ for (i = 0; i < SXE2_FLOW_UDP_TUNNEL_MAX; i++) {
+ if (flow_udp_tunnel_port[i] == 0) {
+ ret = sxe2_drv_get_udp_tunnel_port(adapter, i,
+ &flow_udp_tunnel_port[i]);
+ if (ret != 0) {
+ PMD_LOG_ERR(DRV, "Failed to get udp tunnel port, proto: %d,"
+ "ret: %d", i, ret);
+ goto l_end;
+ }
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_flowlist_add_tunnel_port(struct rte_eth_dev *dev,
+ struct rte_flow *flow_list,
+ struct rte_flow_error *error)
+{
+ struct sxe2_flow_list_t *sxe2_flow_list = &flow_list->sxe2_flow_list;
+ struct sxe2_flow *flow = TAILQ_FIRST(sxe2_flow_list);
+ enum sxe2_flow_tunnel_type tunnel_type = flow->meta.tunnel_type;
+ DECLARE_BITMAP(flow_type, SXE2_EXPANSION_MAX);
+ sxe2_bitmap_zero(flow_type, SXE2_EXPANSION_MAX);
+ sxe2_bitmap_copy(flow_type, flow->flow_type, SXE2_EXPANSION_MAX);
+ int32_t ret = 0;
+
+ if (flow->engine_type == SXE2_FLOW_ENGINE_FNAV)
+ return sxe2_flow_add_tunnel_port(dev, error, flow, flow_type, tunnel_type);
+
+ return ret;
+}
+
static int32_t sxe2_flow_check_item_empty(uint8_t *item, uint16_t size)
{
uint16_t i = 0;
@@ -679,6 +724,10 @@ static int32_t sxe2_flow_post_proc(struct rte_eth_dev *dev,
{
int32_t ret = 0;
+ ret = sxe2_flowlist_add_tunnel_port(dev, flow_list, error);
+ if (ret)
+ goto l_end;
+
ret = sxe2_flowlist_add_proto_type(dev, flow_list, error);
if (ret)
goto l_end;
@@ -1308,6 +1357,11 @@ int32_t sxe2_flow_init(struct rte_eth_dev *dev)
adapter->flow_ctxt.fnav_inited = 1;
rte_spinlock_init(&adapter->flow_ctxt.flow_list_lock);
+
+ ret = sxe2_flow_init_udp_tunnel_port(dev);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Failed to init udp tunnel port, ret: %d.", ret);
+
return ret;
}
diff --git a/drivers/net/sxe2/sxe2_flow.h b/drivers/net/sxe2/sxe2_flow.h
index 9970fddcf0..daaeedd4dc 100644
--- a/drivers/net/sxe2/sxe2_flow.h
+++ b/drivers/net/sxe2/sxe2_flow.h
@@ -8,7 +8,6 @@
#include "sxe2_osal.h"
#include "sxe2_common.h"
-
int32_t sxe2_flow_ops_get(struct rte_eth_dev *dev, const struct rte_flow_ops **ops);
int32_t sxe2_flow_init(struct rte_eth_dev *dev);
@@ -26,4 +25,6 @@ int32_t sxe2_flow_query_mgr(struct sxe2_adapter *adapter,
struct sxe2_flow *flow,
struct sxe2_fnav_cid_mgr **mgr_ptr,
struct rte_flow_error *error);
+
+int32_t sxe2_flow_init_udp_tunnel_port(struct rte_eth_dev *dev);
#endif /* __SXE2_FLOW_H__ */
diff --git a/drivers/net/sxe2/sxe2_flow_define.h b/drivers/net/sxe2/sxe2_flow_define.h
index d2f6000efa..263a573f04 100644
--- a/drivers/net/sxe2/sxe2_flow_define.h
+++ b/drivers/net/sxe2/sxe2_flow_define.h
@@ -119,6 +119,7 @@ struct sxe2_flow_context {
struct rte_flow_list_t rte_flow_list;
rte_spinlock_t flow_list_lock;
struct sxe2_fnav_count_resource hw_res;
+ uint16_t tunnel_port_list[SXE2_FLOW_UDP_TUNNEL_MAX];
uint32_t fnav_inited;
};
#define SXE2_INVALID_RSS_ATTR \
diff --git a/drivers/net/sxe2/sxe2_flow_parse_pattern.c b/drivers/net/sxe2/sxe2_flow_parse_pattern.c
index 189abb1a33..f5bf8922c6 100644
--- a/drivers/net/sxe2/sxe2_flow_parse_pattern.c
+++ b/drivers/net/sxe2/sxe2_flow_parse_pattern.c
@@ -1637,6 +1637,119 @@ static int32_t sxe2_flow_parse_pattern_vxlan_gpe(const struct rte_flow_item *ite
return ret;
}
+static int32_t sxe2_flow_parse_pattern_ipip(struct sxe2_flow *flow, BITMAP_TYPE *flow_type)
+{
+ sxe2_set_bit(SXE2_EXPANSION_IPIP, flow_type);
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV4, flow_type)) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_PROT, flow->pattern_outer.map_spec);
+ if (sxe2_test_bit(SXE2_EXPANSION_IPV4, flow_type))
+ flow->pattern_outer.item_spec.ipv4.protocol = SXE2_FLOW_IP_PROTOCOL_IPV4;
+ if (sxe2_test_bit(SXE2_EXPANSION_IPV6, flow_type))
+ flow->pattern_outer.item_spec.ipv4.protocol = SXE2_FLOW_IP_PROTOCOL_IPV6;
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV6, flow_type)) {
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PROT, flow->pattern_outer.map_spec);
+ if (sxe2_test_bit(SXE2_EXPANSION_ETH, flow_type)) {
+ flow->pattern_outer.item_spec.ipv6.nexthdr = SXE2_FLOW_IP_PROTOCOL_ETH;
+ } else {
+ if (sxe2_test_bit(SXE2_EXPANSION_IPV4, flow_type))
+ flow->pattern_outer.item_spec.ipv6.nexthdr =
+ SXE2_FLOW_IP_PROTOCOL_IPV4;
+ if (sxe2_test_bit(SXE2_EXPANSION_IPV6, flow_type))
+ flow->pattern_outer.item_spec.ipv6.nexthdr =
+ SXE2_FLOW_IP_PROTOCOL_IPV6;
+ }
+ }
+ return 0;
+}
+
+static int32_t sxe2_flow_add_udp_tunnel_port(struct sxe2_adapter *adapter,
+ enum sxe2_flow_udp_tunnel_protocol proto,
+ struct sxe2_flow *flow,
+ BITMAP_TYPE *flow_type)
+{
+ int32_t ret = 0;
+ uint16_t tun_port;
+
+ tun_port = adapter->flow_ctxt.tunnel_port_list[proto];
+ if (tun_port == 0xffff || tun_port == 0) {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "UDP tunnel port not initialized, proto: %d", proto);
+ goto l_end;
+ }
+ if (!sxe2_test_bit(SXE2_EXPANSION_OUTER_UDP, flow_type)) {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "UDP must be over tunnel");
+ goto l_end;
+ }
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_UDP_DST_PORT, flow->pattern_outer.map_spec);
+ flow->pattern_outer.item_spec.udp.dest = rte_cpu_to_be_16(tun_port);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_flow_add_tunnel_port(struct rte_eth_dev *dev,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow, BITMAP_TYPE *flow_type,
+ enum sxe2_flow_tunnel_type tunnel_type)
+{
+ int32_t ret = 0;
+ enum sxe2_flow_udp_tunnel_protocol proto = SXE2_FLOW_UDP_TUNNEL_MAX;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_flow_pattern *pattern = &flow->pattern_outer;
+ switch (tunnel_type) {
+ case SXE2_FLOW_TUNNEL_TYPE_VXLAN:
+ if (sxe2_test_bit(SXE2_EXPANSION_ETH, flow_type)) {
+ proto = SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN;
+ } else if (sxe2_test_bit(SXE2_EXPANSION_IPV4, flow_type) ||
+ sxe2_test_bit(SXE2_EXPANSION_IPV6, flow_type)) {
+ proto = SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN_GPE;
+ }
+ break;
+ case SXE2_FLOW_TUNNEL_TYPE_GTPU:
+ proto = SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GTP_U;
+ break;
+ case SXE2_FLOW_TUNNEL_TYPE_GENEVE:
+ proto = SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GENEVE;
+ break;
+ case SXE2_FLOW_TUNNEL_TYPE_GRE:
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_UDP, flow_type)) {
+ proto = SXE2_FLOW_UDP_TUNNEL_PROTOCOL_NVGRE;
+ } else {
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV4, flow_type)) {
+ pattern->item_spec.ipv4.protocol = SXE2_FLOW_IP_PROTOCOL_GRE;
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV4_PROT, pattern->map_spec);
+ }
+ if (sxe2_test_bit(SXE2_EXPANSION_OUTER_IPV6, flow_type)) {
+ pattern->item_spec.ipv6.nexthdr = SXE2_FLOW_IP_PROTOCOL_GRE;
+ sxe2_set_bit(SXE2_FLOW_FLD_ID_IPV6_PROT, pattern->map_spec);
+ }
+ }
+ break;
+ case SXE2_FLOW_TUNNEL_TYPE_IPIP:
+ ret = sxe2_flow_parse_pattern_ipip(flow, flow_type);
+ break;
+ default:
+ break;
+ }
+ if (proto != SXE2_FLOW_UDP_TUNNEL_MAX) {
+ ret = sxe2_flow_add_udp_tunnel_port(adapter, proto, flow, flow_type);
+ if (ret != 0) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ NULL, "Failed to add udp port for tunnel.");
+ PMD_LOG_ERR(DRV, "Failed to add udp port for tunnel, ret %d.", ret);
+ goto l_end;
+ }
+ }
+ if (tunnel_type != SXE2_FLOW_TUNNEL_TYPE_NONE) {
+ if (!sxe2_test_bit(SXE2_EXPANSION_OUTER_UDP, flow_type))
+ sxe2_set_bit(SXE2_FLOW_HDR_IPV_OTHER, pattern->hdrs);
+ }
+l_end:
+ return ret;
+}
+
struct sxe2_flow_parse_pattern_ops sxe2_flow_parse_pattern_list[] = {
[SXE2_EXPANSION_OUTER_ETH] = {
.is_inner = false,
diff --git a/drivers/net/sxe2/sxe2_flow_parse_pattern.h b/drivers/net/sxe2/sxe2_flow_parse_pattern.h
index 69d83a6ea6..8442c35cae 100644
--- a/drivers/net/sxe2/sxe2_flow_parse_pattern.h
+++ b/drivers/net/sxe2/sxe2_flow_parse_pattern.h
@@ -37,4 +37,10 @@ int32_t sxe2_flow_parse_pattern(struct rte_eth_dev *dev,
struct rte_flow_error *error,
struct sxe2_flow *flow);
+int32_t sxe2_flow_add_tunnel_port(struct rte_eth_dev *dev,
+ struct rte_flow_error *error,
+ struct sxe2_flow *flow,
+ BITMAP_TYPE *flow_type,
+ enum sxe2_flow_tunnel_type tunnel_type);
+
#endif /* SXE2_FLOW_PARSE_PATTERN_H_ */
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index f3c4fa0d91..746f9cc2d5 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -234,6 +234,44 @@ sxe2_tx_pkt_data_desc_count(struct rte_mbuf *tx_pkt)
return count;
}
+static __rte_always_inline void sxe2_tx_desc_tunneling_params_fill(uint64_t offloads,
+ union sxe2_tx_offload_info ol_info,
+ uint32_t *desc_tunneling_params)
+{
+ if (offloads & RTE_MBUF_F_TX_OUTER_IP_CKSUM)
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_EIPT_IPV4;
+ else if (offloads & RTE_MBUF_F_TX_OUTER_IPV4)
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_EIPT_IPV4_NO_CSUM;
+ else if (offloads & RTE_MBUF_F_TX_OUTER_IPV6)
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_EIPT_IPV6;
+
+ *desc_tunneling_params |=
+ SXE2_TX_CTXT_DESC_EIPLEN_VAL(ol_info.outer_l3_len);
+ switch (offloads & RTE_MBUF_F_TX_TUNNEL_MASK) {
+ case RTE_MBUF_F_TX_TUNNEL_IPIP:
+ break;
+ case RTE_MBUF_F_TX_TUNNEL_VXLAN:
+ case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
+ case RTE_MBUF_F_TX_TUNNEL_GTP:
+ case RTE_MBUF_F_TX_TUNNEL_GENEVE:
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_UDP_TUNNE;
+ break;
+ case RTE_MBUF_F_TX_TUNNEL_GRE:
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_GRE_TUNNE;
+ break;
+ default:
+ PMD_LOG_ERR(TX, "Tunnel type [0x%" PRIx64 "] is not supported.",
+ (uint64_t)(offloads & RTE_MBUF_F_TX_TUNNEL_MASK));
+ return;
+ }
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_NATLEN_VAL(ol_info.l2_len);
+ if (!(*desc_tunneling_params & SXE2_TX_CTXT_DESC_EIPT_NONE) &&
+ (*desc_tunneling_params & SXE2_TX_CTXT_DESC_UDP_TUNNE) &&
+ (offloads & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) {
+ *desc_tunneling_params |= SXE2_TX_CTXT_DESC_L4T_CS_MASK;
+ }
+}
+
static __rte_always_inline void
sxe2_tx_desc_checksum_fill(uint64_t offloads, uint32_t *desc_cmd, uint32_t *desc_offset,
union sxe2_tx_offload_info ol_info)
@@ -414,7 +452,13 @@ uint16_t sxe2_tx_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkt
}
}
- desc_offset |= SXE2_TX_DATA_DESC_MACLEN_VAL(ol_info.l2_len);
+ if ((offloads & RTE_MBUF_F_TX_TUNNEL_MASK) && ctxt_desc_num) {
+ desc_offset |= SXE2_TX_DATA_DESC_MACLEN_VAL(ol_info.outer_l2_len);
+ sxe2_tx_desc_tunneling_params_fill(offloads, ol_info,
+ &desc_tunneling_params);
+ } else {
+ desc_offset |= SXE2_TX_DATA_DESC_MACLEN_VAL(ol_info.l2_len);
+ }
if (offloads & SXE2_TX_OFFLOAD_CKSUM_MASK) {
sxe2_tx_desc_checksum_fill(offloads, &desc_cmd,
--
2.47.3
^ permalink raw reply related
* [PATCH v2 09/23] drivers: interrupt handling
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch includes:
- MSI-X vector allocation and mapping logic.
- Support for rx_queue_intr_enable and rx_queue_intr_disable ops.
- Interrupt handler to process admin and queue events.
- Integration with EAL interrupt framework.
RX queue interrupts allow applications to use rte_eth_dev_rx_intr_wait()
for power-efficient packet processing.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_ioctl_chnl.c | 177 +++-
drivers/common/sxe2/sxe2_ioctl_chnl_func.h | 18 +
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 42 +
drivers/net/sxe2/sxe2_cmd_chnl.h | 4 +
drivers/net/sxe2/sxe2_drv_cmd.h | 8 +
drivers/net/sxe2/sxe2_ethdev.c | 93 +-
drivers/net/sxe2/sxe2_ethdev.h | 6 +
drivers/net/sxe2/sxe2_irq.c | 941 +++++++++++++++++++++
drivers/net/sxe2/sxe2_irq.h | 21 +
drivers/net/sxe2/sxe2vf_regs.h | 82 ++
11 files changed, 1389 insertions(+), 4 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_irq.c
create mode 100644 drivers/net/sxe2/sxe2vf_regs.h
diff --git a/drivers/common/sxe2/sxe2_ioctl_chnl.c b/drivers/common/sxe2/sxe2_ioctl_chnl.c
index 2ffbeb9217..173d8d57ae 100644
--- a/drivers/common/sxe2/sxe2_ioctl_chnl.c
+++ b/drivers/common/sxe2/sxe2_ioctl_chnl.c
@@ -67,6 +67,7 @@ sxe2_drv_cmd_exec(struct sxe2_common_device *cdev,
return ret;
}
+
RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_open)
int32_t
sxe2_drv_dev_open(struct sxe2_common_device *cdev, struct rte_pci_device *pci_dev)
@@ -87,7 +88,7 @@ sxe2_drv_dev_open(struct sxe2_common_device *cdev, struct rte_pci_device *pci_de
if (fd < 0) {
ret = -EBADF;
PMD_LOG_ERR(COM, "Fail to open device:%s, ret=%d, err:%s",
- drv_name, ret, strerror(errno));
+ drv_name, ret, strerror(errno));
goto l_end;
}
@@ -159,6 +160,177 @@ sxe2_drv_dev_handshake(struct sxe2_common_device *cdev)
return ret;
}
+RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_rxq_irq_set)
+int32_t
+sxe2_drv_dev_rxq_irq_set(struct sxe2_common_device *cdev,
+ uint16_t base_irq, int32_t *efd, uint16_t nb_irq)
+{
+ struct sxe2_ioctl_irq_set cmd_params;
+ int32_t ret = 0;
+ int32_t cmd_fd = 0;
+
+ if (cdev->config.kernel_reset) {
+ ret = -EPERM;
+ PMD_LOG_WARN(COM, "kernel reset, need restart app.");
+ goto l_end;
+ }
+
+ cmd_fd = SXE2_CDEV_TO_CMD_FD(cdev);
+ if (cmd_fd < 0) {
+ ret = -EBADF;
+ PMD_LOG_ERR(COM, "Failed to exec cmd, fd=%d", cmd_fd);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(COM, "Open fd=%d to set rxq irq, base_queue=%d, efds=%p, nb_irq=%d",
+ cmd_fd, base_irq, efd, nb_irq);
+
+ memset(&cmd_params, 0, sizeof(struct sxe2_ioctl_irq_set));
+ cmd_params.base_irq_in_com = base_irq;
+ cmd_params.cnt = nb_irq;
+ cmd_params.event_fd = efd;
+
+ pthread_mutex_lock(&cdev->config.lock);
+ ret = ioctl(cmd_fd, SXE2_COM_CMD_IO_IRQS_REQ, &cmd_params);
+ if (ret < 0) {
+ PMD_LOG_ERR(COM, "Failed to set io irqs, fd=%d, ret=%d, err:%s",
+ cmd_fd, ret, strerror(errno));
+ pthread_mutex_unlock(&cdev->config.lock);
+ goto l_end;
+ }
+ pthread_mutex_unlock(&cdev->config.lock);
+
+l_end:
+ return ret;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_other_irq_set)
+int32_t
+sxe2_drv_dev_other_irq_set(struct sxe2_common_device *cdev,
+ int32_t efd, uint64_t event)
+{
+ int32_t ret = 0;
+ int32_t cmd_fd = 0;
+ struct sxe2_ioctl_other_evt_set cmd_params;
+
+ if (cdev->config.kernel_reset) {
+ ret = -EPERM;
+ PMD_LOG_WARN(COM, "kernel reset, need restart app.");
+ goto l_end;
+ }
+
+ cmd_fd = SXE2_CDEV_TO_CMD_FD(cdev);
+ if (cmd_fd < 0) {
+ ret = -EBADF;
+ PMD_LOG_ERR(COM, "Failed to exec cmd, fd=%d", cmd_fd);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(COM, "Open fd=%d to set other irq, efd=%d, event=%"PRIu64"",
+ cmd_fd, efd, event);
+
+ memset(&cmd_params, 0, sizeof(struct sxe2_ioctl_other_evt_set));
+ cmd_params.eventfd = efd;
+ cmd_params.filter_table = event;
+
+ pthread_mutex_lock(&cdev->config.lock);
+ ret = ioctl(cmd_fd, SXE2_COM_CMD_EVT_IRQ_REQ, &cmd_params);
+ if (ret < 0) {
+ PMD_LOG_ERR(COM, "Failed to set others evt, fd=%d, ret=%d, err:%s",
+ cmd_fd, ret, strerror(errno));
+ ret = -EIO;
+ pthread_mutex_unlock(&cdev->config.lock);
+ goto l_end;
+ }
+ pthread_mutex_unlock(&cdev->config.lock);
+
+l_end:
+ return ret;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_other_irq_get)
+int32_t
+sxe2_drv_dev_other_irq_get(struct sxe2_common_device *cdev, uint64_t *event)
+{
+ int32_t ret = 0;
+ int32_t cmd_fd = 0;
+ struct sxe2_ioctl_other_evt_get cmd_params;
+
+ if (cdev->config.kernel_reset) {
+ ret = -EPERM;
+ PMD_LOG_WARN(COM, "kernel reset, need restart app.");
+ goto l_end;
+ }
+
+ cmd_fd = SXE2_CDEV_TO_CMD_FD(cdev);
+ if (cmd_fd < 0) {
+ ret = -EBADF;
+ PMD_LOG_ERR(COM, "Failed to exec cmd, fd=%d", cmd_fd);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(COM, "Open fd=%d to get other irq", cmd_fd);
+
+ memset(&cmd_params, 0, sizeof(struct sxe2_ioctl_other_evt_get));
+
+ pthread_mutex_lock(&cdev->config.lock);
+ ret = ioctl(cmd_fd, SXE2_COM_CMD_EVT_CAUSE_GET, &cmd_params);
+ if (ret < 0) {
+ PMD_LOG_ERR(COM, "Failed to set others evt, fd=%d, ret=%d, err:%s",
+ cmd_fd, ret, strerror(errno));
+ ret = -EIO;
+ pthread_mutex_unlock(&cdev->config.lock);
+ goto l_end;
+ }
+ pthread_mutex_unlock(&cdev->config.lock);
+ *event = cmd_params.evt_cause;
+
+l_end:
+ return ret;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_reset_irq_set)
+int32_t
+sxe2_drv_dev_reset_irq_set(struct sxe2_common_device *cdev, int32_t efd)
+{
+ int32_t ret = 0;
+ int32_t cmd_fd = 0;
+ struct sxe2_ioctl_reset_sub_set cmd_params;
+
+ if (cdev->config.kernel_reset) {
+ ret = -EPERM;
+ PMD_LOG_WARN(COM, "kernel reset, need restart app.");
+ goto l_end;
+ }
+
+ cmd_fd = SXE2_CDEV_TO_CMD_FD(cdev);
+ if (cmd_fd < 0) {
+ ret = -EBADF;
+ PMD_LOG_ERR(COM, "Failed to exec cmd, fd=%d", cmd_fd);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(COM, "Open fd=%d to set reset irq, efd=%d",
+ cmd_fd, efd);
+
+ memset(&cmd_params, 0, sizeof(struct sxe2_ioctl_reset_sub_set));
+ cmd_params.eventfd = efd;
+
+ pthread_mutex_lock(&cdev->config.lock);
+ ret = ioctl(cmd_fd, SXE2_COM_CMD_RST_IRQ_REQ, &cmd_params);
+ if (ret < 0) {
+ PMD_LOG_ERR(COM, "Failed to set reset irqs, fd=%d, ret=%d, err:%s",
+ cmd_fd, ret, strerror(errno));
+ ret = -EIO;
+ pthread_mutex_unlock(&cdev->config.lock);
+ goto l_end;
+ }
+ pthread_mutex_unlock(&cdev->config.lock);
+
+l_end:
+ return ret;
+}
+
RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_mmap)
void
*sxe2_drv_dev_mmap(struct sxe2_common_device *cdev, uint8_t bar_idx, uint64_t len, uint64_t offset)
@@ -223,7 +395,7 @@ sxe2_drv_dev_munmap(struct sxe2_common_device *cdev, void *virt, uint64_t len)
RTE_EXPORT_INTERNAL_SYMBOL(sxe2_drv_dev_dma_map)
int32_t
sxe2_drv_dev_dma_map(struct sxe2_common_device *cdev, uint64_t vaddr,
- uint64_t iova, uint64_t size)
+ uint64_t iova, uint64_t size)
{
struct sxe2_ioctl_iommu_dma_map cmd_params;
enum rte_iova_mode iova_mode;
@@ -322,4 +494,3 @@ sxe2_drv_dev_dma_unmap(struct sxe2_common_device *cdev, uint64_t iova)
l_end:
return ret;
}
-
diff --git a/drivers/common/sxe2/sxe2_ioctl_chnl_func.h b/drivers/common/sxe2/sxe2_ioctl_chnl_func.h
index aed5a5b50d..f29194fc9e 100644
--- a/drivers/common/sxe2/sxe2_ioctl_chnl_func.h
+++ b/drivers/common/sxe2/sxe2_ioctl_chnl_func.h
@@ -29,6 +29,7 @@ int32_t
sxe2_drv_dev_open(struct sxe2_common_device *cdev,
struct rte_pci_device *pci_dev);
+
__rte_internal
void
sxe2_drv_dev_close(struct sxe2_common_device *cdev);
@@ -37,6 +38,23 @@ __rte_internal
int32_t
sxe2_drv_dev_handshake(struct sxe2_common_device *cdev);
+__rte_internal
+int32_t
+sxe2_drv_dev_rxq_irq_set(struct sxe2_common_device *cdev, uint16_t base_irq,
+ int32_t *efd, uint16_t nb_irq);
+
+__rte_internal
+int32_t
+sxe2_drv_dev_other_irq_set(struct sxe2_common_device *cdev, int32_t efd, uint64_t event);
+
+__rte_internal
+int32_t
+sxe2_drv_dev_other_irq_get(struct sxe2_common_device *cdev, uint64_t *event);
+
+__rte_internal
+int32_t
+sxe2_drv_dev_reset_irq_set(struct sxe2_common_device *cdev, int32_t fd);
+
__rte_internal
void
*sxe2_drv_dev_mmap(struct sxe2_common_device *cdev, uint8_t bar_idx,
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index d860629def..c73e13bbad 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -68,6 +68,7 @@ sources += files(
'sxe2_security.c',
'sxe2_mp.c',
'sxe2_stats.c',
+ 'sxe2_irq.c',
)
allow_internal_get_api = true
\ No newline at end of file
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index a1fc8a50e3..d1f15084ed 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -348,6 +348,48 @@ int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
return ret;
}
+int32_t sxe2_drv_rxq_bind_irq(struct sxe2_adapter *adapter, uint16_t rxq_idx, uint16_t msix_idx)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_queue_irq_bind_req req = {0};
+ int32_t ret = 0;
+
+ req.msix_idx = msix_idx;
+ req.q_idx = rxq_idx;
+ req.itr_idx = 0;
+ req.bind = true;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_EVT_IRQ_BAND_RXQ,
+ &req, sizeof(req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "rxq bind irq failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_rxq_unbind_irq(struct sxe2_adapter *adapter, uint16_t rxq_idx)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_queue_irq_bind_req req = {0};
+ int32_t ret = 0;
+
+ req.bind = false;
+ req.q_idx = rxq_idx;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_EVT_IRQ_BAND_RXQ,
+ &req, sizeof(req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "rxq unbind irq failed, ret=%d", ret);
+
+ return ret;
+}
+
int32_t sxe2_drv_get_mac_stats(struct sxe2_adapter *adapter)
{
int32_t ret = 0;
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index ff73d2f901..c13653e8af 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -118,4 +118,8 @@ int32_t sxe2_drv_rss_hf_clear(struct sxe2_adapter *adapter);
int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue *rxq);
+int32_t sxe2_drv_rxq_bind_irq(struct sxe2_adapter *adapter, uint16_t rxq_idx, uint16_t msix_idx);
+
+int32_t sxe2_drv_rxq_unbind_irq(struct sxe2_adapter *adapter, uint16_t rxq_idx);
+
#endif /* __SXE2_CMD_CHNL_H__ */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index fe1ca9160a..a839d9dfe0 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -556,6 +556,14 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_tx_map_req {
uint8_t pool_idx;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_irq_bind_req {
+ uint16_t q_idx;
+ uint16_t msix_idx;
+ uint8_t itr_idx;
+ uint8_t bind;
+ uint8_t rsv[2];
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index dbe1a2bce1..f3fee74ddf 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -22,6 +22,7 @@
#include <rte_eal_paging.h>
#include "sxe2_ethdev.h"
+#include "sxe2_irq.h"
#include "sxe2_drv_cmd.h"
#include "sxe2_cmd_chnl.h"
#include "sxe2_tx.h"
@@ -107,6 +108,9 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.rx_queue_release = sxe2_rx_queue_release,
.tx_queue_setup = sxe2_tx_queue_setup,
.tx_queue_release = sxe2_tx_queue_release,
+ .rx_queue_intr_enable = sxe2_rx_queue_intr_enable,
+ .rx_queue_intr_disable = sxe2_rx_queue_intr_disable,
+
.rxq_info_get = sxe2_rx_queue_info_get,
.txq_info_get = sxe2_tx_queue_info_get,
.rx_burst_mode_get = sxe2_rx_burst_mode_get,
@@ -177,6 +181,8 @@ static int32_t sxe2_dev_stop(struct rte_eth_dev *dev)
if (adapter->started == 0)
goto l_end;
+ sxe2_rxq_intr_disable(dev);
+
sxe2_txqs_all_stop(dev);
sxe2_rxqs_all_stop(dev);
@@ -215,6 +221,12 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
goto l_end;
}
+ ret = sxe2_rxq_intr_enable(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to enable rx queue intr");
+ goto l_rxq_intr_err;
+ }
+
ret = sxe2_queues_start(dev);
if (ret) {
PMD_LOG_ERR(INIT, "enable queues failed");
@@ -224,7 +236,10 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
dev->data->dev_started = 1;
adapter->started = 1;
goto l_end;
+
l_start_queues_err:
+ (void)sxe2_rxq_intr_disable(dev);
+l_rxq_intr_err:
(void)sxe2_filter_rule_stop(dev);
l_end:
return ret;
@@ -615,6 +630,8 @@ static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
sxe2_sw_queue_ctx_hw_cap_set(adapter, &dev_caps.queue_caps);
+ sxe2_sw_irq_ctx_hw_cap_set(adapter, &dev_caps.msix_caps);
+
sxe2_sw_rss_ctx_hw_cap_set(adapter, &dev_caps.rss_hash_caps);
sxe2_sw_vsi_ctx_hw_cap_set(adapter, &dev_caps.vsi_caps);
@@ -636,6 +653,41 @@ static int32_t sxe2_dev_caps_get(struct sxe2_adapter *adapter)
return ret;
}
+uint32_t sxe2_pci_map_read_reg(struct sxe2_adapter *adapter,
+ enum sxe2_pci_map_resource res_type, uint16_t idx_in_func)
+{
+ void *reg_addr;
+ uint32_t value;
+
+ reg_addr = sxe2_pci_map_addr_get(adapter, res_type, idx_in_func);
+ if (unlikely(reg_addr == NULL)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "reg addr:0x%p is error.", reg_addr);
+ value = SXE2_PCI_MAP_INVALID_VAL;
+ goto l_ret;
+ }
+
+ value = SXE2_PCI_REG_READ(reg_addr);
+
+l_ret:
+ return value;
+}
+
+void sxe2_pci_map_write_reg(struct sxe2_adapter *adapter,
+ enum sxe2_pci_map_resource res_type, uint16_t idx_in_func, uint32_t value)
+{
+ void *reg_addr;
+
+ reg_addr = sxe2_pci_map_addr_get(adapter, res_type, idx_in_func);
+ if (unlikely(reg_addr == NULL)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "reg addr:0x%p is error.", reg_addr);
+ goto l_ret;
+ }
+
+ SXE2_PCI_REG_WRITE_WC(reg_addr, value);
+l_ret:
+ return;
+}
+
int32_t sxe2_dev_pci_seg_map(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type,
uint64_t org_len,
@@ -715,6 +767,27 @@ static int32_t sxe2_hw_init(struct rte_eth_dev *dev)
return ret;
}
+static int32_t sxe2_sw_init(struct rte_eth_dev *dev)
+{
+ int32_t ret = -1;
+
+ PMD_INIT_FUNC_TRACE();
+
+ ret = sxe2_sw_irq_ctxt_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to sw irq ctxt init, ret=[%d]", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static void sxe2_sw_uninit(struct rte_eth_dev *dev)
+{
+ sxe2_sw_irq_ctxt_uninit(dev);
+}
+
int32_t sxe2_dev_pci_res_seg_map(struct sxe2_adapter *adapter,
uint32_t res_type,
uint32_t item_cnt,
@@ -1065,6 +1138,18 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_dev_info_err;
}
+ ret = sxe2_sw_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize sw parameters, ret=[%d]", ret);
+ goto init_sw_err;
+ }
+
+ ret = sxe2_intr_init(dev);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to initialize interrupt, ret:%d", ret);
+ goto init_irq_err;
+ }
+
ret = sxe2_eth_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to initialize eth parameters, ret=%d", ret);
@@ -1109,6 +1194,10 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
init_rss_err:
sxe2_security_uinit(dev);
init_security_err:
+ sxe2_intr_uninit(dev);
+init_irq_err:
+ sxe2_sw_uninit(dev);
+init_sw_err:
sxe2_eth_uinit(dev);
init_eth_err:
init_dev_info_err:
@@ -1132,8 +1221,10 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
(void)sxe2_sched_uinit(dev);
sxe2_vsi_uninit(dev);
sxe2_security_uinit(dev);
- sxe2_dev_pci_map_uinit(dev);
+ sxe2_intr_uninit(dev);
+ sxe2_sw_uninit(dev);
sxe2_eth_uinit(dev);
+ sxe2_dev_pci_map_uinit(dev);
l_end:
return 0;
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index 32a67ed344..65ada44c12 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -354,6 +354,12 @@ int32_t sxe2_dev_pci_res_seg_map(struct sxe2_adapter *adapter,
uint32_t item_cnt,
uint32_t item_base);
+void sxe2_pci_map_write_reg(struct sxe2_adapter *adapter,
+ enum sxe2_pci_map_resource res_type, uint16_t idx_in_func, uint32_t value);
+
+uint32_t sxe2_pci_map_read_reg(struct sxe2_adapter *adapter,
+ enum sxe2_pci_map_resource res_type, uint16_t idx_in_func);
+
void sxe2_dev_pci_seg_unmap(struct sxe2_adapter *adapter, uint32_t res_type);
int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_irq.c b/drivers/net/sxe2/sxe2_irq.c
new file mode 100644
index 0000000000..13619500ea
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_irq.c
@@ -0,0 +1,941 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <stdint.h>
+#include <sys/eventfd.h>
+#include <unistd.h>
+#include <ethdev_pci.h>
+#include <ethdev_driver.h>
+#include <rte_alarm.h>
+#include <fcntl.h>
+#include <rte_stdatomic.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_irq.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_drv_cmd.h"
+#include "sxe2_ioctl_chnl_func.h"
+#include "sxe2vf_regs.h"
+#include "sxe2_host_regs.h"
+#include "sxe2_cmd_chnl.h"
+
+#define SXE2_INT_EVENT_OICR_ALL (SXE2_PF_INT_OICR_SWINT | \
+ SXE2_PF_INT_OICR_LAN_TX_ERR | \
+ SXE2_PF_INT_OICR_LAN_RX_ERR | \
+ SXE2_PF_INT_OICR_FW)
+
+#define MAX_EVENT_PENDING (16)
+
+struct sxe2_event_element {
+ TAILQ_ENTRY(sxe2_event_element) next;
+ struct rte_eth_dev *dev;
+};
+
+struct sxe2_event_handler {
+ RTE_ATOMIC(uint16_t)ndev;
+ rte_thread_t tid;
+ int32_t fd[2];
+ rte_spinlock_t lock;
+ TAILQ_HEAD(event_list, sxe2_event_element) pending;
+};
+static struct sxe2_event_handler event_handler = {
+ .fd = {-1, -1},
+};
+static volatile int32_t event_thread_run;
+
+
+static void sxe2_event_irq_common_handler(struct sxe2_adapter *adapter, uint64_t oicr)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[adapter->dev_info.dev_data->port_id];
+
+ if (oicr & RTE_BIT32(SXE2_COM_EC_LINK_CHG)) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "OICR=%" PRIu64, oicr);
+ (void)sxe2_drv_mac_link_status_get(adapter);
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ rte_eth_dev_callback_process(dev,
+ RTE_ETH_EVENT_INTR_LSC,
+ NULL);
+ }
+}
+
+static uint32_t sxe2_event_intr_handle(void *param __rte_unused)
+{
+ struct sxe2_event_handler *handler = &event_handler;
+ struct sxe2_adapter *adapter;
+ struct sxe2_event_element *pos;
+ struct sxe2_event_element *tmp;
+ int32_t ret = 0;
+ uint64_t oicr = 0;
+ TAILQ_HEAD(event_list, sxe2_event_element) pending;
+ int8_t unused[MAX_EVENT_PENDING];
+ ssize_t nr;
+
+ while (event_thread_run) {
+ nr = read(handler->fd[0], &unused, sizeof(unused));
+ if (nr <= 0)
+ break;
+
+ rte_spinlock_lock(&handler->lock);
+ TAILQ_INIT(&pending);
+ TAILQ_CONCAT(&pending, &handler->pending, next);
+ rte_spinlock_unlock(&handler->lock);
+
+ TAILQ_FOREACH_SAFE(pos, &pending, next, tmp) {
+ TAILQ_REMOVE(&pending, pos, next);
+ adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(pos->dev);
+
+ ret = sxe2_drv_dev_other_irq_get(adapter->cdev, &oicr);
+ sxe2_event_irq_common_handler(adapter, oicr);
+
+ rte_free(pos);
+ }
+ }
+
+ return ret;
+}
+
+static int32_t sxe2_event_intr_handler_init(void)
+{
+ struct sxe2_event_handler *handler = &event_handler;
+ int32_t ret = 0;
+ int err = 0;
+
+ PMD_INIT_FUNC_TRACE();
+ if (rte_atomic_fetch_add_explicit(&handler->ndev, 1, rte_memory_order_acq_rel) >= 1) {
+ ret = 0;
+ PMD_LOG_DEBUG(INIT, "%s: ndev > 1", __func__);
+ goto l_end;
+ }
+
+#if defined(RTE_EXEC_ENV_IS_WINDOWS) && RTE_EXEC_ENV_IS_WINDOWS != 0
+ err = _pipe(handler->fd, MAX_EVENT_PENDING, O_BINARY);
+#else
+ err = pipe(handler->fd);
+#endif
+ if (err != 0) {
+ ret = -ECHILD;
+ rte_atomic_fetch_sub_explicit(&handler->ndev, 1, rte_memory_order_release);
+ PMD_LOG_ERR(INIT, "%s: pipe failed", __func__);
+ goto l_end;
+ }
+
+ event_thread_run = 1;
+
+ TAILQ_INIT(&handler->pending);
+ rte_spinlock_init(&handler->lock);
+
+ if (rte_thread_create_internal_control(&handler->tid, "sxe2-event",
+ sxe2_event_intr_handle, NULL)) {
+ PMD_LOG_ERR(INIT, "%s: thread create failed", __func__);
+ rte_atomic_fetch_sub_explicit(&handler->ndev, 1, rte_memory_order_release);
+ ret = -ECHILD;
+ goto l_end;
+ }
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static void sxe2_event_intr_handler_uinit(void)
+{
+ struct sxe2_event_handler *handler = &event_handler;
+ struct sxe2_event_element *pos;
+ struct sxe2_event_element *tmp;
+ ssize_t nw = 0;
+ int8_t notify_byte = 0;
+
+ PMD_INIT_FUNC_TRACE();
+ if (rte_atomic_fetch_sub_explicit(&handler->ndev, 1, rte_memory_order_acq_rel) > 1) {
+ PMD_LOG_DEBUG(INIT, "event handler uinit, ndev > 0");
+ return;
+ }
+
+ event_thread_run = 0;
+ nw = write(handler->fd[1], ¬ify_byte, 1);
+ RTE_SET_USED(nw);
+
+ (void)rte_thread_join(handler->tid, NULL);
+
+ if (handler->fd[0] != -1) {
+ close(handler->fd[0]);
+ handler->fd[0] = -1;
+ }
+ if (handler->fd[1] != -1) {
+ close(handler->fd[1]);
+ handler->fd[1] = -1;
+ }
+
+ rte_spinlock_lock(&handler->lock);
+ TAILQ_FOREACH_SAFE(pos, &handler->pending, next, tmp) {
+ TAILQ_REMOVE(&handler->pending, pos, next);
+ rte_free(pos);
+ }
+ rte_spinlock_unlock(&handler->lock);
+}
+
+static void sxe2_event_intr_post(struct rte_eth_dev *dev)
+{
+ struct sxe2_event_handler *handler = &event_handler;
+ struct sxe2_event_element *elem = rte_malloc(NULL, sizeof(struct sxe2_event_element), 0);
+ int8_t notify_byte = 0;
+ ssize_t nw = 0;
+
+ if (!elem)
+ goto l_end;
+
+ elem->dev = dev;
+
+ rte_spinlock_lock(&handler->lock);
+ TAILQ_INSERT_TAIL(&handler->pending, elem, next);
+ rte_spinlock_unlock(&handler->lock);
+
+ nw = write(handler->fd[1], ¬ify_byte, 1);
+ RTE_SET_USED(nw);
+
+l_end:
+ return;
+}
+
+static void sxe2_interrupt_handler_other(void *arg)
+{
+ struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t eventfd = adapter->irq_ctxt.other_event_fd;
+ int32_t ret = 0;
+ uint64_t buf = 0;
+
+ ret = read(eventfd, &buf, sizeof(buf));
+ if (ret != sizeof(buf)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "read eventfd[%d] failed, ret:%d, errno:%d.",
+ eventfd, ret, errno);
+ }
+
+ sxe2_event_intr_post(dev);
+}
+
+static void sxe2_interrupt_handler_reset(void *arg)
+{
+ struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t resetfd = adapter->irq_ctxt.reset_event_fd;
+ int32_t ret = 0;
+ uint64_t buf = 0;
+
+ ret = read(resetfd, &buf, sizeof(buf));
+ if (ret != sizeof(buf)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "read resetfd[%d] failed, ret:%d, errno:%d.",
+ resetfd, ret, errno);
+ }
+
+ sxe2_drv_cmd_close(adapter->cdev);
+
+ rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL);
+}
+
+int32_t sxe2_sw_irq_ctxt_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ int32_t ret = 0;
+
+ irq_ctxt->rxq_avail_cnt = irq_ctxt->max_cnt_hw;
+ irq_ctxt->rxq_base_idx_in_pf = irq_ctxt->base_idx_in_func;
+ irq_ctxt->reset_event_fd = -1;
+ irq_ctxt->other_event_fd = -1;
+
+ return ret;
+}
+
+void sxe2_sw_irq_ctxt_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ memset(&adapter->irq_ctxt, 0, sizeof(adapter->irq_ctxt));
+ adapter->irq_ctxt.reset_event_fd = -1;
+ adapter->irq_ctxt.other_event_fd = -1;
+}
+
+void sxe2_sw_irq_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_msix_caps *msix_caps)
+{
+ adapter->irq_ctxt.max_cnt_hw = msix_caps->msix_vectors_cnt;
+ adapter->irq_ctxt.base_idx_in_func = msix_caps->base_idx_in_func;
+}
+
+static int32_t sxe2_intr_handler_cfg(struct rte_intr_handle *handle,
+ int32_t fd, rte_intr_callback_fn cb, void *cb_arg)
+{
+ int32_t ret = 0;
+
+ ret = rte_intr_fd_set(handle, fd);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to set intr_handle->fd, error %i (%s)",
+ errno, strerror(errno));
+ goto err;
+ }
+
+ ret = rte_intr_type_set(handle, RTE_INTR_HANDLE_EXT);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to set intr_handle->type, error %i (%s)",
+ errno, strerror(errno));
+ goto err;
+ }
+
+ ret = rte_intr_callback_register(handle, cb, cb_arg);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize register intr callback, ret=%d", ret);
+ goto err;
+ }
+err:
+ return ret;
+}
+
+static struct rte_intr_handle *
+sxe2_intr_handler_create(int32_t fd, rte_intr_callback_fn cb, void *cb_arg)
+{
+ struct rte_intr_handle *tmp_intr_handle = NULL;
+ int32_t ret = 0;
+
+ tmp_intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
+ if (!tmp_intr_handle) {
+ PMD_LOG_ERR(INIT, "Failed to alloc memory for intr_handler, error %i (%s)",
+ errno, strerror(errno));
+ goto err;
+ }
+
+ ret = rte_intr_fd_set(tmp_intr_handle, fd);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to set intr_handle->fd, error %i (%s)",
+ errno, strerror(errno));
+ goto err;
+ }
+
+ ret = rte_intr_type_set(tmp_intr_handle, RTE_INTR_HANDLE_EXT);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to set intr_handle->type, error %i (%s)",
+ errno, strerror(errno));
+ goto err;
+ }
+
+ ret = rte_intr_callback_register(tmp_intr_handle, cb, cb_arg);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize register intr callback, ret=%d", ret);
+ goto err;
+ }
+
+ return tmp_intr_handle;
+err:
+ rte_intr_instance_free(tmp_intr_handle);
+ return NULL;
+}
+
+static void sxe2_intr_handler_destroy(struct rte_intr_handle *intr_handle,
+ rte_intr_callback_fn cb, void *cb_arg)
+{
+ if (!intr_handle)
+ return;
+
+ if (rte_intr_fd_get(intr_handle) >= 0)
+ (void)rte_intr_callback_unregister(intr_handle, cb, cb_arg);
+ rte_intr_instance_free(intr_handle);
+}
+
+static int32_t sxe2_event_intr_fd_create(void)
+{
+ int32_t fd = 0;
+
+ fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+ if (fd < 0) {
+ PMD_LOG_ERR(INIT, "Can't setup eventfd, error %i (%s)",
+ errno, strerror(errno));
+ goto err;
+ }
+
+ return fd;
+err:
+ return -EBADF;
+}
+
+static void sxe2_event_intr_fd_destroy(int32_t fd)
+{
+ if (fd >= 0)
+ close(fd);
+}
+
+static int32_t sxe2_other_intr_register(struct rte_eth_dev *dev, int32_t fd)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ uint64_t event = RTE_BIT32(SXE2_COM_EC_LINK_CHG) |
+ RTE_BIT32(SXE2_COM_SW_MODE_LEGACY) |
+ RTE_BIT32(SXE2_COM_SW_MODE_SWITCHDEV) |
+ RTE_BIT32(SXE2_COM_FC_ST_CHANGE);
+
+ ret = sxe2_drv_dev_other_irq_set(adapter->cdev, fd, event);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set other irq, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static void sxe2_other_intr_unregister(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_drv_dev_other_irq_set(adapter->cdev, -1, 0);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set other irq, ret=%d", ret);
+}
+
+static int32_t sxe2_reset_intr_register(struct rte_eth_dev *dev, int32_t fd)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_drv_dev_reset_irq_set(adapter->cdev, fd);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set reset irq, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static void sxe2_reset_intr_unregister(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_drv_dev_reset_irq_set(adapter->cdev, -1);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set reset irq, ret=%d", ret);
+}
+
+int32_t sxe2_intr_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_pci_device *pci_dev = SXE2_DEV_TO_PCI(dev);
+ struct rte_intr_handle *reset_handle = NULL;
+ int32_t ofd = -1;
+ int32_t rfd = -1;
+ int32_t ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ ofd = sxe2_event_intr_fd_create();
+ if (ofd < 0) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to create event intr fd");
+ ret = -EBADF;
+ goto l_end;
+ }
+
+ ret = sxe2_intr_handler_cfg(pci_dev->intr_handle,
+ ofd, sxe2_interrupt_handler_other, dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to create intr handler");
+ goto l_err_create_other_handler;
+ }
+
+ ret = sxe2_event_intr_handler_init();
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Event handler init failed, ret=%d", ret);
+ goto l_err_event_intr_handler_init;
+ }
+
+ ret = sxe2_other_intr_register(dev, ofd);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to register other intr, ret=%d", ret);
+ goto l_err_register_other_intr;
+ }
+ adapter->irq_ctxt.other_event_fd = ofd;
+
+ rfd = sxe2_event_intr_fd_create();
+ if (rfd < 0) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to create event intr fd");
+ ret = -EBADF;
+ goto l_err_create_reset_fd;
+ }
+
+ reset_handle = sxe2_intr_handler_create(rfd, sxe2_interrupt_handler_reset, dev);
+ if (!reset_handle) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to create intr handler");
+ ret = -ENOMEM;
+ goto l_err_create_reset_handler;
+ }
+ adapter->irq_ctxt.reset_handle = reset_handle;
+
+ ret = sxe2_reset_intr_register(dev, rfd);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to register reset intr, ret=%d", ret);
+ goto l_err_register_reset_intr;
+ }
+ adapter->irq_ctxt.reset_event_fd = rfd;
+
+ goto l_end;
+l_err_register_reset_intr:
+ sxe2_intr_handler_destroy(reset_handle, sxe2_interrupt_handler_reset, dev);
+ adapter->irq_ctxt.reset_handle = NULL;
+l_err_create_reset_handler:
+ sxe2_event_intr_fd_destroy(rfd);
+l_err_create_reset_fd:
+ sxe2_other_intr_unregister(dev);
+ adapter->irq_ctxt.other_event_fd = -1;
+l_err_register_other_intr:
+ sxe2_event_intr_handler_uinit();
+l_err_event_intr_handler_init:
+ sxe2_intr_handler_destroy(pci_dev->intr_handle,
+ sxe2_interrupt_handler_other, dev);
+ pci_dev->intr_handle = NULL;
+l_err_create_other_handler:
+ sxe2_event_intr_fd_destroy(ofd);
+l_end:
+ return ret;
+}
+
+void sxe2_intr_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_pci_device *pci_dev = SXE2_DEV_TO_PCI(dev);
+
+ sxe2_reset_intr_unregister(dev);
+ sxe2_intr_handler_destroy(adapter->irq_ctxt.reset_handle,
+ sxe2_interrupt_handler_reset, dev);
+ sxe2_event_intr_fd_destroy(adapter->irq_ctxt.reset_event_fd);
+ sxe2_other_intr_unregister(dev);
+ sxe2_event_intr_handler_uinit();
+ sxe2_intr_handler_destroy(pci_dev->intr_handle,
+ sxe2_interrupt_handler_other, dev);
+ sxe2_event_intr_fd_destroy(adapter->irq_ctxt.other_event_fd);
+
+ adapter->irq_ctxt.other_event_fd = -1;
+ adapter->irq_ctxt.reset_event_fd = -1;
+ pci_dev->intr_handle = NULL;
+ adapter->irq_ctxt.reset_handle = NULL;
+}
+
+static int32_t sxe2_rxq_intr_efd_alloc(struct rte_eth_dev *dev, int32_t *efd)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ int32_t fd = 0;
+
+ fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+ if (fd < 0) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Can't setup eventfd, error %i (%s)",
+ errno, strerror(errno));
+ ret = -EBADF;
+ goto l_end;
+ }
+
+ *efd = fd;
+
+l_end:
+ return ret;
+}
+
+static void sxe2_rxq_intr_efd_free(int32_t efd)
+{
+ close(efd);
+}
+
+static void sxe2_pci_hw_int_itr_set(struct sxe2_adapter *adapter, uint16_t msix_idx, uint16_t itr)
+{
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_ITR, msix_idx, itr);
+}
+
+static void sxe2_pci_hw_irq_disable(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ uint32_t value = (SXE2_ITR_IDX_NONE << SXE2_VF_DYN_CTL_ITR_IDX_S);
+
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_DYN, irq_idx, value);
+}
+
+static void sxe2_pci_hw_irq_enable(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ uint32_t value = SXE2_VF_DYN_CTL_INTENABLE |
+ SXE2_VF_DYN_CTL_CLEARPBA |
+ (SXE2_ITR_IDX_NONE << SXE2_VF_DYN_CTL_ITR_IDX_S);
+
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_DYN, irq_idx, value);
+}
+
+static uint32_t sxe2_pci_hw_irq_dyn_ctl_read(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ return sxe2_pci_map_read_reg(adapter, SXE2_PCI_MAP_RES_IRQ_DYN, irq_idx);
+}
+
+static void sxe2_pci_hw_msix_disable(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_MSIX,
+ irq_idx, SXE2VF_BAR4_MSIX_DISABLE);
+}
+
+static void sxe2_pci_hw_msix_enable(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_MSIX,
+ irq_idx, SXE2VF_BAR4_MSIX_ENABLE);
+}
+
+static void sxe2_pci_hw_irq_trigger(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_DYN, irq_idx,
+ (SXE2VF_ITR_IDX_NONE << SXE2VF_DYN_CTL_ITR_IDX_SHIFT) |
+ SXE2VF_DYN_CTL_SWINT_TRIG | SXE2VF_DYN_CTL_INTENABLE_MSK);
+}
+
+static void sxe2_pci_hw_irq_clear_pba(struct sxe2_adapter *adapter, uint16_t irq_idx)
+{
+ sxe2_pci_map_write_reg(adapter, SXE2_PCI_MAP_RES_IRQ_DYN, irq_idx,
+ (SXE2VF_ITR_IDX_NONE << SXE2VF_DYN_CTL_ITR_IDX_SHIFT) |
+ SXE2VF_DYN_CTL_CLEARPBA | SXE2VF_DYN_CTL_INTENABLE_MSK);
+}
+
+static void sxe2_rxq_msix_cfg_unmap(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ uint16_t rxq_cnt = adapter->q_ctxt.qp_cnt_assign;
+ uint16_t i = 0;
+
+ for (i = 0; i < irq_ctxt->rxq_irq_cnt; i++)
+ sxe2_pci_hw_irq_disable(adapter, irq_ctxt->rxq_msix_idx[i]);
+
+ for (i = 0; i < rxq_cnt; i++)
+ (void)sxe2_drv_rxq_unbind_irq(adapter, i);
+}
+
+static int32_t sxe2_rxq_msix_cfg_map(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ int32_t ret = 0;
+ uint32_t val;
+ uint16_t rxq_cnt = dev->data->nb_rx_queues;
+ uint16_t i = 0;
+ uint8_t rx_low_latency = adapter->devargs.rx_low_latency;
+
+ for (i = 0; i < irq_ctxt->rxq_irq_cnt; i++) {
+ sxe2_pci_hw_irq_disable(adapter, irq_ctxt->rxq_msix_idx[i]);
+ if (rx_low_latency) {
+ sxe2_pci_hw_int_itr_set(adapter, irq_ctxt->rxq_msix_idx[i],
+ SXE2_ITR_INTERVAL_LOW);
+ } else {
+ sxe2_pci_hw_int_itr_set(adapter, irq_ctxt->rxq_msix_idx[i],
+ SXE2_ITR_INTERVAL_NORMAL);
+ }
+ }
+
+ for (i = 0; i < rxq_cnt; i++) {
+ ret = sxe2_drv_rxq_bind_irq(adapter, i, irq_ctxt->rxq_msix_idx[i]);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "RXQ[%u] bind IRQ[%u] failed.",
+ i, irq_ctxt->rxq_msix_idx[i]);
+ goto l_end;
+ }
+ }
+
+ for (i = 0; i < irq_ctxt->rxq_irq_cnt; i++)
+ sxe2_pci_hw_irq_enable(adapter, irq_ctxt->rxq_msix_idx[i]);
+
+ for (i = 0; i < irq_ctxt->rxq_irq_cnt; i++) {
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, i);
+ if ((val & SXE2VF_DYN_CTL_INTENABLE) == 0)
+ continue;
+
+ sxe2_pci_hw_msix_disable(adapter, i);
+ sxe2_pci_hw_irq_trigger(adapter, i);
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, i);
+ sxe2_pci_hw_irq_clear_pba(adapter, i);
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, i);
+ sxe2_pci_hw_msix_enable(adapter, i);
+ }
+
+l_end:
+ if (ret != 0)
+ sxe2_rxq_msix_cfg_unmap(dev);
+ return ret;
+}
+
+static int32_t sxe2_rxq_map_msix_intr(struct rte_eth_dev *dev,
+ uint16_t msix_base __rte_unused, uint16_t nb_msix,
+ uint16_t base_queue, uint16_t nb_queue)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ uint32_t *msix_tbl = NULL;
+ int32_t ret = 0;
+ uint16_t i;
+
+ if (!nb_queue || !nb_msix || nb_queue < nb_msix) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Queue num[%u] or msix num[%u] is invalid.",
+ nb_queue, nb_msix);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ msix_tbl = rte_zmalloc(NULL, sizeof(uint32_t) * nb_queue, 0);
+ if (!msix_tbl) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to alloc msix_tbl memory.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ for (i = 0; i < nb_queue; i++) {
+ msix_tbl[i] = i % nb_msix;
+ PMD_DEV_LOG_INFO(adapter, INIT, "Queue %u is binding to vect %u",
+ base_queue + i, msix_tbl[i]);
+ }
+
+ irq_ctxt->rxq_irq_cnt = nb_msix;
+ irq_ctxt->rxq_msix_idx = msix_tbl;
+
+l_end:
+ return ret;
+}
+
+static void sxe2_rxq_unmap_msix_intr(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+
+ rte_free(irq_ctxt->rxq_msix_idx);
+ irq_ctxt->rxq_msix_idx = NULL;
+ irq_ctxt->rxq_irq_cnt = 0;
+}
+
+static int32_t sxe2_rxq_intr_register(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ struct rte_intr_handle *intr_handle = dev->intr_handle;
+ int32_t *efd_tbl = NULL;
+ uint16_t rxq_cnt = dev->data->nb_rx_queues;
+ uint16_t nb_msix = irq_ctxt->rxq_irq_cnt;
+ uint16_t i;
+ int32_t ret = 0;
+
+ if (rte_intr_type_set(intr_handle, RTE_INTR_HANDLE_EXT)) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set intr_handle->type, error %i (%s)",
+ errno, strerror(errno));
+ ret = -EPERM;
+ goto l_end;
+ }
+
+ efd_tbl = rte_zmalloc(NULL, sizeof(int32_t) * nb_msix, 0);
+ if (!efd_tbl) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to alloc efd_tbl memory.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ for (i = 0; i < nb_msix; i++) {
+ ret = sxe2_rxq_intr_efd_alloc(dev, &efd_tbl[i]);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to alloc efd_tbl, ret=%d", ret);
+ goto l_free_efd_tbl;
+ }
+ }
+
+ if (rte_intr_vec_list_alloc(intr_handle, "sxe2 rxq int", rxq_cnt)) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to allocate %d rx_queues intr_vec",
+ rxq_cnt);
+ ret = -ENOMEM;
+ goto l_free_efd_tbl;
+ }
+
+ for (i = 0; i < rxq_cnt; i++) {
+ ret = rte_intr_vec_list_index_set(intr_handle, i,
+ irq_ctxt->rxq_msix_idx[i] + RTE_INTR_VEC_RXTX_OFFSET);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set msix_tbl, ret=%d", ret);
+ goto l_free_efd_tbl;
+ }
+ }
+
+ for (i = 0; i < irq_ctxt->rxq_irq_cnt; i++) {
+ ret = rte_intr_efds_index_set(intr_handle, i, efd_tbl[i]);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set efd_tbl, ret=%d", ret);
+ goto l_free_efd_tbl;
+ }
+ }
+
+ if (rte_intr_nb_efd_set(intr_handle, rxq_cnt)) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Set intr nb efd failed, error %i (%s)",
+ errno, strerror(errno));
+ ret = -EPERM;
+ goto l_free_efd_tbl;
+ }
+
+ ret = sxe2_drv_dev_rxq_irq_set(adapter->cdev, 0, efd_tbl, nb_msix);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rxq irq, ret=%d", ret);
+ goto l_free_efd_tbl;
+ }
+ irq_ctxt->rxq_event_fd = efd_tbl;
+
+ goto l_end;
+
+l_free_efd_tbl:
+ if (efd_tbl) {
+ for (i = 0; i < nb_msix; i++)
+ if (efd_tbl[i] >= 0)
+ sxe2_rxq_intr_efd_free(efd_tbl[i]);
+ rte_free(efd_tbl);
+ }
+ irq_ctxt->rxq_event_fd = NULL;
+
+ rte_intr_vec_list_free(intr_handle);
+l_end:
+ return ret;
+}
+
+static void sxe2_rxq_intr_unregister(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ struct rte_intr_handle *intr_handle = dev->intr_handle;
+ int32_t efd = -1;
+ uint16_t msix_cnt = irq_ctxt->rxq_irq_cnt;
+ uint16_t i;
+
+ if (irq_ctxt->rxq_event_fd) {
+ for (i = 0; i < msix_cnt; i++) {
+ (void)sxe2_drv_dev_rxq_irq_set(adapter->cdev, i, &efd, 1);
+ sxe2_rxq_intr_efd_free(irq_ctxt->rxq_event_fd[i]);
+ }
+ }
+ rte_free(irq_ctxt->rxq_event_fd);
+ irq_ctxt->rxq_event_fd = NULL;
+
+ rte_intr_vec_list_free(intr_handle);
+
+ rte_intr_nb_efd_set(intr_handle, 0);
+ rte_intr_max_intr_set(intr_handle, 0);
+}
+
+int32_t sxe2_rxq_intr_enable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint16_t msix_vect = adapter->irq_ctxt.rxq_base_idx_in_pf;
+ uint16_t msix_cnt = adapter->irq_ctxt.rxq_avail_cnt;
+ uint16_t rxq_cnt = dev->data->nb_rx_queues;
+ uint16_t rxq_base = adapter->q_ctxt.base_idx_in_pf;
+ int32_t ret = 0;
+
+ if (!rxq_cnt)
+ goto l_end;
+
+ msix_cnt = RTE_MIN(msix_cnt, rxq_cnt);
+
+ ret = sxe2_rxq_map_msix_intr(dev, msix_vect, msix_cnt, rxq_base, rxq_cnt);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to rxq[%d] map msix[%d] intr, cnt=%d, ret=%d",
+ rxq_base, msix_vect, rxq_cnt, ret);
+ goto l_end;
+ }
+
+ if (dev->data->dev_conf.intr_conf.rxq) {
+ ret = sxe2_rxq_intr_register(dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to register rxq[%d] intr, ret=%d",
+ rxq_base, ret);
+ goto l_err_unmap;
+ }
+ }
+
+ ret = sxe2_rxq_msix_cfg_map(dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to rxq[%d] map msix[%d] intr, ret=%d",
+ rxq_base, msix_vect, ret);
+ goto l_err_unregister;
+ }
+
+ goto l_end;
+l_err_unregister:
+ if (dev->data->dev_conf.intr_conf.rxq)
+ sxe2_rxq_intr_unregister(dev);
+l_err_unmap:
+ sxe2_rxq_unmap_msix_intr(dev);
+l_end:
+ return ret;
+}
+
+void sxe2_rxq_intr_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+
+ if (!irq_ctxt->rxq_irq_cnt)
+ goto l_end;
+
+ sxe2_rxq_msix_cfg_unmap(dev);
+
+ if (dev->data->dev_conf.intr_conf.rxq)
+ sxe2_rxq_intr_unregister(dev);
+
+ sxe2_rxq_unmap_msix_intr(dev);
+
+l_end:
+ return;
+}
+
+int32_t sxe2_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_irq_context *irq_ctxt = &adapter->irq_ctxt;
+ uint64_t buf;
+ uint16_t irq_idx = irq_ctxt->rxq_msix_idx[queue_id];
+ size_t read_ret;
+
+ read_ret = read(irq_ctxt->rxq_event_fd[irq_idx], &buf, sizeof(buf));
+ (void)read_ret;
+ sxe2_pci_hw_irq_enable(adapter, irq_idx);
+ return 0;
+}
+
+int32_t sxe2_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct sxe2_adapter *adapter =
+ SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint32_t val;
+ int32_t ret = 0;
+ uint16_t irq_idx = adapter->irq_ctxt.rxq_msix_idx[queue_id];
+
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, irq_idx);
+ if ((val & SXE2VF_DYN_CTL_INTENABLE) == 0) {
+ PMD_DEV_LOG_DEBUG(adapter, INIT, "rxq [%d] interrupt is disabled.", queue_id);
+ goto l_end;
+ }
+
+ sxe2_pci_hw_msix_disable(adapter, irq_idx);
+ sxe2_pci_hw_irq_trigger(adapter, irq_idx);
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, irq_idx);
+ sxe2_pci_hw_irq_clear_pba(adapter, irq_idx);
+ val = sxe2_pci_hw_irq_dyn_ctl_read(adapter, irq_idx);
+ sxe2_pci_hw_msix_enable(adapter, irq_idx);
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_irq.h b/drivers/net/sxe2/sxe2_irq.h
index bb96c6d842..c898c16f84 100644
--- a/drivers/net/sxe2/sxe2_irq.h
+++ b/drivers/net/sxe2/sxe2_irq.h
@@ -45,4 +45,25 @@ struct sxe2_irq_context {
int32_t *rxq_event_fd;
};
+uint32_t sxe2_drv_dev_other_cause_get(struct sxe2_adapter *adapter);
+
+int32_t sxe2_intr_init(struct rte_eth_dev *dev);
+
+void sxe2_intr_uninit(struct rte_eth_dev *dev);
+
+int32_t sxe2_sw_irq_ctxt_init(struct rte_eth_dev *dev);
+
+void sxe2_sw_irq_ctxt_uninit(struct rte_eth_dev *dev);
+
+void sxe2_sw_irq_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_msix_caps *msix_caps);
+
+int32_t sxe2_rxq_intr_enable(struct rte_eth_dev *dev);
+
+void sxe2_rxq_intr_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
+
+int32_t sxe2_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+
#endif /* __SXE2_IRQ_H__ */
diff --git a/drivers/net/sxe2/sxe2vf_regs.h b/drivers/net/sxe2/sxe2vf_regs.h
new file mode 100644
index 0000000000..503c7ce04f
--- /dev/null
+++ b/drivers/net/sxe2/sxe2vf_regs.h
@@ -0,0 +1,82 @@
+
+#ifndef __SXE2VF_REGS_H__
+#define __SXE2VF_REGS_H__
+
+#define SXE2VF_MBX_Q_LEN_M 0x3FF
+#define SXE2VF_MBX_Q_LEN_VFE_M RTE_BIT32(28)
+
+#define SXE2VF_MBX_Q_LEN_OVFL_M RTE_BIT32(29)
+#define SXE2VF_MBX_Q_LEN_CRIT_M RTE_BIT32(30)
+#define SXE2VF_MBX_Q_LEN_ENA_M RTE_BIT32(31)
+
+#define SXE2VF_MBX_RQ_HEAD (0x00008000)
+#define SXE2VF_MBX_RQ_TAIL (0x00008400)
+#define SXE2VF_MBX_RQ_LEN (0x00007C00)
+#define SXE2VF_MBX_RQ_BAH (0x00007800)
+#define SXE2VF_MBX_RQ_BAL (0x00007400)
+
+#define SXE2VF_MBX_TQ_HEAD (0x00006C00)
+#define SXE2VF_MBX_TQ_TAIL (0x00007000)
+#define SXE2VF_MBX_TQ_LEN (0x00006800)
+#define SXE2VF_MBX_TQ_BAH (0x00006400)
+#define SXE2VF_MBX_TQ_BAL (0x00006000)
+
+#define SXE2VF_RXQ_TAIL(_QRX) (0x2000 + ((_QRX) * 4))
+#define SXE2VF_TXQ_TAIL(_QRX) (0x1000 + ((_QRX) * 4))
+
+#define SXE2VF_INT_BASE 0x00002800
+
+#define SXE2VF_DYN_CTL0 (SXE2VF_INT_BASE + 0x0)
+
+#define SXE2VF_DYN_CTL(_idx) (SXE2VF_INT_BASE + 0x4 + ((_idx) * 4))
+#define SXE2VF_VF_DYN_CTL(_idx) (SXE2VF_INT_BASE + ((_idx) * 4))
+
+#define SXE2VF_BAR4_MSIX_BASE 0
+#define SXE2VF_BAR4_MSIX_CTL(_idx) (SXE2VF_BAR4_MSIX_BASE + 0xC + ((_idx) * 0x10))
+#define SXE2VF_BAR4_MSIX_ENABLE 0
+#define SXE2VF_BAR4_MSIX_DISABLE 1
+
+#define SXE2VF_DYN_CTL_INTENABLE RTE_BIT32(0)
+#define SXE2VF_DYN_CTL_CLEARPBA RTE_BIT32(1)
+#define SXE2VF_DYN_CTL_SWINT_TRIG RTE_BIT32(2)
+#define SXE2VF_DYN_CTL_SW_ITR_IDX_ENABLE RTE_BIT32(24)
+
+#define SXE2VF_DYN_CTL_INTENABLE_MSK \
+ RTE_BIT32(31)
+
+#define SXE2VF_DYN_CTL_ITR_IDX_SHIFT 3
+
+enum sxe2vf_itr_idx {
+ SXE2VF_ITR_IDX_0 = 0,
+ SXE2VF_ITR_IDX_1,
+ SXE2VF_ITR_IDX_2,
+ SXE2VF_ITR_IDX_NONE,
+};
+
+#define SXE2VF_INT_ITR0 (0x00002800 + 65 * 0x4)
+#define SXE2VF_INT_ITR(_i, _irq_idx) (SXE2VF_INT_ITR0 + \
+ 0x4 + (_i) * 0x104 + ((_irq_idx) * 4))
+#define SXE2VF_VF_INT_ITR(_itr_idx, _irq_idx) (0x00002800 + \
+ (0x104 * (_itr_idx)) + ((_irq_idx) * 4))
+#define SXE2VF_PFG_INT_CTL_ITR_GRAN_0 (2)
+
+#define SXE2VF_PCIE_SYS_READY 0x38c
+#define SXE2VF_PCIE_SYS_READY_CORER_ASSERT RTE_BIT32(0)
+#define SXE2VF_PCIE_SYS_READY_STOP_DROP_DONE RTE_BIT32(2)
+#define SXE2VF_PCIE_SYS_READY_R5 RTE_BIT32(3)
+#define SXE2VF_PCIE_SYS_READY_STOP_DROP RTE_BIT32(16)
+
+#define SXE2VF_PCIE_DEV_CTRL_DEV_STATUS 0x78
+#define SXE2VF_PCIE_DEV_CTRL_DEV_STATUS_TRANS_PENDING RTE_BIT32(21)
+
+#define SXE2VF_VF_VRC_VFGEN_RSTAT (0x5800)
+#define SXE2VF_VF_VRC_VFGEN_VFRSTAT GENMASK(1, 0)
+#define SXE2VF_VF_VRC_VFGEN_VFRSTAT_VFR (0)
+#define SXE2VF_VF_VRC_VFGEN_VFRSTAT_COMPLETE (RTE_BIT32(0))
+#define SXE2VF_VF_VRC_VFGEN_VFRSTAT_VF_ACTIVE (RTE_BIT32(1))
+
+#define SXE2VF_VF_VRC_VFGEN_VFRSTAT_FORVF_VFR (1)
+#define SXE2VF_VF_VRC_VFGEN_VFRSTAT_FORVF_MASK \
+ (RTE_BIT32(10))
+
+#endif /* SXE2VF_VF_INT_ITR */
--
2.47.3
^ permalink raw reply related
* [PATCH v2 08/23] net/sxe2: support statistics and multi-process
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
- The statistics support includes:
- Basic statistics (ipackets, opackets, ibytes, obytes, etc.)
- Extended statistics (xstats) for detailed hardware counters.
- Per-queue statistics for both RX and TX.
The multi-process support allows secondary processes to retrieve
statistics. Since secondary processes cannot access hardware registers
directly, an IPC mechanism is implemented using the DPDK MP API.
Atomic operations are used when reading 64-bit counters to ensure
data consistency between processes.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 2 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 340 +++++++++++++++++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 23 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 120 +++++++
drivers/net/sxe2/sxe2_ethdev.c | 38 +-
drivers/net/sxe2/sxe2_mp.c | 413 ++++++++++++++++++++++
drivers/net/sxe2/sxe2_mp.h | 73 ++++
drivers/net/sxe2/sxe2_stats.c | 586 +++++++++++++++++++++++++++++++
drivers/net/sxe2/sxe2_stats.h | 39 ++
9 files changed, 1621 insertions(+), 13 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_mp.c
create mode 100644 drivers/net/sxe2/sxe2_mp.h
create mode 100644 drivers/net/sxe2/sxe2_stats.c
create mode 100644 drivers/net/sxe2/sxe2_stats.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index e3bcfc2876..d860629def 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -66,6 +66,8 @@ sources += files(
'sxe2_tm.c',
'sxe2_ipsec.c',
'sxe2_security.c',
+ 'sxe2_mp.c',
+ 'sxe2_stats.c',
)
allow_internal_get_api = true
\ No newline at end of file
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index 7711e8e57d..a1fc8a50e3 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -348,6 +348,184 @@ int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
return ret;
}
+int32_t sxe2_drv_get_mac_stats(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ uint8_t i = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_stats *stats = &vsi->vsi_stats.vsi_hw_stats;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_mac_stats_resp resp = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_STATS_GET,
+ NULL, 0,
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "vsi stats get failed, ret=%d", ret);
+ goto l_end;
+ }
+
+ stats->rx_out_of_buffer = rte_le_to_cpu_64(resp.rx_out_of_buffer);
+ stats->rx_qblock_drop = rte_le_to_cpu_64(resp.rx_qblock_drop);
+ stats->tx_frame_good = rte_le_to_cpu_64(resp.tx_frame_good);
+ stats->rx_frame_good = rte_le_to_cpu_64(resp.rx_frame_good);
+ stats->rx_crc_errors = rte_le_to_cpu_64(resp.rx_crc_errors);
+ stats->tx_bytes_good = rte_le_to_cpu_64(resp.tx_bytes_good);
+ stats->rx_bytes_good = rte_le_to_cpu_64(resp.rx_bytes_good);
+ stats->tx_multicast_good = rte_le_to_cpu_64(resp.tx_multicast_good);
+ stats->tx_broadcast_good = rte_le_to_cpu_64(resp.tx_broadcast_good);
+ stats->rx_multicast_good = rte_le_to_cpu_64(resp.rx_multicast_good);
+ stats->rx_broadcast_good = rte_le_to_cpu_64(resp.rx_broadcast_good);
+ stats->rx_len_errors = rte_le_to_cpu_64(resp.rx_len_errors);
+ stats->rx_out_of_range_errors = rte_le_to_cpu_64(resp.rx_out_of_range_errors);
+ stats->rx_oversize_pkts_phy = rte_le_to_cpu_64(resp.rx_oversize_pkts_phy);
+ stats->rx_symbol_err = rte_le_to_cpu_64(resp.rx_symbol_err);
+ stats->rx_pause_frame = rte_le_to_cpu_64(resp.rx_pause_frame);
+ stats->tx_pause_frame = rte_le_to_cpu_64(resp.tx_pause_frame);
+ stats->rx_discards_phy = rte_le_to_cpu_64(resp.rx_discards_phy);
+ stats->rx_discards_ips_phy = rte_le_to_cpu_64(resp.rx_discards_ips_phy);
+ stats->tx_dropped_link_down = rte_le_to_cpu_64(resp.tx_dropped_link_down);
+ stats->rx_undersize_good = rte_le_to_cpu_64(resp.rx_undersize_good);
+ stats->rx_runt_error = rte_le_to_cpu_64(resp.rx_runt_error);
+ stats->tx_bytes_good_bad = rte_le_to_cpu_64(resp.tx_bytes_good_bad);
+ stats->tx_frame_good_bad = rte_le_to_cpu_64(resp.tx_frame_good_bad);
+ stats->rx_jabbers = rte_le_to_cpu_64(resp.rx_jabbers);
+ stats->rx_size_64 = rte_le_to_cpu_64(resp.rx_size_64);
+ stats->rx_size_65_127 = rte_le_to_cpu_64(resp.rx_size_65_127);
+ stats->rx_size_128_255 = rte_le_to_cpu_64(resp.rx_size_128_255);
+ stats->rx_size_256_511 = rte_le_to_cpu_64(resp.rx_size_256_511);
+ stats->rx_size_512_1023 = rte_le_to_cpu_64(resp.rx_size_512_1023);
+ stats->rx_size_1024_1522 = rte_le_to_cpu_64(resp.rx_size_1024_1522);
+ stats->rx_size_1523_max = rte_le_to_cpu_64(resp.rx_size_1523_max);
+ stats->rx_pcs_symbol_err_phy = rte_le_to_cpu_64(resp.rx_pcs_symbol_err_phy);
+ stats->rx_corrected_bits_phy = rte_le_to_cpu_64(resp.rx_corrected_bits_phy);
+ stats->rx_err_lane_0_phy = rte_le_to_cpu_64(resp.rx_err_lane_0_phy);
+ stats->rx_err_lane_1_phy = rte_le_to_cpu_64(resp.rx_err_lane_1_phy);
+ stats->rx_err_lane_2_phy = rte_le_to_cpu_64(resp.rx_err_lane_2_phy);
+ stats->rx_err_lane_3_phy = rte_le_to_cpu_64(resp.rx_err_lane_3_phy);
+ stats->rx_illegal_bytes = rte_le_to_cpu_64(resp.rx_illegal_bytes);
+ stats->rx_oversize_good = rte_le_to_cpu_64(resp.rx_oversize_good);
+ stats->tx_unicast = rte_le_to_cpu_64(resp.tx_unicast);
+ stats->tx_broadcast = rte_le_to_cpu_64(resp.tx_broadcast);
+ stats->tx_multicast = rte_le_to_cpu_64(resp.tx_multicast);
+ stats->tx_vlan_packet_good = rte_le_to_cpu_64(resp.tx_vlan_packet_good);
+ stats->tx_size_64 = rte_le_to_cpu_64(resp.tx_size_64);
+ stats->tx_size_65_127 = rte_le_to_cpu_64(resp.tx_size_65_127);
+ stats->tx_size_128_255 = rte_le_to_cpu_64(resp.tx_size_128_255);
+ stats->tx_size_256_511 = rte_le_to_cpu_64(resp.tx_size_256_511);
+ stats->tx_size_512_1023 = rte_le_to_cpu_64(resp.tx_size_512_1023);
+ stats->tx_size_1024_1522 = rte_le_to_cpu_64(resp.tx_size_1024_1522);
+ stats->tx_size_1523_max = rte_le_to_cpu_64(resp.tx_size_1523_max);
+ stats->tx_underflow_error = rte_le_to_cpu_64(resp.tx_underflow_error);
+ stats->rx_byte_good_bad = rte_le_to_cpu_64(resp.rx_byte_good_bad);
+ stats->rx_frame_good_bad = rte_le_to_cpu_64(resp.rx_frame_good_bad);
+ stats->rx_unicast_good = rte_le_to_cpu_64(resp.rx_unicast_good);
+ stats->rx_vlan_packets = rte_le_to_cpu_64(resp.rx_vlan_packets);
+
+ for (i = 0; i < SXE2_MAX_USER_PRIORITY; i++) {
+ stats->rx_prio_buf_discard[i] =
+ rte_le_to_cpu_64(resp.rx_prio_buf_discard[i]);
+ stats->prio_xoff_rx[i] =
+ rte_le_to_cpu_64(resp.prio_xoff_rx[i]);
+ stats->prio_xoff_tx[i] =
+ rte_le_to_cpu_64(resp.prio_xoff_tx[i]);
+ stats->prio_xon_rx[i] =
+ rte_le_to_cpu_64(resp.prio_xon_rx[i]);
+ stats->prio_xon_tx[i] =
+ rte_le_to_cpu_64(resp.prio_xon_tx[i]);
+ stats->prio_xon_2_xoff[i] =
+ rte_le_to_cpu_64(resp.prio_xon_2_xoff[i]);
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_mac_stats_reset(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_STATS_CLEAR,
+ NULL, 0,
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "mac stats reset failed, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_get_vsi_stats(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_stats *new_stats = &vsi->vsi_stats.vsi_hw_stats;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_vsi_stats_req req = {0};
+ struct sxe2_drv_vsi_stats_resp resp = {0};
+
+ req.vsi_id = rte_cpu_to_le_16(vsi->vsi_id);
+ req.sw_stats.rx_bytes = rte_cpu_to_le_64(vsi->vsi_stats.stats.ibytes);
+ req.sw_stats.rx_packets = rte_cpu_to_le_64(vsi->vsi_stats.stats.ipackets);
+ req.sw_stats.tx_bytes = rte_cpu_to_le_64(vsi->vsi_stats.stats.obytes);
+ req.sw_stats.tx_packets = rte_cpu_to_le_64(vsi->vsi_stats.stats.opackets);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VSI_STATS_GET,
+ &req, sizeof(req),
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "vsi stats get failed, ret=%d", ret);
+ goto l_end;
+ }
+
+ new_stats->rx_vsi_unicast_packets = rte_le_to_cpu_64(resp.rx_vsi_unicast_packets);
+ new_stats->rx_vsi_bytes = rte_le_to_cpu_64(resp.rx_vsi_bytes);
+ new_stats->tx_vsi_unicast_packets = rte_le_to_cpu_64(resp.tx_vsi_unicast_packets);
+ new_stats->tx_vsi_bytes = rte_le_to_cpu_64(resp.tx_vsi_bytes);
+ new_stats->rx_vsi_multicast_packets = rte_le_to_cpu_64(resp.rx_vsi_multicast_packets);
+ new_stats->tx_vsi_multicast_packets = rte_le_to_cpu_64(resp.tx_vsi_multicast_packets);
+ new_stats->rx_vsi_broadcast_packets = rte_le_to_cpu_64(resp.rx_vsi_broadcast_packets);
+ new_stats->tx_vsi_broadcast_packets = rte_le_to_cpu_64(resp.tx_vsi_broadcast_packets);
+ new_stats->opackets = new_stats->tx_vsi_unicast_packets +
+ new_stats->tx_vsi_multicast_packets +
+ new_stats->tx_vsi_broadcast_packets;
+ new_stats->obytes = new_stats->tx_vsi_bytes;
+ new_stats->ipackets = new_stats->rx_vsi_unicast_packets +
+ new_stats->rx_vsi_multicast_packets +
+ new_stats->rx_vsi_broadcast_packets;
+ new_stats->ibytes = new_stats->rx_vsi_bytes;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_vsi_stats_reset(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VSI_STATS_CLEAR,
+ NULL, 0, NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "vsi stats reset failed, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set)
{
int32_t ret = 0;
@@ -409,8 +587,9 @@ int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *
mac_filter_cfg_req.type = SXE2_MAC_FILTER_TYPE_UC;
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_ADDR_UC,
- &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
- NULL, 0);
+ &mac_filter_cfg_req,
+ sizeof(mac_filter_cfg_req),
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret)
@@ -435,8 +614,8 @@ int32_t sxe2_drv_mc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *
mac_filter_cfg_req.type = SXE2_MAC_FILTER_TYPE_MC;
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_ADDR_MC,
- &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
- NULL, 0);
+ &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret)
@@ -455,7 +634,7 @@ int32_t sxe2_drv_vlan_config_query(struct sxe2_adapter *adapter)
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_CFG_QUERY,
NULL, 0,
&vlan_cfg_query_resp,
- sizeof(vlan_cfg_query_resp));
+ sizeof(vlan_cfg_query_resp));
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret)
@@ -566,7 +745,8 @@ int32_t sxe2_drv_rss_key_set(struct sxe2_adapter *adapter, uint8_t *key, uint16_
rte_memcpy(req->key, key, key_size);
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_KEY_SET,
- req, buf_size, NULL, 0);
+ req, buf_size,
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret) {
@@ -602,7 +782,8 @@ int32_t sxe2_drv_rss_lut_set(struct sxe2_adapter *adapter, uint8_t *lut, uint16_
rte_memcpy(req->lut, lut, lut_size);
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_LUT_SET,
- req, buf_size, NULL, 0);
+ req, buf_size,
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret) {
@@ -629,7 +810,8 @@ int32_t sxe2_drv_rss_hash_ctrl_func(struct sxe2_adapter *adapter, enum sxe2_rss_
req.func = func;
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_FUNC_SET,
- &req, sizeof(req), NULL, 0);
+ &req, sizeof(req),
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret)
@@ -665,7 +847,8 @@ int32_t sxe2_drv_rss_hf_add(struct sxe2_adapter *adapter,
sxe2_drv_flow_bitmap_fill(req.hash_flds, rss_conf->flds);
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_ADD,
- &req, sizeof(req), NULL, 0);
+ &req, sizeof(req),
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret)
@@ -703,7 +886,8 @@ int32_t sxe2_drv_rss_hf_clear(struct sxe2_adapter *adapter)
int32_t ret = 0;
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_CLEAR,
- NULL, 0, NULL, 0);
+ NULL, 0,
+ NULL, 0);
ret = sxe2_drv_cmd_exec(cdev, ¶m);
if (ret)
@@ -878,7 +1062,6 @@ int32_t sxe2_drv_tm_commit(struct sxe2_adapter *adapter)
return ret;
}
-
int32_t sxe2_drv_ipsec_get_capa(struct sxe2_adapter *adapter)
{
int32_t ret = -1;
@@ -1074,3 +1257,138 @@ int32_t sxe2_drv_ipsec_txsa_delete(struct sxe2_adapter *adapter,
return ret;
}
+int32_t sxe2_drv_queue_info_get_update(struct sxe2_adapter *adapter, struct eth_queue_stats *qstats)
+{
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_queue_map_info resp = {0};
+ struct sxe2_common_device *cdev = adapter->cdev;
+ uint8_t pool_idx;
+ uint8_t index;
+ int32_t ret;
+
+ if (!(adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_Q_MAP)) {
+ ret = 0;
+ goto l_end;
+ }
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_TX_RX_MAP_GET,
+ NULL, 0,
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "get queue info map failed, ret=%d", ret);
+ goto l_end;
+ }
+
+ for (pool_idx = 0; pool_idx < SXE2_RXQ_STATS_MAP_MAX_NUM; pool_idx++) {
+ qstats->q_ipackets[pool_idx] = resp.rxq_stats_map_info[pool_idx].rxq_lan_in_pkt_cnt;
+ qstats->q_ibytes[pool_idx] = resp.rxq_stats_map_info[pool_idx].rxq_lan_in_byte_cnt;
+ }
+
+ for (index = 0; index < SXE2_TXQ_STATS_MAP_MAX_NUM; index++) {
+ qstats->q_opackets[index] = resp.txq_stats_map_info[index].txq_lan_pkt_cnt;
+ qstats->q_obytes[index] = resp.txq_stats_map_info[index].txq_lan_byte_cnt;
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_rxq_mapping_set(struct rte_eth_dev *eth_dev, uint16_t queue_id, uint8_t pool_idx)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(eth_dev);
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_rx_map_req req = {0};
+ struct sxe2_rx_queue *rxq = NULL;
+
+ rxq = eth_dev->data->rx_queues[queue_id];
+ if (rxq == NULL) {
+ PMD_LOG_ERR(DRV, "Rx queue %u is not available or setup",
+ queue_id);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ req.queue_id = rxq->queue_id;
+ req.pool_idx = pool_idx;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RX_MAP_SET,
+ &req, sizeof(req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_LOG_ERR(DRV, "get dev caps failed, ret=%d", ret);
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_txq_mapping_set(struct rte_eth_dev *eth_dev, uint16_t queue_id, uint8_t pool_idx)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(eth_dev);
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_tx_map_req req = {0};
+ struct sxe2_tx_queue *txq = NULL;
+
+ txq = eth_dev->data->tx_queues[queue_id];
+ if (txq == NULL) {
+ PMD_LOG_ERR(DRV, "Rx queue %u is not available or setup", queue_id);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ req.queue_id = txq->queue_id;
+ req.pool_idx = pool_idx;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_TX_MAP_SET,
+ &req, sizeof(req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_LOG_ERR(DRV, "get dev caps failed, ret=%d", ret);
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_mapping_reset(struct rte_eth_dev *eth_dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(eth_dev);
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_TX_RX_MAP_RESET,
+ NULL, 0,
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Reset queue mapping failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_mapping_stats_info_clear(struct rte_eth_dev *eth_dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(eth_dev);
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_TX_RX_MAP_INFO_CLEAR,
+ NULL, 0,
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_LOG_ERR(DRV, "Clear map stats info failed, ret=%d", ret);
+
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index d8e09a4453..ff73d2f901 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -64,6 +64,29 @@ int32_t sxe2_drv_ipsec_txsa_delete(struct sxe2_adapter *adapter,
int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_get_mac_stats(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_mac_stats_reset(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_get_vsi_stats(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_vsi_stats_reset(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_queue_info_get_update(struct sxe2_adapter *adapter,
+ struct eth_queue_stats *qstats);
+
+int32_t sxe2_drv_rxq_mapping_set(struct rte_eth_dev *eth_dev, uint16_t queue_id, uint8_t pool_idx);
+
+int32_t sxe2_drv_txq_mapping_set(struct rte_eth_dev *eth_dev, uint16_t queue_id, uint8_t pool_idx);
+
+int32_t sxe2_drv_mapping_reset(struct rte_eth_dev *eth_dev);
+
+int32_t sxe2_drv_mapping_stats_info_clear(struct rte_eth_dev *eth_dev);
+
+int32_t sxe2_drv_rxq_mapping_set(struct rte_eth_dev *eth_dev, uint16_t queue_id, uint8_t pool_idx);
+
int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set);
int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index fac272a91e..fe1ca9160a 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -436,6 +436,126 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_ipsec_rxsa_del_req {
uint8_t drv_id;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_sw_stats {
+ uint64_t rx_packets;
+ uint64_t rx_bytes;
+ uint64_t tx_packets;
+ uint64_t tx_bytes;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_stats_req {
+ uint16_t vsi_id;
+ uint8_t rsv[2];
+ struct sxe2_drv_vsi_sw_stats sw_stats;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_stats_resp {
+ uint64_t rx_vsi_unicast_packets;
+ uint64_t rx_vsi_bytes;
+ uint64_t tx_vsi_unicast_packets;
+ uint64_t tx_vsi_bytes;
+ uint64_t rx_vsi_multicast_packets;
+ uint64_t tx_vsi_multicast_packets;
+ uint64_t rx_vsi_broadcast_packets;
+ uint64_t tx_vsi_broadcast_packets;
+} __rte_packed_end;
+
+#define SXE2_MAX_USER_PRIORITY (8)
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_mac_stats_resp {
+ uint64_t rx_out_of_buffer;
+ uint64_t rx_qblock_drop;
+ uint64_t tx_frame_good;
+ uint64_t rx_frame_good;
+ uint64_t rx_crc_errors;
+ uint64_t tx_bytes_good;
+ uint64_t rx_bytes_good;
+ uint64_t tx_multicast_good;
+ uint64_t tx_broadcast_good;
+ uint64_t rx_multicast_good;
+ uint64_t rx_broadcast_good;
+ uint64_t rx_len_errors;
+ uint64_t rx_out_of_range_errors;
+ uint64_t rx_oversize_pkts_phy;
+ uint64_t rx_symbol_err;
+ uint64_t rx_pause_frame;
+ uint64_t tx_pause_frame;
+ uint64_t rx_discards_phy;
+ uint64_t rx_discards_ips_phy;
+ uint64_t tx_dropped_link_down;
+ uint64_t rx_undersize_good;
+ uint64_t rx_runt_error;
+ uint64_t tx_bytes_good_bad;
+ uint64_t tx_frame_good_bad;
+ uint64_t rx_jabbers;
+ uint64_t rx_size_64;
+ uint64_t rx_size_65_127;
+ uint64_t rx_size_128_255;
+ uint64_t rx_size_256_511;
+ uint64_t rx_size_512_1023;
+ uint64_t rx_size_1024_1522;
+ uint64_t rx_size_1523_max;
+ uint64_t rx_pcs_symbol_err_phy;
+ uint64_t rx_corrected_bits_phy;
+ uint64_t rx_err_lane_0_phy;
+ uint64_t rx_err_lane_1_phy;
+ uint64_t rx_err_lane_2_phy;
+ uint64_t rx_err_lane_3_phy;
+ uint64_t rx_prio_buf_discard[SXE2_MAX_USER_PRIORITY];
+ uint64_t rx_illegal_bytes;
+ uint64_t rx_oversize_good;
+ uint64_t tx_unicast;
+ uint64_t tx_broadcast;
+ uint64_t tx_multicast;
+ uint64_t tx_vlan_packet_good;
+ uint64_t tx_size_64;
+ uint64_t tx_size_65_127;
+ uint64_t tx_size_128_255;
+ uint64_t tx_size_256_511;
+ uint64_t tx_size_512_1023;
+ uint64_t tx_size_1024_1522;
+ uint64_t tx_size_1523_max;
+ uint64_t tx_underflow_error;
+ uint64_t rx_byte_good_bad;
+ uint64_t rx_frame_good_bad;
+ uint64_t rx_unicast_good;
+ uint64_t rx_vlan_packets;
+ uint64_t prio_xoff_rx[SXE2_MAX_USER_PRIORITY];
+ uint64_t prio_xon_rx[SXE2_MAX_USER_PRIORITY];
+ uint64_t prio_xon_tx[SXE2_MAX_USER_PRIORITY];
+ uint64_t prio_xoff_tx[SXE2_MAX_USER_PRIORITY];
+ uint64_t prio_xon_2_xoff[SXE2_MAX_USER_PRIORITY];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_txq_map_info {
+ uint32_t txq_lan_pkt_cnt;
+ uint32_t txq_lan_byte_cnt;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rxq_map_info {
+ uint64_t rxq_lan_in_pkt_cnt;
+ uint64_t rxq_lan_in_byte_cnt;
+ uint64_t rxq_fd_in_pkt_cnt;
+ uint64_t rxq_mng_in_pkt_cnt;
+ uint64_t rxq_mng_in_byte_cnt;
+ uint64_t rxq_mng_out_pkt_cnt;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_queue_map_info {
+ struct sxe2_rxq_map_info rxq_stats_map_info[SXE2_RXQ_STATS_MAP_MAX_NUM];
+ struct sxe2_txq_map_info txq_stats_map_info[SXE2_TXQ_STATS_MAP_MAX_NUM];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rx_map_req {
+ uint16_t queue_id;
+ uint8_t pool_idx;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_tx_map_req {
+ uint16_t queue_id;
+ uint8_t pool_idx;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 00c0552d4a..dbe1a2bce1 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -31,6 +31,8 @@
#include "sxe2_common.h"
#include "sxe2_ptype.h"
#include "sxe2_common_log.h"
+#include "sxe2_mp.h"
+#include "sxe2_stats.h"
#include "sxe2_host_regs.h"
#include "sxe2_ioctl_chnl_func.h"
@@ -132,6 +134,14 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
.tm_ops_get = sxe2_tm_ops_get,
+
+ .stats_get = sxe2_stats_info_get,
+ .stats_reset = sxe2_stats_info_reset,
+ .xstats_get = sxe2_xstats_info_get,
+ .xstats_get_names = sxe2_xstats_names_get,
+ .xstats_reset = sxe2_stats_info_reset,
+
+ .queue_stats_mapping_set = sxe2_queue_stats_mapping_set,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -1023,6 +1033,9 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
sxe2_rx_mode_func_set(dev);
sxe2_tx_mode_func_set(dev);
+ ret = sxe2_mp_init(dev);
+ if (ret != 0)
+ PMD_LOG_ERR(INIT, "Failed to mp init (secondary), ret=%d", ret);
goto l_end;
}
@@ -1076,12 +1089,27 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_sched_err;
}
+ ret = sxe2_stats_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to stats init, ret=%d", ret);
+ goto init_xstats_err;
+ }
+
+ ret = sxe2_mp_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to mp init, ret=%d", ret);
+ goto init_xstats_err;
+ }
+
goto l_end;
-init_security_err:
- sxe2_eth_uinit(dev);
+init_xstats_err:
+ (void)sxe2_sched_uinit(dev);
init_sched_err:
init_rss_err:
+ sxe2_security_uinit(dev);
+init_security_err:
+ sxe2_eth_uinit(dev);
init_eth_err:
init_dev_info_err:
sxe2_vsi_uninit(dev);
@@ -1093,8 +1121,13 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
{
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ sxe2_mp_uninit(dev);
+ goto l_end;
+ }
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
+ sxe2_mp_uninit(dev);
(void)sxe2_rss_disable(dev);
(void)sxe2_sched_uinit(dev);
sxe2_vsi_uninit(dev);
@@ -1102,6 +1135,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
sxe2_dev_pci_map_uinit(dev);
sxe2_eth_uinit(dev);
+l_end:
return 0;
}
diff --git a/drivers/net/sxe2/sxe2_mp.c b/drivers/net/sxe2/sxe2_mp.c
new file mode 100644
index 0000000000..caf044dd24
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_mp.c
@@ -0,0 +1,413 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_eal.h>
+#include <rte_ethdev.h>
+#include <ethdev_driver.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+#include <rte_string_fns.h>
+
+#include "sxe2_mp.h"
+#include "sxe2_stats.h"
+#include "sxe2_common_log.h"
+
+static RTE_ATOMIC(uint16_t)primary_ethdev_cnt;
+static RTE_ATOMIC(uint16_t)secondary_ethdev_cnt;
+static const struct rte_memzone *sxe2_mp_mz;
+
+static int32_t sxe2_mp_acquire_token(void);
+static void sxe2_mp_release_token(void);
+
+static int32_t sxe2_mp_primary_handle(const struct rte_mp_msg *mp_msg,
+ const void *peer);
+
+static int32_t sxe2_mp_secondary_handle(const struct rte_mp_msg *mp_msg,
+ const void *peer);
+
+static int32_t
+sxe2_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
+{
+ struct rte_mp_msg reply;
+ const struct sxe2_mp_param *param =
+ (const struct sxe2_mp_param *)mp_msg->param;
+ struct sxe2_mp_param *reply_param = (struct sxe2_mp_param *)reply.param;
+ struct rte_eth_dev *dev;
+ int32_t ret = 0;
+ struct sxe2_mp_shared_data *mz_data;
+ int32_t send_reply = 0;
+ int32_t cnt = 0;
+
+ if (!rte_eth_dev_is_valid_port(param->port_id)) {
+ PMD_LOG_ERR(DRV, "primary process: invalid port_id %u",
+ param->port_id);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ dev = &rte_eth_devices[param->port_id];
+ sxe2_mp_mz = rte_memzone_lookup(SXE2_MP_MZ_NAME);
+ if (sxe2_mp_mz == NULL) {
+ PMD_LOG_ERR(DRV, "Failed to lookup memzone %s", SXE2_MP_MZ_NAME);
+ ret = -ENOENT;
+ goto out;
+ }
+
+ mz_data = (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+ send_reply = 1;
+
+ memset(&reply, 0, sizeof(reply));
+ (void)strlcpy(reply.name, SXE2_MP_NAME, sizeof(reply.name));
+ reply.len_param = sizeof(*reply_param);
+
+ switch (param->type) {
+ case SXE2_MP_REQ_GET_STATS:
+ ret = sxe2_stats_info_get(dev,
+ &mz_data->payload.stats_blk.stats,
+ &mz_data->payload.stats_blk.qstats);
+ break;
+ case SXE2_MP_REQ_GET_XSTATS:
+ cnt = sxe2_xstats_info_get(dev,
+ mz_data->payload.xstats_blk.xstats,
+ SXE2_MP_MAX_XSTATS);
+
+ if (cnt >= 0) {
+ mz_data->payload.xstats_blk.xstats_num = (uint32_t)cnt;
+ ret = 0;
+ } else {
+ mz_data->payload.xstats_blk.xstats_num = 0;
+ ret = cnt;
+ }
+ break;
+ case SXE2_MP_REQ_RESET_STATS:
+ ret = sxe2_stats_hw_reset(dev);
+ break;
+ default:
+ PMD_LOG_ERR(DRV, "primary process: unrecognized msg type: %d",
+ param->type);
+ send_reply = false;
+ ret = -EINVAL;
+ goto out;
+ }
+out:
+ if (!send_reply)
+ return ret;
+
+ reply_param->result = ret;
+ reply_param->type = param->type;
+ reply_param->port_id = param->port_id;
+
+ return rte_mp_reply(&reply, peer);
+}
+
+static int32_t
+sxe2_mp_secondary_handle(const struct rte_mp_msg *mp_msg __rte_unused,
+ const void *peer __rte_unused)
+{
+ PMD_LOG_WARN(DRV, "the secondary process handler should not be called");
+ return 0;
+}
+
+static int32_t
+sxe2_mp_init_primary(__rte_unused struct rte_eth_dev *dev)
+{
+ int32_t ret;
+
+ if (sxe2_mp_mz == NULL) {
+ sxe2_mp_mz = rte_memzone_reserve(SXE2_MP_MZ_NAME,
+ sizeof(struct sxe2_mp_shared_data),
+ rte_socket_id(), 0);
+ if (sxe2_mp_mz == NULL && rte_errno != EEXIST) {
+ PMD_LOG_ERR(DRV, "Failed to reserve memzone %s, error: %d",
+ SXE2_MP_MZ_NAME, -rte_errno);
+ ret = -rte_errno;
+ goto out;
+ }
+
+ sxe2_mp_mz = rte_memzone_lookup(SXE2_MP_MZ_NAME);
+ if (sxe2_mp_mz == NULL) {
+ PMD_LOG_ERR(DRV, "Failed to lookup memzone %s", SXE2_MP_MZ_NAME);
+ ret = -ENOENT;
+ goto out;
+ }
+
+ struct sxe2_mp_shared_data *mz =
+ (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+ rte_atomic_store_explicit(&mz->in_use, 0, rte_memory_order_release);
+ }
+
+ ret = rte_mp_action_register(SXE2_MP_NAME, sxe2_mp_primary_handle);
+ if (ret && rte_errno == ENOTSUP) {
+ PMD_LOG_INFO(DRV, "Primary not support IPC.");
+ ret = 0;
+ goto out;
+ } else if (ret && rte_errno != EEXIST) {
+ PMD_LOG_ERR(DRV, "Failed to register MP primary handle, error: %d",
+ -rte_errno);
+ goto out;
+ }
+
+ rte_atomic_fetch_add_explicit(&primary_ethdev_cnt, 1, rte_memory_order_relaxed);
+
+ ret = 0;
+out:
+ return ret;
+}
+
+static int32_t
+sxe2_mp_init_secondary(__rte_unused struct rte_eth_dev *dev)
+{
+ int32_t ret;
+
+ sxe2_mp_mz = rte_memzone_lookup(SXE2_MP_MZ_NAME);
+ if (sxe2_mp_mz == NULL) {
+ PMD_LOG_ERR(DRV, "Failed to lookup memzone %s", SXE2_MP_MZ_NAME);
+ ret = -ENOENT;
+ goto out;
+ }
+
+ ret = rte_mp_action_register(SXE2_MP_NAME, sxe2_mp_secondary_handle);
+ if (ret && rte_errno != EEXIST) {
+ PMD_LOG_ERR(DRV, "Failed to register MP secondary handle, error: %d",
+ -rte_errno);
+ goto out;
+ }
+
+ rte_atomic_fetch_add_explicit(&secondary_ethdev_cnt, 1, rte_memory_order_relaxed);
+
+ ret = 0;
+out:
+ return ret;
+}
+
+int32_t
+sxe2_mp_init(struct rte_eth_dev *dev)
+{
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ return sxe2_mp_init_primary(dev);
+ else
+ return sxe2_mp_init_secondary(dev);
+}
+
+void
+sxe2_mp_uninit(__rte_unused struct rte_eth_dev *dev)
+{
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ if (rte_atomic_fetch_sub_explicit(&primary_ethdev_cnt, 1,
+ rte_memory_order_acq_rel) == 1) {
+ rte_mp_action_unregister(SXE2_MP_NAME);
+ if (sxe2_mp_mz != NULL) {
+ rte_memzone_free(sxe2_mp_mz);
+ sxe2_mp_mz = NULL;
+ }
+ }
+ } else {
+ if (rte_atomic_fetch_sub_explicit(&secondary_ethdev_cnt, 1,
+ rte_memory_order_acq_rel) == 1)
+ rte_mp_action_unregister(SXE2_MP_NAME);
+ }
+}
+
+static int32_t sxe2_mp_acquire_token(void)
+{
+ struct sxe2_mp_shared_data *mz;
+ uint16_t expected;
+
+ if (sxe2_mp_mz == NULL)
+ return -EINVAL;
+
+ mz = (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+
+ for (int32_t i = 0; i < SXE2_MP_MAX_SPIN; i++) {
+ expected = 0;
+ if (rte_atomic_compare_exchange_strong_explicit(&mz->in_use, &expected, 1,
+ rte_memory_order_acquire, rte_memory_order_relaxed))
+ return 0;
+
+ rte_pause();
+ }
+ return -EBUSY;
+}
+
+static void sxe2_mp_release_token(void)
+{
+ struct sxe2_mp_shared_data *mz;
+
+ if (sxe2_mp_mz == NULL)
+ return;
+
+ mz = (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+ rte_atomic_store_explicit(&mz->in_use, 0, rte_memory_order_release);
+}
+
+int32_t sxe2_mp_request_simple(struct rte_eth_dev *dev,
+ enum sxe2_mp_req_type type,
+ int32_t *result_out)
+{
+ struct rte_mp_msg msg;
+ struct rte_mp_reply reply = { 0 };
+ struct timespec ts = { .tv_sec = SXE2_MP_MSG_TIMEOUT, .tv_nsec = 0 };
+ struct sxe2_mp_param *param = (struct sxe2_mp_param *)msg.param;
+ struct sxe2_mp_shared_data *mz_data;
+ int32_t ret = 0;
+
+ mz_data = (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+
+ memset(&mz_data->payload, 0, sizeof(mz_data->payload));
+ memset(&msg, 0, sizeof(msg));
+ (void)strlcpy(msg.name, SXE2_MP_NAME, sizeof(msg.name));
+ msg.len_param = sizeof(*param);
+ param->type = type;
+ param->port_id = dev->data->port_id;
+
+ ret = rte_mp_request_sync(&msg, &reply, &ts);
+ if (ret != 0) {
+ PMD_LOG_ERR(DRV,
+ "IPC request(type=%d) failed for port %u: %s",
+ type, dev->data->port_id, rte_strerror(rte_errno));
+ ret = -rte_errno;
+ goto out;
+ }
+
+ if (reply.nb_received == 0) {
+ PMD_LOG_ERR(DRV, "No response received from primary for type=%d, port %u",
+ type, dev->data->port_id);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ *result_out = ((struct sxe2_mp_param *)reply.msgs[0].param)->result;
+
+out:
+ if (reply.msgs != NULL)
+ free(reply.msgs);
+
+ return ret;
+}
+
+int32_t sxe2_mp_req_get_stats(struct rte_eth_dev *dev,
+ struct rte_eth_stats *stats,
+ struct eth_queue_stats *qstats)
+{
+ struct sxe2_mp_shared_data *mz_data;
+ int32_t mp_ret;
+ int32_t ret;
+ int32_t token_acquired = 0;
+
+ if (sxe2_mp_mz == NULL)
+ return -EINVAL;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ PMD_LOG_WARN(DRV, "Primary process direct execution for port %u",
+ dev->data->port_id);
+ return sxe2_stats_info_get(dev, stats, qstats);
+ }
+
+ int32_t token_ret = sxe2_mp_acquire_token();
+ if (token_ret != 0)
+ return token_ret;
+ token_acquired = 1;
+
+ mp_ret = sxe2_mp_request_simple(dev, SXE2_MP_REQ_GET_STATS, &ret);
+ if (mp_ret != 0) {
+ ret = mp_ret;
+ goto out;
+ }
+
+ if (ret != 0) {
+ PMD_LOG_ERR(DRV, "Primary failed to exec request (type=%d), result: %d for port %u",
+ SXE2_MP_REQ_GET_STATS, ret, dev->data->port_id);
+ goto out;
+ }
+
+ mz_data = (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+ memcpy(stats, &mz_data->payload.stats_blk.stats, sizeof(*stats));
+ memcpy(qstats, &mz_data->payload.stats_blk.qstats, sizeof(*qstats));
+ PMD_LOG_DEBUG(DRV, "sxe2_mp: stats received via IPC for port %u",
+ dev->data->port_id);
+ ret = 0;
+out:
+ if (token_acquired)
+ sxe2_mp_release_token();
+ return ret;
+}
+
+int32_t sxe2_mp_req_get_xstats(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *xstats, uint32_t usr_cnt)
+{
+ struct sxe2_mp_shared_data *mz_data;
+ int32_t ret;
+ int32_t mp_ret;
+ int32_t token_acquired = 0;
+
+ if (sxe2_mp_mz == NULL)
+ return -EINVAL;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ PMD_LOG_WARN(DRV, "Primary process direct execution for port %u",
+ dev->data->port_id);
+ return sxe2_xstats_info_get(dev, xstats, usr_cnt);
+ }
+
+ int32_t token_ret = sxe2_mp_acquire_token();
+ if (token_ret != 0)
+ return token_ret;
+ token_acquired = 1;
+
+ mp_ret = sxe2_mp_request_simple(dev, SXE2_MP_REQ_GET_XSTATS, &ret);
+ if (mp_ret != 0) {
+ ret = mp_ret;
+ goto out;
+ }
+
+ if (ret != 0) {
+ PMD_LOG_ERR(DRV, "Primary failed to exec request (type=%d), result: %d for port %u",
+ SXE2_MP_REQ_GET_XSTATS, ret, dev->data->port_id);
+ goto out;
+ }
+
+ mz_data = (struct sxe2_mp_shared_data *)sxe2_mp_mz->addr;
+ if (usr_cnt < mz_data->payload.xstats_blk.xstats_num) {
+ PMD_LOG_ERR(DRV, "user usr_cnt:%u less than xstats cnt:%u.",
+ usr_cnt, mz_data->payload.xstats_blk.xstats_num);
+ ret = (int32_t)mz_data->payload.xstats_blk.xstats_num;
+ goto out;
+ }
+
+ memcpy(xstats, mz_data->payload.xstats_blk.xstats,
+ mz_data->payload.xstats_blk.xstats_num *
+ sizeof(struct rte_eth_xstat));
+ ret = (int32_t)mz_data->payload.xstats_blk.xstats_num;
+
+ PMD_LOG_DEBUG(DRV,
+ "xstats received via IPC for port %u (cnt=%d)",
+ dev->data->port_id, ret);
+out:
+ if (token_acquired)
+ sxe2_mp_release_token();
+ return ret;
+}
+
+int32_t
+sxe2_mp_req_reset_stats(struct rte_eth_dev *dev)
+{
+ int32_t mp_ret;
+ int32_t ret = 0;
+
+ if (sxe2_mp_mz == NULL)
+ return -EINVAL;
+
+ mp_ret = sxe2_mp_request_simple(dev, SXE2_MP_REQ_RESET_STATS, &ret);
+ if (mp_ret != 0)
+ return mp_ret;
+
+ if (ret != 0) {
+ PMD_LOG_ERR(DRV,
+ "Primary failed SXE2_MP_REQ_RESET_STATS, result: %d for port %u",
+ ret, dev->data->port_id);
+ return ret;
+ }
+ return 0;
+}
diff --git a/drivers/net/sxe2/sxe2_mp.h b/drivers/net/sxe2/sxe2_mp.h
new file mode 100644
index 0000000000..0e97655718
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_mp.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_MP_H
+#define SXE2_MP_H
+
+#include <rte_eal.h>
+#include <rte_ethdev.h>
+#include <rte_memzone.h>
+#include <rte_stdatomic.h>
+
+#define SXE2_MP_NAME "sxe2_mp_msg"
+#define SXE2_MP_MZ_NAME "sxe2_stats_mz"
+
+#define SXE2_MP_MSG_TIMEOUT 30
+
+#define SXE2_MP_MAX_XSTATS 128
+
+#define SXE2_MP_MAX_SPIN 100000
+
+enum sxe2_mp_req_type {
+ SXE2_MP_REQ_GET_STATS = 1,
+ SXE2_MP_REQ_GET_XSTATS,
+ SXE2_MP_REQ_RESET_STATS,
+};
+
+struct sxe2_mp_param {
+ enum sxe2_mp_req_type type;
+ uint32_t port_id;
+ int result;
+};
+
+union sxe2_mp_shared_payload {
+ struct {
+ struct rte_eth_stats stats;
+ struct eth_queue_stats qstats;
+ } stats_blk;
+ struct {
+ struct rte_eth_xstat xstats[SXE2_MP_MAX_XSTATS];
+ uint32_t xstats_num;
+ } xstats_blk;
+};
+
+struct sxe2_mp_shared_data {
+ RTE_ATOMIC(uint16_t)in_use;
+ union sxe2_mp_shared_payload payload;
+};
+
+static inline void sxe2_unlock_auto(rte_spinlock_t **lock)
+{
+ if (lock && *lock)
+ rte_spinlock_unlock(*lock);
+}
+
+int sxe2_mp_init(struct rte_eth_dev *dev);
+
+void sxe2_mp_uninit(struct rte_eth_dev *dev);
+
+int sxe2_mp_request_simple(struct rte_eth_dev *dev,
+ enum sxe2_mp_req_type type,
+ int *result_out);
+
+int sxe2_mp_req_get_stats(struct rte_eth_dev *dev,
+ struct rte_eth_stats *stats,
+ struct eth_queue_stats *qstats);
+
+int sxe2_mp_req_get_xstats(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *xstats, uint32_t usr_cnt);
+
+int sxe2_mp_req_reset_stats(struct rte_eth_dev *dev);
+
+#endif
diff --git a/drivers/net/sxe2/sxe2_stats.c b/drivers/net/sxe2/sxe2_stats.c
new file mode 100644
index 0000000000..7ea2815fa3
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_stats.c
@@ -0,0 +1,586 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_eal.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_vsi.h"
+#include "sxe2_common_log.h"
+#include "sxe2_stats.h"
+#include "sxe2_queue.h"
+#include "sxe2_cmd_chnl.h"
+#include "sxe2_mp.h"
+
+#define SXE2_XSTAT_CNT_PF RTE_DIM(sxe2_xstats_field_pf)
+#define SXE2_XSTAT_CNT_VF RTE_DIM(sxe2_xstats_field_vf)
+
+static struct sxe2_stats_field sxe2_xstats_field_pf[] = {
+ {"rx_qblock_drop", offsetof(struct sxe2_stats, rx_qblock_drop)},
+ {"rx_out_of_buffer", offsetof(struct sxe2_stats, rx_out_of_buffer)},
+ {"tx_packets_phy", offsetof(struct sxe2_stats, tx_frame_good)},
+ {"rx_packets_phy", offsetof(struct sxe2_stats, rx_frame_good)},
+ {"rx_crc_errors_phy", offsetof(struct sxe2_stats, rx_crc_errors)},
+ {"tx_bytes_phy", offsetof(struct sxe2_stats, tx_bytes_good)},
+ {"rx_bytes_phy", offsetof(struct sxe2_stats, rx_bytes_good)},
+ {"tx_multicast_phy", offsetof(struct sxe2_stats, tx_multicast_good)},
+ {"tx_broadcast_phy", offsetof(struct sxe2_stats, tx_broadcast_good)},
+ {"rx_multicast_phy", offsetof(struct sxe2_stats, rx_multicast_good)},
+ {"rx_broadcast_phy", offsetof(struct sxe2_stats, rx_broadcast_good)},
+ {"rx_in_range_len_errs_phy", offsetof(struct sxe2_stats, rx_len_errors)},
+ {"rx_out_of_range_len_phy", offsetof(struct sxe2_stats, rx_out_of_range_errors)},
+ {"rx_oversize_pkts_phy", offsetof(struct sxe2_stats, rx_oversize_pkts_phy)},
+ {"rx_symbol_err_phy", offsetof(struct sxe2_stats, rx_symbol_err)},
+ {"rx_pause_ctrl_phy", offsetof(struct sxe2_stats, rx_pause_frame)},
+ {"tx_pause_ctrl_phy", offsetof(struct sxe2_stats, tx_pause_frame)},
+ {"rx_discards_phy", offsetof(struct sxe2_stats, rx_discards_phy)},
+ {"rx_discards_ips_phy", offsetof(struct sxe2_stats, rx_discards_ips_phy)},
+ {"tx_dropped_link_down_phy", offsetof(struct sxe2_stats, tx_dropped_link_down)},
+ {"rx_undersize_pkts_phy", offsetof(struct sxe2_stats, rx_undersize_good)},
+ {"rx_fragments_phy", offsetof(struct sxe2_stats, rx_runt_error)},
+ {"tx_bytes_all_phy", offsetof(struct sxe2_stats, tx_bytes_good_bad)},
+ {"tx_packets_all_phy", offsetof(struct sxe2_stats, tx_frame_good_bad)},
+ {"rx_jabbers_phy", offsetof(struct sxe2_stats, rx_jabbers)},
+ {"rx_64_bytes_phy", offsetof(struct sxe2_stats, rx_size_64)},
+ {"rx_65_to_127_bytes_phy", offsetof(struct sxe2_stats, rx_size_65_127)},
+ {"rx_128_to_255_bytes_phy", offsetof(struct sxe2_stats, rx_size_128_255)},
+ {"rx_256_to_511_bytes_phy", offsetof(struct sxe2_stats, rx_size_256_511)},
+ {"rx_512_to_1023_bytes_phy", offsetof(struct sxe2_stats, rx_size_512_1023)},
+ {"rx_1024_to_1522_bytes_phy", offsetof(struct sxe2_stats, rx_size_1024_1522)},
+ {"rx_1523_to_max_bytes_phy", offsetof(struct sxe2_stats, rx_size_1523_max)},
+ {"rx_pcs_symbol_err_phy", offsetof(struct sxe2_stats, rx_pcs_symbol_err_phy)},
+ {"rx_corrected_bits_phy", offsetof(struct sxe2_stats, rx_corrected_bits_phy)},
+ {"rx_err_lane_0_phy", offsetof(struct sxe2_stats, rx_err_lane_0_phy)},
+ {"rx_err_lane_1_phy", offsetof(struct sxe2_stats, rx_err_lane_1_phy)},
+ {"rx_err_lane_2_phy", offsetof(struct sxe2_stats, rx_err_lane_2_phy)},
+ {"rx_err_lane_3_phy", offsetof(struct sxe2_stats, rx_err_lane_3_phy)},
+ {"rx_illegal_bytes_phy", offsetof(struct sxe2_stats, rx_illegal_bytes)},
+ {"rx_oversize_good_phy", offsetof(struct sxe2_stats, rx_oversize_good)},
+ {"tx_unicast_all_phy", offsetof(struct sxe2_stats, tx_unicast)},
+ {"tx_broadcast_all_phy", offsetof(struct sxe2_stats, tx_broadcast)},
+ {"tx_multicast_all_phy", offsetof(struct sxe2_stats, tx_multicast)},
+ {"tx_vlan_packets_good_phy", offsetof(struct sxe2_stats, tx_vlan_packet_good)},
+ {"tx_64_bytes_phy", offsetof(struct sxe2_stats, tx_size_64)},
+ {"tx_65_to_127_bytes_phy", offsetof(struct sxe2_stats, tx_size_65_127)},
+ {"tx_128_to_255_bytes_phy", offsetof(struct sxe2_stats, tx_size_128_255)},
+ {"tx_256_to_511_bytes_phy", offsetof(struct sxe2_stats, tx_size_256_511)},
+ {"tx_512_to_1023_bytes_phy", offsetof(struct sxe2_stats, tx_size_512_1023)},
+ {"tx_1024_to_1522_bytes_phy", offsetof(struct sxe2_stats, tx_size_1024_1522)},
+ {"tx_1523_to_max_bytes_phy", offsetof(struct sxe2_stats, tx_size_1523_max)},
+ {"tx_underflow_error_phy", offsetof(struct sxe2_stats, tx_underflow_error)},
+ {"rx_bytes_all_phy", offsetof(struct sxe2_stats, rx_byte_good_bad)},
+ {"rx_packets_all_phy", offsetof(struct sxe2_stats, rx_frame_good_bad)},
+ {"rx_unicast_phy", offsetof(struct sxe2_stats, rx_unicast_good)},
+ {"rx_vlan_packets_phy", offsetof(struct sxe2_stats, rx_vlan_packets)},
+
+ {"rx_vport_bytes", offsetof(struct sxe2_stats, rx_vsi_bytes)},
+ {"rx_vport_unicast_packets", offsetof(struct sxe2_stats, rx_vsi_unicast_packets)},
+ {"rx_vport_broadcast_packets", offsetof(struct sxe2_stats, rx_vsi_broadcast_packets)},
+ {"rx_vport_multicast_packets", offsetof(struct sxe2_stats, rx_vsi_multicast_packets)},
+ {"rx_sw_unicast_packets", offsetof(struct sxe2_stats, rx_sw_unicast_packets)},
+ {"rx_sw_broadcast_packets", offsetof(struct sxe2_stats, rx_sw_broadcast_packets)},
+ {"rx_sw_multicast_packets", offsetof(struct sxe2_stats, rx_sw_multicast_packets)},
+ {"rx_sw_drop_packets", offsetof(struct sxe2_stats, rx_sw_drop_packets)},
+ {"rx_sw_drop_bytes", offsetof(struct sxe2_stats, rx_sw_drop_bytes)},
+
+ {"tx_vport_bytes", offsetof(struct sxe2_stats, tx_vsi_bytes)},
+ {"tx_vport_unicast_packets", offsetof(struct sxe2_stats, tx_vsi_unicast_packets)},
+ {"tx_vport_broadcast_packets", offsetof(struct sxe2_stats, tx_vsi_broadcast_packets)},
+ {"tx_vport_multicast_packets", offsetof(struct sxe2_stats, tx_vsi_multicast_packets)},
+};
+
+static struct sxe2_stats_field sxe2_xstats_field_vf[] = {
+ {"rx_vport_bytes", offsetof(struct sxe2_stats, rx_vsi_bytes)},
+ {"rx_vport_unicast_packets", offsetof(struct sxe2_stats, rx_vsi_unicast_packets)},
+ {"rx_vport_broadcast_packets", offsetof(struct sxe2_stats, rx_vsi_broadcast_packets)},
+ {"rx_vport_multicast_packets", offsetof(struct sxe2_stats, rx_vsi_multicast_packets)},
+ {"rx_sw_unicast_packets", offsetof(struct sxe2_stats, rx_sw_unicast_packets)},
+ {"rx_sw_broadcast_packets", offsetof(struct sxe2_stats, rx_sw_broadcast_packets)},
+ {"rx_sw_multicast_packets", offsetof(struct sxe2_stats, rx_sw_multicast_packets)},
+ {"rx_sw_drop_packets", offsetof(struct sxe2_stats, rx_sw_drop_packets)},
+ {"rx_sw_drop_bytes", offsetof(struct sxe2_stats, rx_sw_drop_bytes)},
+
+ {"tx_vport_bytes", offsetof(struct sxe2_stats, tx_vsi_bytes)},
+ {"tx_vport_unicast_packets", offsetof(struct sxe2_stats, tx_vsi_unicast_packets)},
+ {"tx_vport_broadcast_packets", offsetof(struct sxe2_stats, tx_vsi_broadcast_packets)},
+ {"tx_vport_multicast_packets", offsetof(struct sxe2_stats, tx_vsi_multicast_packets)},
+};
+
+static int32_t sxe2_xstat_pf_offset_get(uint32_t id, uint32_t *offset)
+{
+ int32_t ret = 0;
+ uint32_t size = SXE2_XSTAT_CNT_PF;
+
+ if (id < size) {
+ *offset = sxe2_xstats_field_pf[id].offset;
+ } else {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "invalid id:%u exceed stats size cnt:%u", id, size);
+ }
+ return ret;
+}
+
+static int32_t sxe2_xstat_vf_offset_get(uint32_t id, uint32_t *offset)
+{
+ int32_t ret = 0;
+ uint32_t size = SXE2_XSTAT_CNT_VF;
+
+ if (id < size) {
+ *offset = sxe2_xstats_field_vf[id].offset;
+ } else {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "invalid id:%u exceed stats size cnt:%u", id, size);
+ }
+ return ret;
+}
+
+static int32_t sxe2_mac_hw_stats_get_update(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ ret = sxe2_drv_get_mac_stats(adapter);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "get mac stats failed, ret:%d.", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_vsi_hw_stats_get_update(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ ret = sxe2_drv_get_vsi_stats(adapter);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "get vsi stats failed, ret:%d.", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_vsi_sw_stats_get_update(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_stats *sw_stats = &vsi->vsi_stats.vsi_sw_stats;
+ struct sxe2_rx_queue *rxq;
+ uint32_t rx_queue_id;
+ memset(sw_stats, 0, sizeof(struct sxe2_stats));
+
+ for (rx_queue_id = 0; rx_queue_id < adapter->dev_info.dev_data->nb_rx_queues;
+ rx_queue_id++) {
+ rxq = adapter->dev_info.dev_data->rx_queues[rx_queue_id];
+ if (rxq) {
+ sw_stats->ipackets += rxq->sw_stats.pkts;
+ sw_stats->ierrors += rxq->sw_stats.drop_pkts;
+ sw_stats->ibytes += rxq->sw_stats.bytes;
+
+ sw_stats->rx_sw_unicast_packets += rxq->sw_stats.unicast_pkts;
+ sw_stats->rx_sw_broadcast_packets += rxq->sw_stats.broadcast_pkts;
+ sw_stats->rx_sw_multicast_packets += rxq->sw_stats.multicast_pkts;
+ sw_stats->rx_sw_drop_packets += rxq->sw_stats.drop_pkts;
+ sw_stats->rx_sw_drop_bytes += rxq->sw_stats.drop_bytes;
+ }
+ }
+
+ return ret;
+}
+
+static void sxe2_stats_update(struct sxe2_adapter *adapter)
+{
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_stats *stats = &vsi->vsi_stats.stats;
+ struct sxe2_stats *hw_stats = &vsi->vsi_stats.vsi_hw_stats;
+ struct sxe2_stats *sw_stats = &vsi->vsi_stats.vsi_sw_stats;
+ struct sxe2_stats *sw_stats_prev = &vsi->vsi_stats.vsi_sw_stats_prev;
+ uint8_t i = 0;
+
+ memset(stats, 0, sizeof(struct sxe2_stats));
+
+ stats->opackets = hw_stats->opackets;
+ stats->obytes = hw_stats->tx_vsi_bytes;
+ stats->tx_vsi_bytes = hw_stats->tx_vsi_bytes;
+ stats->tx_vsi_unicast_packets = hw_stats->tx_vsi_unicast_packets;
+ stats->tx_vsi_broadcast_packets = hw_stats->tx_vsi_broadcast_packets;
+ stats->tx_vsi_multicast_packets = hw_stats->tx_vsi_multicast_packets;
+
+ stats->ierrors = sw_stats->ierrors + sw_stats_prev->ierrors;
+ if (adapter->devargs.sw_stats_en) {
+ stats->ipackets = sw_stats->ipackets + sw_stats_prev->ipackets;
+ stats->ibytes = sw_stats->ibytes + sw_stats_prev->ibytes;
+ } else {
+ stats->ipackets = hw_stats->ipackets;
+ stats->ibytes = hw_stats->rx_vsi_bytes;
+ }
+ stats->rx_vsi_bytes = hw_stats->rx_vsi_bytes;
+ stats->rx_vsi_unicast_packets = hw_stats->rx_vsi_unicast_packets;
+ stats->rx_vsi_broadcast_packets = hw_stats->rx_vsi_broadcast_packets;
+ stats->rx_vsi_multicast_packets = hw_stats->rx_vsi_multicast_packets;
+ stats->rx_sw_unicast_packets = sw_stats->rx_sw_unicast_packets +
+ sw_stats_prev->rx_sw_unicast_packets;
+ stats->rx_sw_broadcast_packets = sw_stats->rx_sw_broadcast_packets +
+ sw_stats_prev->rx_sw_broadcast_packets;
+ stats->rx_sw_multicast_packets = sw_stats->rx_sw_multicast_packets +
+ sw_stats_prev->rx_sw_multicast_packets;
+ stats->rx_sw_drop_packets = sw_stats->rx_sw_drop_packets +
+ sw_stats_prev->rx_sw_drop_packets;
+ stats->rx_sw_drop_bytes = sw_stats->rx_sw_drop_bytes +
+ sw_stats_prev->rx_sw_drop_bytes;
+
+ if (adapter->dev_type != SXE2_DEV_T_VF) {
+ stats->rx_out_of_buffer = hw_stats->rx_out_of_buffer;
+ stats->rx_qblock_drop = hw_stats->rx_qblock_drop;
+ stats->tx_frame_good = hw_stats->tx_frame_good;
+ stats->rx_frame_good = hw_stats->rx_frame_good;
+ stats->rx_crc_errors = hw_stats->rx_crc_errors;
+ stats->tx_bytes_good = hw_stats->tx_bytes_good;
+ stats->rx_bytes_good = hw_stats->rx_bytes_good;
+ stats->tx_multicast_good = hw_stats->tx_multicast_good;
+ stats->tx_broadcast_good = hw_stats->tx_broadcast_good;
+ stats->rx_multicast_good = hw_stats->rx_multicast_good;
+ stats->rx_broadcast_good = hw_stats->rx_broadcast_good;
+ stats->rx_len_errors = hw_stats->rx_len_errors;
+ stats->rx_out_of_range_errors = hw_stats->rx_out_of_range_errors;
+ stats->rx_oversize_pkts_phy = hw_stats->rx_oversize_pkts_phy;
+ stats->rx_symbol_err = hw_stats->rx_symbol_err;
+ stats->rx_pause_frame = hw_stats->rx_pause_frame;
+ stats->tx_pause_frame = hw_stats->tx_pause_frame;
+ stats->rx_discards_phy = hw_stats->rx_discards_phy;
+ stats->rx_discards_ips_phy = hw_stats->rx_discards_ips_phy;
+ stats->tx_dropped_link_down = hw_stats->tx_dropped_link_down;
+ stats->rx_undersize_good = hw_stats->rx_undersize_good;
+ stats->rx_runt_error = hw_stats->rx_runt_error;
+ stats->tx_bytes_good_bad = hw_stats->tx_bytes_good_bad;
+ stats->tx_frame_good_bad = hw_stats->tx_frame_good_bad;
+ stats->rx_jabbers = hw_stats->rx_jabbers;
+ stats->rx_size_64 = hw_stats->rx_size_64;
+ stats->rx_size_65_127 = hw_stats->rx_size_65_127;
+ stats->rx_size_128_255 = hw_stats->rx_size_128_255;
+ stats->rx_size_256_511 = hw_stats->rx_size_256_511;
+ stats->rx_size_512_1023 = hw_stats->rx_size_512_1023;
+ stats->rx_size_1024_1522 = hw_stats->rx_size_1024_1522;
+ stats->rx_size_1523_max = hw_stats->rx_size_1523_max;
+ stats->rx_pcs_symbol_err_phy = hw_stats->rx_pcs_symbol_err_phy;
+ stats->rx_corrected_bits_phy = hw_stats->rx_corrected_bits_phy;
+ stats->rx_err_lane_0_phy = hw_stats->rx_err_lane_0_phy;
+ stats->rx_err_lane_1_phy = hw_stats->rx_err_lane_1_phy;
+ stats->rx_err_lane_2_phy = hw_stats->rx_err_lane_2_phy;
+ stats->rx_err_lane_3_phy = hw_stats->rx_err_lane_3_phy;
+ stats->rx_illegal_bytes = hw_stats->rx_illegal_bytes;
+ stats->rx_oversize_good = hw_stats->rx_oversize_good;
+ stats->tx_unicast = hw_stats->tx_unicast;
+ stats->tx_broadcast = hw_stats->tx_broadcast;
+ stats->tx_multicast = hw_stats->tx_multicast;
+ stats->tx_vlan_packet_good = hw_stats->tx_vlan_packet_good;
+ stats->tx_size_64 = hw_stats->tx_size_64;
+ stats->tx_size_65_127 = hw_stats->tx_size_65_127;
+ stats->tx_size_128_255 = hw_stats->tx_size_128_255;
+ stats->tx_size_256_511 = hw_stats->tx_size_256_511;
+ stats->tx_size_512_1023 = hw_stats->tx_size_512_1023;
+ stats->tx_size_1024_1522 = hw_stats->tx_size_1024_1522;
+ stats->tx_size_1523_max = hw_stats->tx_size_1523_max;
+ stats->tx_underflow_error = hw_stats->tx_underflow_error;
+ stats->rx_byte_good_bad = hw_stats->rx_byte_good_bad;
+ stats->rx_frame_good_bad = hw_stats->rx_frame_good_bad;
+ stats->rx_unicast_good = hw_stats->rx_unicast_good;
+ stats->rx_vlan_packets = hw_stats->rx_vlan_packets;
+ rte_memcpy(stats->rx_prio_buf_discard, hw_stats->rx_prio_buf_discard,
+ sizeof(hw_stats->rx_prio_buf_discard));
+ rte_memcpy(stats->prio_xoff_rx, hw_stats->prio_xoff_rx,
+ sizeof(hw_stats->prio_xoff_rx));
+ rte_memcpy(stats->prio_xon_rx, hw_stats->prio_xon_rx,
+ sizeof(hw_stats->prio_xon_rx));
+ rte_memcpy(stats->prio_xon_tx, hw_stats->prio_xon_tx,
+ sizeof(hw_stats->prio_xon_tx));
+ rte_memcpy(stats->prio_xoff_tx, hw_stats->prio_xoff_tx,
+ sizeof(hw_stats->prio_xoff_tx));
+ rte_memcpy(stats->prio_xon_2_xoff, hw_stats->prio_xon_2_xoff,
+ sizeof(hw_stats->prio_xon_2_xoff));
+
+ stats->imissed = hw_stats->rx_out_of_buffer +
+ hw_stats->rx_qblock_drop;
+ for (i = 0; i < SXE2_MAX_USER_PRIORITY; i++)
+ stats->imissed += hw_stats->rx_prio_buf_discard[i];
+ }
+}
+
+int32_t sxe2_stats_info_get(struct rte_eth_dev *dev,
+ struct rte_eth_stats *stats,
+ struct eth_queue_stats *qstats)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_stats *stats_out = &vsi->vsi_stats.stats;
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+ return sxe2_mp_req_get_stats(dev, stats, qstats);
+
+ ret = sxe2_vsi_hw_stats_get_update(adapter);
+ if (ret)
+ goto end;
+
+ ret = sxe2_vsi_sw_stats_get_update(adapter);
+ if (ret)
+ goto end;
+
+ ret = sxe2_drv_queue_info_get_update(adapter, qstats);
+ if (ret)
+ goto end;
+
+ sxe2_stats_update(adapter);
+
+ stats->ipackets = stats_out->ipackets;
+ stats->ibytes = stats_out->ibytes;
+ stats->ierrors = stats_out->ierrors;
+ stats->imissed = stats_out->imissed;
+ stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
+
+ stats->opackets = stats_out->opackets;
+ stats->obytes = stats_out->obytes;
+
+ ret = 0;
+
+end:
+ return ret;
+}
+
+int32_t sxe2_xstats_info_get(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *xstats, uint32_t usr_cnt)
+{
+ uint32_t i = 0;
+ uint32_t cnt = 0;
+ int32_t ret = 0;
+ uint32_t offset = 0;
+ uint32_t xstats_cnt = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_vsi_stats *xstats_out = &vsi->vsi_stats;
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+ return sxe2_mp_req_get_xstats(dev, xstats, usr_cnt);
+
+ if (adapter->dev_type == SXE2_DEV_T_VF)
+ xstats_cnt = SXE2_XSTAT_CNT_VF;
+ else
+ xstats_cnt = SXE2_XSTAT_CNT_PF;
+
+ if (usr_cnt < xstats_cnt) {
+ ret = xstats_cnt;
+ PMD_LOG_ERR(DRV, "user usr_cnt:%u less than stats cnt:%u", usr_cnt, ret);
+ goto end;
+ }
+
+ ret = sxe2_vsi_hw_stats_get_update(adapter);
+ if (ret) {
+ ret = xstats_cnt;
+ goto end;
+ }
+
+ ret = sxe2_vsi_sw_stats_get_update(adapter);
+ if (ret) {
+ ret = xstats_cnt;
+ goto end;
+ }
+
+ if (adapter->dev_type == SXE2_DEV_T_VF) {
+ sxe2_stats_update(adapter);
+ for (i = 0; i < xstats_cnt; i++) {
+ (void)sxe2_xstat_vf_offset_get(i, &offset);
+ xstats[cnt].value = *(uint64_t *)(((int8_t *)(&xstats_out->stats)) +
+ offset);
+ xstats[cnt].id = cnt;
+ cnt++;
+ }
+ } else {
+ ret = sxe2_mac_hw_stats_get_update(adapter);
+ if (ret) {
+ ret = xstats_cnt;
+ goto end;
+ }
+
+ sxe2_stats_update(adapter);
+
+ for (i = 0; i < xstats_cnt; i++) {
+ (void)sxe2_xstat_pf_offset_get(i, &offset);
+ xstats[cnt].value = *(uint64_t *)(((int8_t *)(&xstats_out->stats)) +
+ offset);
+ xstats[cnt].id = cnt;
+ cnt++;
+ }
+ }
+ ret = cnt;
+ PMD_LOG_DEBUG(DRV, "usr_cnt:%u stats cnt:%u stats done", usr_cnt, cnt);
+
+end:
+ return ret;
+}
+
+int32_t sxe2_xstats_names_get(__rte_unused struct rte_eth_dev *dev,
+ struct rte_eth_xstat_name *xstats_names,
+ __rte_unused unsigned int usr_cnt)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_stats_field *field = NULL;
+ uint32_t i = 0;
+ uint32_t cnt = 0;
+ int32_t ret = -1;
+ uint32_t xstats_cnt = 0;
+
+ if (adapter->dev_type == SXE2_DEV_T_VF) {
+ field = sxe2_xstats_field_vf;
+ xstats_cnt = SXE2_XSTAT_CNT_VF;
+ } else {
+ field = sxe2_xstats_field_pf;
+ xstats_cnt = SXE2_XSTAT_CNT_PF;
+ }
+
+ if (!xstats_names) {
+ ret = xstats_cnt;
+ PMD_LOG_DEBUG(DRV, "xstats field size:%u", ret);
+ goto l_out;
+ }
+
+ if (usr_cnt < xstats_cnt) {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "max:%d usr_cnt:%u invalid (err:%d)", xstats_cnt, usr_cnt, ret);
+ goto l_out;
+ }
+
+ for (i = 0; i < xstats_cnt; i++) {
+ (void)strlcpy(xstats_names[cnt].name, field[i].name,
+ sizeof(xstats_names[cnt].name));
+ cnt++;
+ }
+
+ ret = cnt;
+
+l_out:
+ return ret;
+}
+
+int32_t sxe2_stats_hw_reset(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vsi *vsi = adapter->vsi_ctxt.main_vsi;
+ struct sxe2_rx_queue *rxq;
+ uint32_t rx_queue_id;
+
+ ret = sxe2_drv_vsi_stats_reset(adapter);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "reset vsi stats failed, ret:%d.", ret);
+ goto l_end;
+ }
+ if (adapter->dev_type != SXE2_DEV_T_VF) {
+ ret = sxe2_drv_mac_stats_reset(adapter);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "reset mac stats failed, ret:%d.", ret);
+ goto l_end;
+ }
+ }
+
+ memset(&vsi->vsi_stats, 0, sizeof(vsi->vsi_stats));
+ for (rx_queue_id = 0; rx_queue_id < dev->data->nb_rx_queues; rx_queue_id++) {
+ rxq = dev->data->rx_queues[rx_queue_id];
+ if (rxq)
+ memset(&rxq->sw_stats, 0, sizeof(rxq->sw_stats));
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_stats_info_reset(struct rte_eth_dev *dev)
+{
+ int32_t ret;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+ return sxe2_mp_req_reset_stats(dev);
+
+ if (adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_Q_MAP) {
+ ret = sxe2_drv_mapping_stats_info_clear(dev);
+ if (ret)
+ goto l_end;
+ }
+
+ ret = sxe2_stats_hw_reset(dev);
+ if (ret)
+ goto l_end;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_stats_init(struct rte_eth_dev *dev)
+{
+ PMD_INIT_FUNC_TRACE();
+ int32_t ret;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return 0;
+
+ ret = sxe2_queue_stats_map_init(dev);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_stats_hw_reset(dev);
+ if (ret)
+ goto l_end;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
+ uint16_t queue_id, uint8_t pool_idx, uint8_t is_rx)
+{
+ int32_t ret = -1;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(eth_dev);
+
+ if (!(adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_Q_MAP)) {
+ PMD_LOG_ERR(DRV, "VF does not support queue mapping! ");
+ goto l_end;
+ }
+
+ if (is_rx)
+ ret = sxe2_drv_rxq_mapping_set(eth_dev, queue_id, pool_idx);
+ else
+ ret = sxe2_drv_txq_mapping_set(eth_dev, queue_id, pool_idx);
+
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Queue stats mapping failed ! "
+ "queue_id:%u pool_idx:%u", queue_id, pool_idx);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(DRV, "port %u %s queue_id %d stat map to pool[%u] ",
+ (uint16_t)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
+ queue_id, pool_idx);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_queue_stats_map_init(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_Q_MAP) {
+ dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+
+ ret = sxe2_drv_mapping_reset(dev);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Queue stats mapping init failed !");
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_stats.h b/drivers/net/sxe2/sxe2_stats.h
new file mode 100644
index 0000000000..64ac2bb11d
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_stats.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_STATS_H__
+#define __SXE2_STATS_H__
+
+#define SXE2_STATS_FIELD_NAME_SIZE 50
+
+struct sxe2_stats_field {
+ char name[SXE2_STATS_FIELD_NAME_SIZE];
+ uint32_t offset;
+};
+
+struct sxe2_adapter;
+
+int32_t sxe2_stats_info_get(struct rte_eth_dev *dev,
+ struct rte_eth_stats *stats,
+ struct eth_queue_stats *qstats);
+
+int32_t sxe2_xstats_info_get(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *xstats, uint32_t usr_cnt);
+
+int32_t sxe2_xstats_names_get(__rte_unused struct rte_eth_dev *dev,
+ struct rte_eth_xstat_name *xstats_names,
+ __rte_unused unsigned int usr_cnt);
+
+int32_t sxe2_stats_hw_reset(struct rte_eth_dev *dev);
+
+int32_t sxe2_stats_info_reset(struct rte_eth_dev *dev);
+
+int32_t sxe2_stats_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
+ uint16_t queue_id, uint8_t pool_idx, uint8_t is_rx);
+
+int32_t sxe2_queue_stats_map_init(struct rte_eth_dev *dev);
+
+#endif
--
2.47.3
^ permalink raw reply related
* [PATCH v2 05/23] drivers: support RSS feature
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add support for Receive Side Scaling (RSS) to distribute incoming
traffic across multiple receive queues.
- Implement rss_hash_update and rss_hash_conf_get.
- Implement reta_update and reta_query.
- Support RSS hash configuration for IPv4, IPv6, TCP and UDP.
- Default hash key is initialized during port start.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++++++++++++++++++++
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 173 +++++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 16 +
drivers/net/sxe2/sxe2_drv_cmd.h | 29 ++
drivers/net/sxe2/sxe2_ethdev.c | 37 ++
drivers/net/sxe2/sxe2_ethdev.h | 8 +
drivers/net/sxe2/sxe2_flow_define.h | 143 ++++++
drivers/net/sxe2/sxe2_queue.c | 11 +
| 584 +++++++++++++++++++++++
| 81 ++++
drivers/net/sxe2/sxe2_txrx.h | 4 +
drivers/net/sxe2/sxe2_txrx_poll.c | 85 +++-
13 files changed, 1804 insertions(+), 1 deletion(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
diff --git a/drivers/common/sxe2/sxe2_flow_public.h b/drivers/common/sxe2/sxe2_flow_public.h
new file mode 100644
index 0000000000..32ab2a9713
--- /dev/null
+++ b/drivers/common/sxe2/sxe2_flow_public.h
@@ -0,0 +1,633 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FLOW_PUBLIC_H__
+#define __SXE2_FLOW_PUBLIC_H__
+#include "sxe2_osal.h"
+
+enum sxe2_flow_type {
+ SXE2_FLOW_TYPE_NONE = 0,
+ SXE2_FLOW_MAC_PAY = 1,
+ SXE2_FLOW_MAC_IPV4_FRAG_PAY = 22,
+ SXE2_FLOW_MAC_IPV4_PAY = 23,
+ SXE2_FLOW_MAC_IPV4_UDP_PAY = 24,
+ SXE2_FLOW_MAC_IPV4_TCP_PAY = 26,
+ SXE2_FLOW_MAC_IPV4_SCTP_PAY = 27,
+ SXE2_FLOW_MAC_IPV4_IPV4_FRAG_PAY = 29,
+ SXE2_FLOW_MAC_IPV4_IPV4_PAY = 30,
+ SXE2_FLOW_MAC_IPV4_IPV4_UDP_PAY = 31,
+ SXE2_FLOW_MAC_IPV4_IPV4_TCP_PAY = 33,
+ SXE2_FLOW_MAC_IPV4_IPV4_SCTP_PAY = 34,
+ SXE2_FLOW_MAC_IPV4_IPV6_FRAG_PAY = 36,
+ SXE2_FLOW_MAC_IPV4_IPV6_PAY = 37,
+ SXE2_FLOW_MAC_IPV4_IPV6_UDP_PAY = 38,
+ SXE2_FLOW_MAC_IPV4_IPV6_TCP_PAY = 40,
+ SXE2_FLOW_MAC_IPV4_IPV6_SCTP_PAY = 41,
+ SXE2_FLOW_MAC_IPV4_GRE_PAY = 43,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_FRAG_PAY = 44,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_PAY = 45,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_UDP_PAY = 46,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_TCP_PAY = 48,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_SCTP_PAY = 49,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_FRAG_PAY = 51,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_PAY = 52,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_UDP_PAY = 53,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_TCP_PAY = 55,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_SCTP_PAY = 56,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_PAY = 58,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_FRAG_PAY = 59,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_PAY = 60,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_UDP_PAY = 61,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_TCP_PAY = 63,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_SCTP_PAY = 64,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_FRAG_PAY = 66,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_PAY = 67,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_UDP_PAY = 68,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_TCP_PAY = 70,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_SCTP_PAY = 71,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_PAY = 73,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_FRAG_PAY = 74,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_PAY = 75,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_UDP_PAY = 76,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_TCP_PAY = 78,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_SCTP_PAY = 79,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_FRAG_PAY = 81,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_PAY = 82,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_UDP_PAY = 83,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_TCP_PAY = 85,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_SCTP_PAY = 86,
+ SXE2_FLOW_MAC_IPV6_FRAG_PAY = 88,
+ SXE2_FLOW_MAC_IPV6_PAY = 89,
+ SXE2_FLOW_MAC_IPV6_UDP_PAY = 90,
+ SXE2_FLOW_MAC_IPV6_TCP_PAY = 92,
+ SXE2_FLOW_MAC_IPV6_SCTP_PAY = 93,
+ SXE2_FLOW_MAC_IPV6_IPV4_FRAG_PAY = 95,
+ SXE2_FLOW_MAC_IPV6_IPV4_PAY = 96,
+ SXE2_FLOW_MAC_IPV6_IPV4_UDP_PAY = 97,
+ SXE2_FLOW_MAC_IPV6_IPV4_TCP_PAY = 99,
+ SXE2_FLOW_MAC_IPV6_IPV4_SCTP_PAY = 100,
+ SXE2_FLOW_MAC_IPV6_IPV6_FRAG_PAY = 102,
+ SXE2_FLOW_MAC_IPV6_IPV6_PAY = 103,
+ SXE2_FLOW_MAC_IPV6_IPV6_UDP_PAY = 104,
+ SXE2_FLOW_MAC_IPV6_IPV6_TCP_PAY = 106,
+ SXE2_FLOW_MAC_IPV6_IPV6_SCTP_PAY = 107,
+ SXE2_FLOW_MAC_IPV6_GRE_PAY = 109,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_FRAG_PAY = 110,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_PAY = 111,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_UDP_PAY = 112,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_TCP_PAY = 114,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_SCTP_PAY = 115,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_FRAG_PAY = 117,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_PAY = 118,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_UDP_PAY = 119,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_TCP_PAY = 121,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_SCTP_PAY = 122,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_PAY = 124,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_FRAG_PAY = 125,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_PAY = 126,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_UDP_PAY = 127,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_TCP_PAY = 129,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_SCTP_PAY = 130,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_FRAG_PAY = 132,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_PAY = 133,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_UDP_PAY = 134,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_TCP_PAY = 136,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_SCTP_PAY = 137,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_PAY = 139,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_FRAG_PAY = 140,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_PAY = 141,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_UDP_PAY = 142,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_TCP_PAY = 144,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_SCTP_PAY = 145,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_FRAG_PAY = 147,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_PAY = 148,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_UDP_PAY = 149,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_TCP_PAY = 151,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_SCTP_PAY = 152,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_PAY = 329,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_PAY = 330,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_FRAG_PAY = 331,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_PAY = 332,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_UDP_PAY = 333,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_TCP_PAY = 334,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_SCTP_PAY = 335,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_FRAG_PAY = 336,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_PAY = 337,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_UDP_PAY = 338,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_TCP_PAY = 339,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_SCTP_PAY = 340,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_FRAG_PAY = 341,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_PAY = 342,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_UDP_PAY = 343,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_TCP_PAY = 344,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_SCTP_PAY = 345,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_FRAG_PAY = 346,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_PAY = 347,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_UDP_PAY = 348,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_TCP_PAY = 349,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_SCTP_PAY = 350,
+ SXE2_FLOW_MAC_IPV6_MAC_PAY = 820,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_FRAG_PAY = 821,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_PAY = 822,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_UDP_PAY = 823,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_TCP_PAY = 824,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_SCTP_PAY = 825,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_FRAG_PAY = 827,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_PAY = 828,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_UDP_PAY = 829,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_TCP_PAY = 830,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_SCTP_PAY = 831,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_PAY = 835,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_FRAG_PAY = 836,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_PAY = 837,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_UDP_PAY = 838,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_TCP_PAY = 839,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_SCTP_PAY = 840,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_FRAG_PAY = 842,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_PAY = 843,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_UDP_PAY = 844,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_TCP_PAY = 845,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_SCTP_PAY = 846,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_PAY = 878,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_FRAG_PAY = 877,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_PAY = 876,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_UDP_PAY = 879,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_TCP_PAY = 880,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_SCTP_PAY = 875,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_FRAG_PAY = 871,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_PAY = 870,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_UDP_PAY = 872,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_TCP_PAY = 873,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_SCTP_PAY = 869,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_PAY = 891,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_FRAG_PAY = 890,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_PAY = 889,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_UDP_PAY = 892,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_TCP_PAY = 893,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_SCTP_PAY = 888,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_FRAG_PAY = 884,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_PAY = 883,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_UDP_PAY = 885,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_TCP_PAY = 886,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_SCTP_PAY = 882,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_PAY = 904,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_FRAG_PAY = 903,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_PAY = 902,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_UDP_PAY = 905,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_TCP_PAY = 906,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_SCTP_PAY = 901,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_FRAG_PAY = 897,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_PAY = 896,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_UDP_PAY = 898,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_TCP_PAY = 899,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_SCTP_PAY = 895,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_PAY = 917,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_FRAG_PAY = 916,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_PAY = 915,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_UDP_PAY = 918,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_TCP_PAY = 919,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_SCTP_PAY = 914,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_FRAG_PAY = 910,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_PAY = 909,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_UDP_PAY = 911,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_TCP_PAY = 912,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_SCTP_PAY = 908,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_PAY = 930,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY = 929,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_PAY = 928,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY = 931,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY = 932,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY = 927,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY = 923,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_PAY = 922,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY = 924,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY = 925,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY = 921,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_PAY = 943,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY = 942,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_PAY = 941,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY = 944,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY = 945,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY = 940,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY = 936,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_PAY = 935,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY = 937,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY = 938,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY = 934,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_PAY = 956,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY = 955,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_PAY = 954,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY = 957,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY = 958,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY = 953,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY = 949,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_PAY = 948,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY = 950,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY = 951,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY = 947,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_PAY = 969,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY = 968,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_PAY = 967,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY = 970,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY = 971,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY = 966,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY = 962,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_PAY = 961,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY = 963,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY = 964,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY = 960,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_PAY = 982,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_FRAG_PAY = 981,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_PAY = 980,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_UDP_PAY = 983,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_TCP_PAY = 984,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_SCTP_PAY = 979,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_FRAG_PAY = 975,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_PAY = 974,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_UDP_PAY = 976,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_TCP_PAY = 977,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_SCTP_PAY = 973,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_PAY = 995,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_FRAG_PAY = 994,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_PAY = 993,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_UDP_PAY = 996,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_TCP_PAY = 997,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_SCTP_PAY = 992,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_FRAG_PAY = 988,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_PAY = 987,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_UDP_PAY = 989,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_TCP_PAY = 990,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_SCTP_PAY = 986,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_PAY = 1008,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_FRAG_PAY = 1007,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_PAY = 1006,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_UDP_PAY = 1009,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_TCP_PAY = 1010,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_SCTP_PAY = 1005,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_FRAG_PAY = 1001,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_PAY = 1000,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_UDP_PAY = 1002,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_TCP_PAY = 1003,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_SCTP_PAY = 999,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_PAY = 1021,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_FRAG_PAY = 1020,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_PAY = 1019,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_UDP_PAY = 1022,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_TCP_PAY = 1023,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_SCTP_PAY = 1018,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_FRAG_PAY = 1014,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_PAY = 1013,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_UDP_PAY = 1015,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_TCP_PAY = 1016,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_SCTP_PAY = 1012,
+ SXE2_FLOW_TYPE_MAX = 2048,
+};
+
+enum sxe2_rss_cfg_hdr_type {
+ SXE2_RSS_OUTER_HEADERS,
+ SXE2_RSS_INNER_HEADERS,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_VXLAN,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_VXLAN,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GENEVE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GENEVE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GTPU,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GTPU,
+ SXE2_RSS_ANY_HEADERS
+};
+
+enum sxe2_flow_hdr {
+ SXE2_FLOW_HDR_ETH = 0,
+ SXE2_FLOW_HDR_VLAN,
+ SXE2_FLOW_HDR_QINQ,
+ SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_ICMP = 5,
+ SXE2_FLOW_HDR_TCP,
+ SXE2_FLOW_HDR_UDP,
+ SXE2_FLOW_HDR_SCTP,
+ SXE2_FLOW_HDR_GRE,
+ SXE2_FLOW_HDR_VXLAN = 10,
+ SXE2_FLOW_HDR_GENEVE,
+ SXE2_FLOW_HDR_GTPU,
+
+ SXE2_FLOW_HDR_IPV_FRAG,
+
+ SXE2_FLOW_HDR_IPV_OTHER,
+
+ SXE2_FLOW_HDR_ETH_NON_IP = 15,
+ SXE2_FLOW_HDR_MAX = 128,
+};
+
+enum sxe2_flow_fld_id {
+ SXE2_FLOW_FLD_ID_ETH_DA = 0,
+ SXE2_FLOW_FLD_ID_ETH_SA,
+ SXE2_FLOW_FLD_ID_S_TCI,
+ SXE2_FLOW_FLD_ID_C_TCI,
+ SXE2_FLOW_FLD_ID_S_TPID,
+ SXE2_FLOW_FLD_ID_C_TPID = 5,
+ SXE2_FLOW_FLD_ID_S_VID,
+ SXE2_FLOW_FLD_ID_C_VID,
+ SXE2_FLOW_FLD_ID_ETH_TYPE,
+
+ SXE2_FLOW_FLD_ID_IPV4_TOS,
+ SXE2_FLOW_FLD_ID_IPV6_DSCP = 10,
+ SXE2_FLOW_FLD_ID_IPV4_TTL,
+ SXE2_FLOW_FLD_ID_IPV4_PROT,
+ SXE2_FLOW_FLD_ID_IPV6_TTL,
+ SXE2_FLOW_FLD_ID_IPV6_PROT,
+ SXE2_FLOW_FLD_ID_IPV4_SA = 15,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_IPV4_CHKSUM,
+ SXE2_FLOW_FLD_ID_IPV4_ID = 20,
+ SXE2_FLOW_FLD_ID_IPV6_ID,
+ SXE2_FLOW_FLD_ID_IPV6_PRE32_SA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE32_DA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE48_SA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE48_DA = 25,
+ SXE2_FLOW_FLD_ID_IPV6_PRE64_SA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE64_DA,
+
+ SXE2_FLOW_FLD_ID_TCP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_TCP_DST_PORT,
+ SXE2_FLOW_FLD_ID_UDP_SRC_PORT = 30,
+ SXE2_FLOW_FLD_ID_UDP_DST_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_DST_PORT,
+ SXE2_FLOW_FLD_ID_TCP_FLAGS,
+ SXE2_FLOW_FLD_ID_TCP_CHKSUM = 35,
+ SXE2_FLOW_FLD_ID_UDP_CHKSUM,
+ SXE2_FLOW_FLD_ID_SCTP_CHKSUM,
+
+ SXE2_FLOW_FLD_ID_VXLAN_VNI,
+
+ SXE2_FLOW_FLD_ID_GENEVE_VNI,
+
+ SXE2_FLOW_FLD_ID_GTPU_TEID = 40,
+
+ SXE2_FLOW_FLD_ID_NVGRE_TNI,
+
+ SXE2_FLOW_FLD_ID_MAX = 128,
+};
+
+struct sxe2_ether_hdr {
+ uint8_t dst_addr[SXE2_ETH_ALEN];
+ uint8_t src_addr[SXE2_ETH_ALEN];
+ uint16_t ether_type;
+};
+
+struct sxe2_vlan_hdr {
+ uint16_t type;
+ uint16_t vlan;
+};
+
+struct sxe2_ipv4_hdr {
+ uint8_t ver_ihl;
+ uint8_t tos;
+ uint16_t tot_len;
+ uint16_t id;
+ uint16_t frag_off;
+ uint8_t ttl;
+ uint8_t protocol;
+ uint16_t check;
+ uint32_t saddr;
+ uint32_t daddr;
+};
+#define SXE2_IPV6_ADDR_LENGTH (16)
+#define SXE2_IPV6_TC_SHIFT (20)
+#define SXE2_IPV6_TC_MASK (0xFF)
+
+struct sxe2_ipv6_hdr {
+ uint32_t pri_ver_flow;
+ uint16_t payload_len;
+ uint8_t nexthdr;
+ uint8_t hop_limit;
+ union {
+ uint8_t saddr[16];
+ uint16_t saddr16[8];
+ uint32_t saddr32[4];
+ };
+ union {
+ uint8_t daddr[16];
+ uint16_t daddr16[8];
+ uint32_t daddr32[4];
+ };
+};
+
+struct sxe2_tcp_hdr {
+ uint16_t source;
+ uint16_t dest;
+ uint32_t seq;
+ uint32_t ack_seq;
+ uint16_t flag;
+ uint16_t window;
+ uint16_t check;
+ uint16_t urg_ptr;
+};
+
+struct sxe2_udp_hdr {
+ uint16_t source;
+ uint16_t dest;
+ uint16_t len;
+ uint16_t check;
+};
+
+struct sxe2_sctp_hdr {
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+struct sxe2_nvgre_hdr {
+ uint16_t flags;
+ uint16_t protocol;
+ uint32_t tni;
+};
+struct sxe2_geneve_hdr {
+ uint16_t flags;
+ uint16_t protocol;
+ uint32_t vni;
+};
+struct sxe2_gtpu_hdr {
+ uint8_t flag;
+ uint8_t msg_type;
+ uint16_t msg_len;
+ uint32_t teid;
+};
+struct sxe2_vxlan_hdr {
+ uint8_t flag;
+ uint8_t resvd0;
+ uint8_t resvd1;
+ uint8_t protocol;
+ uint32_t vni;
+};
+
+enum sxe2_flow_act_type {
+ SXE2_FLOW_ACTION_DROP = 0,
+ SXE2_FLOW_ACTION_TC_REDIRECT,
+ SXE2_FLOW_ACTION_TO_VSI,
+ SXE2_FLOW_ACTION_TO_VSI_LIST,
+ SXE2_FLOW_ACTION_PASSTHRU,
+ SXE2_FLOW_ACTION_QUEUE,
+ SXE2_FLOW_ACTION_Q_REGION,
+ SXE2_FLOW_ACTION_MARK,
+ SXE2_FLOW_ACTION_COUNT,
+ SXE2_FLOW_ACTION_RSS,
+ SXE2_FLOW_ACTION_MAX = 32,
+};
+
+enum sxe2_rss_hash_key_func {
+ SXE2_RSS_HASH_FUNC_TOEPLITZ = 0,
+ SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ = 1,
+ SXE2_RSS_HASH_FUNC_XOR = 2,
+ SXE2_RSS_HASH_FUNC_JEKINS = 3
+};
+
+struct sxe2_flow_action_rss {
+ DECLARE_BITMAP(hdr_out, SXE2_FLOW_HDR_MAX);
+ DECLARE_BITMAP(hdr_in, SXE2_FLOW_HDR_MAX);
+ DECLARE_BITMAP(fld, SXE2_FLOW_FLD_ID_MAX);
+ uint8_t is_inner;
+ uint8_t func;
+ uint8_t hdr_type;
+};
+
+struct sxe2_flow_action_queue {
+ uint16_t vsi_index;
+ uint16_t q_index;
+};
+
+struct sxe2_flow_action_queue_region {
+ uint16_t vsi_index;
+ uint16_t q_index;
+ uint8_t region;
+};
+
+struct sxe2_flow_action_passthru {
+ uint16_t vsi_index;
+};
+
+struct sxe2_flow_action_mark {
+ uint32_t mark_id;
+};
+
+#define SXE2_VSI_MAX (2048)
+struct sxe2_flow_action_vsi {
+ uint16_t vsi_index;
+};
+
+struct sxe2_flow_action_vsi_list {
+ DECLARE_BITMAP(vsi_list_map, SXE2_VSI_MAX);
+ uint16_t vsi_cnt;
+};
+
+enum sxe2_fnav_stat_ctrl_type {
+ SXE2_FNAV_STAT_ENA_NONE = 0,
+ SXE2_FNAV_STAT_ENA_PKTS,
+ SXE2_FNAV_STAT_ENA_BYTES,
+ SXE2_FNAV_STAT_ENA_ALL,
+};
+
+struct sxe2_flow_action_count {
+ uint32_t user_id;
+ uint32_t driver_id;
+ uint32_t stat_index;
+ uint32_t stat_ctrl;
+};
+
+enum sxe2_flow_engine_type {
+ SXE2_FLOW_ENGINE_ACL,
+ SXE2_FLOW_ENGINE_SWITCH,
+ SXE2_FLOW_ENGINE_FNAV,
+ SXE2_FLOW_ENGINE_RSS,
+ SXE2_FLOW_ENGINE_MAX,
+};
+
+struct sxe2_flow_item {
+ struct sxe2_ether_hdr eth;
+ struct sxe2_vlan_hdr vlan;
+ struct sxe2_vlan_hdr qinq;
+ struct sxe2_ipv4_hdr ipv4;
+ struct sxe2_ipv6_hdr ipv6;
+ struct sxe2_udp_hdr udp;
+ struct sxe2_tcp_hdr tcp;
+ struct sxe2_sctp_hdr sctp;
+ struct sxe2_gtpu_hdr gtpu;
+ struct sxe2_vxlan_hdr vxlan;
+ struct sxe2_nvgre_hdr nvgre;
+ struct sxe2_geneve_hdr geneve;
+};
+
+enum sxe2_flow_sw_direct_type {
+ SXE2_FLOW_SW_DIRECT_TX,
+ SXE2_FLOW_SW_DIRECT_RX,
+ SXE2_FLOW_SW_DIRECT_MAX,
+};
+enum sxe2_flow_sw_pattern_type {
+ SXE2_FLOW_SW_PATTERN_ONLY,
+ SXE2_FLOW_SW_PATTERN_LAST,
+ SXE2_FLOW_SW_PATTERN_FIRST,
+ SXE2_FLOW_SW_PATTERN_MAX,
+};
+
+enum sxe2_flow_tunnel_type {
+ SXE2_FLOW_TUNNEL_TYPE_NONE,
+ SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ SXE2_FLOW_TUNNEL_TYPE_VXLAN,
+ SXE2_FLOW_TUNNEL_TYPE_GTPU,
+ SXE2_FLOW_TUNNEL_TYPE_GENEVE,
+ SXE2_FLOW_TUNNEL_TYPE_GRE,
+ SXE2_FLOW_TUNNEL_TYPE_IPIP,
+};
+
+struct sxe2_flow_meta {
+ uint8_t switch_pattern_dup_allow;
+ uint8_t switch_src_direct;
+ uint16_t flow_src_vsi;
+ uint16_t flow_rule_vsi;
+ uint32_t flow_prio;
+ uint16_t flow_type;
+ uint8_t tunnel_type;
+ uint8_t rsv;
+};
+
+struct sxe2_flow_pattern {
+ DECLARE_BITMAP(hdrs, SXE2_FLOW_HDR_MAX);
+ DECLARE_BITMAP(map_spec, SXE2_FLOW_FLD_ID_MAX);
+ DECLARE_BITMAP(map_mask, SXE2_FLOW_FLD_ID_MAX);
+ struct sxe2_flow_item item_spec;
+ struct sxe2_flow_item item_mask;
+ uint64_t rss_type_allow;
+};
+
+struct sxe2_flow_action {
+ DECLARE_BITMAP(act_types, SXE2_FLOW_ACTION_MAX);
+ struct sxe2_flow_action_rss rss;
+ struct sxe2_flow_action_queue queue;
+ struct sxe2_flow_action_queue_region q_region;
+ struct sxe2_flow_action_passthru passthru;
+ struct sxe2_flow_action_vsi vsi;
+ struct sxe2_flow_action_vsi_list vsi_list;
+ struct sxe2_flow_action_mark mark;
+ struct sxe2_flow_action_count count;
+};
+#endif /* __SXE2_FLOW_PUBLIC_H__ */
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 8ff74e5233..dfd31bfc97 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -62,6 +62,7 @@ sources += files(
'sxe2_txrx_vec.c',
'sxe2_mac.c',
'sxe2_filter.c',
+ 'sxe2_rss.c',
)
allow_internal_get_api = true
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index 1fa9ad718e..b997e7b044 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -541,3 +541,176 @@ int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on)
return ret;
}
+
+int32_t sxe2_drv_rss_key_set(struct sxe2_adapter *adapter, uint8_t *key, uint16_t key_size)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_key_req *req = NULL;
+ int32_t ret = 0;
+ uint16_t buf_size = sizeof(*req) + key_size;
+
+ req = rte_zmalloc("drv_cmd_rss_key", buf_size, 0);
+ if (!req) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to alloc rss key");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ req->vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req->key_size = rte_cpu_to_le_16(key_size);
+ rte_memcpy(req->key, key, key_size);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_KEY_SET,
+ req, buf_size, NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd set rss key, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ if (req) {
+ rte_free(req);
+ req = NULL;
+ }
+ return ret;
+}
+
+int32_t sxe2_drv_rss_lut_set(struct sxe2_adapter *adapter, uint8_t *lut, uint16_t lut_size)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_lut_req *req = NULL;
+ int32_t ret = 0;
+ uint16_t buf_size = sizeof(struct sxe2_rss_lut_req) + lut_size;
+
+ req = rte_zmalloc("drv_cmd_rss_lut", buf_size, 0);
+ if (!req) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to alloc rss lut");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ req->vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req->lut_size = rte_cpu_to_le_16(lut_size);
+ rte_memcpy(req->lut, lut, lut_size);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_LUT_SET,
+ req, buf_size, NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd set rss lut, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ if (req) {
+ rte_free(req);
+ req = NULL;
+ }
+ return ret;
+}
+
+int32_t sxe2_drv_rss_hash_ctrl_func(struct sxe2_adapter *adapter, enum sxe2_rss_hash_key_func func)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_func_req req = {0};
+ int32_t ret = 0;
+
+ req.vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req.func = func;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_FUNC_SET,
+ &req, sizeof(req), NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd set rss func, ret=%d", ret);
+ return ret;
+}
+
+static void sxe2_drv_flow_bitmap_fill(uint32_t *bitmap, uint16_t *list)
+{
+ uint16_t index = 0;
+ uint16_t i = 0;
+ uint16_t map_size = sizeof(*bitmap) * SXE2_BITS_PER_BYTE;
+
+ while (list[i] != SXE2_FLOW_END) {
+ index = list[i] / map_size;
+ bitmap[index] |= (1UL << (list[i] % map_size));
+ i++;
+ }
+}
+
+int32_t sxe2_drv_rss_hf_add(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_hf_req req = {0};
+ int32_t ret = 0;
+
+ req.vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req.symm = rss_conf->symm;
+ req.hdr_type = rte_cpu_to_le_32(SXE2_RSS_OUTER_HEADERS);
+ sxe2_drv_flow_bitmap_fill(req.headers, rss_conf->hdrs);
+ sxe2_drv_flow_bitmap_fill(req.hash_flds, rss_conf->flds);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_ADD,
+ &req, sizeof(req), NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd add rss hf, ret=%d", ret);
+ return ret;
+}
+
+int32_t sxe2_drv_rss_hf_del(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_hf_req req = {0};
+ int32_t ret = 0;
+
+ req.vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req.symm = rss_conf->symm;
+ req.hdr_type = rte_cpu_to_le_32(SXE2_RSS_OUTER_HEADERS);
+ sxe2_drv_flow_bitmap_fill(req.headers, rss_conf->hdrs);
+ sxe2_drv_flow_bitmap_fill(req.hash_flds, rss_conf->flds);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_DEL,
+ &req, sizeof(req), NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd del rss hf, ret=%d", ret);
+ return ret;
+}
+
+int32_t sxe2_drv_rss_hf_clear(struct sxe2_adapter *adapter)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ int32_t ret = 0;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_CLEAR,
+ NULL, 0, NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd clear rss hf, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue *rxq)
+{
+ (void)adapter;
+ (void)rxq;
+ return 0;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index fb01c41aad..3274db6551 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -53,4 +53,20 @@ int32_t sxe2_drv_vlan_insert_strip_cfg(struct sxe2_adapter *adapter);
int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on);
+int32_t sxe2_drv_rss_key_set(struct sxe2_adapter *adapter, uint8_t *key, uint16_t key_size);
+
+int32_t sxe2_drv_rss_lut_set(struct sxe2_adapter *adapter, uint8_t *lut, uint16_t lut_size);
+
+int32_t sxe2_drv_rss_hash_ctrl_func(struct sxe2_adapter *adapter, enum sxe2_rss_hash_key_func func);
+
+int32_t sxe2_drv_rss_hf_add(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf);
+
+int32_t sxe2_drv_rss_hf_del(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf);
+
+int32_t sxe2_drv_rss_hf_clear(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue *rxq);
+
#endif /* __SXE2_CMD_CHNL_H__ */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index f4d8069edb..93e83c3181 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -6,6 +6,7 @@
#define __SXE2_DRV_CMD_H__
#include "sxe2_osal.h"
+#include "sxe2_flow_public.h"
#define SXE2_DRV_CMD_MODULE_S (16)
#define SXE2_MK_DRV_CMD(module, cmd) (((module) << SXE2_DRV_CMD_MODULE_S) | ((cmd) & 0xFFFF))
@@ -320,6 +321,34 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_vlan_filter_switch_req {
uint8_t rsv;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_key_req {
+ uint16_t vsi_id;
+ uint16_t key_size;
+ uint8_t key[];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_lut_req {
+ uint16_t vsi_id;
+ uint16_t lut_size;
+ uint8_t lut[];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_func_req {
+ uint16_t vsi_id;
+ uint8_t func;
+ uint8_t rsv[1];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_hf_req {
+ uint16_t vsi_id;
+ uint8_t rsv[2];
+ uint32_t headers[BITS_TO_U32(SXE2_FLOW_HDR_MAX)];
+ uint32_t hash_flds[BITS_TO_U32(SXE2_FLOW_FLD_ID_MAX)];
+ uint32_t hdr_type;
+ uint8_t symm;
+ uint8_t rsv1[3];
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 9b117f097e..d48841b8e4 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -125,6 +125,11 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.vlan_filter_set = sxe2_dev_vlan_filter_set,
.vlan_offload_set = sxe2_dev_vlan_offload_set,
+
+ .reta_update = sxe2_dev_rss_reta_update,
+ .reta_query = sxe2_dev_rss_reta_query,
+ .rss_hash_update = sxe2_dev_rss_hash_update,
+ .rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -141,6 +146,12 @@ static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
goto end;
}
+ ret = sxe2_rss_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init rss, ret=%d", ret);
+ goto end;
+ }
+
end:
return ret;
}
@@ -281,6 +292,22 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO |
RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO;
+
+ if (adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_PTP)
+ dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TIMESTAMP;
+
+ if (adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_RSS) {
+ dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
+ dev_info->flow_type_rss_offloads |= SXE2_RSS_HF_SUPPORT_ALL;
+ dev_info->reta_size = adapter->rss_ctxt.rss_lut_size;
+ dev_info->hash_key_size = adapter->rss_ctxt.rss_key_size;
+ dev_info->rss_algo_capa =
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_DEFAULT) |
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_TOEPLITZ) |
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) |
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_SIMPLE_XOR);
+ }
+
dev_info->default_rxconf = (struct rte_eth_rxconf) {
.rx_thresh = {
.pthresh = SXE2_DEFAULT_RX_PTHRESH,
@@ -563,6 +590,8 @@ static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
sxe2_sw_queue_ctx_hw_cap_set(adapter, &dev_caps.queue_caps);
+ sxe2_sw_rss_ctx_hw_cap_set(adapter, &dev_caps.rss_hash_caps);
+
sxe2_sw_vsi_ctx_hw_cap_set(adapter, &dev_caps.vsi_caps);
l_end:
@@ -950,8 +979,15 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_eth_err;
}
+ ret = sxe2_rss_disable(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to disable rss, ret=%d", ret);
+ goto init_rss_err;
+ }
+
goto l_end;
+init_rss_err:
init_eth_err:
init_dev_info_err:
sxe2_vsi_uninit(dev);
@@ -965,6 +1001,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
{
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
+ (void)sxe2_rss_disable(dev);
sxe2_vsi_uninit(dev);
sxe2_dev_pci_map_uinit(dev);
sxe2_eth_uinit(dev);
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index cc8a84c0a0..556a11cc77 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -13,6 +13,7 @@
#include "sxe2_common.h"
#include "sxe2_vsi.h"
+#include "sxe2_rss.h"
#include "sxe2_irq.h"
#include "sxe2_queue.h"
#include "sxe2_mac.h"
@@ -120,6 +121,11 @@ enum {
SXE2_FLAGS_NBITS
};
+struct sxe2_ptp_context {
+ uint64_t mbuf_rx_ts_flag;
+ int32_t mbuf_rx_ts_offset;
+};
+
struct sxe2_devargs {
uint8_t flow_dup_pattern_mode;
uint8_t func_flow_direct_en;
@@ -298,7 +304,9 @@ struct sxe2_adapter {
struct sxe2_queue_context q_ctxt;
struct sxe2_vsi_context vsi_ctxt;
struct sxe2_filter_context filter_ctxt;
+ struct sxe2_rss_context rss_ctxt;
struct sxe2_link_context link_ctxt;
+ struct sxe2_ptp_context ptp_ctxt;
struct sxe2_devargs devargs;
struct sxe2_switchdev_info switchdev_info;
bool rule_started;
diff --git a/drivers/net/sxe2/sxe2_flow_define.h b/drivers/net/sxe2/sxe2_flow_define.h
new file mode 100644
index 0000000000..d2f6000efa
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_define.h
@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FLOW_DEFINE_H__
+#define __SXE2_FLOW_DEFINE_H__
+#include <rte_tailq.h>
+#include <rte_eal.h>
+#include <rte_flow_driver.h>
+
+#include "sxe2_osal.h"
+#include "sxe2_flow_public.h"
+
+#define SXE2_FLOW_ETH_TYPE_MIN (1500)
+
+enum sxe2_expansion {
+ SXE2_EXPANSION_ERROR = 0,
+ SXE2_EXPANSION_OUTER_ETH,
+ SXE2_EXPANSION_OUTER_VLAN,
+ SXE2_EXPANSION_OUTER_QINQ,
+ SXE2_EXPANSION_OUTER_IPV4,
+ SXE2_EXPANSION_OUTER_IPV4_FRAG_EXT,
+ SXE2_EXPANSION_OUTER_IPV6,
+ SXE2_EXPANSION_OUTER_IPV6_FRAG_EXT,
+ SXE2_EXPANSION_OUTER_UDP,
+ SXE2_EXPANSION_OUTER_TCP,
+ SXE2_EXPANSION_OUTER_SCTP,
+
+ SXE2_EXPANSION_VXLAN,
+ SXE2_EXPANSION_VXLAN_GPE,
+ SXE2_EXPANSION_GRE,
+ SXE2_EXPANSION_NVGRE,
+ SXE2_EXPANSION_GENEVE,
+ SXE2_EXPANSION_GTPU,
+ SXE2_EXPANSION_IPIP,
+ SXE2_EXPANSION_OUTER_END,
+
+ SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_VLAN,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV4_FRAG_EXT,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_IPV6_FRAG_EXT,
+ SXE2_EXPANSION_UDP,
+ SXE2_EXPANSION_TCP,
+ SXE2_EXPANSION_SCTP,
+
+ SXE2_EXPANSION_END,
+ SXE2_EXPANSION_MAX,
+};
+
+enum sxe2_flow_udp_tunnel_protocol {
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN_GPE,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GENEVE,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GTP_U,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_NVGRE,
+ SXE2_FLOW_UDP_TUNNEL_MAX,
+};
+
+enum {
+ SXE2_FLOW_ETH_TYPE_IPV4 = 0x0800,
+ SXE2_FLOW_ETH_TYPE_IPV6 = 0x86DD,
+ SXE2_FLOW_IP_PROTOCOL_GRE = 0x2F,
+ SXE2_FLOW_IP_PROTOCOL_IPV4 = 0x04,
+ SXE2_FLOW_IP_PROTOCOL_IPV6 = 0x29,
+ SXE2_FLOW_IP_PROTOCOL_ETH = 0x3B,
+ SXE2_FLOW_IP_PROTOCOL_UDP = 0x11,
+ SXE2_FLOW_IP_PROTOCOL_TCP = 0x06,
+ SXE2_FLOW_IP_PROTOCOL_SCTP = 0x84,
+};
+
+union sxe2_flow_item_raw {
+ struct sxe2_flow_item item;
+ uint8_t raw[sizeof(struct sxe2_flow_item)];
+};
+
+struct sxe2_flow {
+ TAILQ_ENTRY(sxe2_flow) next;
+ enum sxe2_flow_engine_type engine_type;
+ struct sxe2_flow_pattern pattern_outer;
+ struct sxe2_flow_pattern pattern_inner;
+ uint8_t has_mask;
+ uint8_t has_spec;
+ uint8_t has_hdr;
+ struct sxe2_flow_meta meta;
+ struct sxe2_flow_action action;
+ uint32_t flow_id;
+ int32_t create_err;
+ DECLARE_BITMAP(flow_type, SXE2_EXPANSION_MAX);
+};
+
+TAILQ_HEAD(sxe2_flow_list_t, sxe2_flow);
+
+struct rte_flow {
+ TAILQ_ENTRY(rte_flow) next;
+ struct sxe2_flow_list_t sxe2_flow_list;
+};
+TAILQ_HEAD(rte_flow_list_t, rte_flow);
+
+struct sxe2_fnav_cid_mgr {
+ TAILQ_ENTRY(sxe2_fnav_cid_mgr) next;
+ uint16_t stat_index;
+ uint32_t user_id;
+ uint32_t driver_id;
+ uint32_t count_type;
+ uint64_t hits;
+ uint64_t bytes;
+};
+TAILQ_HEAD(sxe2_fnav_cid_mgr_list_t, sxe2_fnav_cid_mgr);
+
+struct sxe2_fnav_count_resource {
+ uint32_t count_type;
+ uint32_t global_index;
+ struct sxe2_fnav_cid_mgr_list_t fnav_cid_mgr_list;
+};
+
+struct sxe2_flow_context {
+ struct rte_flow_list_t rte_flow_list;
+ rte_spinlock_t flow_list_lock;
+ struct sxe2_fnav_count_resource hw_res;
+ uint32_t fnav_inited;
+};
+#define SXE2_INVALID_RSS_ATTR \
+ (RTE_ETH_RSS_L3_PRE40 | RTE_ETH_RSS_L3_PRE56 | RTE_ETH_RSS_L3_PRE96)
+#define SXE2_VALID_RSS_IPV4_L4 \
+ (RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_NONFRAG_IPV4_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_SCTP)
+
+#define SXE2_VALID_RSS_IPV6_L4 \
+ (RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_NONFRAG_IPV6_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP)
+#define SXE2_VALID_RSS_IPV4 \
+ (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | \
+ RTE_ETH_RSS_NONFRAG_IPV4_OTHER | SXE2_VALID_RSS_IPV4_L4)
+#define SXE2_VALID_RSS_IPV6 \
+ (RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | \
+ RTE_ETH_RSS_NONFRAG_IPV6_OTHER | SXE2_VALID_RSS_IPV6_L4)
+
+#define SXE2_VALID_RSS_L3 (SXE2_VALID_RSS_IPV4 | SXE2_VALID_RSS_IPV6)
+#define SXE2_VALID_RSS_L4 (SXE2_VALID_RSS_IPV4_L4 | SXE2_VALID_RSS_IPV6_L4)
+
+#endif /* __SXE2_FLOW_DEFINE_H__ */
diff --git a/drivers/net/sxe2/sxe2_queue.c b/drivers/net/sxe2/sxe2_queue.c
index 1786d6ea4f..220cab6fce 100644
--- a/drivers/net/sxe2/sxe2_queue.c
+++ b/drivers/net/sxe2/sxe2_queue.c
@@ -17,6 +17,7 @@ void sxe2_sw_queue_ctx_hw_cap_set(struct sxe2_adapter *adapter,
int32_t sxe2_queues_init(struct rte_eth_dev *dev)
{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
int32_t ret = 0;
uint16_t buf_size;
uint16_t frame_size;
@@ -36,6 +37,16 @@ int32_t sxe2_queues_init(struct rte_eth_dev *dev)
dev->data->scattered_rx = 1;
}
+ adapter->ptp_ctxt.mbuf_rx_ts_offset = -1;
+ adapter->ptp_ctxt.mbuf_rx_ts_flag = 0;
+ if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
+ ret = rte_mbuf_dyn_rx_timestamp_register
+ (&adapter->ptp_ctxt.mbuf_rx_ts_offset,
+ (uint64_t *)&adapter->ptp_ctxt.mbuf_rx_ts_flag);
+ if (ret)
+ PMD_LOG_ERR(INIT, "Failed to enable timestamp offloads, ret=%d", ret);
+ }
+
return ret;
}
--git a/drivers/net/sxe2/sxe2_rss.c b/drivers/net/sxe2/sxe2_rss.c
new file mode 100644
index 0000000000..1d56613043
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_rss.c
@@ -0,0 +1,584 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include "sxe2_rss.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+
+void sxe2_sw_rss_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_rss_hash_caps *rss_caps)
+{
+ adapter->rss_ctxt.rss_key_size = rss_caps->hash_key_size;
+ adapter->rss_ctxt.rss_lut_size = rss_caps->lut_key_size;
+}
+
+int32_t sxe2_rss_hash_key_init(struct rte_eth_dev *dev)
+{
+ struct rte_eth_rss_conf *rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i = 0;
+
+ if (rss_ctxt->rss_key == NULL) {
+ rss_ctxt->rss_key = (uint8_t *)rte_zmalloc("rss_key", rss_ctxt->rss_key_size, 0);
+ if (rss_ctxt->rss_key == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to allocate rss key");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ }
+
+ if (!rss_conf->rss_key) {
+ for (i = 0; i < rss_ctxt->rss_key_size; i++)
+ rss_ctxt->rss_key[i] = (uint8_t)rte_rand();
+ } else {
+ rte_memcpy(rss_ctxt->rss_key, rss_conf->rss_key,
+ RTE_MIN(rss_conf->rss_key_len, rss_ctxt->rss_key_size));
+ }
+
+ ret = sxe2_drv_rss_key_set(adapter, rss_ctxt->rss_key,
+ rss_ctxt->rss_key_size);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss key, ret:%d", ret);
+ rte_free(rss_ctxt->rss_key);
+ rss_ctxt->rss_key = NULL;
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+void sxe2_rss_hash_key_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+
+ if (rss_ctxt->rss_key) {
+ rte_free(rss_ctxt->rss_key);
+ rss_ctxt->rss_key = NULL;
+ }
+}
+
+int32_t sxe2_rss_lut_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i;
+
+ if (rss_ctxt->rss_lut == NULL) {
+ rss_ctxt->rss_lut = (uint8_t *)rte_zmalloc("rss_lut", rss_ctxt->rss_lut_size, 0);
+ if (rss_ctxt->rss_lut == NULL) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to allocate rss lut");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ }
+
+ for (i = 0; i < rss_ctxt->rss_lut_size; i++)
+ rss_ctxt->rss_lut[i] = (uint8_t)(i % dev->data->nb_rx_queues);
+
+ ret = sxe2_drv_rss_lut_set(adapter, rss_ctxt->rss_lut, rss_ctxt->rss_lut_size);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss lut, ret:%d", ret);
+ rte_free(rss_ctxt->rss_lut);
+ rss_ctxt->rss_lut = NULL;
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+void sxe2_rss_lut_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+
+ if (rss_ctxt->rss_lut) {
+ rte_free(rss_ctxt->rss_lut);
+ rss_ctxt->rss_lut = NULL;
+ }
+}
+
+static struct sxe2_rss_hf_config sxe2_rss_default_hf_config[] = {
+ {
+ .rss_hf = RTE_ETH_RSS_L2_PAYLOAD,
+ .hdrs = {SXE2_FLOW_HDR_ETH,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_ETH_TYPE,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_IPV4,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_IPV6,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_FRAG_IPV4,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_IPV_FRAG,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_FRAG_IPV6,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_IPV_FRAG,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_IPV_OTHER,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_IPV_OTHER,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_UDP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_UDP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_UDP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_UDP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_UDP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_UDP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_TCP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_TCP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_TCP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_TCP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_TCP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_TCP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_SCTP,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_SCTP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_SCTP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_SCTP,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_SCTP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_SCTP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+};
+
+int32_t sxe2_rss_hf_type_set(struct rte_eth_dev *dev, uint64_t rss_hf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint32_t i;
+ uint8_t symm = 0;
+
+ if (0 == (rss_hf & SXE2_RSS_HF_SUPPORT_ALL) && rss_hf != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to set unsupported rss_hf:0x%016" PRIx64,
+ rss_hf);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < RTE_DIM(sxe2_rss_default_hf_config); i++) {
+ if (rss_ctxt->rss_hf & sxe2_rss_default_hf_config[i].rss_hf) {
+ sxe2_rss_default_hf_config[i].symm = rss_ctxt->symm;
+ ret = sxe2_drv_rss_hf_del(adapter, &sxe2_rss_default_hf_config[i]);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT,
+ "Failed to del rss hf cfg[%d], ret:%d", i, ret);
+ goto l_end;
+ }
+ }
+ }
+
+ if (rss_ctxt->hash_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
+ symm = 1;
+
+ for (i = 0; i < RTE_DIM(sxe2_rss_default_hf_config); i++) {
+ if (rss_hf & sxe2_rss_default_hf_config[i].rss_hf) {
+ sxe2_rss_default_hf_config[i].symm = symm;
+ ret = sxe2_drv_rss_hf_add(adapter, &sxe2_rss_default_hf_config[i]);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT,
+ "Failed to add rss hf cfg[%d], ret:%d", i, ret);
+ goto l_end;
+ }
+ }
+ }
+
+ rss_ctxt->rss_hf = rss_hf & SXE2_RSS_HF_SUPPORT_ALL;
+ rss_ctxt->symm = symm;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_rss_hash_function_set(struct rte_eth_dev *dev, enum rte_eth_hash_function func)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ enum sxe2_rss_hash_key_func hash_func = SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ;
+ int32_t ret = 0;
+
+ switch (func) {
+ case RTE_ETH_HASH_FUNCTION_DEFAULT:
+ case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
+ case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ:
+ hash_func = SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ;
+ break;
+ case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR:
+ hash_func = SXE2_RSS_HASH_FUNC_XOR;
+ break;
+ default:
+ PMD_DEV_LOG_ERR(adapter, DRV, "RSS hash function[%d] not support.", func);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_drv_rss_hash_ctrl_func(adapter, hash_func);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss hash function, ret=[%d]", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_rss_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_eth_rss_conf *rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
+ enum rte_eth_hash_function rss_func = RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
+ int32_t ret = 0;
+
+ adapter->rss_ctxt.inited = false;
+
+ if (dev->data->nb_rx_queues <= 1) {
+ PMD_DEV_LOG_DEBUG(adapter, INIT, "No need to init rss, rx queues %d.",
+ dev->data->nb_rx_queues);
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_RSS) == 0) {
+ PMD_DEV_LOG_WARN(adapter, INIT, "RSS not supported");
+ goto l_end;
+ }
+
+ ret = sxe2_rss_hash_key_init(dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init rss key");
+ goto l_end;
+ }
+
+ ret = sxe2_rss_lut_init(dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init rss lut");
+ goto l_err_key;
+ }
+
+ rss_func = rss_conf->algorithm;
+ ret = sxe2_rss_hash_function_set(dev, rss_func);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init rss hash function");
+ goto l_err_lut;
+ }
+ ret = sxe2_rss_hf_type_set(dev, rss_conf->rss_hf);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss hf type");
+ goto l_err_lut;
+ }
+ adapter->rss_ctxt.inited = true;
+ goto l_end;
+
+l_err_lut:
+ sxe2_rss_lut_uninit(dev);
+l_err_key:
+ sxe2_rss_hash_key_uninit(dev);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_rss_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_RSS) == 0)
+ goto l_end;
+
+ ret = sxe2_drv_rss_hf_clear(adapter);
+ if (ret)
+ PMD_LOG_ERR(INIT, "Failed to clear rss hf");
+
+ sxe2_rss_hash_key_uninit(dev);
+
+ sxe2_rss_lut_uninit(dev);
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_rss_reta_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ uint8_t *lut_tmp = NULL;
+ int32_t ret = 0;
+ uint16_t i;
+ uint16_t shift;
+ uint16_t idx;
+
+ if (!adapter->rss_ctxt.inited) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (reta_size != rss_ctxt->rss_lut_size) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "The size of hash lookup table configured "
+ "(%d) doesn't match the number of hardware can "
+ "support (%d)", reta_size, rss_ctxt->rss_lut_size);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ lut_tmp = rte_zmalloc("rss_lut_temp", reta_size, 0);
+ if (!lut_tmp) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "No memory can be allocated");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ rte_memcpy(lut_tmp, rss_ctxt->rss_lut, reta_size);
+
+ for (i = 0; i < reta_size; i++) {
+ idx = i / RTE_ETH_RETA_GROUP_SIZE;
+ shift = i % RTE_ETH_RETA_GROUP_SIZE;
+ if (reta_conf[idx].mask & (1ULL << shift))
+ lut_tmp[i] = reta_conf[idx].reta[shift];
+ }
+
+ ret = sxe2_drv_rss_lut_set(adapter, lut_tmp, reta_size);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss lut");
+ goto l_end;
+ }
+
+ rte_memcpy(rss_ctxt->rss_lut, lut_tmp, reta_size);
+
+l_end:
+ if (lut_tmp)
+ rte_free(lut_tmp);
+ return ret;
+}
+
+int32_t sxe2_dev_rss_reta_query(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i;
+ uint16_t shift;
+ uint16_t idx;
+
+ if (!adapter->rss_ctxt.inited) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (reta_size != rss_ctxt->rss_lut_size) {
+ PMD_LOG_ERR(INIT, "The size of hash lookup table configured "
+ "(%d) doesn't match the number of hardware can "
+ "support (%d)", reta_size, rss_ctxt->rss_lut_size);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < reta_size; i++) {
+ idx = i / RTE_ETH_RETA_GROUP_SIZE;
+ shift = i % RTE_ETH_RETA_GROUP_SIZE;
+ if (reta_conf[idx].mask & (1ULL << shift))
+ reta_conf[idx].reta[shift] = rss_ctxt->rss_lut[i];
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_rss_hash_key_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i;
+
+ if (rss_conf->rss_key_len == 0 || rss_conf->rss_key == NULL)
+ goto l_end;
+
+ if (rss_conf->rss_key_len != rss_ctxt->rss_key_size) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "The size of hash key configured "
+ "(%d) doesn't match the size of hardware can "
+ "support (%d)", rss_conf->rss_key_len,
+ rss_ctxt->rss_key_size);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < rss_conf->rss_key_len; i++) {
+ if (rss_conf->rss_key[i] != rss_ctxt->rss_key[i])
+ break;
+ }
+ if (i == rss_conf->rss_key_len)
+ goto l_end;
+
+ ret = sxe2_drv_rss_key_set(adapter, rss_conf->rss_key,
+ rss_conf->rss_key_len);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss key");
+ goto l_end;
+ }
+
+ rte_memcpy(rss_ctxt->rss_key, rss_conf->rss_key, rss_conf->rss_key_len);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_rss_hash_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = -1;
+
+ if (!adapter->rss_ctxt.inited) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ ret = sxe2_rss_hash_key_update(dev, rss_conf);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss hash key");
+ goto l_end;
+ }
+
+ if (rss_conf->algorithm != rss_ctxt->hash_func) {
+ ret = sxe2_rss_hash_function_set(dev, rss_conf->algorithm);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss hash function");
+ goto l_end;
+ }
+ rss_ctxt->hash_func = rss_conf->algorithm;
+ }
+
+ if ((rss_conf->rss_hf & SXE2_RSS_HF_SUPPORT_ALL)
+ != rss_ctxt->rss_hf) {
+ ret = sxe2_rss_hf_type_set(dev, rss_conf->rss_hf);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss hf type");
+ goto l_end;
+ }
+ }
+ ret = 0;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+
+ if (adapter->rss_ctxt.inited == 0) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (rss_conf->rss_key) {
+ rss_conf->rss_key_len = rss_ctxt->rss_key_size;
+ rte_memcpy(rss_conf->rss_key, rss_ctxt->rss_key, rss_ctxt->rss_key_size);
+ }
+ rss_conf->rss_hf = rss_ctxt->rss_hf;
+ rss_conf->algorithm = rss_ctxt->hash_func;
+l_end:
+ return ret;
+}
--git a/drivers/net/sxe2/sxe2_rss.h b/drivers/net/sxe2/sxe2_rss.h
new file mode 100644
index 0000000000..2a454ac1b3
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_rss.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_RSS_H__
+#define __SXE2_RSS_H__
+#include <ethdev_driver.h>
+#include "sxe2_osal.h"
+#include "sxe2_drv_cmd.h"
+#include "sxe2_flow_define.h"
+
+#define SXE2_FLOW_END (0xFFFF)
+
+struct sxe2_rss_context {
+ enum rte_eth_hash_function hash_func;
+ uint16_t rss_key_size;
+ uint16_t rss_lut_size;
+ uint8_t *rss_key;
+ uint8_t *rss_lut;
+ uint64_t rss_hf;
+ uint8_t symm;
+ bool inited;
+};
+
+struct sxe2_rss_hf_config {
+ uint64_t rss_hf;
+ uint16_t hdrs[SXE2_FLOW_HDR_MAX];
+ uint16_t flds[SXE2_FLOW_FLD_ID_MAX];
+ uint8_t symm;
+};
+
+#define SXE2_RSS_HF_SUPPORT_ALL ( \
+ RTE_ETH_RSS_IPV4 | \
+ RTE_ETH_RSS_FRAG_IPV4 | \
+ RTE_ETH_RSS_NONFRAG_IPV4_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_UDP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_SCTP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_OTHER | \
+ RTE_ETH_RSS_IPV6 | \
+ RTE_ETH_RSS_FRAG_IPV6 | \
+ RTE_ETH_RSS_NONFRAG_IPV6_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_UDP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_OTHER | \
+ RTE_ETH_RSS_L2_PAYLOAD)
+
+struct sxe2_adapter;
+
+void sxe2_sw_rss_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_rss_hash_caps *rss_caps);
+
+int32_t sxe2_rss_hash_key_init(struct rte_eth_dev *dev);
+
+void sxe2_rss_hash_key_uninit(struct rte_eth_dev *dev);
+
+int32_t sxe2_rss_lut_init(struct rte_eth_dev *dev);
+
+void sxe2_rss_lut_uninit(struct rte_eth_dev *dev);
+
+int32_t sxe2_rss_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_rss_hash_function_set(struct rte_eth_dev *dev, enum rte_eth_hash_function func);
+
+int32_t sxe2_rss_hf_type_set(struct rte_eth_dev *dev, uint64_t rss_hf);
+
+int32_t sxe2_rss_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_dev_rss_reta_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size);
+
+int32_t sxe2_dev_rss_reta_query(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size);
+
+int32_t sxe2_dev_rss_hash_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf);
+
+int32_t sxe2_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf);
+#endif /* __SXE2_RSS_H__ */
diff --git a/drivers/net/sxe2/sxe2_txrx.h b/drivers/net/sxe2/sxe2_txrx.h
index 6d3d7455c2..a642a077bf 100644
--- a/drivers/net/sxe2/sxe2_txrx.h
+++ b/drivers/net/sxe2/sxe2_txrx.h
@@ -20,4 +20,8 @@ int32_t sxe2_tx_burst_mode_get(struct rte_eth_dev *dev,
__rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode);
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
__rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode);
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+int32_t sxe2_rx_update_ptp_time(struct sxe2_rx_queue *rxq);
+#endif
+
#endif /* __SXE2_TXRX_H__ */
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index 21d5c38725..3c6fe37404 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -17,6 +17,7 @@
#include "sxe2_queue.h"
#include "sxe2_ethdev.h"
#include "sxe2_common_log.h"
+#include "sxe2_cmd_chnl.h"
static __rte_always_inline int32_t
sxe2_tx_bufs_free(struct sxe2_tx_queue *txq)
@@ -282,6 +283,30 @@ sxe2_tx_desc_checksum_fill(uint64_t offloads, uint32_t *desc_cmd, uint32_t *desc
return;
}
+static __rte_always_inline void sxe2_desc_tso_fill(struct rte_mbuf *tx_pkt,
+ uint64_t *desc_type_cmd_tso_mss,
+ union sxe2_tx_offload_info ol_info)
+{
+ uint32_t hdr_len;
+ uint32_t tso_len;
+
+ if (!ol_info.l4_len) {
+ PMD_LOG_DEBUG(TX, "TSO ERROR: L4 length is 0");
+ goto l_end;
+ }
+ hdr_len = ol_info.l2_len + ol_info.l3_len + ol_info.l4_len;
+ if (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
+ hdr_len += ol_info.outer_l2_len + ol_info.outer_l3_len;
+
+ tso_len = tx_pkt->pkt_len - hdr_len;
+ *desc_type_cmd_tso_mss |=
+ ((uint64_t)SXE2_TX_CTXT_DESC_CMD_TSO << SXE2_TX_CTXT_DESC_CMD_SHIFT) |
+ ((uint64_t)tso_len << SXE2_TX_CTXT_DESC_TSO_LEN_SHIFT) |
+ ((uint64_t)tx_pkt->tso_segsz << SXE2_TX_CTXT_DESC_MSS_SHIFT);
+l_end:
+ return;
+}
+
static __rte_always_inline uint64_t
sxe2_tx_data_desc_build_cobt(uint32_t cmd, uint32_t offset, uint16_t buf_size, uint16_t l2tag)
{
@@ -395,6 +420,11 @@ uint16_t sxe2_tx_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkt
rte_pktmbuf_free_seg(buffer->mbuf);
buffer->mbuf = NULL;
}
+ if (offloads & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
+ sxe2_desc_tso_fill(tx_pkt,
+ &desc_type_cmd_tso_mss, ol_info);
+ else if (offloads & RTE_MBUF_F_TX_IEEE1588_TMST)
+ desc_type_cmd_tso_mss |= SXE2_TX_CTXT_DESC_CMD_TSYN_MASK;
if (offloads & RTE_MBUF_F_TX_QINQ) {
desc_l2tag2 = tx_pkt->vlan_tci_outer;
@@ -707,6 +737,57 @@ sxe2_rx_desc_filter_para_fill(struct sxe2_rx_queue *rxq __rte_unused,
#endif
}
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+int32_t sxe2_rx_update_ptp_time(struct sxe2_rx_queue *rxq)
+{
+ struct sxe2_adapter *adapter;
+ uint64_t cur_time_ms;
+ int32_t ret = 0;
+ cur_time_ms = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
+
+ if (likely((cur_time_ms - rxq->update_time) < SXE2_RX_PKTS_TS_TIMEOUT_VAL))
+ goto l_end;
+ rxq->update_time = cur_time_ms;
+ adapter = rxq->vsi->adapter;
+ rxq->ts_need_update = true;
+ ret = sxe2_drv_ptp_gettime(adapter, rxq);
+ if (rxq->desc_ts < rxq->ts_low)
+ rxq->ts_need_update = false;
+
+ PMD_LOG_INFO(RX, "rxq update time ret=%d, cur time=%" PRIu64 ", rxqh=%" PRIu64 ", rxql=%d",
+ ret, cur_time_ms, rxq->ts_high, rxq->ts_low);
+l_end:
+ return ret;
+}
+
+static inline void sxe2_rx_desc_ptp_para_fill(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf *mbuf,
+ union sxe2_rx_desc *desc)
+{
+ struct sxe2_adapter *adapter = rxq->vsi->adapter;
+ uint64_t ts_ns;
+
+ if (adapter->ptp_ctxt.mbuf_rx_ts_flag != 0 &&
+ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) &&
+ SXE2_RX_DESC_RXDID_VAL_GET(desc->wb.rxdid_src) == SXE2_RX_DESC_RXDID_1588) {
+ rxq->desc_ts = rte_le_to_cpu_32(desc->wb_ts.ts_h);
+ (void)sxe2_rx_update_ptp_time(rxq);
+ if (rxq->ts_need_update && rxq->desc_ts < rxq->ts_low)
+ rxq->ts_high += 1;
+
+ rxq->ts_need_update = true;
+ rxq->ts_low = rxq->desc_ts;
+ rxq->update_time = rte_get_timer_cycles() /
+ (rte_get_timer_hz() / 1000);
+ ts_ns = rxq->ts_high * NSEC_PER_SEC + rxq->ts_low;
+ *RTE_MBUF_DYNFIELD(mbuf, adapter->ptp_ctxt.mbuf_rx_ts_offset, uint64_t *) = ts_ns;
+ mbuf->ol_flags |= adapter->ptp_ctxt.mbuf_rx_ts_flag;
+ PMD_LOG_INFO(RX, "receive ptp pkt,ts_s=%" PRIu64 ", ts_ns=%d", rxq->ts_high,
+ rxq->ts_low);
+ }
+}
+#endif
+
static __rte_always_inline void
sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf,
union sxe2_rx_desc *rxd)
@@ -718,10 +799,12 @@ sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf
mbuf->ol_flags = 0;
mbuf->packet_type = ptype_tbl[SXE2_RX_DESC_PTYPE_VAL_GET(qword1)];
-
pkt_flags = sxe2_rx_desc_error_para(rxq, rxd);
sxe2_rx_desc_vlan_para_fill(mbuf, rxd);
sxe2_rx_desc_filter_para_fill(rxq, mbuf, rxd);
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ sxe2_rx_desc_ptp_para_fill(rxq, mbuf, rxd);
+#endif
mbuf->ol_flags |= pkt_flags;
}
--
2.47.3
^ permalink raw reply related
* [PATCH v2 03/23] drivers: add supported packet types get callback
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Implement dev_supported_ptypes_get ethdev callback for sxe2 PMD.
This allows applications to query the packet types the driver
is capable of identifying, such as L2, L3 (IPv4/IPv6), and
L4 (TCP/UDP/SCTP) layers.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_ptype.h | 1793 ++++++++++++++++++++++++++++++
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 22 +
drivers/net/sxe2/sxe2_cmd_chnl.h | 2 +
drivers/net/sxe2/sxe2_drv_cmd.h | 6 +
drivers/net/sxe2/sxe2_ethdev.c | 169 ++-
drivers/net/sxe2/sxe2_ethdev.h | 19 +-
drivers/net/sxe2/sxe2_mac.c | 103 ++
drivers/net/sxe2/sxe2_mac.h | 50 +
9 files changed, 2126 insertions(+), 39 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_ptype.h
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
diff --git a/drivers/common/sxe2/sxe2_ptype.h b/drivers/common/sxe2/sxe2_ptype.h
new file mode 100644
index 0000000000..9c19570979
--- /dev/null
+++ b/drivers/common/sxe2/sxe2_ptype.h
@@ -0,0 +1,1793 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef _SXE2_PTYPE_H_
+#define _SXE2_PTYPE_H_
+#include <rte_ethdev.h>
+
+static inline const uint32_t *
+sxe2_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements)
+{
+ const uint32_t *ret = NULL;
+
+ static const uint32_t ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER_TIMESYNC,
+ RTE_PTYPE_L2_ETHER_LLDP,
+ RTE_PTYPE_L2_ETHER_ARP,
+ RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L4_FRAG,
+ RTE_PTYPE_L4_ICMP,
+ RTE_PTYPE_L4_NONFRAG,
+ RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT,
+ RTE_PTYPE_TUNNEL_IP,
+ RTE_PTYPE_INNER_L2_ETHER,
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_INNER_L4_FRAG,
+ RTE_PTYPE_INNER_L4_ICMP,
+ RTE_PTYPE_INNER_L4_NONFRAG,
+ RTE_PTYPE_INNER_L4_SCTP,
+ RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_UNKNOWN
+ };
+
+ if (dev->rx_pkt_burst != NULL) {
+ *no_of_elements = RTE_DIM(ptypes);
+ ret = ptypes;
+ } else {
+ ret = NULL;
+ }
+
+ return ret;
+}
+
+static inline void sxe2_init_ptype_list(uint32_t *ptype)
+{
+ /* ptype[0] reserved */
+ ptype[1] = RTE_PTYPE_L2_ETHER;
+ ptype[2] = RTE_PTYPE_L2_ETHER_TIMESYNC;
+ /* ptype[3] - ptype[5] reserved */
+ ptype[6] = RTE_PTYPE_L2_ETHER_LLDP;
+ /* ECP */
+ ptype[7] = RTE_PTYPE_UNKNOWN;
+ /* ptype[8] - ptype[9] reserved */
+ /* EAPol */
+ ptype[10] = RTE_PTYPE_UNKNOWN;
+ ptype[11] = RTE_PTYPE_L2_ETHER_ARP;
+ /* ptype[12] - ptype[21] reserved */
+
+ /* Non tunneled IPv4 */
+ ptype[22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_FRAG;
+ ptype[23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[25] reserved */
+ ptype[26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_SCTP;
+ ptype[28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_ICMP;
+
+ /* IPv4 --> IPv4 */
+ ptype[29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[32] reserved */
+ ptype[33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> IPv6 */
+ ptype[36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[39] reserved */
+ ptype[40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN */
+ ptype[43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> IPv4 */
+ ptype[44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[47] reserved */
+ ptype[48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> IPv6 */
+ ptype[51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[54] reserved */
+ ptype[55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC */
+ ptype[58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC --> IPv4 */
+ ptype[59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[62] reserved */
+ ptype[63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC --> IPv6 */
+ ptype[66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[69] reserved */
+ ptype[70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC/VLAN */
+ ptype[73] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv4 */
+ ptype[74] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[75] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[76] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[77] reserved */
+ ptype[78] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[79] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv6 */
+ ptype[81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_FRAG;
+ ptype[82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[64] reserved */
+ ptype[85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[87] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+ /* Non tunneled IPv6 */
+ ptype[88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_FRAG;
+ ptype[89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[91] reserved */
+ ptype[92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_SCTP;
+ ptype[94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_ICMP;
+
+ /* IPv6 --> IPv4 */
+ ptype[95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[98] reserved */
+ ptype[99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> IPv6 */
+ ptype[102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[105] reserved */
+ ptype[106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN */
+ ptype[109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> IPv4 */
+ ptype[110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[113] reserved */
+ ptype[114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> IPv6 */
+ ptype[117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[120] reserved */
+ ptype[121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC */
+ ptype[124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC --> IPv4 */
+ ptype[125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[128] reserved */
+ ptype[129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC --> IPv6 */
+ ptype[132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[135] reserved */
+ ptype[136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC/VLAN */
+ ptype[139] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv4 */
+ ptype[140] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[141] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[142] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[143] reserved */
+ ptype[144] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[145] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[146] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv6 */
+ ptype[147] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[148] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[149] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[150] reserved */
+ ptype[151] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[152] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[153] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+ /* ptype[154] - ptype[159] reserved */
+ /* IPSec */
+ ptype[160] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_ESP;
+ ptype[161] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_ESP;
+ /* AH */
+ ptype[162] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[163] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* NAT-T-ESP */
+ ptype[164] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ ptype[165] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ /* SDN-ESP */
+ ptype[166] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ ptype[167] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ /* ptype[168] - ptype[271] reserved */
+ /* IPV4 --> VRRP */
+ ptype[272] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPV4 --> OSPF */
+ ptype[273] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPV6 --> VRRP */
+ ptype[274] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPV6 --> VRRP */
+ ptype[275] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* ATAoE */
+ ptype[276] = RTE_PTYPE_UNKNOWN;
+ /* Control */
+ ptype[278] = RTE_PTYPE_UNKNOWN;
+ /* ptype[279] - ptype[324] reserved */
+ /* GTP */
+ ptype[325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU;
+ ptype[330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU;
+ ptype[331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ ptype[336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ ptype[341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ ptype[346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* PFCP */
+ ptype[351] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[352] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[353] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[354] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[355] - ptype[359] reserved */
+ /* L2TPv3 */
+ ptype[360] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_L2TP;
+ ptype[361] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_L2TP;
+ /* ptype[362] - ptype[370] reserved */
+ /* eCPRI */
+ ptype[371] = RTE_PTYPE_UNKNOWN;
+ ptype[381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[396] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, IGMP */
+ ptype[397] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, EIGRP */
+ ptype[398] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, EIGRP */
+ ptype[399] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, PIM */
+ ptype[400] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, PIM */
+ ptype[401] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, IGMP */
+ ptype[402] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, IGMP */
+ ptype[403] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, EIGRP */
+ ptype[404] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, EIGRP */
+ ptype[405] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, PIM */
+ ptype[406] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, PIM */
+ ptype[407] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, VRRP */
+ ptype[408] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, VRRP */
+ ptype[409] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, VRRP */
+ ptype[410] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, VRRP */
+ ptype[411] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, OSPF */
+ ptype[412] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, OSPF */
+ ptype[413] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, OSPF */
+ ptype[414] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, OSPF */
+ ptype[415] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, L2_TP_V3 */
+ ptype[416] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, L2_TP_V3 */
+ ptype[417] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, L2_TP_V3 */
+ ptype[418] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, L2_TP_V3 */
+ ptype[419] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, AH */
+ ptype[420] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, AH */
+ ptype[421] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, AH */
+ ptype[422] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, AH */
+ ptype[423] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, ESP */
+ ptype[424] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, ESP */
+ ptype[425] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, ESP */
+ ptype[426] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, ESP */
+ ptype[427] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* TP-TUN GTPU */
+ ptype[450] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[451] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[452] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[453] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[454] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[455] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[456] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[457] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[458] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[459] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[460] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[461] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[462] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[463] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[464] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[465] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[466] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[467] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[468] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[469] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[470] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[471] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[472] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[473] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[474] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[475] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[476] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[477] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[478] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[479] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[480] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[481] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[482] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[483] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[484] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[485] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[486] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[487] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[488] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[489] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[490] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[491] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[492] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[493] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[494] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[495] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[496] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[497] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[498] - ptype[767] reserved */
+ /* L2(NETWORK CPU) */
+ /* ISIS */
+ ptype[768] = RTE_PTYPE_UNKNOWN;
+ /* SDF */
+ ptype[769] = RTE_PTYPE_UNKNOWN;
+ /* PPoE_NEGO */
+ ptype[770] = RTE_PTYPE_L2_ETHER_PPPOE;
+ /* PPoE_PROTOCOL */
+ ptype[771] = RTE_PTYPE_L2_ETHER_PPPOE;
+ ptype[772] = RTE_PTYPE_L2_ETHER_PPPOE;
+ /* LACP */
+ ptype[773] = RTE_PTYPE_UNKNOWN;
+ /* ptype[774] - ptype[775] reserved */
+ /* IPv4 L3(NETWORK CPU) */
+ ptype[776] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_IGMP;
+ /* EIGRP */
+ ptype[777] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* PIM */
+ ptype[778] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[779] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[780] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[781] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ /* ptype[782] - ptype[783] reserved */
+ /* IPv6 L3(NETWORK CPU) */
+ ptype[784] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_IGMP;
+ /* EIGRP */
+ ptype[785] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* PIM */
+ ptype[786] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[787] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[788] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[789] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ /* ptype[790] - ptype[791] reserved */
+ /* IPv4 L4(NETWORK CPU) */
+ ptype[792] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[793] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[794] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[795] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[796] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[797] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[798] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[799] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[800] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[801] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[802] - ptype[807] reserved */
+ /* IPv6 L4(NETWORK CPU) */
+ ptype[808] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[809] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[810] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[811] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[812] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[813] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[814] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[815] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[816] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[817] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[818] - ptype[819] reserved */
+ /* IPv6 -> MAC */
+ ptype[820] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC -> IPv4*/
+ ptype[821] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[822] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[823] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[824] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[825] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[826] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC -> IPv4*/
+ ptype[827] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[828] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[829] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[830] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[831] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[832] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* ptype[833] - ptype[834] reserved */
+ /* IPv6 -> MAC/VLAN */
+ ptype[835] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC/VLAN -> IPv4 */
+ ptype[836] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[837] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[838] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[839] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[840] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[841] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC/VLAN -> IPv6 */
+ ptype[842] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[843] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[844] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[845] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[846] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[847] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> PAY */
+ ptype[878] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> IPv4 */
+ ptype[877] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[876] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[879] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[880] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[875] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[874] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> IPv6 */
+ ptype[871] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[870] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[872] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[873] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[869] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[868] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> PAY */
+ ptype[891] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> IPv4 */
+ ptype[890] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[889] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[892] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[893] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[888] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[887] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> IPv6 */
+ ptype[884] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[883] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[885] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[886] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[882] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[881] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> GRE -> PAY */
+ ptype[904] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv6 -> UDP -> GRE -> IPv4 */
+ ptype[903] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[902] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[905] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[906] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[901] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[900] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv6 -> UDP -> GRE -> IPv6 */
+ ptype[897] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[896] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[898] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[899] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[895] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[894] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> GRE -> PAY */
+ ptype[917] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv4 -> UDP -> GRE -> IPv4 */
+ ptype[916] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[915] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[918] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[919] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[914] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[913] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> GRE -> IPv6 */
+ ptype[910] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[909] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[911] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[912] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[908] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[907] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MACVLAN -> PAY */
+ ptype[930] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv4 */
+ ptype[929] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[928] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[931] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[932] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[927] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[926] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv6 */
+ ptype[923] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[922] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[924] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[925] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[921] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[920] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MACVLAN -> PAY */
+ ptype[943] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv4 */
+ ptype[942] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[941] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[944] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[945] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[940] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[939] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv6 */
+ ptype[936] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[935] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[937] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[938] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[934] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[933] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 > UDP -> GRE -> MACVLAN -> PAY */
+ ptype[956] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv6 -> UDP -> GRE -> MACVLAN -> IPv4 */
+ ptype[955] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[954] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[957] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[958] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[953] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[952] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv6 -> UDP -> GRE -> MACVLAN -> IPv6 */
+ ptype[949] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[948] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[950] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[951] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[947] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[946] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> GRE -> MACVLAN -> PAY */
+ ptype[969] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv4 -> UDP -> GRE -> MACVLAN -> IPv4 */
+ ptype[968] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[967] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[970] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[971] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[966] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[965] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> GRE -> MACVLAN -> IPv6 */
+ ptype[962] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[961] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[963] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[964] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[960] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[959] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MAC -> PAY */
+ ptype[982] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MAC -> IPv4 */
+ ptype[981] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[980] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[983] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[984] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[979] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[978] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MAC -> IPv6 */
+ ptype[975] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[974] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[976] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[977] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[973] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[972] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MAC -> PAY */
+ ptype[995] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MAC -> IPv4 */
+ ptype[994] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[993] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[996] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[997] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[992] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[991] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MAC -> IPv6 */
+ ptype[988] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[987] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[989] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[990] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[986] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[985] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 > UDP -> GRE -> MAC -> PAY */
+ ptype[1008] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+ /* IPv6 -> UDP -> GRE -> MAC -> IPv4 */
+ ptype[1007] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1006] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1009] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1010] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[1005] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[1004] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv6 -> UDP -> GRE -> MAC -> IPv6 */
+ ptype[1001] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1000] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1002] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1003] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[999] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[998] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> GRE -> MAC -> PAY */
+ ptype[1021] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+ /* IPv4 -> UDP -> GRE -> MAC -> IPv4 */
+ ptype[1020] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1019] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1022] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1023] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[1018] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[1017] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> GRE -> MAC -> IPv6 */
+ ptype[1014] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1013] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1015] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1016] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[1012] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[1011] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+}
+
+#endif /* _RTE_PTYPE_TUNNEL_GRENAT_H_ */
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 98dd8bcec7..e22204e850 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -60,6 +60,7 @@ sources += files(
'sxe2_txrx_poll.c',
'sxe2_txrx.c',
'sxe2_txrx_vec.c',
+ 'sxe2_mac.c',
)
allow_internal_get_api = true
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index d16b6528d0..07eeb7f38c 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -321,3 +321,25 @@ int32_t sxe2_drv_txq_switch(struct sxe2_adapter *adapter, struct sxe2_tx_queue *
return ret;
}
+
+int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_link_info_resp resp = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_LINK_STATUS_GET,
+ NULL, 0,
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "link status get failed, ret=%d", ret);
+ goto l_end;
+ }
+ adapter->link_ctxt.speed = resp.speed;
+ adapter->link_ctxt.link_up = resp.status;
+
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index cd41cd9e8d..34004d37e2 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -34,4 +34,6 @@ int32_t sxe2_drv_txq_ctxt_cfg(struct sxe2_adapter *adapter,
struct sxe2_tx_queue *txq,
uint16_t txq_cnt);
+int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
+
#endif /* __SXE2_CMD_CHNL_H__ */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index a9b678f3c5..ad1c1c9603 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -227,6 +227,12 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
struct sxe2_drv_msix_caps used_msix;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_link_info_resp {
+ uint32_t speed;
+ uint8_t status;
+ uint8_t rsv[3];
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index e0f7002138..01552a8202 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -27,7 +27,9 @@
#include "sxe2_tx.h"
#include "sxe2_rx.h"
#include "sxe2_txrx.h"
+#include "sxe2_mac.h"
#include "sxe2_common.h"
+#include "sxe2_ptype.h"
#include "sxe2_common_log.h"
#include "sxe2_host_regs.h"
#include "sxe2_ioctl_chnl_func.h"
@@ -78,6 +80,41 @@ static struct sxe2_pci_map_addr_info sxe2_net_map_addr_info_pf[SXE2_PCI_MAP_RES_
.reg_width = 10},
};
+static int32_t sxe2_dev_configure(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_start(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_stop(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_close(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info);
+static const uint32_t *sxe2_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev
+ __rte_unused, size_t *no_of_elements __rte_unused);
+
+static const struct eth_dev_ops sxe2_eth_dev_ops = {
+ .dev_configure = sxe2_dev_configure,
+ .dev_start = sxe2_dev_start,
+ .dev_stop = sxe2_dev_stop,
+ .dev_close = sxe2_dev_close,
+ .dev_infos_get = sxe2_dev_infos_get,
+ .dev_supported_ptypes_get = sxe2_dev_supported_ptypes_get,
+ .link_update = sxe2_link_update,
+
+ .rx_queue_start = sxe2_rx_queue_start,
+ .rx_queue_stop = sxe2_rx_queue_stop,
+ .tx_queue_start = sxe2_tx_queue_start,
+ .tx_queue_stop = sxe2_tx_queue_stop,
+ .rx_queue_setup = sxe2_rx_queue_setup,
+ .rx_queue_release = sxe2_rx_queue_release,
+ .tx_queue_setup = sxe2_tx_queue_setup,
+ .tx_queue_release = sxe2_tx_queue_release,
+ .rxq_info_get = sxe2_rx_queue_info_get,
+ .txq_info_get = sxe2_tx_queue_info_get,
+ .rx_burst_mode_get = sxe2_rx_burst_mode_get,
+ .tx_burst_mode_get = sxe2_tx_burst_mode_get,
+ .tx_done_cleanup = sxe2_tx_done_cleanup,
+
+ .mtu_set = sxe2_mtu_set,
+ .buffer_split_supported_hdr_ptypes_get = sxe2_buffer_split_supported_hdr_ptypes_get,
+};
+
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
{
int32_t ret = 0;
@@ -122,6 +159,12 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
sxe2_rx_mode_func_set(dev);
sxe2_tx_mode_func_set(dev);
+ ret = sxe2_link_update_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
+ goto l_end;
+ }
+
ret = sxe2_queues_start(dev);
if (ret) {
PMD_LOG_ERR(INIT, "enable queues failed");
@@ -136,16 +179,6 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
return ret;
}
-static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
-{
- (void)sxe2_dev_stop(dev);
- (void)sxe2_queues_release(dev);
- sxe2_vsi_uninit(dev);
- sxe2_dev_pci_map_uinit(dev);
-
- return 0;
-}
-
static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
@@ -270,28 +303,59 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
return 0;
}
-static const struct eth_dev_ops sxe2_eth_dev_ops = {
- .dev_configure = sxe2_dev_configure,
- .dev_start = sxe2_dev_start,
- .dev_stop = sxe2_dev_stop,
- .dev_close = sxe2_dev_close,
- .dev_infos_get = sxe2_dev_infos_get,
+static const uint32_t *
+sxe2_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev __rte_unused,
+ size_t *no_of_elements __rte_unused)
+{
+ static const uint32_t ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_SCTP,
+
+ RTE_PTYPE_TUNNEL_GRENAT,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP,
+
+ RTE_PTYPE_UNKNOWN
+ };
- .rx_queue_start = sxe2_rx_queue_start,
- .rx_queue_stop = sxe2_rx_queue_stop,
- .tx_queue_start = sxe2_tx_queue_start,
- .tx_queue_stop = sxe2_tx_queue_stop,
- .rx_queue_setup = sxe2_rx_queue_setup,
- .rx_queue_release = sxe2_rx_queue_release,
- .tx_queue_setup = sxe2_tx_queue_setup,
- .tx_queue_release = sxe2_tx_queue_release,
+ return ptypes;
+}
- .rxq_info_get = sxe2_rx_queue_info_get,
- .txq_info_get = sxe2_tx_queue_info_get,
- .rx_burst_mode_get = sxe2_rx_burst_mode_get,
- .tx_burst_mode_get = sxe2_tx_burst_mode_get,
- .tx_done_cleanup = sxe2_tx_done_cleanup,
-};
+static inline void sxe2_init_ptype_tbl(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint32_t *ptype = adapter->ptype_tbl;
+
+ PMD_INIT_FUNC_TRACE();
+
+ sxe2_init_ptype_list(ptype);
+}
struct sxe2_pci_map_bar_info *sxe2_dev_get_bar_info(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type)
@@ -360,6 +424,29 @@ void *sxe2_pci_map_addr_get(struct sxe2_adapter *adapter,
return addr;
}
+static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+
+ ret = sxe2_link_update_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_mtu_set(dev, dev->data->mtu);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to set mtu, ret=%d", ret);
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static void sxe2_eth_uinit(struct rte_eth_dev *dev __rte_unused)
+{
+}
+
static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
struct sxe2_drv_dev_caps_resp *dev_caps)
{
@@ -765,6 +852,8 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto l_end;
}
+ sxe2_init_ptype_tbl(dev);
+
ret = sxe2_hw_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to initialize hw, ret=[%d]", ret);
@@ -789,18 +878,38 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_dev_info_err;
}
+ ret = sxe2_eth_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize eth parameters, ret=%d", ret);
+ goto init_eth_err;
+ }
+
goto l_end;
+init_eth_err:
init_dev_info_err:
sxe2_vsi_uninit(dev);
init_vsi_err:
+ sxe2_dev_pci_map_uinit(dev);
l_end:
return ret;
}
+static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
+{
+ (void)sxe2_dev_stop(dev);
+ (void)sxe2_queues_release(dev);
+ sxe2_vsi_uninit(dev);
+ sxe2_dev_pci_map_uinit(dev);
+ sxe2_eth_uinit(dev);
+
+ return 0;
+}
+
static int32_t sxe2_dev_uninit(struct rte_eth_dev *dev)
{
int32_t ret = 0;
+
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
goto l_end;
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index 001413e75a..66f49ac0cc 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -118,12 +118,6 @@ enum {
SXE2_FLAGS_NBITS
};
-struct sxe2_link_context {
- rte_spinlock_t link_lock;
- bool link_up;
- uint32_t speed;
-};
-
struct sxe2_devargs {
uint8_t flow_dup_pattern_mode;
uint8_t func_flow_direct_en;
@@ -265,6 +259,12 @@ struct sxe2_sched_hw_cap {
uint8_t adj_lvl;
};
+struct sxe2_link_context {
+ rte_spinlock_t link_lock;
+ bool link_up;
+ uint32_t speed;
+};
+
struct sxe2_adapter {
struct sxe2_common_device *cdev;
struct sxe2_dev_info dev_info;
@@ -274,9 +274,10 @@ struct sxe2_adapter {
struct sxe2_irq_context irq_ctxt;
struct sxe2_queue_context q_ctxt;
struct sxe2_vsi_context vsi_ctxt;
- struct sxe2_devargs devargs;
- uint16_t dev_port_id;
- uint64_t cap_flags;
+ struct sxe2_link_context link_ctxt;
+ struct sxe2_devargs devargs;
+ uint16_t dev_port_id;
+ uint64_t cap_flags;
enum sxe2_dev_type dev_type;
uint32_t ptype_tbl[SXE2_MAX_PTYPE_NUM];
struct rte_ether_addr mac_addr;
diff --git a/drivers/net/sxe2/sxe2_mac.c b/drivers/net/sxe2/sxe2_mac.c
new file mode 100644
index 0000000000..3c2f909002
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_mac.c
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_os.h>
+#include "sxe2_osal.h"
+#include "sxe2_mac.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+#include "sxe2_host_regs.h"
+
+int32_t sxe2_link_update_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret;
+
+ PMD_INIT_FUNC_TRACE();
+
+ rte_spinlock_init(&adapter->link_ctxt.link_lock);
+
+ ret = sxe2_drv_mac_link_status_get(adapter);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to get link status, ret=%d", ret);
+ goto l_end;
+ }
+
+ (void)sxe2_link_update(dev, 0);
+
+l_end:
+ return ret;
+}
+int32_t sxe2_link_update(struct rte_eth_dev *dev, __rte_unused int32_t wait_to_complete)
+{
+ struct rte_eth_link new_link;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ memset(&new_link, 0, sizeof(new_link));
+
+ switch (adapter->link_ctxt.speed) {
+ case 0:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_NONE;
+ break;
+ case 10:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_10M;
+ break;
+ case 100:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_100M;
+ break;
+ case 1000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_1G;
+ break;
+ case 10000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_10G;
+ break;
+ case 20000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_20G;
+ break;
+ case 25000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_25G;
+ break;
+ case 40000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_40G;
+ break;
+ case 50000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_50G;
+ break;
+ case 100000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_100G;
+ break;
+ default:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_NONE;
+ break;
+ }
+
+ new_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
+ new_link.link_status = adapter->link_ctxt.link_up ? RTE_ETH_LINK_UP :
+ RTE_ETH_LINK_DOWN;
+ new_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
+ RTE_ETH_LINK_SPEED_FIXED);
+
+ return rte_eth_linkstatus_set(dev, &new_link);
+}
+
+int32_t sxe2_mtu_set(struct rte_eth_dev *dev, uint16_t mtu __rte_unused)
+{
+ int32_t ret = -1;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (dev->data->dev_started != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "port %d must be stopped before configuration",
+ dev->data->port_id);
+ ret = -1;
+ goto l_end;
+ }
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_mac.h b/drivers/net/sxe2/sxe2_mac.h
new file mode 100644
index 0000000000..f2f3edaeff
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_mac.h
@@ -0,0 +1,50 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_MAC_H__
+#define __SXE2_MAC_H__
+#include <ethdev_driver.h>
+#include "sxe2_host_regs.h"
+
+#define SXE2_NUM_MACADDR_MAX 64
+
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_8021Q SXE2_VSI_L2TAGSTXVALID_ID_OUT_VLAN1
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_8021AD SXE2_VSI_L2TAGSTXVALID_ID_STAG
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_QINQ1 SXE2_VSI_L2TAGSTXVALID_ID_OUT_VLAN2
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_VLAN SXE2_VSI_L2TAGSTXVALID_ID_VLAN
+
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_ENABLE SXE2_VSI_L2TAGSTXVALID_L2TAG1_VALID
+
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021Q SXE2_VSI_TSR_ID_OUT_VLAN1
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021AD SXE2_VSI_TSR_ID_STAG
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_QINQ1 SXE2_VSI_TSR_ID_OUT_VLAN2
+
+#define SXE2_DPDK_OFFLOAD_INNER_INSERT_QINQ1 SXE2_VSI_L2TAGSTXVALID_ID_VLAN
+#define SXE2_DPDK_OFFLOAD_INNER_INSERT_ENABLE SXE2_VSI_L2TAGSTXVALID_L2TAG2_VALID
+
+#define SXE2_DPDK_OFFLOAD_INNER_STRIP_QINQ1 SXE2_VSI_TSR_ID_VLAN
+
+#define SXE2_DPDK_OFFLOAD_FIELD (0X0F)
+#define SXE2_DPDK_OFFLOAD_TAGID_FIELD (0X07)
+
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_MASK (SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021Q | \
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021AD | \
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_QINQ1)
+#define SXE2_DPDK_OFFLOAD_STRIP_OFFSET SXE2_VSI_TSR_SHOW_TAG_S
+
+#define SXE2_DPDK_OFFLOAD_INSERT_ENABLE (RTE_BIT32(3))
+
+struct sxe2_mac_mc_list {
+ uint32_t count;
+ struct rte_ether_addr addr[SXE2_NUM_MACADDR_MAX];
+};
+
+int32_t sxe2_link_update_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_link_update(struct rte_eth_dev *dev, __rte_unused int32_t wait_to_complete);
+
+int32_t sxe2_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
+
+#endif /* __SXE2_MAC_H__ */
--
2.47.3
^ permalink raw reply related
* [PATCH v2 06/23] net/sxe2: support TM hierarchy and shaping
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch implements the traffic management ops for example PMD.
It supports a 4-level hierarchy: port, vsi, queue group and queue.
- Support node add/delete and hierarchy commit.
- Support private shaper and rate limiting on each node.
The hardware requires all nodes to be configured before the hierarchy
is committed to the global registers.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 3 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 163 +++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 6 +
drivers/net/sxe2/sxe2_drv_cmd.h | 26 +
drivers/net/sxe2/sxe2_ethdev.c | 82 +++
drivers/net/sxe2/sxe2_ethdev.h | 6 +
drivers/net/sxe2/sxe2_tm.c | 1151 ++++++++++++++++++++++++++++++
drivers/net/sxe2/sxe2_tm.h | 76 ++
drivers/net/sxe2/sxe2_tx.c | 1 -
9 files changed, 1512 insertions(+), 2 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index dfd31bfc97..d0aa7fecf0 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -63,6 +63,7 @@ sources += files(
'sxe2_mac.c',
'sxe2_filter.c',
'sxe2_rss.c',
+ 'sxe2_tm.c',
)
-allow_internal_get_api = true
+allow_internal_get_api = true
\ No newline at end of file
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index b997e7b044..19323ffcc4 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -230,6 +230,7 @@ static void sxe2_txq_ctxt_cfg_fill(struct sxe2_tx_queue *txq,
struct sxe2_drv_txq_cfg_req *req,
uint16_t txq_cnt)
{
+ struct sxe2_adapter *adapter = txq->vsi->adapter;
struct sxe2_drv_txq_ctxt *ctxt = req->cfg;
uint16_t q_idx = 0;
@@ -241,6 +242,8 @@ static void sxe2_txq_ctxt_cfg_fill(struct sxe2_tx_queue *txq,
ctxt->depth = txq[q_idx].ring_depth;
ctxt->dma_addr = txq[q_idx].base_addr;
ctxt->queue_id = txq[q_idx].queue_id;
+
+ ctxt->sched_mode = sxe2_sched_mode_get(adapter);
}
}
@@ -310,6 +313,7 @@ int32_t sxe2_drv_txq_switch(struct sxe2_adapter *adapter, struct sxe2_tx_queue *
req.q_idx = txq->queue_id;
req.is_enable = (uint8_t)enable;
+ req.sched_mode = sxe2_sched_mode_get(adapter);
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_TXQ_DISABLE,
&req, sizeof(req), NULL, 0);
@@ -714,3 +718,162 @@ int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue
(void)rxq;
return 0;
}
+
+int32_t sxe2_drv_root_tree_alloc(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_tm_res tm_resp;
+ int32_t ret;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SCHED_ROOT_TREE_ALLOC,
+ NULL, 0,
+ &tm_resp, sizeof(tm_resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "add sched root failed, ret:%d", ret);
+ goto l_end;
+ }
+
+ tm_ctxt->root_teid = tm_resp.teid;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_root_tree_release(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_tm_res tm_res = {0};
+ int32_t ret;
+
+ tm_res.teid = adapter->tm_ctxt.root_teid;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SCHED_ROOT_TREE_RELEASE,
+ &tm_res, sizeof(tm_res),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "release sched root failed, ret:%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static void sxe2_drv_tm_node_to_info(struct sxe2_adapter *adapter,
+ struct sxe2_tm_node *node, struct sxe2_tm_info *info)
+{
+ uint32_t rate = 0;
+
+ if (node->shaper_profile->profile.committed.rate == UINT64_MAX)
+ rate = UINT32_MAX;
+ else
+ rate = (uint32_t)(node->shaper_profile->profile.committed.rate * 8 / 1000);
+
+ info->committed = rte_cpu_to_le_32(rate);
+
+ if (node->shaper_profile->profile.peak.rate == UINT64_MAX)
+ rate = UINT32_MAX;
+ else
+ rate = (uint32_t)(node->shaper_profile->profile.peak.rate * 8 / 1000);
+
+ info->peak = rte_cpu_to_le_32(rate);
+
+ info->priority = (adapter->tm_ctxt.prio_max - 1 - node->priority);
+
+ info->weight = rte_cpu_to_le_16(node->hw_weight);
+}
+
+static int32_t sxe2_drv_tm_commit_node(struct sxe2_adapter *adapter,
+ struct sxe2_tm_node *tm_node)
+{
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_tm_add_mid_msg msg_mid = {0};
+ struct sxe2_tm_add_queue_msg msg_queue = {0};
+ struct sxe2_tm_res res = {0};
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret;
+ uint32_t i;
+
+ if (tm_node->type == SXE2_TM_NODE_TYPE_VSIG) {
+ goto l_add;
+ } else if (tm_node->type == SXE2_TM_NODE_TYPE_MID) {
+ sxe2_drv_tm_node_to_info(adapter, tm_node, &msg_mid.info);
+ msg_mid.parent_teid = rte_cpu_to_le_16(tm_node->parent->teid);
+ msg_mid.adj_lvl = adapter->sched_ctxt.adj_lvl;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd,
+ SXE2_DRV_CMD_SCHED_TM_ADD_MID_NODE,
+ &msg_mid, sizeof(msg_mid),
+ &res, sizeof(res));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "add tm mid node failed, ret:%d", ret);
+ goto l_end;
+ }
+ } else if (tm_node->type == SXE2_TM_NODE_TYPE_QUEUE) {
+ sxe2_drv_tm_node_to_info(adapter, tm_node, &msg_queue.info);
+ msg_queue.parent_teid = rte_cpu_to_le_16(tm_node->parent->teid);
+ msg_queue.queue_id = rte_cpu_to_le_16(tm_node->id);
+ msg_queue.adj_lvl = adapter->sched_ctxt.adj_lvl;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd,
+ SXE2_DRV_CMD_SCHED_TM_ADD_QUEUE_NODE,
+ &msg_queue, sizeof(msg_queue),
+ &res, sizeof(res));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "add tm queue failed, ret:%d", ret);
+ goto l_end;
+ }
+ } else {
+ PMD_LOG_ERR(DRV, "commit tm node failed, type:%d", tm_node->type);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ tm_node->teid = rte_le_to_cpu_16(res.teid);
+
+l_add:
+ for (i = 0; i < tm_node->child_cnt; i++) {
+ ret = sxe2_drv_tm_commit_node(adapter, tm_node->children[i]);
+ if (ret)
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_tm_commit(struct sxe2_adapter *adapter)
+{
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_tm_res tm_res = {0};
+ int32_t ret;
+
+ tm_res.teid = adapter->tm_ctxt.root_teid;
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_SCHED_ROOT_CHILDREN_DELETE,
+ &tm_res, sizeof(tm_res),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "del tm root failed, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_drv_tm_commit_node(adapter, adapter->tm_ctxt.root);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "commit tm node failed, ret:%d", ret);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(DRV, "commit tm success");
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index 3274db6551..6e209377c7 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -38,6 +38,12 @@ int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+int32_t sxe2_drv_root_tree_release(struct rte_eth_dev *dev);
+
+int32_t sxe2_drv_root_tree_alloc(struct rte_eth_dev *dev);
+
+int32_t sxe2_drv_tm_commit(struct sxe2_adapter *adapter);
+
int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set);
int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index 93e83c3181..81401bcada 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -349,6 +349,32 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_rss_hf_req {
uint8_t rsv1[3];
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_res {
+ uint16_t teid;
+ uint8_t rsv[2];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_info {
+ uint32_t committed;
+ uint32_t peak;
+ uint8_t priority;
+ uint8_t reserve;
+ uint16_t weight;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_add_mid_msg {
+ uint16_t parent_teid;
+ uint8_t adj_lvl;
+ struct sxe2_tm_info info;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_add_queue_msg {
+ uint16_t parent_teid;
+ uint16_t queue_id;
+ uint8_t adj_lvl;
+ struct sxe2_tm_info info;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index d48841b8e4..a095888c00 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -130,6 +130,8 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.reta_query = sxe2_dev_rss_reta_query,
.rss_hash_update = sxe2_dev_rss_hash_update,
.rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
+
+ .tm_ops_get = sxe2_tm_ops_get,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -575,6 +577,14 @@ static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
adapter->cap_flags |= SXE2_DEV_CAPS_OFFLOAD_FC_STATE;
}
+static void sxe2_sw_sched_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_txsch_caps *txsch_caps)
+{
+ adapter->sched_ctxt.tm_layers = txsch_caps->layer_cap;
+ adapter->sched_ctxt.root_max_children = txsch_caps->tm_mid_node_num;
+ adapter->sched_ctxt.prio_max = txsch_caps->prio_num;
+}
+
static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
{
int32_t ret = -1;
@@ -594,6 +604,8 @@ static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
sxe2_sw_vsi_ctx_hw_cap_set(adapter, &dev_caps.vsi_caps);
+ sxe2_sw_sched_hw_cap_set(adapter, &dev_caps.txsch_caps);
+
l_end:
return ret;
}
@@ -930,6 +942,68 @@ void sxe2_dev_pci_map_uinit(struct rte_eth_dev *dev)
adapter->dev_info.dev_data = NULL;
}
+uint32_t sxe2_sched_mode_get(struct sxe2_adapter *adapter)
+{
+ uint32_t ret_mode = SXE2_SCHED_MODE_INVALID;
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+
+ if (adapter->devargs.high_performance_mode)
+ ret_mode = SXE2_SCHED_MODE_HIGH_PERFORMANCE;
+ else if (tm_ctxt->committed)
+ ret_mode = SXE2_SCHED_MODE_TM;
+ else
+ ret_mode = SXE2_SCHED_MODE_DEFAULT;
+
+ return ret_mode;
+}
+
+static int32_t sxe2_sched_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ adapter->sched_ctxt.adj_lvl = adapter->devargs.sched_layer_mode;
+
+ if (adapter->devargs.high_performance_mode) {
+ PMD_LOG_DEBUG(DRV, "TM feature will be disabled in high-performance mode.");
+ adapter->cap_flags &= ~(SXE2_DEV_CAPS_OFFLOAD_TM);
+ } else {
+ ret = sxe2_tm_init(dev);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_drv_root_tree_alloc(dev);
+ if (ret)
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_sched_uinit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ if (adapter->devargs.high_performance_mode == 0) {
+ ret = sxe2_tm_uninit(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to uninit tm, ret=%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_drv_root_tree_release(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to release root tree, ret=%d", ret);
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
struct sxe2_dev_kvargs_info *kvargs __rte_unused)
{
@@ -985,8 +1059,15 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_rss_err;
}
+ ret = sxe2_sched_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init sched, ret=%d", ret);
+ goto init_sched_err;
+ }
+
goto l_end;
+init_sched_err:
init_rss_err:
init_eth_err:
init_dev_info_err:
@@ -1002,6 +1083,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
(void)sxe2_rss_disable(dev);
+ (void)sxe2_sched_uinit(dev);
sxe2_vsi_uninit(dev);
sxe2_dev_pci_map_uinit(dev);
sxe2_eth_uinit(dev);
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index 556a11cc77..609e1e92ba 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -1,6 +1,7 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
*/
+
#ifndef __SXE2_ETHDEV_H__
#define __SXE2_ETHDEV_H__
#include <rte_compat.h>
@@ -18,6 +19,7 @@
#include "sxe2_queue.h"
#include "sxe2_mac.h"
#include "sxe2_osal.h"
+#include "sxe2_tm.h"
#include "sxe2_filter.h"
struct sxe2_link_msg {
@@ -307,6 +309,8 @@ struct sxe2_adapter {
struct sxe2_rss_context rss_ctxt;
struct sxe2_link_context link_ctxt;
struct sxe2_ptp_context ptp_ctxt;
+ struct sxe2_sched_hw_cap sched_ctxt;
+ struct sxe2_tm_context tm_ctxt;
struct sxe2_devargs devargs;
struct sxe2_switchdev_info switchdev_info;
bool rule_started;
@@ -333,6 +337,8 @@ void *sxe2_pci_map_addr_get(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type,
uint16_t idx_in_func);
+uint32_t sxe2_sched_mode_get(struct sxe2_adapter *adapter);
+
struct sxe2_pci_map_bar_info *sxe2_dev_get_bar_info(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type);
diff --git a/drivers/net/sxe2/sxe2_tm.c b/drivers/net/sxe2/sxe2_tm.c
new file mode 100644
index 0000000000..4c4f793cd5
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_tm.c
@@ -0,0 +1,1151 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_ethdev.h>
+#include <rte_tm_driver.h>
+
+#include "sxe2_tm.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_cmd_chnl.h"
+
+static uint16_t sxe2_tm_level_node_num_get(uint8_t level)
+{
+ uint16_t node_num = 0;
+
+ switch (level) {
+ case 0:
+ node_num = SXE2_TM_1L_NODE_NUM_MAX;
+ break;
+ case 1:
+ node_num = SXE2_TM_2L_NODE_NUM_MAX;
+ break;
+ case 2:
+ node_num = SXE2_TM_3L_NODE_NUM_MAX;
+ break;
+ case 3:
+ node_num = SXE2_TM_4L_NODE_NUM_MAX;
+ break;
+ }
+ return node_num;
+}
+
+static int32_t sxe2_tm_capabilities_get(struct rte_eth_dev *dev,
+ struct rte_tm_capabilities *cap,
+ struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ uint32_t i;
+
+ if (!cap || !error) {
+ PMD_LOG_ERR(DRV, "sxe2 get tm cap failed, cap or error is NULL.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ error->type = RTE_TM_ERROR_TYPE_NONE;
+ memset(cap, 0, sizeof(struct rte_tm_capabilities));
+
+ for (i = 0; i < adapter->tm_ctxt.tm_layers; i++)
+ cap->n_nodes_max += sxe2_tm_level_node_num_get(i);
+
+ cap->n_levels_max = adapter->tm_ctxt.tm_layers;
+
+ cap->non_leaf_nodes_identical = 1;
+
+ cap->leaf_nodes_identical = 1;
+
+ cap->shaper_n_max = cap->n_nodes_max;
+
+ cap->shaper_private_n_max = cap->n_nodes_max;
+
+ cap->shaper_private_dual_rate_n_max = cap->n_nodes_max;
+
+ cap->shaper_private_rate_min = 0;
+
+ cap->shaper_private_rate_max = 12500000000ull;
+
+ cap->shaper_private_packet_mode_supported = 0;
+ cap->shaper_private_byte_mode_supported = 1;
+
+ cap->shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD;
+
+ cap->shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
+
+ cap->shaper_shared_n_max = 0;
+ cap->shaper_shared_n_nodes_per_shaper_max = 0;
+ cap->shaper_shared_n_shapers_per_node_max = 0;
+ cap->shaper_shared_dual_rate_n_max = 0;
+ cap->shaper_shared_rate_min = 0;
+ cap->shaper_shared_rate_max = 0;
+ cap->shaper_shared_packet_mode_supported = 0;
+ cap->shaper_shared_byte_mode_supported = 0;
+
+ cap->sched_n_children_max = dev->data->nb_tx_queues;
+
+ cap->sched_sp_n_priorities_max = 7;
+
+ cap->sched_wfq_n_children_per_group_max = 1;
+ cap->sched_wfq_n_groups_max = 0;
+ cap->sched_wfq_weight_max = 0;
+ cap->sched_wfq_packet_mode_supported = 0;
+ cap->sched_wfq_byte_mode_supported = 0;
+
+ cap->cman_wred_packet_mode_supported = 0;
+ cap->cman_wred_byte_mode_supported = 0;
+ cap->cman_head_drop_supported = 0;
+ cap->cman_wred_context_n_max = 0;
+ cap->cman_wred_context_private_n_max = 0;
+ cap->cman_wred_context_shared_n_max = 0;
+ cap->cman_wred_context_shared_n_nodes_per_context_max = 0;
+ cap->cman_wred_context_shared_n_contexts_per_node_max = 0;
+
+ cap->dynamic_update_mask = 0;
+
+ cap->stats_mask = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_level_capabilities_get(struct rte_eth_dev *dev,
+ uint32_t level_id, struct rte_tm_level_capabilities *cap,
+ struct rte_tm_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!cap || !error) {
+ PMD_LOG_ERR(DRV, "sxe2 get tm cap failed, cap or error is NULL.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (level_id >= adapter->tm_ctxt.tm_layers) {
+ ret = -EINVAL;
+ error->type = RTE_TM_ERROR_TYPE_LEVEL_ID;
+ error->message = "too deep level";
+ goto l_end;
+ }
+
+ cap->n_nodes_max = sxe2_tm_level_node_num_get(level_id);
+
+ cap->non_leaf_nodes_identical = true;
+
+ cap->leaf_nodes_identical = true;
+
+ if (level_id != adapter->tm_ctxt.tm_layers - 1) {
+ cap->n_nodes_nonleaf_max = cap->n_nodes_max;
+ cap->n_nodes_leaf_max = 0;
+
+ cap->nonleaf.shaper_private_supported = true;
+ cap->nonleaf.shaper_private_dual_rate_supported = true;
+ cap->nonleaf.shaper_private_rate_min = 0;
+
+ cap->nonleaf.shaper_private_rate_max = 12500000000ull;
+ cap->nonleaf.shaper_private_packet_mode_supported = 0;
+ cap->nonleaf.shaper_private_byte_mode_supported = 1;
+
+ cap->nonleaf.shaper_shared_n_max = 0;
+ cap->nonleaf.shaper_shared_packet_mode_supported = 0;
+ cap->nonleaf.shaper_shared_byte_mode_supported = 0;
+
+ cap->nonleaf.sched_n_children_max = SXE2_TM_MAX_CHILDREN_COUNT;
+ cap->nonleaf.sched_sp_n_priorities_max = 7;
+
+ cap->nonleaf.sched_wfq_n_children_per_group_max = 0;
+ cap->nonleaf.sched_wfq_n_groups_max = 0;
+ cap->nonleaf.sched_wfq_weight_max = 0;
+ cap->nonleaf.sched_wfq_packet_mode_supported = 0;
+ cap->nonleaf.sched_wfq_byte_mode_supported = 0;
+
+ cap->nonleaf.stats_mask = 0;
+ } else {
+ cap->n_nodes_nonleaf_max = 0;
+ cap->n_nodes_leaf_max = cap->n_nodes_max;
+
+ cap->leaf.shaper_private_supported = true;
+ cap->leaf.shaper_private_dual_rate_supported = true;
+ cap->leaf.shaper_private_rate_min = 0;
+ cap->leaf.shaper_private_rate_max = 12500000000ull;
+ cap->leaf.shaper_private_packet_mode_supported = 0;
+ cap->leaf.shaper_private_byte_mode_supported = 1;
+
+ cap->leaf.shaper_shared_n_max = 0;
+ cap->leaf.shaper_shared_packet_mode_supported = 0;
+ cap->leaf.shaper_shared_byte_mode_supported = 0;
+
+ cap->leaf.cman_head_drop_supported = false;
+ cap->leaf.cman_wred_context_private_supported = false;
+ cap->leaf.cman_wred_context_shared_n_max = 0;
+
+ cap->leaf.stats_mask = 0;
+ }
+
+l_end:
+ return ret;
+}
+
+static struct sxe2_tm_node *sxe2_tm_find_node(struct sxe2_tm_node *parent, uint32_t id)
+{
+ struct sxe2_tm_node *node = NULL;
+ uint32_t i;
+
+ if (parent == NULL || parent->id == id) {
+ node = parent;
+ goto l_end;
+ }
+
+ for (i = 0; i < parent->child_cnt; i++) {
+ node = sxe2_tm_find_node(parent->children[i], id);
+ if (node)
+ goto l_end;
+ }
+
+l_end:
+ return node;
+}
+
+static int32_t sxe2_node_capabilities_get(struct rte_eth_dev *dev, uint32_t node_id,
+ struct rte_tm_node_capabilities *cap,
+ struct rte_tm_error *error)
+{
+ int32_t ret = -EINVAL;
+ struct sxe2_tm_node *tm_node;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!cap || !error) {
+ PMD_LOG_ERR(DRV, "sxe2 get tm cap failed, cap or error is NULL.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "invalid node id");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ tm_node = sxe2_tm_find_node(adapter->tm_ctxt.root, node_id);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "no such node");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no such node";
+ goto l_end;
+ }
+
+ cap->shaper_private_supported = true;
+ cap->shaper_private_dual_rate_supported = true;
+ cap->shaper_private_rate_min = 0;
+ cap->shaper_private_rate_max = 12500000000ull;
+ cap->shaper_private_packet_mode_supported = 0;
+ cap->shaper_private_byte_mode_supported = 1;
+
+ cap->shaper_shared_n_max = 0;
+ cap->shaper_shared_packet_mode_supported = 0;
+ cap->shaper_shared_byte_mode_supported = 0;
+
+ if (tm_node->level == adapter->tm_ctxt.tm_layers - 1) {
+ cap->leaf.cman_head_drop_supported = false;
+ cap->leaf.cman_wred_context_private_supported = false;
+ cap->leaf.cman_wred_context_shared_n_max = 0;
+ } else {
+ cap->nonleaf.sched_n_children_max = SXE2_TM_MAX_CHILDREN_COUNT;
+ cap->nonleaf.sched_sp_n_priorities_max = 7;
+ cap->nonleaf.sched_wfq_n_children_per_group_max = 0;
+ cap->nonleaf.sched_wfq_n_groups_max = 0;
+ cap->nonleaf.sched_wfq_weight_max = 0;
+ cap->nonleaf.sched_wfq_packet_mode_supported = 0;
+ cap->nonleaf.sched_wfq_byte_mode_supported = 0;
+ }
+ cap->stats_mask = 0;
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_shaper_profile_param_check(const struct rte_tm_shaper_params *profile,
+ struct rte_tm_error *error)
+{
+ int32_t ret = 0;
+
+ if (profile->committed.size) {
+ PMD_LOG_ERR(DRV, "committed bucket size not supported.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE;
+ error->message = "committed bucket size not supported";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->peak.size) {
+ PMD_LOG_ERR(DRV, "peak bucket size not supported.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE;
+ error->message = "peak bucket size not supported";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->pkt_length_adjust) {
+ PMD_LOG_ERR(DRV, "packet length adjustment not supported.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN;
+ error->message = "packet length adjustment not supported";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->committed.rate > SXE2_HW_RATE_MAX ||
+ profile->committed.rate < SXE2_HW_RATE_MIN) {
+ PMD_LOG_ERR(DRV, "The committed rate limit value is required to be in "
+ "the range [%" PRIu64 ", %" PRIu64 "].",
+ (uint64_t)SXE2_HW_RATE_MIN, (uint64_t)SXE2_HW_RATE_MAX);
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
+ error->message = "invalid rate limit: value out of range.";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->peak.rate > SXE2_HW_RATE_MAX ||
+ profile->peak.rate < SXE2_HW_RATE_MIN) {
+ PMD_LOG_ERR(DRV, "The peak rate limit value is required to be in "
+ "the range [%" PRIu64 ", %" PRIu64 "].",
+ (uint64_t)SXE2_HW_RATE_MIN, (uint64_t)SXE2_HW_RATE_MAX);
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE;
+ error->message = "invalid rate limit: value out of range.";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->committed.rate > profile->peak.rate) {
+ PMD_LOG_ERR(DRV, "committed rate can't be greater than peak rate.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
+ error->message = "committed rate can't be greater than peak rate.";
+ ret = -EINVAL;
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static inline struct sxe2_tm_shaper_profile *
+sxe2_tm_shaper_profile_search(struct rte_eth_dev *dev, uint32_t id)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_shaper_profile_list *shaper_profile_list =
+ &adapter->tm_ctxt.profile_list;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+
+ TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) {
+ if (id == shaper_profile->id)
+ goto l_end;
+ }
+
+ shaper_profile = NULL;
+
+l_end:
+ return shaper_profile;
+}
+
+static int32_t sxe2_tm_shaper_profile_add(struct rte_eth_dev *dev, uint32_t shaper_profile_id,
+ const struct rte_tm_shaper_params *profile,
+ struct rte_tm_error *error)
+{
+ int32_t ret = -EINVAL;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!profile || !error) {
+ PMD_LOG_ERR(DRV, "Invalid input: profile:0x%p or error:0x%p is null.",
+ profile, error);
+ if (error) {
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Invalid input: profile or error is null.";
+ }
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ ret = sxe2_tm_shaper_profile_param_check(profile, error);
+ if (ret)
+ goto l_end;
+
+ shaper_profile = sxe2_tm_shaper_profile_search(dev, shaper_profile_id);
+ if (shaper_profile) {
+ PMD_LOG_ERR(DRV, "profile ID exist.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
+ error->message = "profile ID exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ shaper_profile = rte_zmalloc("sxe2_tm_shaper_profile",
+ sizeof(struct sxe2_tm_shaper_profile), 0);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "Alloc shaper_profile memory failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Alloc shaper_profile memory failed";
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ rte_memcpy(&shaper_profile->profile, profile,
+ sizeof(struct rte_tm_shaper_params));
+ shaper_profile->id = shaper_profile_id;
+
+ TAILQ_INSERT_TAIL(&adapter->tm_ctxt.profile_list, shaper_profile, node);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_shaper_profile_del(struct rte_eth_dev *dev,
+ uint32_t id, struct rte_tm_error *error)
+{
+ int32_t ret = -EINVAL;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!error) {
+ PMD_LOG_ERR(DRV, "Error param is null.");
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ shaper_profile = sxe2_tm_shaper_profile_search(dev, id);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "profile ID not exist.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
+ error->message = "profile ID not exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (shaper_profile->ref_cnt) {
+ PMD_LOG_ERR(DRV, "profile in use.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE;
+ error->message = "profile in use";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ TAILQ_REMOVE(&adapter->tm_ctxt.profile_list, shaper_profile, node);
+ rte_free(shaper_profile);
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_node_param_check(struct rte_eth_dev *dev,
+ uint32_t parent_node_type,
+ uint32_t node_id, uint32_t priority, uint32_t weight,
+ const struct rte_tm_node_params *params,
+ bool is_leaf, struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ int32_t ret = -EINVAL;
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "Invalid node id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ if (parent_node_type == SXE2_TM_NODE_TYPE_VSIG &&
+ priority >= tm_ctxt->prio_max) {
+ PMD_LOG_ERR(DRV, "Priority should be less than %u.", tm_ctxt->prio_max);
+ error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
+ error->message = "The priority is too high.";
+ goto l_end;
+ }
+
+ if (priority > SXE2_TM_PRIO_MAX) {
+ PMD_LOG_ERR(DRV, "Priority should be less than %u.", SXE2_TM_PRIO_MAX);
+ error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
+ error->message = "The priority is too high.";
+ goto l_end;
+ }
+
+ if (weight > SXE2_TM_WEIGHT_MAX || weight < SXE2_TM_WEIGHT_MIN) {
+ PMD_LOG_ERR(DRV, "Weight must be between 1 and 200.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT;
+ error->message = "weight must be between 1 and 200";
+ goto l_end;
+ }
+
+ if (params->shared_shaper_id) {
+ PMD_LOG_ERR(DRV, "Shared shaper not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID;
+ error->message = "shared shaper not supported";
+ goto l_end;
+ }
+ if (params->n_shared_shapers) {
+ PMD_LOG_ERR(DRV, "Shared shaper not supported..");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS;
+ error->message = "shared shaper not supported";
+ goto l_end;
+ }
+
+ if (!is_leaf) {
+ if (node_id <= dev->data->nb_tx_queues) {
+ PMD_LOG_ERR(DRV, "no leaf node id must bigger than queue id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no leaf node id must bigger than queue id.";
+ goto l_end;
+ }
+
+ if (params->nonleaf.wfq_weight_mode) {
+ PMD_LOG_ERR(DRV, "WFQ not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE;
+ error->message = "WFQ not supported";
+ goto l_end;
+ }
+
+ if (params->nonleaf.n_sp_priorities != 1) {
+ PMD_LOG_ERR(DRV, "SP priority not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES;
+ error->message = "SP priority not supported";
+ goto l_end;
+ }
+ } else {
+ if (node_id >= dev->data->nb_tx_queues) {
+ PMD_LOG_ERR(DRV, "leaf node id must be queue id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "leaf node id must be queue id.";
+ goto l_end;
+ }
+
+ if (params->leaf.cman) {
+ PMD_LOG_ERR(DRV, "Congestion management not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN;
+ error->message = "Congestion management not supported";
+ goto l_end;
+ }
+ if (params->leaf.wred.wred_profile_id != RTE_TM_WRED_PROFILE_ID_NONE) {
+ PMD_LOG_ERR(DRV, "WRED not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID;
+ error->message = "WRED not supported";
+ goto l_end;
+ }
+ if (params->leaf.wred.shared_wred_context_id) {
+ PMD_LOG_ERR(DRV, "WRED not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID;
+ error->message = "WRED not supported";
+ goto l_end;
+ }
+ if (params->leaf.wred.n_shared_wred_contexts) {
+ PMD_LOG_ERR(DRV, "WRED not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS;
+ error->message = "WRED not supported";
+ goto l_end;
+ }
+ }
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_add_child(struct sxe2_tm_node *parent,
+ struct sxe2_tm_node *child)
+{
+ int32_t ret = -1;
+ uint32_t i;
+ for (i = 0; i < SXE2_TM_MAX_CHILDREN_COUNT; i++) {
+ if (parent->children[i] == NULL) {
+ parent->children[i] = child;
+ child->index_in_parent = i;
+ parent->child_cnt++;
+ ret = 0;
+ break;
+ }
+ }
+ return ret;
+}
+
+static int32_t sxe2_tm_node_add(struct rte_eth_dev *dev, uint32_t node_id, uint32_t parent_node_id,
+ uint32_t priority, uint32_t weight, uint32_t level_id,
+ const struct rte_tm_node_params *params,
+ struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_node *tm_node = NULL;
+ struct sxe2_tm_node *parent_node = NULL;
+ int32_t ret = -EINVAL;
+ bool is_leaf;
+
+ if (!params || !error) {
+ PMD_LOG_ERR(DRV, "Invalid input: params:0x%p or error:0x%p is null.",
+ params, error);
+ if (error) {
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Invalid input: params or error is null.";
+ }
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ shaper_profile = sxe2_tm_shaper_profile_search(dev, params->shaper_profile_id);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "Shaper profile does not exist.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID;
+ error->message = "shaper profile does not exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (parent_node_id == RTE_TM_NODE_ID_NULL) {
+ if (level_id != 0) {
+ PMD_LOG_ERR(DRV, "Wrong level, root node (NULL parent) must be at level 0.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "Wrong level, root node (NULL parent) must be at level 0";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (tm_ctxt->root) {
+ PMD_LOG_ERR(DRV, "Already have a root.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
+ error->message = "already have a root";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_tm_node_param_check(dev, SXE2_TM_NODE_TYPE_INVALID, node_id, priority,
+ weight, params, false, error);
+ if (ret)
+ goto l_end;
+
+ tm_node = rte_zmalloc("tm_node_root", sizeof(struct sxe2_tm_node), 0);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "Alloc tm_node memory failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Alloc tm_node memory failed";
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ tm_node->id = node_id;
+ tm_node->level = 0;
+ tm_node->parent = NULL;
+ tm_node->child_cnt = 0;
+ tm_node->weight = weight;
+ tm_node->hw_weight = SXE2_TM_WEIGHT_SUM;
+ tm_node->type = SXE2_TM_NODE_TYPE_VSIG;
+ tm_node->priority = priority;
+ tm_node->shaper_profile = shaper_profile;
+
+ tm_node->teid = tm_ctxt->root_teid;
+
+ shaper_profile->ref_cnt++;
+ tm_ctxt->root = tm_node;
+ ret = 0;
+ goto l_end;
+ }
+
+ parent_node = sxe2_tm_find_node(tm_ctxt->root, parent_node_id);
+ if (!parent_node) {
+ PMD_LOG_ERR(DRV, "Parent not exist.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
+ error->message = "parent not exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (parent_node->child_cnt >= SXE2_TM_MAX_CHILDREN_COUNT ||
+ (parent_node->type == SXE2_TM_NODE_TYPE_VSIG &&
+ parent_node->child_cnt >= tm_ctxt->root_max_children)) {
+ PMD_LOG_ERR(DRV, "Parent node is full.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "parent node is full";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (level_id == RTE_TM_NODE_LEVEL_ID_ANY) {
+ level_id = parent_node->level + 1;
+ } else if (level_id != parent_node->level + 1) {
+ PMD_LOG_ERR(DRV, "Wrong level.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "Wrong level";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (level_id >= tm_ctxt->tm_layers) {
+ PMD_LOG_ERR(DRV, "The maximum number of TM configuration levels is %d",
+ tm_ctxt->tm_layers);
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "TM level exceeds supported hardware limit";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (level_id + 1 == tm_ctxt->tm_layers)
+ is_leaf = true;
+ else
+ is_leaf = false;
+
+ ret = sxe2_tm_node_param_check(dev, parent_node->type, node_id, priority, weight,
+ params, is_leaf, error);
+ if (ret)
+ goto l_end;
+
+ if (sxe2_tm_find_node(tm_ctxt->root, node_id)) {
+ PMD_LOG_ERR(DRV, "Node id already used.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "node id already used";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ tm_node = rte_zmalloc("tm_node_no_root", sizeof(struct sxe2_tm_node), 0);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "Alloc tm_node memory failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Alloc tm_node memory failed";
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ if (level_id + 1 != tm_ctxt->tm_layers)
+ tm_node->type = SXE2_TM_NODE_TYPE_MID;
+ else
+ tm_node->type = SXE2_TM_NODE_TYPE_QUEUE;
+ tm_node->id = node_id;
+ tm_node->level = level_id;
+ tm_node->parent = parent_node;
+ tm_node->child_cnt = 0;
+ tm_node->weight = weight;
+ tm_node->priority = priority;
+ tm_node->shaper_profile = shaper_profile;
+ shaper_profile->ref_cnt++;
+
+ ret = sxe2_tm_add_child(parent_node, tm_node);
+ if (ret) {
+ shaper_profile->ref_cnt--;
+ rte_free(tm_node);
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_tree_delete(struct sxe2_tm_node *tm_node)
+{
+ int32_t ret = 0;
+ uint32_t i, j;
+ struct sxe2_tm_node *parent = NULL;
+
+ if (!tm_node)
+ goto l_end;
+
+ parent = tm_node->parent;
+
+ if (tm_node->child_cnt != 0) {
+ for (i = SXE2_TM_MAX_CHILDREN_COUNT; i > 0; i--) {
+ if (tm_node->children[i - 1])
+ ret = sxe2_tm_tree_delete(tm_node->children[i - 1]);
+ }
+ }
+
+ if (tm_node->shaper_profile)
+ tm_node->shaper_profile->ref_cnt--;
+
+ if (tm_node->type != SXE2_TM_NODE_TYPE_VSIG && parent) {
+ for (i = 0; i < parent->child_cnt; i++) {
+ if (parent->children[i] == tm_node)
+ break;
+ }
+ for (j = i; j < parent->child_cnt - 1; j++)
+ parent->children[j] = parent->children[j + 1];
+
+ parent->children[parent->child_cnt - 1] = NULL;
+ parent->child_cnt--;
+ }
+ rte_free(tm_node);
+ tm_node = NULL;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_node_delete(struct rte_eth_dev *dev, uint32_t node_id,
+ struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_node *tm_node = NULL;
+ int32_t ret = -EINVAL;
+
+ if (!error) {
+ PMD_LOG_ERR(DRV, "Invalid input: error is null.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "Invalid node id. node_id = %u", node_id);
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ tm_node = sxe2_tm_find_node(tm_ctxt->root, node_id);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "No such node.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no such node";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_tm_tree_delete(tm_node);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Delete node failed.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "Delete node failed";
+ goto l_end;
+ }
+
+ if (tm_node == tm_ctxt->root)
+ tm_ctxt->root = NULL;
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_node_type_get(struct rte_eth_dev *dev, uint32_t node_id,
+ int32_t *is_leaf, struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_node *tm_node = NULL;
+ int32_t ret = -EINVAL;
+
+ if (!is_leaf || !error) {
+ PMD_LOG_ERR(DRV, "Invalid input: is_leaf:0x%p or error:0x%p is null.",
+ is_leaf, error);
+ if (error) {
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Invalid input: is_leaf or error is null";
+ }
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "Invalid node id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ tm_node = sxe2_tm_find_node(tm_ctxt->root, node_id);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "No such node.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no such node";
+ goto l_end;
+ }
+
+ if (tm_node->level + 1 == tm_ctxt->tm_layers)
+ *is_leaf = true;
+ else
+ *is_leaf = false;
+ ret = 0;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_tm_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ int32_t ret = 0;
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ ret = 0;
+ goto l_end;
+ }
+
+ tm_ctxt->tm_layers = 0;
+ tm_ctxt->root_max_children = 0;
+ tm_ctxt->committed = false;
+
+ (void)sxe2_tm_tree_delete(tm_ctxt->root);
+
+ while ((shaper_profile = TAILQ_FIRST(&tm_ctxt->profile_list))) {
+ TAILQ_REMOVE(&tm_ctxt->profile_list, shaper_profile, node);
+ rte_free(shaper_profile);
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_tm_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_sched_hw_cap *sched_ctxt = &adapter->sched_ctxt;
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ int32_t ret = 0;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0)
+ goto l_end;
+
+ tm_ctxt->tm_layers = sched_ctxt->tm_layers;
+ tm_ctxt->root_max_children = sched_ctxt->root_max_children;
+ tm_ctxt->prio_max = sched_ctxt->prio_max;
+ tm_ctxt->committed = false;
+ TAILQ_INIT(&tm_ctxt->profile_list);
+ tm_ctxt->root = NULL;
+
+ shaper_profile = rte_zmalloc("sxe2_tm_shaper_profile",
+ sizeof(struct sxe2_tm_shaper_profile), 0);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "Alloc shaper_profile memory failed.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ shaper_profile->id = RTE_TM_SHAPER_PROFILE_ID_NONE;
+ shaper_profile->ref_cnt = 1;
+ shaper_profile->profile.committed.rate = UINT64_MAX;
+ shaper_profile->profile.peak.rate = UINT64_MAX;
+ TAILQ_INSERT_TAIL(&tm_ctxt->profile_list, shaper_profile, node);
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_chk_all_leaf(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ uint32_t i = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ if (!sxe2_tm_find_node(adapter->tm_ctxt.root, i)) {
+ ret = -1;
+ break;
+ }
+ }
+ return ret;
+}
+
+static int32_t sxe2_tm_weight_calc(struct sxe2_tm_node *tm_node)
+{
+ int32_t ret = 0;
+ int32_t total_weight = 0;
+ int32_t total_weight2 = 0;
+ uint32_t i = 0;
+ uint32_t j = 0;
+ uint32_t k = 0;
+ uint32_t maxindex = 0;
+ uint32_t maxweight = 0;
+ struct sxe2_tm_node *cacl_node[SXE2_TM_MAX_CHILDREN_COUNT] = {NULL};
+
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "Invalid input: tm_node is null.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (tm_node->child_cnt == 0)
+ goto l_end;
+
+ for (j = SXE2_TM_PRIO_MIN; j <= SXE2_TM_PRIO_MAX; j++) {
+ k = 0;
+ total_weight = 0;
+ total_weight2 = 0;
+ maxindex = 0;
+ maxweight = 0;
+
+ for (i = 0; i < tm_node->child_cnt; i++) {
+ if (tm_node->children[i]->priority == j)
+ cacl_node[k++] = tm_node->children[i];
+ }
+ if (k == 0)
+ continue;
+
+ for (i = 0; i < k; i++)
+ total_weight += cacl_node[i]->weight;
+
+ for (i = 0; i < k; i++) {
+ cacl_node[i]->hw_weight = cacl_node[i]->weight *
+ SXE2_TM_WEIGHT_SUM / total_weight;
+ total_weight2 += cacl_node[i]->hw_weight;
+ if (cacl_node[i]->hw_weight > maxweight) {
+ maxweight = cacl_node[i]->hw_weight;
+ maxindex = i;
+ }
+ }
+
+ cacl_node[maxindex]->hw_weight += SXE2_TM_WEIGHT_SUM - total_weight2;
+ }
+
+ for (i = 0; i < tm_node->child_cnt; i++) {
+ ret = sxe2_tm_weight_calc(tm_node->children[i]);
+ if (ret)
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_hierarchy_commit(struct rte_eth_dev *dev,
+ int32_t clear_on_fail, struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = -EINVAL;
+
+ if (!error) {
+ PMD_LOG_ERR(DRV, "Invalid input: error is null.");
+ ret = -EINVAL;
+ goto l_clear_on_fail;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_clear_on_fail;
+ }
+
+ if (dev->data->dev_started) {
+ PMD_LOG_ERR(DRV, "Device failed to Stop.");
+ error->message = "Device failed to Stop";
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ ret = -EPERM;
+ goto l_clear_on_fail;
+ }
+
+ ret = sxe2_tm_chk_all_leaf(dev);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "All tx queues need config.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "All tx queues need config.";
+ goto l_clear_on_fail;
+ }
+
+ ret = sxe2_tm_weight_calc(adapter->tm_ctxt.root);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "The weight in tree is wrong.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The weight in tree is wrong.";
+ goto l_clear_on_fail;
+ }
+
+ ret = sxe2_drv_tm_commit(adapter);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Commit tree to fw failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Commit tree to fw failed.";
+ goto l_clear_on_fail;
+ }
+
+ adapter->tm_ctxt.committed = true;
+ ret = 0;
+ goto l_end;
+
+l_clear_on_fail:
+ if (clear_on_fail) {
+ (void)sxe2_tm_uninit(dev);
+ (void)sxe2_tm_init(dev);
+ }
+
+l_end:
+ return ret;
+}
+
+static const struct rte_tm_ops sxe2_tm_ops = {
+ .capabilities_get = sxe2_tm_capabilities_get,
+ .level_capabilities_get = sxe2_level_capabilities_get,
+ .node_capabilities_get = sxe2_node_capabilities_get,
+ .shaper_profile_add = sxe2_tm_shaper_profile_add,
+ .shaper_profile_delete = sxe2_tm_shaper_profile_del,
+ .node_add = sxe2_tm_node_add,
+ .node_delete = sxe2_tm_node_delete,
+ .node_type_get = sxe2_tm_node_type_get,
+
+ .hierarchy_commit = sxe2_tm_hierarchy_commit,
+};
+
+int32_t sxe2_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
+{
+ int32_t ret = 0;
+
+ if (!arg) {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "%s failed because arg is NULL", __func__);
+ goto l_end;
+ }
+ *(const void **)arg = &sxe2_tm_ops;
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_tm.h b/drivers/net/sxe2/sxe2_tm.h
new file mode 100644
index 0000000000..c4f8da6a8e
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_tm.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_TM_H__
+#define __SXE2_TM_H__
+#include <ethdev_driver.h>
+#include <rte_flow_driver.h>
+#include "sxe2_osal.h"
+
+#define SXE2_TM_MAX_LEVEL 7
+#define SXE2_TM_1L_NODE_NUM_MAX 1
+#define SXE2_TM_2L_NODE_NUM_MAX 8
+#define SXE2_TM_3L_NODE_NUM_MAX 64
+#define SXE2_TM_4L_NODE_NUM_MAX 256
+
+#define SXE2_TM_MAX_CHILDREN_COUNT 8
+
+#define SXE2_TM_WEIGHT_MAX (200)
+#define SXE2_TM_WEIGHT_MIN (1)
+#define SXE2_TM_WEIGHT_SUM (32768)
+
+#define SXE2_HW_RATE_MIN 62500ull
+#define SXE2_HW_RATE_MAX 12500000000ull
+
+#define SXE2_TM_PRIO_MAX (7)
+#define SXE2_TM_PRIO_MIN (0)
+
+enum sxe2_tm_node_type {
+ SXE2_TM_NODE_TYPE_VSIG = 0,
+ SXE2_TM_NODE_TYPE_MID,
+ SXE2_TM_NODE_TYPE_QUEUE,
+ SXE2_TM_NODE_TYPE_INVALID,
+};
+
+struct sxe2_tm_shaper_profile {
+ TAILQ_ENTRY(sxe2_tm_shaper_profile) node;
+ uint32_t id;
+ uint32_t ref_cnt;
+ struct rte_tm_shaper_params profile;
+};
+
+TAILQ_HEAD(sxe2_shaper_profile_list, sxe2_tm_shaper_profile);
+
+struct sxe2_tm_node {
+ uint16_t id;
+ uint16_t teid;
+ uint32_t level;
+ uint32_t child_cnt;
+ uint32_t type;
+ uint16_t hw_weight;
+ uint16_t weight;
+ uint8_t priority;
+ struct sxe2_tm_node *parent;
+ uint8_t index_in_parent;
+ struct sxe2_tm_node *children[SXE2_TM_MAX_CHILDREN_COUNT];
+ struct sxe2_tm_shaper_profile *shaper_profile;
+};
+
+struct sxe2_tm_context {
+ uint32_t tm_layers;
+ uint16_t root_teid;
+ uint8_t root_max_children;
+ uint8_t prio_max;
+ bool committed;
+ struct sxe2_tm_node *root;
+ struct sxe2_shaper_profile_list profile_list;
+};
+
+int32_t sxe2_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg);
+
+int32_t sxe2_tm_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_tm_uninit(struct rte_eth_dev *dev);
+
+#endif /* __SXE2_TM_H__ */
diff --git a/drivers/net/sxe2/sxe2_tx.c b/drivers/net/sxe2/sxe2_tx.c
index a05beb8c7a..a280edc9c5 100644
--- a/drivers/net/sxe2/sxe2_tx.c
+++ b/drivers/net/sxe2/sxe2_tx.c
@@ -304,7 +304,6 @@ int32_t __rte_cold sxe2_tx_queue_setup(struct rte_eth_dev *dev,
}
offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
-
txq = sxe2_tx_queue_alloc(dev, queue_idx, nb_desc, socket_id);
if (txq == NULL) {
PMD_LOG_ERR(TX, "failed to alloc sxe2vf tx queue:%u resource", queue_idx);
--
2.47.3
^ permalink raw reply related
* [PATCH v2 01/23] net/sxe2: support AVX512 vectorized path for Rx and Tx
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add AVX512 vector data path for Rx and Tx burst functions.
The decision to use AVX512 is based on:
1. CPU hardware flags (AVX512F, AVX512BW).
2. Compiler support (CC_AVX512_SUPPORT).
3. Max SIMD bitwidth configuration.
Performance shows approximately X% improvement in small packet
forwarding scenarios.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 24 +
drivers/net/sxe2/sxe2_drv_cmd.h | 80 +--
drivers/net/sxe2/sxe2_ethdev.c | 2 +-
drivers/net/sxe2/sxe2_txrx.c | 91 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 18 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 ++++++++++++++++++++++++
7 files changed, 1103 insertions(+), 55 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 3df57aee8c..30f2c7d816 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -15,6 +15,30 @@ includes += include_directories('../../common/sxe2')
if arch_subdir == 'x86'
sources += files('sxe2_txrx_vec_sse.c')
+
+ sxe2_avx512_cpu_support =(
+ cc.get_define('__AVX512F__', args: machine_args) != '' and
+ cc.get_define('__AVX512BW__', args: machine_args) != '')
+
+ sxe2_avx512_cc_support = (
+ not machine_args.contains('-mno-avx512f') and
+ cc.has_argument('-mavx512f') and
+ cc.has_argument('-mavx512bw'))
+
+ if sxe2_avx512_cpu_support == true or sxe2_avx512_cc_support == true
+ cflags += ['-DCC_AVX512_SUPPORT']
+ avx512_args = [cflags, '-mavx512f', '-mavx512bw']
+ if cc.has_argument('-march=skylake-avx512')
+ avx512_args += '-march=skylake-avx512'
+ endif
+ sxe2_avx512_lib = static_library('sxe2_avx512_lib', 'sxe2_txrx_vec_avx512.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev, static_rte_bus_pci],
+ include_directories: includes,
+ c_args: avx512_args)
+ objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
+ endif
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index a16087c6bf..a9b678f3c5 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -67,20 +67,20 @@ enum sxe2_dev_type {
SXE2_DEV_T_MAX,
};
-struct sxe2_drv_queue_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_caps {
uint16_t queues_cnt;
uint16_t base_idx_in_pf;
-};
+} __rte_packed_end;
-struct sxe2_drv_msix_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_msix_caps {
uint16_t msix_vectors_cnt;
uint16_t base_idx_in_func;
-};
+} __rte_packed_end;
-struct sxe2_drv_rss_hash_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rss_hash_caps {
uint16_t hash_key_size;
uint16_t lut_key_size;
-};
+} __rte_packed_end;
enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_BOTH = 0,
@@ -89,18 +89,18 @@ enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_MAX,
};
-struct sxe2_drv_vsi_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_caps {
uint16_t func_id;
uint16_t dpdk_vsi_id;
uint16_t kernel_vsi_id;
uint16_t vsi_type;
-};
+} __rte_packed_end;
-struct sxe2_drv_representor_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_representor_caps {
uint16_t cnt_repr_vf;
uint8_t rsv[2];
struct sxe2_drv_vsi_caps repr_vf_id[256];
-};
+} __rte_packed_end;
enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_NOTSET = 0,
@@ -111,25 +111,25 @@ enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_UNKNOWN,
};
-struct sxe2_switchdev_mode_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_mode_info {
uint8_t pf_id;
uint8_t is_switchdev;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_switchdev_cpvsi_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_cpvsi_info {
uint16_t cp_vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_txsch_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_txsch_caps {
uint8_t layer_cap;
uint8_t tm_mid_node_num;
uint8_t prio_num;
uint8_t rev;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_caps_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_caps_resp {
struct sxe2_drv_queue_caps queue_caps;
struct sxe2_drv_msix_caps msix_caps;
struct sxe2_drv_rss_hash_caps rss_hash_caps;
@@ -141,24 +141,24 @@ struct sxe2_drv_dev_caps_resp {
uint8_t dev_type;
uint8_t rev;
uint32_t cap_flags;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_info_resp {
uint64_t dsn;
uint16_t vsi_id;
uint8_t rsv[2];
uint8_t mac_addr[SXE2_ETH_ALEN];
uint8_t rsv2[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_fw_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_fw_info_resp {
uint8_t main_version_id;
uint8_t sub_version_id;
uint8_t fix_version_id;
uint8_t build_id;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_ctxt {
uint64_t dma_addr;
uint32_t max_lro_size;
uint32_t split_type_mask;
@@ -170,62 +170,62 @@ struct sxe2_drv_rxq_ctxt {
uint8_t keep_crc_en;
uint8_t split_en;
uint8_t desc_size;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
uint16_t max_frame_size;
uint8_t rsv[2];
struct sxe2_drv_rxq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_ctxt {
uint64_t dma_addr;
uint32_t sched_mode;
uint16_t queue_id;
uint16_t depth;
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
struct sxe2_drv_txq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_q_switch_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_q_switch_req {
uint16_t q_idx;
uint16_t vsi_id;
uint8_t is_enable;
uint8_t sched_mode;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_create_req_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_create_req_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_free_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_free_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 8d66e5d8c5..e0f7002138 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -891,7 +891,7 @@ static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
static int32_t sxe2_parse_eth_devargs(struct rte_device *dev,
struct rte_eth_devargs *eth_da)
{
- int ret = 0;
+ int32_t ret = 0;
if (dev->devargs == NULL)
return 0;
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 8bd5f2eca4..ee70a2a431 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -158,6 +158,19 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
tx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
+#endif
+ }
+ if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
if (ret != 0)
@@ -173,7 +186,6 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
tx_mode_flags = adapter->q_ctxt.tx_mode_flags;
}
-#ifdef RTE_ARCH_X86
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
@@ -182,6 +194,27 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_prepare = NULL;
dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
}
+ }
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
+ dev->tx_pkt_prepare = NULL;
+#ifdef RTE_ARCH_X86
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
+ }
+#endif
+ } else {
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ }
+ }
} else {
#endif
if (tx_mode_flags & SXE2_TX_MODE_SIMPLE_BATCH) {
@@ -202,8 +235,16 @@ static const struct {
} sxe2_tx_burst_infos[] = {
{ sxe2_tx_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
- { sxe2_tx_pkts_vec_sse, "Vector SSE" },
- { sxe2_tx_pkts_vec_sse_simple, "Vector SSE Simple" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_tx_pkts_vec_avx512,
+ "Vector AVX512" },
+ { sxe2_tx_pkts_vec_avx512_simple,
+ "Vector AVX512 Simple" },
+#endif
+ { sxe2_tx_pkts_vec_sse,
+ "Vector SSE" },
+ { sxe2_tx_pkts_vec_sse_simple,
+ "Vector SSE Simple" },
#endif
};
@@ -289,6 +330,20 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
rx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
+#endif
+ }
+ if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
+ rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
if (ret != 0)
@@ -302,7 +357,16 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
#ifdef RTE_ARCH_X86
if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
- dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
+#endif
+ } else {
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ }
return;
}
#endif
@@ -316,19 +380,30 @@ static const struct {
eth_rx_burst_t rx_burst;
const char *info;
} sxe2_rx_burst_infos[] = {
- { sxe2_rx_pkts_scattered, "Scalar Scattered" },
- { sxe2_rx_pkts_scattered_split, "Scalar Scattered split" },
+ { sxe2_rx_pkts_scattered,
+ "Scalar Scattered" },
+ { sxe2_rx_pkts_scattered_split,
+ "Scalar Scattered split" },
#ifdef RTE_ARCH_X86
- { sxe2_rx_pkts_scattered_vec_sse_offload, "Vector SSE Scattered" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_rx_pkts_scattered_vec_avx512,
+ "Vector AVX512 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx512_offload,
+ "Offload Vector AVX512 Scattered" },
+#endif
+ { sxe2_rx_pkts_scattered_vec_sse_offload,
+ "Vector SSE Scattered" },
#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
- __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode)
+ __rte_unused uint16_t queue_id,
+ struct rte_eth_burst_mode *mode)
{
eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
int32_t ret = -EINVAL;
uint32_t i, size;
+
size = RTE_DIM(sxe2_rx_burst_infos);
for (i = 0; i < size; ++i) {
if (pkt_burst == sxe2_rx_burst_infos[i].rx_burst) {
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index 8df4954d86..cf004f5eb2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -165,16 +165,54 @@ static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
return;
}
i = txq->next_dd - (txq->rs_thresh - 1);
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
+#ifdef CC_AVX512_SUPPORT
+ struct rte_eth_dev *dev;
+ struct sxe2_tx_buffer_vec *buffer_vec;
+
+ dev = &rte_eth_devices[txq->port_id];
+
+ if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
+ dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
+ buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for ( ; i < txq->next_use; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ } else {
+#endif
+ buffer = txq->buffer_ring;
+ buffer = txq->buffer_ring;
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer[i].mbuf);
+ buffer[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
buffer[i].mbuf = NULL;
}
}
- i = 0;
+#ifdef CC_AVX512_SUPPORT
}
+#endif
+
for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 4aef93d140..62a5b1f3f5 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -10,15 +10,19 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
- SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE)
+ SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
+ SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
- SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE)
+ SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
+ SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -53,6 +57,16 @@ uint16_t sxe2_tx_pkts_vec_sse(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_
uint16_t sxe2_tx_pkts_vec_sse_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_sse_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_ctx_offload(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
new file mode 100644
index 0000000000..2aec8037dd
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -0,0 +1,897 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_TEST
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_queue *txq)
+{
+ struct sxe2_tx_buffer_vec *buffer;
+ struct rte_mbuf *mbuf;
+ struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
+ struct rte_mempool *mp;
+ struct rte_mempool_cache *cache;
+ void **cache_objs;
+ uint32_t copied;
+ uint32_t i;
+ int32_t ret;
+ uint16_t rs_thresh;
+ uint16_t free_num;
+
+ if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
+ (txq->desc_ring[txq->next_dd].wb.dd &
+ rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
+ ret = 0;
+ goto l_end;
+ }
+
+ rs_thresh = txq->rs_thresh;
+
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += txq->next_dd - (rs_thresh - 1);
+
+ if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
+ (rs_thresh & 31) == 0) {
+ mp = buffer[0].mbuf->pool;
+ cache = rte_mempool_default_cache(mp, rte_lcore_id());
+
+ if (cache == NULL || cache->len)
+ goto normal;
+
+ if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer, rs_thresh);
+ goto done;
+ }
+ cache_objs = &cache->objs[cache->len];
+
+ copied = 0;
+ while (copied < rs_thresh) {
+ const __m512i objs0 = _mm512_loadu_si512(&buffer[copied]);
+ const __m512i objs1 = _mm512_loadu_si512(&buffer[copied + 8]);
+ const __m512i objs2 = _mm512_loadu_si512(&buffer[copied + 16]);
+ const __m512i objs3 = _mm512_loadu_si512(&buffer[copied + 24]);
+
+ _mm512_storeu_si512(&cache_objs[copied], objs0);
+ _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
+ _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
+ _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
+ copied += 32;
+ }
+ cache->len += rs_thresh;
+
+ if (cache->len >= cache->flushthresh) {
+ (void)rte_mempool_ops_enqueue_bulk(mp,
+ &cache->objs[cache->size], cache->len - cache->size);
+ cache->len = cache->size;
+ }
+ goto done;
+ }
+
+normal:
+ mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
+
+ if (likely(mbuf)) {
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+
+ if (likely(mbuf)) {
+ if (likely(mbuf->pool == mbuf_free_arr[0]->pool)) {
+ mbuf_free_arr[free_num] = mbuf;
+ free_num++;
+ } else {
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+ }
+ }
+ }
+
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+ } else {
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+ if (mbuf != NULL)
+ rte_mempool_put(mbuf->pool, mbuf);
+ }
+ }
+
+done:
+ txq->desc_free_num += txq->rs_thresh;
+ txq->next_dd += txq->rs_thresh;
+ if (txq->next_dd >= txq->ring_depth)
+ txq->next_dd = txq->rs_thresh - 1;
+ ret = rs_thresh;
+
+l_end:
+ return ret;
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline
+void sxe2_tx_desc_fill_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m512i desc_group;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ while (pkts_num > 3) {
+ desc3_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group =
+ _mm512_set_epi64(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]),
+ desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, desc), desc_group);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx512(desc, *pkts, desc_cmd, with_offloads);
+
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline void
+sxe2_tx_pkts_mbuf_fill_avx512(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ uint16_t i;
+
+ for (i = 0; i < nb_pkts; ++i)
+ buffer[i].mbuf = tx_pkts[i];
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer_vec *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec_avx512(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx512 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += next_use;
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx512(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = txq->desc_ring;
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ }
+
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num =
+ sxe2_tx_pkts_vec_avx512_batch(txq, tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx512(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read),
+ dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ? (rxq->ring_depth - 1) :
+ (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m512i mbufs4_7;
+ __m512i mbufs0_3;
+ __m256i mbufs6_7;
+ __m256i mbufs4_5;
+ __m256i mbufs2_3;
+ __m256i mbufs0_1;
+ uint32_t bit_num = 0;
+ uint16_t done_num = 0;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx512(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) & SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m512i crc_adjust =
+ _mm512_set4_epi32(0, -rxq->crc_len, -rxq->crc_len, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+
+ const __m512i rvp_shuf_mask =
+ _mm512_set4_epi32((7 << 24) | (6 << 16) | (5 << 8) | 4,
+ (3 << 24) | (2 << 16) | (13 << 8) | 12,
+ (0xFFU << 24) | (0xFF << 16) | (13 << 8) | 12,
+ 0xFFFFFFFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 8, 0, 10, 2, 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ const __m512i descs4_7 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs4_5), descs6_7, 1);
+ const __m512i descs0_3 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs0_1), descs2_3, 1);
+
+ if (split_rxe_flags != NULL) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs4_7 = _mm512_shuffle_epi8(descs4_7, rvp_shuf_mask);
+ mbufs0_3 = _mm512_shuffle_epi8(descs0_3, rvp_shuf_mask);
+
+ mbufs4_7 = _mm512_add_epi32(mbufs4_7, crc_adjust);
+ mbufs0_3 = _mm512_add_epi32(mbufs0_3, crc_adjust);
+
+ const __m512i ptype_mask = _mm512_set1_epi64(SXE2_RX_FLEX_DESC_PTYPE_M <<
+ SXE2_RX_FLEX_DESC_PTYPE_S);
+
+ __m512i ptypes4_7 = _mm512_and_si512(descs4_7, ptype_mask);
+ __m512i ptypes0_3 = _mm512_and_si512(descs0_3, ptype_mask);
+
+ const __m256i ptypes6_7 = _mm512_extracti64x4_epi64(ptypes4_7, 1);
+ const __m256i ptypes4_5 = _mm512_extracti64x4_epi64(ptypes4_7, 0);
+ const __m256i ptypes2_3 = _mm512_extracti64x4_epi64(ptypes0_3, 1);
+ const __m256i ptypes0_1 = _mm512_extracti64x4_epi64(ptypes0_3, 0);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 13);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 5);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 13);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 5);
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 13);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 5);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 13);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 5);
+
+ const __m512i ptype_mask4_7 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype7],
+ 0, 0, 0, ptype_tbl[ptype6],
+ 0, 0, 0, ptype_tbl[ptype5],
+ 0, 0, 0, ptype_tbl[ptype4]);
+ const __m512i ptype_mask0_3 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype3],
+ 0, 0, 0, ptype_tbl[ptype2],
+ 0, 0, 0, ptype_tbl[ptype1],
+ 0, 0, 0, ptype_tbl[ptype0]);
+
+ mbufs4_7 = _mm512_or_si512(mbufs4_7, ptype_mask4_7);
+ mbufs0_3 = _mm512_or_si512(mbufs0_3, ptype_mask0_3);
+
+ mbufs6_7 = _mm512_extracti64x4_epi64(mbufs4_7, 1);
+ mbufs4_5 = _mm512_extracti64x4_epi64(mbufs4_7, 0);
+ mbufs2_3 = _mm512_extracti64x4_epi64(mbufs0_3, 1);
+ mbufs0_1 = _mm512_extracti64x4_epi64(mbufs0_3, 0);
+
+ const __m512i staterr_per_mask =
+ _mm512_set_epi32(0x17, 0x1F, 0x07, 0x0F,
+ 0x13, 0x1B, 0x03, 0x0B,
+ 0x16, 0x1E, 0x06, 0x0E,
+ 0x12, 0x1A, 0x02, 0x0A);
+ __m512i qw1_0_7 = _mm512_permutex2var_epi32(descs4_7,
+ staterr_per_mask,
+ descs0_3);
+
+ __m256i staterrs0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 0);
+
+ __m256i stu_len0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 1);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0xC0001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0,
+ 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+
+ const __m256i vlan_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 8), 0x04);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 4), 0x04);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(mbuf_flags, 4), 0x04);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] = _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 =
+ _mm256_and_si256(staterrs0_7, umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+ __m128i umbcast_bits_128 =
+ _mm_packs_epi32(_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256(umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags) {
+ const __m256i eop_rxe_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+ const __m128i eop_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits =
+ _mm_or_si128(not_eop_bits,
+ _mm_srli_epi16(_mm_and_si128(eop_rxe_bits,
+ rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((rxq->processing_idx & 1) == 1 && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ !split_rxe_flags64[0] && !split_rxe_flags64[1] &&
+ !split_rxe_flags64[2] && !split_rxe_flags64[3]) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num = 0;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue, rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM, offload);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue,
+ rx_pkts + done_num, nb_pkts, offload);
+
+end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
+
+#endif
--
2.47.3
^ permalink raw reply related
* [PATCH v2 02/23] net/sxe2: add AVX2 vector data path for Rx and Tx
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Added AVX256 vectorized versions of Rx and Tx data path functions to
improve packet processing performance.
The vector path uses AVX2 SIMD instructions to process multiple
descriptors per loop, significantly reducing the per-packet overhead.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
net/sxe2: support AVX512 vectorized path for Rx and Tx
---
drivers/net/sxe2/meson.build | 9 +
drivers/net/sxe2/sxe2_txrx.c | 38 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 12 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 777 ++++++++++++++++++++++++++
4 files changed, 831 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 30f2c7d816..98dd8bcec7 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -39,6 +39,15 @@ if arch_subdir == 'x86'
c_args: avx512_args)
objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
endif
+ sxe2_avx2_lib = static_library('sxe2_avx2_lib',
+ 'sxe2_txrx_vec_avx2.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev,
+ static_rte_bus_pci],
+ include_directories: includes,
+ c_args: [cflags, '-mavx2'])
+ objs += sxe2_avx2_lib.extract_objects('sxe2_txrx_vec_avx2.c')
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index ee70a2a431..dcfaf7278d 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -168,8 +168,14 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
#endif
}
- if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
- tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+ if (((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX2;
+
+ if ((0 == (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK)))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
@@ -207,6 +213,13 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
}
#endif
+ } else if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX2) {
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2_simple;
+ }
} else {
if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
@@ -241,6 +254,10 @@ static const struct {
{ sxe2_tx_pkts_vec_avx512_simple,
"Vector AVX512 Simple" },
#endif
+ { sxe2_tx_pkts_vec_avx2,
+ "Vector AVX2" },
+ { sxe2_tx_pkts_vec_avx2_simple,
+ "Vector AVX2 Simple" },
{ sxe2_tx_pkts_vec_sse,
"Vector SSE" },
{ sxe2_tx_pkts_vec_sse_simple,
@@ -340,7 +357,13 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
#endif
}
- if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX2;
+
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
#endif
@@ -364,6 +387,11 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
else
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
#endif
+ } else if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX2) {
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2;
} else {
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
}
@@ -391,6 +419,10 @@ static const struct {
{ sxe2_rx_pkts_scattered_vec_avx512_offload,
"Offload Vector AVX512 Scattered" },
#endif
+ { sxe2_rx_pkts_scattered_vec_avx2,
+ "Vector AVX2 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx2_offload,
+ "Offload Vector AVX2 Scattered" },
{ sxe2_rx_pkts_scattered_vec_sse_offload,
"Vector SSE Scattered" },
#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 62a5b1f3f5..d7a0ce6ca5 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -10,19 +10,21 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
- SXE2_RX_MODE_VEC_AVX512)
+ SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
- SXE2_TX_MODE_VEC_AVX512)
+ SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -67,6 +69,12 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
new file mode 100644
index 0000000000..5d57ebf9e2
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
@@ -0,0 +1,777 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m256i desc_group0;
+ __m256i desc_group1;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ if (((uint64_t)desc & 0x1F) != 0 && pkts_num != 0) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+
+ while (pkts_num > 3) {
+ desc3_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group1 = _mm256_set_epi64x(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]));
+
+ desc_group0 = _mm256_set_epi64x(desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc + 2), desc_group1);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc), desc_group0);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx2 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx2(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+ }
+
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num = sxe2_tx_pkts_vec_avx2_batch(txq,
+ tx_pkts + tx_done_num, tx_need_num, with_offloads);
+
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx2(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read), dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ?
+ (rxq->ring_depth - 1) : (rxq->realloc_start - 1);
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+l_end:
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx2(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m256i mbufs6_7, mbufs4_5, mbufs2_3, mbufs0_1;
+ uint32_t bit_num;
+ uint16_t done_num;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+ done_num = 0;
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx2(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m256i crc_adjust =
+ _mm256_set_epi16(0, 0, 0, -rxq->crc_len,
+ 0, -rxq->crc_len, 0,
+ 0, 0, 0, 0,
+ -rxq->crc_len, 0, -rxq->crc_len, 0, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+ const __m256i rvp_shuf_mask =
+ _mm256_set_epi8(7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 8, 0, 10, 2,
+ 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ if (split_rxe_flags) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs6_7 = _mm256_shuffle_epi8(descs6_7, rvp_shuf_mask);
+ mbufs4_5 = _mm256_shuffle_epi8(descs4_5, rvp_shuf_mask);
+
+ mbufs6_7 = _mm256_add_epi16(mbufs6_7, crc_adjust);
+ mbufs4_5 = _mm256_add_epi16(mbufs4_5, crc_adjust);
+
+ const __m256i ptype_mask = _mm256_set1_epi32(SXE2_RX_DESC_PTYPE_MASK);
+
+ const __m256i staterrs4_7 = _mm256_unpackhi_epi32(descs6_7, descs4_5);
+
+ __m256i ptypes4_7 = _mm256_and_si256(staterrs4_7, ptype_mask);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes4_7, 9);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes4_7, 1);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_7, 11);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_7, 3);
+
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype7], 4);
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype6], 0);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype5], 4);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype4], 0);
+
+ mbufs2_3 = _mm256_shuffle_epi8(descs2_3, rvp_shuf_mask);
+ mbufs0_1 = _mm256_shuffle_epi8(descs0_1, rvp_shuf_mask);
+
+ mbufs2_3 = _mm256_add_epi16(mbufs2_3, crc_adjust);
+ mbufs0_1 = _mm256_add_epi16(mbufs0_1, crc_adjust);
+
+ const __m256i staterrs0_3 = _mm256_unpackhi_epi32(descs2_3, descs0_1);
+
+ __m256i ptypes0_3 = _mm256_and_si256(staterrs0_3, ptype_mask);
+
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes0_3, 9);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes0_3, 1);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_3, 11);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_3, 3);
+
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype3], 4);
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype2], 0);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype1], 4);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype0], 0);
+
+ __m256i staterrs0_7 = _mm256_unpacklo_epi64(staterrs4_7, staterrs0_3);
+
+ __m256i stu_len0_7 = _mm256_unpackhi_epi64(staterrs4_7, staterrs0_3);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0x00001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+ const __m256i vlan_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 8), 4);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 4), 4);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 4);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(mbuf_flags, 4), 4);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] =
+ _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] =
+ _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags != NULL) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 = _mm256_and_si256(staterrs0_7,
+ umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+
+ __m128i umbcast_bits_128 = _mm_packs_epi32
+ (_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256
+ (umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags != NULL) {
+ const __m256i eop_rxe_mask = _mm256_set1_epi32
+ (SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m128i eop_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits = _mm_or_si128
+ (not_eop_bits,
+ _mm_srli_epi16
+ (_mm_and_si128(eop_rxe_bits, rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((1 == (rxq->processing_idx & 1)) && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ split_rxe_flags64[0] == 0 && split_rxe_flags64[1] == 0 &&
+ split_rxe_flags64[2] == 0 && split_rxe_flags64[3] == 0) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num =
+ sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM,
+ do_offload);
+ done_num += once_num;
+ nb_pkts -= once_num;
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num, nb_pkts, do_offload);
+l_end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
--
2.47.3
^ permalink raw reply related
* [PATCH v2 04/23] net/sxe2: support L2 filtering and MAC config
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260530140904.157099-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
- Support primary/secondary MAC address setup.
- Enable L2 broadcast/multicast filter bits.
- Add multicast address update logic.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 198 ++++++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 17 +
drivers/net/sxe2/sxe2_drv_cmd.h | 87 ++++
drivers/net/sxe2/sxe2_ethdev.c | 70 ++-
drivers/net/sxe2/sxe2_ethdev.h | 43 +-
drivers/net/sxe2/sxe2_filter.c | 784 ++++++++++++++++++++++++++++++
drivers/net/sxe2/sxe2_filter.h | 98 ++++
drivers/net/sxe2/sxe2_mac.c | 432 ++++++++++++++++
drivers/net/sxe2/sxe2_mac.h | 34 ++
drivers/net/sxe2/sxe2_txrx_poll.c | 49 ++
11 files changed, 1807 insertions(+), 6 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index e22204e850..8ff74e5233 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -61,6 +61,7 @@ sources += files(
'sxe2_txrx.c',
'sxe2_txrx_vec.c',
'sxe2_mac.c',
+ 'sxe2_filter.c',
)
allow_internal_get_api = true
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index 07eeb7f38c..1fa9ad718e 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -343,3 +343,201 @@ int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
l_end:
return ret;
}
+
+int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_promisc_filter_cfg_req promisc_filter_cfg_req = {0};
+
+ promisc_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ promisc_filter_cfg_req.is_add = set;
+ promisc_filter_cfg_req.type = SXE2_PROMISC_FILTER_TYPE_PROMISC;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_PROMISC_CFG,
+ &promisc_filter_cfg_req,
+ sizeof(promisc_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "promic config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_promisc_filter_cfg_req promisc_filter_cfg_req = {0};
+
+ promisc_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ promisc_filter_cfg_req.is_add = set;
+ promisc_filter_cfg_req.type = SXE2_PROMISC_FILTER_TYPE_ALLMULTI;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_ALLMULTI_CFG,
+ &promisc_filter_cfg_req,
+ sizeof(promisc_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "allmulti config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add)
+{
+ int32_t ret = 0;
+ int32_t i;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_mac_filter_cfg_req mac_filter_cfg_req = {0};
+
+ mac_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ for (i = 0; i < SXE2_ETH_ALEN; i++)
+ mac_filter_cfg_req.addr[i] = addr->addr_bytes[i];
+ mac_filter_cfg_req.is_add = add;
+ mac_filter_cfg_req.type = SXE2_MAC_FILTER_TYPE_UC;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_ADDR_UC,
+ &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "uc config query failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_mc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add)
+{
+ int32_t ret = 0;
+ int32_t i;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_mac_filter_cfg_req mac_filter_cfg_req = {0};
+
+ mac_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ for (i = 0; i < SXE2_ETH_ALEN; i++)
+ mac_filter_cfg_req.addr[i] = addr->addr_bytes[i];
+
+ mac_filter_cfg_req.is_add = add;
+ mac_filter_cfg_req.type = SXE2_MAC_FILTER_TYPE_MC;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_MAC_ADDR_MC,
+ &mac_filter_cfg_req, sizeof(mac_filter_cfg_req),
+ NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "mac config query failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_config_query(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_vlan_cfg_query_resp vlan_cfg_query_resp = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_CFG_QUERY,
+ NULL, 0,
+ &vlan_cfg_query_resp,
+ sizeof(vlan_cfg_query_resp));
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config query failed, ret=%d", ret);
+
+ adapter->filter_ctxt.vlan_info.port_vlan_exist = vlan_cfg_query_resp.port_vlan_exist;
+ adapter->filter_ctxt.vlan_info.is_switchdev = vlan_cfg_query_resp.is_switchdev;
+
+
+ adapter->filter_ctxt.vlan_info.tpid = vlan_cfg_query_resp.tpid;
+ adapter->filter_ctxt.vlan_info.vid = vlan_cfg_query_resp.vid;
+
+ adapter->filter_ctxt.vlan_info.outer_insert = vlan_cfg_query_resp.outer_insert;
+ adapter->filter_ctxt.vlan_info.outer_strip = vlan_cfg_query_resp.outer_strip;
+ adapter->filter_ctxt.vlan_info.inner_insert = vlan_cfg_query_resp.inner_insert;
+ adapter->filter_ctxt.vlan_info.inner_strip = vlan_cfg_query_resp.inner_strip;
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_filter_id_config(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool on)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_vlan_filter_cfg_req vlan_filter_cfg_req = {0};
+
+ vlan_filter_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ vlan_filter_cfg_req.tpid_id = vlan->tpid;
+ vlan_filter_cfg_req.vlan_id = vlan->vid;
+ vlan_filter_cfg_req.prio = vlan->prio;
+ vlan_filter_cfg_req.is_add = on;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_FILTER_ADD_DEL,
+ &vlan_filter_cfg_req, sizeof(vlan_filter_cfg_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_insert_strip_cfg(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_vlan_offload_cfg_req vlan_offload_cfg_req = {0};
+
+ vlan_offload_cfg_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ vlan_offload_cfg_req.tpid = adapter->filter_ctxt.vlan_info.tpid;
+ vlan_offload_cfg_req.outer_insert = adapter->filter_ctxt.vlan_info.outer_insert;
+ vlan_offload_cfg_req.outer_strip = adapter->filter_ctxt.vlan_info.outer_strip;
+ vlan_offload_cfg_req.inner_insert = adapter->filter_ctxt.vlan_info.inner_insert;
+ vlan_offload_cfg_req.inner_strip = adapter->filter_ctxt.vlan_info.inner_strip;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_OFFLOAD_CFG,
+ &vlan_offload_cfg_req,
+ sizeof(vlan_offload_cfg_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config query failed, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_vlan_filter_switch_req vlan_filter_switch_req = {0};
+
+ vlan_filter_switch_req.vsi_id = adapter->vsi_ctxt.dpdk_vsi_id;
+ vlan_filter_switch_req.is_oper_enable = on;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_VLAN_FILTER_SWITCH,
+ &vlan_filter_switch_req,
+ sizeof(vlan_filter_switch_req),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_WARN(adapter, DRV, "vlan config filter failed, ret=%d", ret);
+
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index 34004d37e2..fb01c41aad 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -36,4 +36,21 @@ int32_t sxe2_drv_txq_ctxt_cfg(struct sxe2_adapter *adapter,
int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
+int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+
+int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set);
+
+int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
+
+int32_t sxe2_drv_mc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
+
+int32_t sxe2_drv_vlan_config_query(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_vlan_filter_id_config(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool on);
+
+int32_t sxe2_drv_vlan_insert_strip_cfg(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on);
+
#endif /* __SXE2_CMD_CHNL_H__ */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index ad1c1c9603..f4d8069edb 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -233,6 +233,93 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_link_info_resp {
uint8_t rsv[3];
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_info {
+ uint8_t is_switchdev;
+ uint8_t master;
+ uint8_t representor;
+ uint8_t port_name_type;
+ uint32_t ctrl_num;
+ uint32_t pf_num;
+ uint32_t vf_num;
+ uint32_t mpesw_owner;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vlan_cfg_query_resp {
+ uint16_t vsi_id;
+ uint8_t port_vlan_exist;
+ uint8_t is_switchdev;
+ uint16_t tpid;
+ uint16_t vid;
+ uint8_t outer_insert;
+ uint8_t outer_strip;
+ uint8_t inner_insert;
+ uint8_t inner_strip;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vlan_offload_cfg_req {
+ uint16_t vsi_id;
+ uint16_t tpid;
+ uint8_t outer_insert;
+ uint8_t outer_strip;
+ uint8_t inner_insert;
+ uint8_t inner_strip;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_port_vlan_cfg_req {
+ uint16_t vsi_id;
+ uint16_t tpid;
+ uint16_t vid;
+ uint8_t prio;
+ uint8_t rsv;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_mac_filter_cfg_req {
+ uint16_t vsi_id;
+ uint8_t addr[SXE2_ETH_ALEN];
+ uint8_t type;
+ uint8_t is_add;
+ uint8_t rsv[2];
+} __rte_packed_end;
+
+enum sxe2_promisc_filter_type {
+ SXE2_PROMISC_FILTER_TYPE_PROMISC = 0,
+ SXE2_PROMISC_FILTER_TYPE_ALLMULTI,
+ SXE2_PROMISC_FILTER_TYPE_MAX,
+};
+
+enum sxe2_mac_filter_type {
+ SXE2_MAC_FILTER_TYPE_UC = 0,
+ SXE2_MAC_FILTER_TYPE_MC,
+ SXE2_MAC_FILTER_TYPE_MAX,
+};
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_promisc_filter_cfg_req {
+ uint16_t vsi_id;
+ uint8_t type;
+ uint8_t is_add;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_srcvsi_ext_cfg_req {
+ uint16_t vsi_id;
+ uint16_t srcvsi_list[SXE2_SRCVSI_PRUNE_MAX_NUM];
+ uint8_t srcvsi_cnt;
+ uint8_t is_add;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_vlan_filter_cfg_req {
+ uint16_t vsi_id;
+ uint16_t vlan_id;
+ uint16_t tpid_id;
+ uint8_t prio;
+ uint8_t is_add;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_vlan_filter_switch_req {
+ uint16_t vsi_id;
+ uint8_t is_oper_enable;
+ uint8_t rsv;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 01552a8202..9b117f097e 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -111,8 +111,20 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.tx_burst_mode_get = sxe2_tx_burst_mode_get,
.tx_done_cleanup = sxe2_tx_done_cleanup,
+ .promiscuous_enable = sxe2_promisc_enable,
+ .promiscuous_disable = sxe2_promisc_disable,
+ .allmulticast_enable = sxe2_allmulti_enable,
+ .allmulticast_disable = sxe2_allmulti_disable,
+
+ .mac_addr_add = sxe2_mac_addr_add,
+ .mac_addr_remove = sxe2_mac_addr_del,
+ .mac_addr_set = sxe2_mac_addr_set,
+ .set_mc_addr_list = sxe2_set_mc_addr_list,
.mtu_set = sxe2_mtu_set,
.buffer_split_supported_hdr_ptypes_get = sxe2_buffer_split_supported_hdr_ptypes_get,
+
+ .vlan_filter_set = sxe2_dev_vlan_filter_set,
+ .vlan_offload_set = sxe2_dev_vlan_offload_set,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -123,6 +135,13 @@ static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
+ ret = sxe2_vlan_default_cfg(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init vlan, ret=%d", ret);
+ goto end;
+ }
+
+end:
return ret;
}
@@ -138,6 +157,8 @@ static int32_t sxe2_dev_stop(struct rte_eth_dev *dev)
sxe2_txqs_all_stop(dev);
sxe2_rxqs_all_stop(dev);
+ (void)sxe2_filter_rule_stop(dev);
+
dev->data->dev_started = 0;
adapter->started = 0;
l_end:
@@ -165,16 +186,23 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
goto l_end;
}
+ ret = sxe2_filter_rule_start(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to add all mc addr to fw.");
+ goto l_end;
+ }
+
ret = sxe2_queues_start(dev);
if (ret) {
PMD_LOG_ERR(INIT, "enable queues failed");
- goto l_end;
+ goto l_start_queues_err;
}
dev->data->dev_started = 1;
adapter->started = 1;
goto l_end;
-
+l_start_queues_err:
+ (void)sxe2_filter_rule_stop(dev);
l_end:
return ret;
}
@@ -194,6 +222,7 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
dev_info->min_mtu = RTE_ETHER_MIN_MTU;
dev_info->rx_offload_capa =
+ RTE_ETH_RX_OFFLOAD_VLAN_STRIP |
RTE_ETH_RX_OFFLOAD_KEEP_CRC |
RTE_ETH_RX_OFFLOAD_SCATTER |
RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
@@ -202,9 +231,15 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
RTE_ETH_RX_OFFLOAD_SCTP_CKSUM |
RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM |
RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT |
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ RTE_ETH_RX_OFFLOAD_QINQ_STRIP |
+#endif
+ RTE_ETH_RX_OFFLOAD_VLAN_EXTEND |
RTE_ETH_RX_OFFLOAD_TCP_LRO;
dev_info->tx_offload_capa =
+ RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
+ RTE_ETH_TX_OFFLOAD_QINQ_INSERT |
RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE |
RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
@@ -428,6 +463,12 @@ static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
{
int32_t ret = 0;
+ ret = sxe2_filter_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize l2 filter, ret:%d", ret);
+ goto l_end;
+ }
+
ret = sxe2_link_update_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
@@ -439,12 +480,37 @@ static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
PMD_LOG_ERR(INIT, "Failed to set mtu, ret=%d", ret);
goto l_end;
}
+
+ ret = sxe2_mac_addr_init(dev);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to initialize mac address, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_mac_default_cfg(dev);
+ if (ret != 0) {
+ PMD_LOG_ERR(INIT, "Failed to configure default mac address, ret:%d", ret);
+ goto l_err;
+ }
+
+ ret = sxe2_vlan_cfg_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize vlan config, ret:%d", ret);
+ goto l_err;
+ }
+ goto l_end;
+
+l_err:
+ sxe2_mac_addr_uinit(dev);
+ (void)sxe2_filter_uinit(dev);
l_end:
return ret;
}
static void sxe2_eth_uinit(struct rte_eth_dev *dev __rte_unused)
{
+ sxe2_mac_addr_uinit(dev);
+ (void)sxe2_filter_uinit(dev);
}
static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index 66f49ac0cc..cc8a84c0a0 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -13,9 +13,11 @@
#include "sxe2_common.h"
#include "sxe2_vsi.h"
-#include "sxe2_queue.h"
#include "sxe2_irq.h"
+#include "sxe2_queue.h"
+#include "sxe2_mac.h"
#include "sxe2_osal.h"
+#include "sxe2_filter.h"
struct sxe2_link_msg {
uint32_t speed;
@@ -33,7 +35,7 @@ enum sxe2_fnav_tunnel_flag_type {
#define SXE2_FRAME_SIZE_MAX 9832
#define SXE2_VLAN_TAG_SIZE 4
#define SXE2_ETH_OVERHEAD \
- (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + SXE2_VLAN_TAG_SIZE)
+ (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 2 * SXE2_VLAN_TAG_SIZE)
#define SXE2_ETH_MAX_LEN (RTE_ETHER_MTU + SXE2_ETH_OVERHEAD)
#ifdef SXE2_TEST
@@ -265,6 +267,27 @@ struct sxe2_link_context {
uint32_t speed;
};
+struct sxe2_filter_context {
+ rte_spinlock_t filter_lock;
+ struct sxe2_vlan_info vlan_info;
+ struct sxe2_uc_filter_list_head uc_list;
+ struct sxe2_mc_filter_list_head mc_list;
+ struct sxe2_vlan_filter_list_head vlan_list;
+ uint8_t uc_num;
+ uint8_t mc_num;
+ uint8_t vlan_num;
+ uint8_t rsv;
+ uint32_t hw_promisc_flags;
+ uint32_t cur_promisc_flags;
+
+ bool hw_uplink_config;
+ bool cur_uplink_config;
+ bool hw_repr_config;
+ bool cur_repr_config;
+ bool hw_l2_config;
+ bool cur_l2_config;
+};
+
struct sxe2_adapter {
struct sxe2_common_device *cdev;
struct sxe2_dev_info dev_info;
@@ -274,10 +297,14 @@ struct sxe2_adapter {
struct sxe2_irq_context irq_ctxt;
struct sxe2_queue_context q_ctxt;
struct sxe2_vsi_context vsi_ctxt;
+ struct sxe2_filter_context filter_ctxt;
struct sxe2_link_context link_ctxt;
struct sxe2_devargs devargs;
- uint16_t dev_port_id;
- uint64_t cap_flags;
+ struct sxe2_switchdev_info switchdev_info;
+ bool rule_started;
+ bool flow_isolated;
+ uint16_t dev_port_id;
+ uint64_t cap_flags;
enum sxe2_dev_type dev_type;
uint32_t ptype_tbl[SXE2_MAX_PTYPE_NUM];
struct rte_ether_addr mac_addr;
@@ -316,4 +343,12 @@ int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev);
void sxe2_dev_pci_map_uinit(struct rte_eth_dev *dev);
+static inline bool
+sxe2_dev_port_vlan_check(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *ad = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ return ad->filter_ctxt.vlan_info.port_vlan_exist;
+}
+
#endif /* __SXE2_ETHDEV_H__ */
diff --git a/drivers/net/sxe2/sxe2_filter.c b/drivers/net/sxe2/sxe2_filter.c
new file mode 100644
index 0000000000..cfeeb7a6c3
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_filter.c
@@ -0,0 +1,784 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_os.h>
+#include <rte_tailq.h>
+#include "sxe2_osal.h"
+#include "sxe2_mac.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+
+static struct sxe2_mac_filter *sxe2_uc_filter_find(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *macaddr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ struct sxe2_mac_filter *entry = NULL;
+ struct sxe2_mac_filter *next_entry = NULL;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.uc_list, next, next_entry) {
+ if (rte_is_same_ether_addr(macaddr, &entry->mac_addr)) {
+ filter = entry;
+ break;
+ }
+ }
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ return filter;
+}
+
+int32_t sxe2_uc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ bool hw_config = false;
+ int32_t ret = 0;
+
+ filter = sxe2_uc_filter_find(adapter, mac_addr);
+ if (filter) {
+ if (default_config && !filter->default_config)
+ filter->default_config = true;
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter already exists.");
+ goto l_end;
+ }
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add hw uc addr in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw uc addr in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw uc addr in switchdev mode");
+ } else {
+ ret = sxe2_drv_uc_config(adapter, mac_addr, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add uc rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ hw_config = true;
+ }
+
+ filter = rte_zmalloc("sxe2_uc_filter",
+ sizeof(struct sxe2_mac_filter), 0);
+ if (!filter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to allocate memory");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ filter->hw_config = hw_config;
+ filter->default_config = default_config;
+ rte_ether_addr_copy(mac_addr, &filter->mac_addr);
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_INSERT_TAIL(&adapter->filter_ctxt.uc_list, filter, next);
+ adapter->filter_ctxt.uc_num++;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ PMD_DEV_LOG_INFO(adapter, DRV, "add mac rule, mac num %u.", adapter->filter_ctxt.uc_num);
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_uc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ int32_t ret = -1;
+
+ filter = sxe2_uc_filter_find(adapter, mac_addr);
+ if (!filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter not exists.");
+ ret = 0;
+ goto l_end;
+ }
+ if (filter->hw_config) {
+ ret = sxe2_drv_uc_config(adapter, mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mac rule");
+ if (ret == -EPERM)
+ goto l_free;
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ PMD_DEV_LOG_INFO(adapter, DRV, "remove mac rule, uc num %u.", adapter->filter_ctxt.uc_num);
+ ret = 0;
+
+l_free:
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_REMOVE(&adapter->filter_ctxt.uc_list, filter, next);
+ adapter->filter_ctxt.uc_num--;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+ rte_free(filter);
+ filter = NULL;
+l_end:
+ return ret;
+}
+
+void sxe2_uc_filter_clear(struct sxe2_adapter *adapter, bool default_config)
+{
+ struct sxe2_mac_filter *entry;
+ struct sxe2_mac_filter *next_entry;
+
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.uc_list, next, next_entry) {
+ if (entry->default_config && !default_config)
+ continue;
+
+ if (sxe2_uc_filter_del(adapter, &entry->mac_addr))
+ PMD_DEV_LOG_ERR(adapter, DRV, "This MAC filter delete fail.");
+ }
+}
+
+static struct sxe2_mac_filter *sxe2_mc_filter_find(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *macaddr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ struct sxe2_mac_filter *entry = NULL;
+ struct sxe2_mac_filter *next_entry = NULL;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.mc_list, next, next_entry) {
+ if (rte_is_same_ether_addr(macaddr, &entry->mac_addr)) {
+ filter = entry;
+ break;
+ }
+ }
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ return filter;
+}
+
+int32_t sxe2_mc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ bool hw_config = false;
+ int32_t ret = 0;
+
+ filter = sxe2_mc_filter_find(adapter, mac_addr);
+ if (filter) {
+ if (default_config && !filter->default_config)
+ filter->default_config = true;
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter already exists.");
+ goto l_end;
+ }
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add hw mc addr in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw mc addr in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add hw mc addr in switchdev mode");
+ } else {
+ ret = sxe2_drv_mc_config(adapter, mac_addr, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add mac rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ hw_config = true;
+ }
+
+ filter = rte_zmalloc("sxe2_mc_filter",
+ sizeof(struct sxe2_mac_filter), 0);
+ if (!filter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to allocate memory");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ filter->hw_config = hw_config;
+ filter->default_config = default_config;
+ rte_ether_addr_copy(mac_addr, &filter->mac_addr);
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_INSERT_TAIL(&adapter->filter_ctxt.mc_list, filter, next);
+ adapter->filter_ctxt.mc_num++;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ PMD_DEV_LOG_INFO(adapter, DRV, "add mc rule, mc num %u.", adapter->filter_ctxt.mc_num);
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_mc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr)
+{
+ struct sxe2_mac_filter *filter = NULL;
+ int32_t ret = -1;
+
+ filter = sxe2_mc_filter_find(adapter, mac_addr);
+ if (!filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This MAC filter not exists.");
+ ret = 0;
+ goto l_end;
+ }
+
+ if (filter->hw_config) {
+ ret = sxe2_drv_mc_config(adapter, mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mc rule");
+ if (ret == -EPERM)
+ goto l_free;
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ PMD_DEV_LOG_INFO(adapter, DRV, "remove mc rule, mc num %u.", adapter->filter_ctxt.mc_num);
+ ret = 0;
+
+l_free:
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_REMOVE(&adapter->filter_ctxt.mc_list, filter, next);
+ adapter->filter_ctxt.mc_num--;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+ rte_free(filter);
+ filter = NULL;
+l_end:
+ return ret;
+}
+
+void sxe2_mc_filter_clear(struct sxe2_adapter *adapter, bool default_config)
+{
+ struct sxe2_mac_filter *entry;
+ struct sxe2_mac_filter *next_entry;
+
+ RTE_TAILQ_FOREACH_SAFE(entry, &adapter->filter_ctxt.mc_list, next, next_entry) {
+ if (entry->default_config && !default_config)
+ continue;
+ if (sxe2_mc_filter_del(adapter, &entry->mac_addr))
+ PMD_DEV_LOG_ERR(adapter, DRV, "This MAC filter delete fail.");
+ }
+}
+
+static struct sxe2_vlan_filter *sxe2_vlan_filter_find(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan)
+{
+ struct sxe2_vlan_filter *f;
+ struct sxe2_vlan_filter *save_f = NULL;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_FOREACH(f, &adapter->filter_ctxt.vlan_list, next)
+ {
+ if (vlan->tpid == f->vlan_info.tpid &&
+ vlan->vid == f->vlan_info.vid) {
+ save_f = f;
+ break;
+ }
+ }
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ return save_f;
+}
+
+int32_t sxe2_vlan_filter_add(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool default_config)
+{
+ struct sxe2_vlan_filter *filter = NULL;
+ bool hw_config = false;
+ int32_t ret = 0;
+
+ if (!vlan || vlan->vid > RTE_ETHER_MAX_VLAN_ID) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "This vlan filter is invalid.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ filter = sxe2_vlan_filter_find(adapter, vlan);
+ if (filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This vlan filter already exists.");
+ ret = 0;
+ goto l_end;
+ }
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add vlan in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan in switchdev mode");
+ } else {
+ ret = sxe2_drv_vlan_filter_id_config(adapter, vlan, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ hw_config = true;
+ }
+
+ filter = rte_zmalloc("sxe2_vlan_filter", sizeof(*filter), 0);
+ if (!filter) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to allocate memory");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ filter->hw_config = hw_config;
+ filter->default_config = default_config;
+
+ filter->vlan_info.tpid = vlan->tpid;
+ filter->vlan_info.vid = vlan->vid;
+ filter->vlan_info.prio = vlan->prio;
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_INSERT_TAIL(&adapter->filter_ctxt.vlan_list, filter, next);
+ adapter->filter_ctxt.vlan_num++;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_vlan_filter_del(struct sxe2_adapter *adapter, struct sxe2_vlan *vlan)
+{
+ struct sxe2_vlan_filter *filter = NULL;
+ int32_t ret = -1;
+
+ if (!vlan || vlan->vid > RTE_ETHER_MAX_VLAN_ID) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This vlan filter is invalid.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ filter = sxe2_vlan_filter_find(adapter, vlan);
+ if (!filter) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "This vlan filter not exists.");
+ ret = 0;
+ goto l_end;
+ }
+
+ if (filter->hw_config) {
+ ret = sxe2_drv_vlan_filter_id_config(adapter, vlan, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete vlan rule");
+ if (ret == -EPERM)
+ goto l_free;
+ ret = -EINVAL;
+ goto l_end;
+ }
+ }
+ ret = 0;
+
+l_free:
+
+ rte_spinlock_lock(&adapter->filter_ctxt.filter_lock);
+ TAILQ_REMOVE(&adapter->filter_ctxt.vlan_list, filter, next);
+ adapter->filter_ctxt.vlan_num--;
+ rte_spinlock_unlock(&adapter->filter_ctxt.filter_lock);
+ rte_free(filter);
+ filter = NULL;
+l_end:
+ return ret;
+}
+
+void sxe2_vlan_filters_clear(struct sxe2_adapter *adapter, bool default_config)
+{
+ int32_t ret = 0;
+ struct sxe2_vlan_filter *v_f;
+ void *temp;
+
+ if (adapter->filter_ctxt.vlan_num == 0)
+ goto l_end;
+
+ RTE_TAILQ_FOREACH_SAFE(v_f, &adapter->filter_ctxt.vlan_list, next, temp)
+ {
+ if (v_f->default_config && !default_config)
+ continue;
+ ret = sxe2_vlan_filter_del(adapter, &v_f->vlan_info);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "This vlan filter delete fail.");
+ }
+
+l_end:
+}
+
+int32_t sxe2_vlan_filter_ctrl(struct sxe2_adapter *adapter, bool flag)
+{
+ struct sxe2_vlan_info *vlan_info = &adapter->filter_ctxt.vlan_info;
+ int32_t ret = 0;
+
+ if (vlan_info->filter_on == flag)
+ goto l_end;
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot add vlan filter ctrl in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan filter ctrl in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot add vlan filter ctrl in switchdev mode");
+ } else {
+ ret = sxe2_drv_vlan_filter_switch(adapter, flag);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan filter ctrl");
+ goto l_end;
+ }
+ vlan_info->hw_filter_on = flag;
+ }
+ vlan_info->filter_on = flag;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_promisc_add(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot enable promiscuous in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable promiscuous in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable promiscuous in switchdev mode");
+ } else if (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC)) {
+ ret = sxe2_drv_promisc_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg promiscuous, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC;
+ }
+ adapter->filter_ctxt.cur_promisc_flags |= SXE2_PROMISC;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_promisc_del(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated &&
+ (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC)) {
+ ret = sxe2_drv_promisc_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg promiscuous, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC;
+ }
+
+ adapter->filter_ctxt.cur_promisc_flags &= ~SXE2_PROMISC;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_allmulti_add(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->rule_started) {
+ PMD_DEV_LOG_DEBUG(adapter, DRV, "cannot enable allmulticast in port stop status");
+ } else if (adapter->flow_isolated) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable allmulticast in flow isolation mode");
+ } else if (adapter->switchdev_info.is_switchdev) {
+ PMD_DEV_LOG_WARN(adapter, DRV, "cannot enable allmulticast in switchdev mode");
+ } else if (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST)) {
+ ret = sxe2_drv_allmulti_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg allmulticast, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC_MULTICAST;
+ }
+ adapter->filter_ctxt.cur_promisc_flags |= SXE2_PROMISC_MULTICAST;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_allmulti_del(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated &&
+ (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST)) {
+ ret = sxe2_drv_allmulti_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg allmulticast, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC_MULTICAST;
+ }
+
+ adapter->filter_ctxt.cur_promisc_flags &= ~SXE2_PROMISC_MULTICAST;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_all_filter_hw_clear(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_mac_filter *mac_entry;
+ struct sxe2_mac_filter *next_mac_entry;
+ struct sxe2_vlan_filter *vlan_entry;
+ struct sxe2_vlan_filter *next_vlan_entry;
+
+ if (adapter->filter_ctxt.uc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.uc_list, next,
+ next_mac_entry) {
+ if (mac_entry->hw_config) {
+ ret = sxe2_drv_uc_config(adapter, &mac_entry->mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mac rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = false;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.mc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.mc_list, next,
+ next_mac_entry) {
+ if (mac_entry->hw_config) {
+ ret = sxe2_drv_mc_config(adapter, &mac_entry->mac_addr, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete mc rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = false;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(vlan_entry, &adapter->filter_ctxt.vlan_list, next,
+ next_vlan_entry) {
+ if (vlan_entry->hw_config) {
+ ret = sxe2_drv_vlan_filter_id_config(adapter,
+ &vlan_entry->vlan_info, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete vlan rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ vlan_entry->hw_config = false;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_info.hw_filter_on) {
+ ret = sxe2_drv_vlan_filter_switch(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to delete vlan rule");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ adapter->filter_ctxt.vlan_info.hw_filter_on = false;
+ }
+
+ if (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC) {
+ ret = sxe2_drv_promisc_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg promiscuous, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC;
+ }
+
+ if (adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST) {
+ ret = sxe2_drv_allmulti_config(adapter, false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "failed to cfg allmulticast, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags &= ~SXE2_PROMISC_MULTICAST;
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_all_filter_hw_set(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_mac_filter *mac_entry;
+ struct sxe2_mac_filter *next_mac_entry;
+ struct sxe2_vlan_filter *vlan_entry;
+ struct sxe2_vlan_filter *next_vlan_entry;
+
+ if (adapter->filter_ctxt.uc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.uc_list, next,
+ next_mac_entry) {
+ if (!mac_entry->hw_config) {
+ ret = sxe2_drv_uc_config(adapter, &mac_entry->mac_addr,
+ true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add uc rule, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = true;
+ ret = 0;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.mc_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(mac_entry, &adapter->filter_ctxt.mc_list, next,
+ next_mac_entry) {
+ if (!mac_entry->hw_config) {
+ ret = sxe2_drv_mc_config(adapter, &mac_entry->mac_addr, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add mc rule, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ mac_entry->hw_config = true;
+ ret = 0;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_num > 0) {
+ RTE_TAILQ_FOREACH_SAFE(vlan_entry, &adapter->filter_ctxt.vlan_list, next,
+ next_vlan_entry) {
+ if (!vlan_entry->hw_config) {
+ ret = sxe2_drv_vlan_filter_id_config(adapter,
+ &vlan_entry->vlan_info, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add vlan rule, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ vlan_entry->hw_config = true;
+ ret = 0;
+ }
+ }
+ }
+
+ if (adapter->filter_ctxt.vlan_info.filter_on) {
+ if (!(adapter->filter_ctxt.vlan_info.hw_filter_on)) {
+ ret = sxe2_drv_vlan_filter_switch(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to add vlan ctrl, ret:%d", ret);
+ ret = -EINVAL;
+ goto l_end;
+ }
+ adapter->filter_ctxt.vlan_info.hw_filter_on = true;
+ ret = 0;
+ }
+ }
+
+ if ((adapter->filter_ctxt.cur_promisc_flags & SXE2_PROMISC) &&
+ (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC))) {
+ ret = sxe2_drv_promisc_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to set promisc, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC;
+ ret = 0;
+ }
+
+ if ((adapter->filter_ctxt.cur_promisc_flags & SXE2_PROMISC_MULTICAST) &&
+ (!(adapter->filter_ctxt.hw_promisc_flags & SXE2_PROMISC_MULTICAST))) {
+ ret = sxe2_drv_allmulti_config(adapter, true);
+ if (ret && ret != -EEXIST) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to set allmulti, ret:%d", ret);
+ goto l_end;
+ }
+ adapter->filter_ctxt.hw_promisc_flags |= SXE2_PROMISC_MULTICAST;
+ ret = 0;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+
+ if (!adapter->flow_isolated && !adapter->switchdev_info.is_switchdev &&
+ adapter->rule_started) {
+ adapter->filter_ctxt.cur_l2_config = true;
+ } else {
+ adapter->filter_ctxt.cur_l2_config = false;
+ }
+
+ if (adapter->filter_ctxt.cur_l2_config !=
+ adapter->filter_ctxt.hw_l2_config) {
+ if (adapter->filter_ctxt.cur_l2_config) {
+ ret = sxe2_all_filter_hw_set(adapter);
+ if (!ret)
+ adapter->filter_ctxt.hw_l2_config = true;
+ } else {
+ ret = sxe2_all_filter_hw_clear(adapter);
+ if (!ret)
+ adapter->filter_ctxt.hw_l2_config = false;
+ }
+ }
+ return ret;
+}
+
+int32_t sxe2_filter_rule_stop(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ adapter->rule_started = 0;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ return ret;
+}
+
+int32_t sxe2_filter_rule_start(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ adapter->rule_started = 1;
+
+ ret = sxe2_l2_rule_update(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to update l2 rule");
+
+ return ret;
+}
+
+int32_t sxe2_filter_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ rte_spinlock_init(&adapter->filter_ctxt.filter_lock);
+
+ TAILQ_INIT(&adapter->filter_ctxt.uc_list);
+ adapter->filter_ctxt.uc_num = 0;
+
+ TAILQ_INIT(&adapter->filter_ctxt.mc_list);
+ adapter->filter_ctxt.mc_num = 0;
+
+ TAILQ_INIT(&adapter->filter_ctxt.vlan_list);
+ adapter->filter_ctxt.vlan_num = 0;
+ return 0;
+}
+
+int32_t sxe2_filter_uinit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ sxe2_uc_filter_clear(adapter, true);
+ adapter->filter_ctxt.uc_num = 0;
+
+ sxe2_mc_filter_clear(adapter, true);
+ adapter->filter_ctxt.mc_num = 0;
+
+ sxe2_vlan_filters_clear(adapter, true);
+ adapter->filter_ctxt.vlan_num = 0;
+ return 0;
+}
diff --git a/drivers/net/sxe2/sxe2_filter.h b/drivers/net/sxe2/sxe2_filter.h
new file mode 100644
index 0000000000..6262e8c845
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_filter.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FILTER_H__
+#define __SXE2_FILTER_H__
+#include <ethdev_driver.h>
+
+#define SXE2_PROMISC (1UL << 0UL)
+#define SXE2_PROMISC_MULTICAST (1UL << 1UL)
+
+struct sxe2_vlan_info {
+ uint8_t port_vlan_exist;
+ uint8_t is_switchdev;
+ uint16_t max_cnt;
+ uint16_t cnt;
+
+ bool filter_on;
+ bool hw_filter_on;
+
+ uint16_t tpid;
+ uint16_t vid;
+
+ uint8_t outer_insert;
+ uint8_t outer_strip;
+ uint8_t inner_insert;
+ uint8_t inner_strip;
+};
+
+struct sxe2_vlan {
+ uint16_t tpid;
+ uint16_t vid;
+ uint8_t prio;
+};
+
+struct sxe2_vlan_filter {
+ TAILQ_ENTRY(sxe2_vlan_filter) next;
+ bool hw_config;
+ bool default_config;
+ struct sxe2_vlan vlan_info;
+};
+
+TAILQ_HEAD(sxe2_vlan_filter_list_head, sxe2_vlan_filter);
+
+struct sxe2_mac_filter {
+ TAILQ_ENTRY(sxe2_mac_filter) next;
+ bool hw_config;
+ bool default_config;
+ struct rte_ether_addr mac_addr;
+};
+
+TAILQ_HEAD(sxe2_uc_filter_list_head, sxe2_mac_filter);
+TAILQ_HEAD(sxe2_mc_filter_list_head, sxe2_mac_filter);
+
+int32_t sxe2_uc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config);
+
+int32_t sxe2_uc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr);
+
+void sxe2_uc_filter_clear(struct sxe2_adapter *adapter, bool default_config);
+
+int32_t sxe2_mc_filter_add(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr, bool default_config);
+
+int32_t sxe2_mc_filter_del(struct sxe2_adapter *adapter,
+ struct rte_ether_addr *mac_addr);
+
+void sxe2_mc_filter_clear(struct sxe2_adapter *adapter, bool default_config);
+
+int32_t sxe2_vlan_filter_add(struct sxe2_adapter *adapter,
+ struct sxe2_vlan *vlan, bool default_config);
+
+int32_t sxe2_vlan_filter_del(struct sxe2_adapter *adapter, struct sxe2_vlan *vlan);
+
+void sxe2_vlan_filters_clear(struct sxe2_adapter *adapter, bool default_config);
+
+int32_t sxe2_vlan_filter_ctrl(struct sxe2_adapter *adapter, bool flag);
+
+int32_t sxe2_promisc_add(struct sxe2_adapter *adapter);
+
+int32_t sxe2_promisc_del(struct sxe2_adapter *adapter);
+
+int32_t sxe2_allmulti_add(struct sxe2_adapter *adapter);
+
+int32_t sxe2_allmulti_del(struct sxe2_adapter *adapter);
+
+int32_t sxe2_l2_rule_update(struct sxe2_adapter *adapter);
+
+int32_t sxe2_filter_rule_stop(struct rte_eth_dev *dev);
+
+int32_t sxe2_filter_rule_start(struct rte_eth_dev *dev);
+
+int32_t sxe2_filter_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_filter_uinit(struct rte_eth_dev *dev);
+
+#endif /* __SXE2_FILTER_H__ */
diff --git a/drivers/net/sxe2/sxe2_mac.c b/drivers/net/sxe2/sxe2_mac.c
index 3c2f909002..d94936a742 100644
--- a/drivers/net/sxe2/sxe2_mac.c
+++ b/drivers/net/sxe2/sxe2_mac.c
@@ -10,6 +10,438 @@
#include "sxe2_cmd_chnl.h"
#include "sxe2_host_regs.h"
+int32_t sxe2_mac_default_cfg(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret;
+ struct rte_ether_addr broadcast = {
+ .addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff} };
+ struct rte_ether_addr mac_addr;
+
+ rte_ether_addr_copy((struct rte_ether_addr *)
+ adapter->dev_info.mac.perm_addr, &mac_addr);
+ ret = sxe2_uc_filter_add(adapter, &mac_addr, true);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add default MAC filter");
+ goto l_end;
+ }
+
+ rte_ether_addr_copy(&broadcast, &mac_addr);
+ ret = sxe2_mc_filter_add(adapter, &mac_addr, true);
+ if (ret != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add broadcast MAC filter");
+ goto l_end;
+ }
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_mac_addr_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = -1;
+ PMD_INIT_FUNC_TRACE();
+
+ if (!rte_is_unicast_ether_addr
+ ((struct rte_ether_addr *)adapter->dev_info.mac.perm_addr)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Invalid MAC address");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ dev->data->mac_addrs = rte_zmalloc("sxe2_mac_adds",
+ sizeof(struct rte_ether_addr) * SXE2_NUM_MACADDR_MAX, 0);
+ if (!dev->data->mac_addrs) {
+ PMD_LOG_ERR(DRV, "Failed to allocate memory to store mac address");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ rte_ether_addr_copy((struct rte_ether_addr *)adapter->dev_info.mac.perm_addr,
+ &dev->data->mac_addrs[0]);
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
+
+void sxe2_mac_addr_uinit(struct rte_eth_dev *dev)
+{
+ PMD_INIT_FUNC_TRACE();
+ if (dev != NULL && dev->data->mac_addrs != NULL) {
+ rte_free(dev->data->mac_addrs);
+ dev->data->mac_addrs = NULL;
+ }
+}
+
+int32_t sxe2_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
+ __rte_unused uint32_t index, __rte_unused uint32_t pool)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = -1;
+
+ if (rte_is_zero_ether_addr(mac_addr)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Invalid MAC Address");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (rte_is_multicast_ether_addr(mac_addr))
+ ret = sxe2_mc_filter_add(adapter, mac_addr, true);
+ else
+ ret = sxe2_uc_filter_add(adapter, mac_addr, false);
+
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add MAC filter");
+
+l_end:
+ return ret;
+}
+
+void sxe2_mac_addr_del(struct rte_eth_dev *dev, uint32_t index)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_ether_addr *mac_addr = &dev->data->mac_addrs[index];
+ int32_t ret = -1;
+
+ if (rte_is_multicast_ether_addr(mac_addr))
+ ret = sxe2_mc_filter_del(adapter, mac_addr);
+ else
+ ret = sxe2_uc_filter_del(adapter, mac_addr);
+
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to remove MAC filter");
+}
+
+int32_t sxe2_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ struct rte_ether_addr *old_addr = (struct rte_ether_addr *)&adapter->dev_info.mac.perm_addr;
+ struct rte_ether_addr temp_addr;
+
+ if (rte_is_same_ether_addr(old_addr, mac_addr))
+ goto l_end;
+
+ if (rte_is_multicast_ether_addr(mac_addr)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set multicast addr");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_uc_filter_del(adapter, old_addr);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to remove MAC filter");
+ goto l_end;
+ }
+
+ rte_ether_addr_copy(old_addr, &temp_addr);
+
+ rte_ether_addr_copy(mac_addr, old_addr);
+
+ ret = sxe2_uc_filter_add(adapter, mac_addr, true);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add MAC filter");
+ rte_ether_addr_copy(&temp_addr, old_addr);
+ (void)sxe2_uc_filter_add(adapter, old_addr, true);
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_set_mc_addr_list(struct rte_eth_dev *dev,
+ struct rte_ether_addr *mc_addrs,
+ uint32_t mc_addrs_num)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ uint32_t i;
+ const uint8_t *mac;
+
+ if (mc_addrs_num > SXE2_NUM_MACADDR_MAX) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Too many multicast MAC addresses, ");
+ ret = -1;
+ goto l_end;
+ }
+
+ sxe2_mc_filter_clear(adapter, false);
+
+ for (i = 0; i < mc_addrs_num; i++) {
+ if (!rte_is_multicast_ether_addr(&mc_addrs[i])) {
+ mac = mc_addrs[i].addr_bytes;
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Invalid mac: %02x:%02x:%02x:%02x:%02x:%02x",
+ mac[0], mac[1], mac[2], mac[3], mac[4],
+ mac[5]);
+ ret = -EINVAL;
+ goto add_err;
+ }
+
+ ret = sxe2_mc_filter_add(adapter, &mc_addrs[i], false);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to remove old multicast MAC filter list");
+ goto add_err;
+ }
+ }
+ goto l_end;
+add_err:
+ sxe2_mc_filter_clear(adapter, false);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int32_t on)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_vlan vlan = {
+ .tpid = RTE_ETHER_TYPE_VLAN,
+ .vid = vlan_id,
+ .prio = 0
+ };
+ int32_t ret = 0;
+
+ if (sxe2_dev_port_vlan_check(dev)) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Filter not supported with Port VLAN");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (vlan_id == 0)
+ goto l_end;
+
+ if (on) {
+ ret = sxe2_vlan_filter_add(adapter, &vlan, false);
+ if (ret < 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan filter");
+ goto l_end;
+ }
+ } else {
+ ret = sxe2_vlan_filter_del(adapter, &vlan);
+ if (ret < 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to remove vlan filter");
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_vlan_offload_set(struct rte_eth_dev *dev, int32_t mask)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
+ struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
+ struct sxe2_vlan_info new_info = adapter->filter_ctxt.vlan_info;
+ bool port_vlan = new_info.port_vlan_exist;
+
+ uint8_t out_strip_mask = SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021Q |
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021AD |
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_QINQ1;
+
+ if (txmode->offloads & RTE_ETH_TX_OFFLOAD_QINQ_INSERT) {
+ if (!(txmode->offloads & RTE_ETH_TX_OFFLOAD_VLAN_INSERT)) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "VLAN INSERT must be enabled when QinQ INSERT is enabled");
+ return -EINVAL;
+ }
+ if (port_vlan) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "QINQ INSERT not supported with Port VLAN");
+ return -EINVAL;
+ }
+ }
+
+ if (mask & RTE_ETH_QINQ_STRIP_MASK) {
+ if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP) {
+ if (port_vlan) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "QinQ strip not supported with Port VLAN");
+ return -EINVAL;
+ }
+ new_info.inner_strip = SXE2_VSI_TSR_ID_VLAN;
+ } else {
+ new_info.inner_strip = 0;
+ }
+ }
+
+ if (mask & RTE_ETH_VLAN_STRIP_MASK) {
+ if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) {
+ new_info.outer_strip =
+ port_vlan ? 0 : out_strip_mask;
+ new_info.inner_strip =
+ port_vlan ? new_info.inner_strip : new_info.inner_strip;
+ } else {
+ if (new_info.inner_strip != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Must disable QinQ strip before disabling VLAN strip");
+ return -EINVAL;
+ }
+ new_info.outer_strip = 0;
+ }
+ }
+
+ if (mask & (RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_QINQ_STRIP_MASK)) {
+ struct sxe2_vlan_info old_info = adapter->filter_ctxt.vlan_info;
+ adapter->filter_ctxt.vlan_info = new_info;
+
+ ret = sxe2_drv_vlan_insert_strip_cfg(adapter);
+ if (ret) {
+ adapter->filter_ctxt.vlan_info = old_info;
+ return ret;
+ }
+ }
+ if (mask & RTE_ETH_VLAN_FILTER_MASK) {
+ if (adapter->filter_ctxt.vlan_info.port_vlan_exist) {
+ ret = 0;
+ PMD_DEV_LOG_INFO(adapter, INIT, "vlan filter is not support when port vlan is enabled");
+ goto l_end;
+ }
+
+ ret = sxe2_vlan_filter_ctrl(adapter,
+ !!(rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER));
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "sxe2_drv_vlan_filter_switch failed ret:%d", ret);
+ goto l_end;
+ }
+ }
+
+ PMD_DEV_LOG_DEBUG(adapter, DRV,
+ "mask:0x%x rx mode offload:0x%" PRIx64 " vlan offload set done",
+ mask, rxmode->offloads);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_vlan_filter_zero(struct sxe2_adapter *adapter)
+{
+ struct sxe2_vlan vlan;
+ int32_t ret;
+ uint16_t tpids[] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, RTE_ETHER_TYPE_QINQ1};
+ uint8_t i;
+
+ vlan = (struct sxe2_vlan){0, 0, 0};
+ ret = sxe2_vlan_filter_add(adapter, &vlan, true);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add VLAN ID 0");
+ goto l_end;
+ }
+
+ for (i = 0; i < RTE_DIM(tpids); i++) {
+ vlan = (struct sxe2_vlan){tpids[i], 0, 0};
+ ret = sxe2_vlan_filter_add(adapter, &vlan, true);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add VLAN ID 0 when tpid:0x%x",
+ tpids[i]);
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_vlan_cfg_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_drv_vlan_config_query(adapter);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to query vlan config, ret=%d", ret);
+ goto l_end;
+ }
+
+ if (!sxe2_dev_port_vlan_check(dev))
+ adapter->filter_ctxt.vlan_info.outer_insert =
+ SXE2_DPDK_OFFLOAD_OUTER_INSERT_8021Q |
+ SXE2_DPDK_OFFLOAD_INSERT_ENABLE;
+ else
+ adapter->filter_ctxt.vlan_info.outer_insert = 0;
+
+ adapter->filter_ctxt.vlan_info.inner_insert =
+ SXE2_DPDK_OFFLOAD_INNER_INSERT_QINQ1 | SXE2_DPDK_OFFLOAD_INSERT_ENABLE;
+
+ if (!sxe2_dev_port_vlan_check(dev)) {
+ ret = sxe2_vlan_filter_zero(adapter);
+ if (ret != 0)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to add vlan filter switch:0 "
+ "for port:%d", adapter->port_idx);
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_vlan_default_cfg(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ ret = sxe2_dev_vlan_offload_set(dev, RTE_ETH_VLAN_STRIP_MASK |
+ RTE_ETH_QINQ_STRIP_MASK |
+ RTE_ETH_VLAN_FILTER_MASK);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cfg vlan offload, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_promisc_enable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_promisc_add(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to enable promisc, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_promisc_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_promisc_del(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to disable promisc, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_allmulti_enable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_allmulti_add(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to enable allmulti, ret:%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_allmulti_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ ret = sxe2_allmulti_del(adapter);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to disable allmulti, ret:%d", ret);
+
+ return ret;
+}
+
int32_t sxe2_link_update_init(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
diff --git a/drivers/net/sxe2/sxe2_mac.h b/drivers/net/sxe2/sxe2_mac.h
index f2f3edaeff..55fd1829a0 100644
--- a/drivers/net/sxe2/sxe2_mac.h
+++ b/drivers/net/sxe2/sxe2_mac.h
@@ -43,6 +43,40 @@ struct sxe2_mac_mc_list {
int32_t sxe2_link_update_init(struct rte_eth_dev *dev);
+int32_t sxe2_mac_default_cfg(struct rte_eth_dev *dev);
+
+int32_t sxe2_vlan_cfg_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_mac_addr_init(struct rte_eth_dev *dev);
+
+void sxe2_mac_addr_uinit(struct rte_eth_dev *dev);
+
+int32_t sxe2_mac_addr_add(struct rte_eth_dev *dev,
+ struct rte_ether_addr *mac_addr,
+ __rte_unused uint32_t index, __rte_unused uint32_t pool);
+
+void sxe2_mac_addr_del(struct rte_eth_dev *dev, uint32_t index);
+
+int32_t sxe2_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
+
+int32_t sxe2_set_mc_addr_list(struct rte_eth_dev *dev,
+ struct rte_ether_addr *mc_addrs,
+ uint32_t mc_addrs_num);
+
+int32_t sxe2_promisc_enable(struct rte_eth_dev *dev);
+
+int32_t sxe2_promisc_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_allmulti_enable(struct rte_eth_dev *dev);
+
+int32_t sxe2_allmulti_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int32_t on);
+
+int32_t sxe2_dev_vlan_offload_set(struct rte_eth_dev *dev, int32_t mask);
+
+int32_t sxe2_vlan_default_cfg(struct rte_eth_dev *dev);
+
int32_t sxe2_link_update(struct rte_eth_dev *dev, __rte_unused int32_t wait_to_complete);
int32_t sxe2_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index b9d34afb31..21d5c38725 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -660,6 +660,53 @@ sxe2_rx_desc_error_para(__rte_unused struct sxe2_rx_queue *rxq,
return flags;
}
+static inline void sxe2_rx_desc_vlan_para_fill(struct rte_mbuf *mbuf,
+ union sxe2_rx_desc *desc)
+{
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_L2TAG1_P_MASK)) {
+ mbuf->vlan_tci = 0;
+ } else {
+ mbuf->ol_flags |= (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
+ mbuf->vlan_tci = rte_le_to_cpu_16(desc->wb.l2tag1);
+ PMD_LOG_DEBUG(RX, "Rx desc mbuf vlan, vlan_tci:%u",
+ mbuf->vlan_tci);
+ }
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (0 == (rte_le_to_cpu_32(desc->wb.status_lrocnt_fdpf_id) &
+ SXE2_RX_DESC_EXT_STATUS_L2TAG2P_MASK)) {
+ mbuf->vlan_tci_outer = 0;
+ } else {
+ mbuf->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED | RTE_MBUF_F_RX_QINQ |
+ RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN;
+ mbuf->vlan_tci_outer = mbuf->vlan_tci;
+ mbuf->vlan_tci = rte_le_to_cpu_16(desc->wb.l2tag2_2nd);
+ PMD_LOG_DEBUG(RX, "Rx desc out vlan, l2tag2_1st:%u l2tag2_2nd:%u.",
+ rte_le_to_cpu_16(desc->wb.l2tag2_1st),
+ rte_le_to_cpu_16(desc->wb.l2tag2_2nd));
+ }
+#endif
+}
+
+static inline void
+sxe2_rx_desc_filter_para_fill(struct sxe2_rx_queue *rxq __rte_unused,
+ struct rte_mbuf *mbuf, union sxe2_rx_desc *desc)
+{
+ if (SXE2_RX_DESC_STATUS_RSS_VLD_MASK &
+ rte_le_to_cpu_64(desc->wb.status_err_ptype_len)) {
+ mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+ mbuf->hash.rss = rte_le_to_cpu_32(desc->wb.filter_status);
+ PMD_LOG_DEBUG(RX, "rss id:%u", mbuf->hash.rss);
+ }
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (SXE2_RX_DESC_FD_VLD_MASK & desc->wb.rxdid_src) {
+ mbuf->ol_flags |= (RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
+ mbuf->hash.fdir.hi = rte_le_to_cpu_32(desc->wb.fd_filter_id);
+ PMD_LOG_DEBUG(RX, "fdir id:%u", mbuf->hash.fdir.hi);
+ }
+#endif
+}
+
static __rte_always_inline void
sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf,
union sxe2_rx_desc *rxd)
@@ -673,6 +720,8 @@ sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf
mbuf->packet_type = ptype_tbl[SXE2_RX_DESC_PTYPE_VAL_GET(qword1)];
pkt_flags = sxe2_rx_desc_error_para(rxq, rxd);
+ sxe2_rx_desc_vlan_para_fill(mbuf, rxd);
+ sxe2_rx_desc_filter_para_fill(rxq, mbuf, rxd);
mbuf->ol_flags |= pkt_flags;
}
--
2.47.3
^ permalink raw reply related
* [PATCH v2 00/23] net/sxe: added Linkdata sxe ethernet driver
From: liujie5 @ 2026-05-30 14:08 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260524093259.397506-21-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch set implements core functionality for the SXE PMD,
including basic driver framework, data path setup, and advanced
offload features (VLAN, RSS, DCB, PTP etc.).
Jie Liu (23):
net/sxe2: support AVX512 vectorized path for Rx and Tx
net/sxe2: add AVX2 vector data path for Rx and Tx
drivers: add supported packet types get callback
net/sxe2: support L2 filtering and MAC config
drivers: support RSS feature
net/sxe2: support TM hierarchy and shaping
net/sxe2: support IPsec inline protocol offload
net/sxe2: support statistics and multi-process
drivers: interrupt handling
net/sxe2: add NEON vec Rx/Tx burst functions
drivers: add support for VF representors
net/sxe2: add support for custom UDP tunnel ports
net/sxe2: support firmware version reading
net/sxe2: implement get monitor address
common/sxe2: add shared SFP module definitions
net/sxe2: support SFP module info and EEPROM access
net/sxe2: implement private dump info
net/sxe2: add mbuf validation in Tx debug mode
net/sxe2: add testpmd commands for private features
net/sxe2: add private devargs parsing
net/sxe2: support flow control status interrupt notification
net/sxe2: update sxe2 feature matrix docs
common/sxe2: add memseg walk callback
doc/guides/nics/features/sxe2.ini | 66 +
drivers/common/sxe2/sxe2_common.c | 156 ++
drivers/common/sxe2/sxe2_common.h | 4 +
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++
drivers/common/sxe2/sxe2_ioctl_chnl.c | 179 +-
drivers/common/sxe2/sxe2_ioctl_chnl_func.h | 18 +
drivers/common/sxe2/sxe2_msg.h | 118 ++
drivers/common/sxe2/sxe2_ptype.h | 1793 ++++++++++++++++++
drivers/net/sxe2/meson.build | 56 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 1588 +++++++++++++++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 139 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 521 +++++-
drivers/net/sxe2/sxe2_dump.c | 304 +++
drivers/net/sxe2/sxe2_dump.h | 12 +
drivers/net/sxe2/sxe2_ethdev.c | 1526 ++++++++++++++-
drivers/net/sxe2/sxe2_ethdev.h | 113 +-
drivers/net/sxe2/sxe2_ethdev_repr.c | 610 ++++++
drivers/net/sxe2/sxe2_ethdev_repr.h | 32 +
drivers/net/sxe2/sxe2_filter.c | 897 +++++++++
drivers/net/sxe2/sxe2_filter.h | 100 +
drivers/net/sxe2/sxe2_flow.c | 1391 ++++++++++++++
drivers/net/sxe2/sxe2_flow.h | 30 +
drivers/net/sxe2/sxe2_flow_define.h | 144 ++
drivers/net/sxe2/sxe2_flow_parse_action.c | 1182 ++++++++++++
drivers/net/sxe2/sxe2_flow_parse_action.h | 23 +
drivers/net/sxe2/sxe2_flow_parse_engine.c | 106 ++
drivers/net/sxe2/sxe2_flow_parse_engine.h | 13 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 1935 ++++++++++++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 46 +
drivers/net/sxe2/sxe2_ipsec.c | 1565 ++++++++++++++++
drivers/net/sxe2/sxe2_ipsec.h | 254 +++
drivers/net/sxe2/sxe2_irq.c | 1024 +++++++++++
drivers/net/sxe2/sxe2_irq.h | 25 +
drivers/net/sxe2/sxe2_mac.c | 535 ++++++
drivers/net/sxe2/sxe2_mac.h | 84 +
drivers/net/sxe2/sxe2_mp.c | 413 +++++
drivers/net/sxe2/sxe2_mp.h | 73 +
drivers/net/sxe2/sxe2_queue.c | 17 +-
drivers/net/sxe2/sxe2_rss.c | 584 ++++++
drivers/net/sxe2/sxe2_rss.h | 81 +
drivers/net/sxe2/sxe2_rx.c | 38 +
drivers/net/sxe2/sxe2_rx.h | 2 +
drivers/net/sxe2/sxe2_security.c | 335 ++++
drivers/net/sxe2/sxe2_security.h | 77 +
drivers/net/sxe2/sxe2_stats.c | 591 ++++++
drivers/net/sxe2/sxe2_stats.h | 39 +
drivers/net/sxe2/sxe2_switchdev.c | 332 ++++
drivers/net/sxe2/sxe2_switchdev.h | 33 +
drivers/net/sxe2/sxe2_testpmd.c | 733 ++++++++
drivers/net/sxe2/sxe2_testpmd_lib.c | 969 ++++++++++
drivers/net/sxe2/sxe2_testpmd_lib.h | 142 ++
drivers/net/sxe2/sxe2_tm.c | 1169 ++++++++++++
drivers/net/sxe2/sxe2_tm.h | 78 +
drivers/net/sxe2/sxe2_tx.c | 7 +
drivers/net/sxe2/sxe2_txrx.c | 174 +-
drivers/net/sxe2/sxe2_txrx.h | 4 +
drivers/net/sxe2/sxe2_txrx_check_mbuf.c | 595 ++++++
drivers/net/sxe2/sxe2_txrx_check_mbuf.h | 38 +
drivers/net/sxe2/sxe2_txrx_poll.c | 243 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 38 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 777 ++++++++
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 +++++++++
drivers/net/sxe2/sxe2_txrx_vec_common.h | 1 -
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 707 +++++++
drivers/net/sxe2/sxe2_vsi.c | 146 ++
drivers/net/sxe2/sxe2_vsi.h | 12 +-
drivers/net/sxe2/sxe2vf_regs.h | 82 +
68 files changed, 26576 insertions(+), 119 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/common/sxe2/sxe2_msg.h
create mode 100644 drivers/common/sxe2/sxe2_ptype.h
create mode 100644 drivers/net/sxe2/sxe2_dump.c
create mode 100644 drivers/net/sxe2/sxe2_dump.h
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.c
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.h
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
create mode 100644 drivers/net/sxe2/sxe2_flow.c
create mode 100644 drivers/net/sxe2/sxe2_flow.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.h
create mode 100644 drivers/net/sxe2/sxe2_ipsec.c
create mode 100644 drivers/net/sxe2/sxe2_ipsec.h
create mode 100644 drivers/net/sxe2/sxe2_irq.c
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
create mode 100644 drivers/net/sxe2/sxe2_mp.c
create mode 100644 drivers/net/sxe2/sxe2_mp.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
create mode 100644 drivers/net/sxe2/sxe2_security.c
create mode 100644 drivers/net/sxe2/sxe2_security.h
create mode 100644 drivers/net/sxe2/sxe2_stats.c
create mode 100644 drivers/net/sxe2/sxe2_stats.h
create mode 100644 drivers/net/sxe2/sxe2_switchdev.c
create mode 100644 drivers/net/sxe2/sxe2_switchdev.h
create mode 100644 drivers/net/sxe2/sxe2_testpmd.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.h
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
create mode 100644 drivers/net/sxe2/sxe2vf_regs.h
--
2.47.3
^ permalink raw reply
* [RFC v4 3/3] app/test: add fastmem test suite
From: Mattias Rönnblom @ 2026-05-30 9:26 UTC (permalink / raw)
To: dev
Cc: Morten Brørup, Konstantin Ananyev, Mattias Rönnblom,
Yogaraj Baskaravel, Stephen Hemminger, Bruce Richardson,
Mattias Rönnblom
In-Reply-To: <20260530092634.46218-1-hofors@lysator.liu.se>
Add functional, performance, and profiling test suites for the
fastmem library.
--
RFC v4:
* Add tests for handle alloc/free from uncached lcores and
non-EAL threads.
* Add tests that statistics survive cache flush.
* Add test for shared-cache statistics.
* Refactor tests to use per-test setup/teardown.
RFC v3:
* Add realloc test cases (same class, grow, shrink, NULL ptr,
zero size, too big, invalid align).
* Merge lifecycle and functional test suites into one.
* Suppress -Wuse-after-free in test_alloc_reuse (intentional
pointer comparison after free).
RFC v2:
* Add test_alloc_cross_socket_deinit exercising cross-socket
teardown path.
* Remove trailing double blank lines in test_fastmem.c.
Signed-off-by: Mattias Rönnblom <hofors@lysator.liu.se>
---
app/test/meson.build | 3 +
app/test/test_fastmem.c | 2111 +++++++++++++++++++++++++++++++
app/test/test_fastmem_perf.c | 1040 +++++++++++++++
app/test/test_fastmem_profile.c | 157 +++
4 files changed, 3311 insertions(+)
create mode 100644 app/test/test_fastmem.c
create mode 100644 app/test/test_fastmem_perf.c
create mode 100644 app/test/test_fastmem_profile.c
diff --git a/app/test/meson.build b/app/test/meson.build
index 3f9340f2f5..fe375e97f3 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -82,6 +82,9 @@ source_file_deps = {
'test_event_vector_adapter.c': ['eventdev', 'bus_vdev'],
'test_eventdev.c': ['eventdev', 'bus_vdev'],
'test_external_mem.c': [],
+ 'test_fastmem.c': ['fastmem'],
+ 'test_fastmem_perf.c': ['fastmem', 'mempool'],
+ 'test_fastmem_profile.c': ['fastmem'],
'test_fbarray.c': [],
'test_fib.c': ['net', 'fib'],
'test_fib6.c': ['rib', 'fib'],
diff --git a/app/test/test_fastmem.c b/app/test/test_fastmem.c
new file mode 100644
index 0000000000..24ba1e671a
--- /dev/null
+++ b/app/test/test_fastmem.c
@@ -0,0 +1,2111 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 Ericsson AB
+ */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_thread.h>
+
+#include <rte_fastmem.h>
+
+#include "test.h"
+
+#define FASTMEM_MEMZONE_SIZE (128U << 20)
+
+/*
+ * Count memzones whose names begin with the fastmem prefix.
+ * Used to verify that rte_fastmem_reserve() really did reserve
+ * backing memzones.
+ */
+static int fastmem_memzone_count;
+
+static void
+count_fastmem_memzones_walk(const struct rte_memzone *mz, void *arg)
+{
+ RTE_SET_USED(arg);
+
+ if (strncmp(mz->name, "fastmem_", strlen("fastmem_")) == 0)
+ fastmem_memzone_count++;
+}
+
+static unsigned int
+count_fastmem_memzones(void)
+{
+ fastmem_memzone_count = 0;
+ rte_memzone_walk(count_fastmem_memzones_walk, NULL);
+ return fastmem_memzone_count;
+}
+
+static int
+test_init_deinit(void)
+{
+ int rc;
+
+ rc = rte_fastmem_init();
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_init() failed: %d", rc);
+
+ rte_fastmem_deinit();
+
+ /* A subsequent init/deinit cycle must succeed. */
+ rc = rte_fastmem_init();
+ TEST_ASSERT_EQUAL(rc, 0, "second rte_fastmem_init() failed: %d", rc);
+
+ rte_fastmem_deinit();
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_init_is_not_idempotent(void)
+{
+ int rc;
+
+ rc = rte_fastmem_init();
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_init() failed: %d", rc);
+
+ rc = rte_fastmem_init();
+ TEST_ASSERT_EQUAL(rc, -EBUSY,
+ "expected -EBUSY on re-init, got %d", rc);
+
+ rte_fastmem_deinit();
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_deinit_without_init(void)
+{
+ /* Must be a no-op, not a crash. */
+ rte_fastmem_deinit();
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_max_size(void)
+{
+ size_t max;
+
+ max = rte_fastmem_max_size();
+ TEST_ASSERT(max >= (1U << 20),
+ "max_size=%zu below required 1 MiB minimum", max);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_reserve_small(void)
+{
+ int socket_id;
+ unsigned int before, after;
+ int rc;
+
+ socket_id = rte_socket_id_by_idx(0);
+ TEST_ASSERT(socket_id >= 0, "no available sockets");
+
+ before = count_fastmem_memzones();
+
+ /*
+ * A small reserve request (1 byte) must result in exactly
+ * one memzone reservation: the internal rounding is to
+ * memzone granularity.
+ */
+ rc = rte_fastmem_reserve(1, socket_id);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_reserve() failed: %d", rc);
+
+ after = count_fastmem_memzones();
+ TEST_ASSERT_EQUAL(after - before, 1,
+ "expected 1 new memzone, got %u", after - before);
+
+ rte_fastmem_deinit();
+
+ /* After deinit the memzones must be released. */
+ TEST_ASSERT_EQUAL(count_fastmem_memzones(), 0,
+ "%u fastmem memzones leaked after deinit",
+ count_fastmem_memzones());
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_reserve_multiple_memzones(void)
+{
+ int socket_id;
+ unsigned int before, after;
+ size_t reserve_size;
+ int rc;
+
+ socket_id = rte_socket_id_by_idx(0);
+ TEST_ASSERT(socket_id >= 0, "no available sockets");
+
+ before = count_fastmem_memzones();
+
+ /*
+ * Request just over one memzone's worth; this must force
+ * a second memzone to be reserved.
+ */
+ reserve_size = FASTMEM_MEMZONE_SIZE + 1;
+ rc = rte_fastmem_reserve(reserve_size, socket_id);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_reserve(%zu) failed: %d",
+ reserve_size, rc);
+
+ after = count_fastmem_memzones();
+ TEST_ASSERT_EQUAL(after - before, 2,
+ "expected 2 new memzones for %zu-byte reserve, got %u",
+ reserve_size, after - before);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_reserve_cumulative(void)
+{
+ int socket_id;
+ unsigned int after_first, after_second;
+ int rc;
+
+ socket_id = rte_socket_id_by_idx(0);
+ TEST_ASSERT(socket_id >= 0, "no available sockets");
+
+ rc = rte_fastmem_reserve(FASTMEM_MEMZONE_SIZE, socket_id);
+ TEST_ASSERT_EQUAL(rc, 0, "first reserve failed: %d", rc);
+
+ after_first = count_fastmem_memzones();
+
+ /*
+ * A second call requesting the same amount that's already
+ * reserved must not trigger any new memzone reservation.
+ */
+ rc = rte_fastmem_reserve(FASTMEM_MEMZONE_SIZE, socket_id);
+ TEST_ASSERT_EQUAL(rc, 0, "second reserve failed: %d", rc);
+
+ after_second = count_fastmem_memzones();
+ TEST_ASSERT_EQUAL(after_first, after_second,
+ "reserve of already-reserved amount added memzones (%u -> %u)",
+ after_first, after_second);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_reserve_invalid_socket(void)
+{
+ int rc;
+
+ rc = rte_fastmem_reserve(1, RTE_MAX_NUMA_NODES);
+ TEST_ASSERT_EQUAL(rc, -EINVAL,
+ "expected -EINVAL for out-of-range socket, got %d", rc);
+
+ rc = rte_fastmem_reserve(1, -2);
+ TEST_ASSERT_EQUAL(rc, -EINVAL,
+ "expected -EINVAL for negative socket, got %d", rc);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_reserve_without_init(void)
+{
+ int rc;
+
+ rc = rte_fastmem_reserve(1, SOCKET_ID_ANY);
+ TEST_ASSERT(rc < 0,
+ "expected failure without init, got %d", rc);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_reserve_any_socket(void)
+{
+ unsigned int before, after;
+ int rc;
+
+ before = count_fastmem_memzones();
+
+ /*
+ * SOCKET_ID_ANY should succeed on any system with at least
+ * one configured socket. The allocator picks the caller's
+ * socket first and falls back to other sockets if needed.
+ */
+ rc = rte_fastmem_reserve(1, SOCKET_ID_ANY);
+ TEST_ASSERT_EQUAL(rc, 0,
+ "rte_fastmem_reserve(SOCKET_ID_ANY) failed: %d", rc);
+
+ after = count_fastmem_memzones();
+ TEST_ASSERT_EQUAL(after - before, 1,
+ "expected 1 new memzone, got %u", after - before);
+
+ return TEST_SUCCESS;
+}
+
+/*
+ * Stage 2 tests: allocation and free.
+ */
+
+static int
+test_alloc_too_big(void)
+{
+ void *p;
+ rte_errno = 0;
+ p = rte_fastmem_alloc(rte_fastmem_max_size() + 1, 0, 0);
+ TEST_ASSERT_NULL(p, "alloc above max_size returned non-NULL");
+ TEST_ASSERT_EQUAL(rte_errno, E2BIG,
+ "expected rte_errno=E2BIG, got %d", rte_errno);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_invalid_align(void)
+{
+ void *p;
+ rte_errno = 0;
+ p = rte_fastmem_alloc(16, 3, 0); /* 3 is not a power of 2 */
+ TEST_ASSERT_NULL(p, "alloc with align=3 returned non-NULL");
+ TEST_ASSERT_EQUAL(rte_errno, EINVAL,
+ "expected rte_errno=EINVAL, got %d", rte_errno);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_free_small(void)
+{
+ void *p;
+ p = rte_fastmem_alloc(8, 0, 0);
+ TEST_ASSERT_NOT_NULL(p, "alloc(8) failed: rte_errno=%d", rte_errno);
+
+ /* Writing into the object must not crash. */
+ memset(p, 0xa5, 8);
+
+ rte_fastmem_free(p);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_free_various_sizes(void)
+{
+ static const size_t sizes[] = {
+ 1, 8, 16, 17, 63, 64, 128, 1024, 4096,
+ 64 * 1024, 256 * 1024, 1024 * 1024,
+ };
+ void *ptrs[RTE_DIM(sizes)];
+ unsigned int i;
+ for (i = 0; i < RTE_DIM(sizes); i++) {
+ ptrs[i] = rte_fastmem_alloc(sizes[i], 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i],
+ "alloc(%zu) failed: rte_errno=%d",
+ sizes[i], rte_errno);
+ memset(ptrs[i], 0x5a, sizes[i]);
+ }
+
+ for (i = 0; i < RTE_DIM(sizes); i++)
+ rte_fastmem_free(ptrs[i]);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_alignment(void)
+{
+ static const size_t aligns[] = {
+ 8, 16, 64, 256, 4096, 65536,
+ };
+ unsigned int i;
+ for (i = 0; i < RTE_DIM(aligns); i++) {
+ void *p = rte_fastmem_alloc(1, aligns[i], 0);
+
+ TEST_ASSERT_NOT_NULL(p,
+ "alloc(1, align=%zu) failed: rte_errno=%d",
+ aligns[i], rte_errno);
+ TEST_ASSERT((uintptr_t)p % aligns[i] == 0,
+ "pointer %p not aligned on %zu",
+ p, aligns[i]);
+ rte_fastmem_free(p);
+ }
+
+ /* Default (align=0) gives at least RTE_CACHE_LINE_SIZE. */
+ {
+ void *p = rte_fastmem_alloc(1, 0, 0);
+
+ TEST_ASSERT_NOT_NULL(p,
+ "alloc(1, align=0) failed: rte_errno=%d", rte_errno);
+ TEST_ASSERT((uintptr_t)p % RTE_CACHE_LINE_SIZE == 0,
+ "default-align pointer %p not cache-line aligned",
+ p);
+ rte_fastmem_free(p);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_zero_flag(void)
+{
+ uint8_t *p;
+ unsigned int i;
+ bool all_zero = true;
+
+ /*
+ * Dirty a slab first by allocating without F_ZERO, writing
+ * a non-zero pattern, and freeing. A subsequent F_ZERO
+ * allocation on the same slab must return zeroed memory.
+ */
+ p = rte_fastmem_alloc(128, 0, 0);
+ TEST_ASSERT_NOT_NULL(p, "priming alloc failed");
+ memset(p, 0xff, 128);
+ rte_fastmem_free(p);
+
+ p = rte_fastmem_alloc(128, 0, RTE_FASTMEM_F_ZERO);
+ TEST_ASSERT_NOT_NULL(p, "F_ZERO alloc failed");
+ for (i = 0; i < 128; i++) {
+ if (p[i] != 0) {
+ all_zero = false;
+ break;
+ }
+ }
+ TEST_ASSERT(all_zero, "F_ZERO returned non-zero byte at offset %u", i);
+
+ rte_fastmem_free(p);
+
+ return TEST_SUCCESS;
+}
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wuse-after-free"
+#endif
+static int
+test_alloc_reuse(void)
+{
+ void *first, *second;
+
+ first = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(first, "first alloc failed");
+ rte_fastmem_free(first);
+
+ second = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(second, "second alloc failed");
+
+ /*
+ * The slab's free list is LIFO, so the most recently freed
+ * object is at the head of the list. A subsequent alloc in
+ * the same class returns it.
+ */
+ TEST_ASSERT_EQUAL(first, second,
+ "free + alloc did not reuse: first=%p second=%p",
+ first, second);
+
+ rte_fastmem_free(second);
+
+ return TEST_SUCCESS;
+}
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+static int
+test_alloc_many_in_class(void)
+{
+ /*
+ * Allocate more objects in one class than fit in a single
+ * slab, forcing the bin to pull a second block. This
+ * exercises the partial->full transition and the cross-slab
+ * allocation path.
+ */
+ enum { CLASS_SIZE = 8, COUNT = 300000 };
+ void **ptrs;
+ unsigned int i;
+
+ ptrs = calloc(COUNT, sizeof(*ptrs));
+ TEST_ASSERT_NOT_NULL(ptrs, "calloc for test ptrs failed");
+
+ for (i = 0; i < COUNT; i++) {
+ ptrs[i] = rte_fastmem_alloc(CLASS_SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i],
+ "alloc[%u] failed: rte_errno=%d",
+ i, rte_errno);
+ }
+
+ for (i = 0; i < COUNT; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ free(ptrs);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_socket(void)
+{
+ void *p;
+ int socket_id;
+ socket_id = rte_socket_id_by_idx(0);
+ TEST_ASSERT(socket_id >= 0, "no available sockets");
+
+ p = rte_fastmem_alloc_socket(64, 0, 0, socket_id);
+ TEST_ASSERT_NOT_NULL(p,
+ "alloc_socket(%d) failed: rte_errno=%d",
+ socket_id, rte_errno);
+
+ rte_fastmem_free(p);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_block_repurposing(void)
+{
+ void *small, *large;
+
+ /*
+ * Allocate and free a small object, forcing a block to be
+ * assigned to the small class and then returned to the
+ * free-block pool. A subsequent allocation in a different
+ * class must be able to reuse that block.
+ */
+ small = rte_fastmem_alloc(8, 0, 0);
+ TEST_ASSERT_NOT_NULL(small, "small alloc failed");
+ rte_fastmem_free(small);
+
+ large = rte_fastmem_alloc(256 * 1024, 0, 0);
+ TEST_ASSERT_NOT_NULL(large, "large alloc failed");
+ rte_fastmem_free(large);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_block_repurposing_no_growth(void)
+{
+ struct rte_fastmem_stats stats;
+ void *small, *large;
+ uint64_t after_small;
+ int rc;
+
+ /*
+ * Stronger version of test_alloc_block_repurposing: assert
+ * that the cross-class allocation does not grow the
+ * backing memory (bytes_backing stays flat). Because the
+ * free-block pool is shared across size classes — not
+ * partitioned per class — the block freed from the small
+ * class must serve the large allocation without triggering
+ * a new memzone reservation.
+ */
+ rc = rte_fastmem_stats(&stats);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_stats() failed: %d", rc);
+ TEST_ASSERT_EQUAL(stats.bytes_backing, (uint64_t)0,
+ "unexpected pre-alloc bytes_backing: %" PRIu64,
+ stats.bytes_backing);
+
+ small = rte_fastmem_alloc(8, 0, 0);
+ TEST_ASSERT_NOT_NULL(small, "small alloc failed");
+
+ rc = rte_fastmem_stats(&stats);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_stats() failed: %d", rc);
+ TEST_ASSERT(stats.bytes_backing > 0,
+ "bytes_backing did not grow on first alloc");
+ after_small = stats.bytes_backing;
+
+ rte_fastmem_free(small);
+ rte_fastmem_cache_flush();
+
+ large = rte_fastmem_alloc(256 * 1024, 0, 0);
+ TEST_ASSERT_NOT_NULL(large,
+ "large alloc failed: rte_errno=%d", rte_errno);
+
+ rc = rte_fastmem_stats(&stats);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_fastmem_stats() failed: %d", rc);
+ TEST_ASSERT_EQUAL(stats.bytes_backing, after_small,
+ "cross-class alloc grew backing memory from %" PRIu64
+ " to %" PRIu64,
+ after_small, stats.bytes_backing);
+
+ rte_fastmem_free(large);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_free_null(void)
+{
+ /* Must be a no-op, not a crash. */
+ rte_fastmem_free(NULL);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_content_integrity(void)
+{
+ /*
+ * Allocate a batch of objects, fill each with a distinct
+ * byte pattern, then verify none of the patterns overlap.
+ * This catches header overwrites (slab header corrupted by
+ * object access) and slot-overlap bugs (two pointers pointing
+ * at overlapping slots).
+ */
+ enum { N = 256, SIZE = 128 };
+ uint8_t *ptrs[N];
+ unsigned int i, j;
+ for (i = 0; i < N; i++) {
+ ptrs[i] = rte_fastmem_alloc(SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "alloc[%u] failed", i);
+ memset(ptrs[i], (int)i, SIZE);
+ }
+
+ for (i = 0; i < N; i++)
+ for (j = 0; j < SIZE; j++)
+ TEST_ASSERT_EQUAL(ptrs[i][j], (uint8_t)i,
+ "corruption at ptrs[%u][%u]: got 0x%x, want 0x%x",
+ i, j, ptrs[i][j], (uint8_t)i);
+
+ for (i = 0; i < N; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_align_too_big(void)
+{
+ void *p;
+ /*
+ * A small size with an alignment larger than the maximum
+ * size class cannot be served. The class selected must be
+ * large enough for the alignment, but no such class exists.
+ */
+ rte_errno = 0;
+ p = rte_fastmem_alloc(1, rte_fastmem_max_size() * 2, 0);
+ TEST_ASSERT_NULL(p,
+ "alloc with align>max_size returned non-NULL");
+ TEST_ASSERT_EQUAL(rte_errno, E2BIG,
+ "expected rte_errno=E2BIG, got %d", rte_errno);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_align_one(void)
+{
+ void *p;
+ /* align=1 is a valid power of 2 and must be accepted. */
+ p = rte_fastmem_alloc(8, 1, 0);
+ TEST_ASSERT_NOT_NULL(p, "alloc(8, 1) failed: rte_errno=%d",
+ rte_errno);
+ rte_fastmem_free(p);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_socket_numa_placement(void)
+{
+ void *p;
+ int socket_id;
+ struct rte_memseg *ms;
+ socket_id = rte_socket_id_by_idx(0);
+ TEST_ASSERT(socket_id >= 0, "no available sockets");
+
+ p = rte_fastmem_alloc_socket(64, 0, 0, socket_id);
+ TEST_ASSERT_NOT_NULL(p,
+ "alloc_socket(%d) failed: rte_errno=%d",
+ socket_id, rte_errno);
+
+ /*
+ * Walk the memory to find the memseg for this pointer and
+ * verify its socket. Skip the check if lookup fails (e.g.,
+ * --no-huge mode may not populate memsegs for fastmem's
+ * allocations in a way that rte_mem_virt2memseg can find).
+ */
+ ms = rte_mem_virt2memseg(p, NULL);
+ if (ms != NULL) {
+ TEST_ASSERT_EQUAL(ms->socket_id, socket_id,
+ "alloc on socket %d landed on socket %d",
+ socket_id, ms->socket_id);
+ }
+
+ rte_fastmem_free(p);
+
+ return TEST_SUCCESS;
+}
+
+/*
+ * Allocate from a socket different from the calling lcore's socket,
+ * triggering a cross-socket cache allocation. Then deinit to exercise
+ * the teardown path where a cache's backing memory lives on a
+ * different socket than the one it serves.
+ */
+static int
+test_alloc_cross_socket_deinit(void)
+{
+ int local_sid, remote_sid;
+ unsigned int i, n_sockets;
+ void *p;
+
+ local_sid = (int)rte_socket_id();
+ if (local_sid < 0 || (unsigned int)local_sid >= RTE_MAX_NUMA_NODES)
+ local_sid = rte_socket_id_by_idx(0);
+
+ n_sockets = rte_socket_count();
+ if (n_sockets < 2)
+ return TEST_SKIPPED;
+
+ /* Find a socket different from the local one. */
+ remote_sid = -1;
+ for (i = 0; i < n_sockets; i++) {
+ int sid = rte_socket_id_by_idx(i);
+ if (sid >= 0 && sid != local_sid) {
+ remote_sid = sid;
+ break;
+ }
+ }
+ if (remote_sid < 0)
+ return TEST_SKIPPED;
+
+ p = rte_fastmem_alloc_socket(64, 0, 0, remote_sid);
+ TEST_ASSERT_NOT_NULL(p,
+ "cross-socket alloc(socket %d) failed: rte_errno=%d",
+ remote_sid, rte_errno);
+
+ rte_fastmem_free(p);
+
+ /* Teardown and re-init to exercise the deinit path with
+ * cross-socket caches.
+ */
+ rte_fastmem_deinit();
+
+ TEST_ASSERT_EQUAL(rte_fastmem_init(), 0,
+ "re-init after cross-socket deinit failed");
+
+ return TEST_SUCCESS;
+}
+
+/*
+ * Stage 3 tests: per-lcore caches.
+ */
+
+static int
+test_cache_flush(void)
+{
+ void *p;
+ /*
+ * Alloc and free one object, leaving it in the cache. Then
+ * flush and verify that a subsequent alloc may or may not
+ * return the same pointer (not asserting same/different —
+ * just checking that flush does not crash and a follow-up
+ * alloc still works).
+ */
+ p = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(p, "first alloc failed");
+ rte_fastmem_free(p);
+
+ rte_fastmem_cache_flush();
+
+ /* Flush again — must be idempotent. */
+ rte_fastmem_cache_flush();
+
+ p = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(p, "post-flush alloc failed");
+ rte_fastmem_free(p);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_cache_flush_without_init(void)
+{
+ /* Must be a no-op, not a crash. */
+ rte_fastmem_cache_flush();
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_cache_exceeds_capacity(void)
+{
+ /*
+ * Free more objects at a single size class than the cache
+ * capacity (64 for classes <= 4 KiB). This forces the
+ * cache-drain slow path and verifies no corruption.
+ */
+ enum { COUNT = 200, SIZE = 64 };
+ void *ptrs[COUNT];
+ unsigned int i;
+
+ for (i = 0; i < COUNT; i++) {
+ ptrs[i] = rte_fastmem_alloc(SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i],
+ "alloc[%u] failed: rte_errno=%d", i, rte_errno);
+ }
+
+ for (i = 0; i < COUNT; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ /* Re-alloc the same count should still work. */
+ for (i = 0; i < COUNT; i++) {
+ ptrs[i] = rte_fastmem_alloc(SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i],
+ "re-alloc[%u] failed: rte_errno=%d", i, rte_errno);
+ }
+
+ for (i = 0; i < COUNT; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ return TEST_SUCCESS;
+}
+
+struct non_eal_args {
+ int ok;
+ char pad[64];
+};
+
+static uint32_t
+non_eal_thread_main(void *arg)
+{
+ struct non_eal_args *args = arg;
+ uint8_t *p;
+
+ p = rte_fastmem_alloc(128, 0, 0);
+ if (p == NULL)
+ return 1;
+
+ memset(p, 0x7e, 128);
+
+ rte_fastmem_free(p);
+
+ args->ok = 1;
+ return 0;
+}
+
+static int
+test_non_eal_thread(void)
+{
+ rte_thread_t thread_id;
+ struct non_eal_args args = { 0 };
+ int rc;
+
+ rc = rte_thread_create(&thread_id, NULL, non_eal_thread_main, &args);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_create() failed: %d", rc);
+
+ rc = rte_thread_join(thread_id, NULL);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_join() failed: %d", rc);
+
+ TEST_ASSERT_EQUAL(args.ok, 1,
+ "non-EAL thread did not complete alloc/free successfully");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_cache_flush_returns_memory(void)
+{
+ /*
+ * When an entire slab's worth of objects is freed, the
+ * slab's block is returned to the free-block pool and can
+ * be reassigned to another size class. Verify the cache
+ * does not permanently hold objects that prevent this.
+ *
+ * Allocate enough objects in one class to force multiple
+ * slabs, free them all, then flush the cache. After the
+ * flush, all cached objects are drained to their bins and
+ * empty slabs are returned to the block pool.
+ */
+ enum { N = 200, SIZE = 64 };
+ void *ptrs[N];
+ unsigned int i;
+
+ for (i = 0; i < N; i++) {
+ ptrs[i] = rte_fastmem_alloc(SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "alloc[%u] failed", i);
+ }
+ for (i = 0; i < N; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ rte_fastmem_cache_flush();
+
+ /*
+ * An allocation in a completely different class should
+ * succeed now, having access to any blocks freed by the
+ * flush.
+ */
+ {
+ void *other = rte_fastmem_alloc(65536, 0, 0);
+
+ TEST_ASSERT_NOT_NULL(other,
+ "post-flush cross-class alloc failed");
+ rte_fastmem_free(other);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_bulk_basic(void)
+{
+ enum { N = 32 };
+ void *ptrs[N];
+ int rc;
+
+ rc = rte_fastmem_alloc_bulk(ptrs, N, 64, 0, 0);
+ TEST_ASSERT_EQUAL(rc, 0, "alloc_bulk failed: %d", rc);
+
+ /* Verify all pointers are non-NULL and distinct. */
+ for (unsigned int i = 0; i < N; i++) {
+ TEST_ASSERT_NOT_NULL(ptrs[i], "ptrs[%u] is NULL", i);
+ for (unsigned int j = 0; j < i; j++)
+ TEST_ASSERT(ptrs[i] != ptrs[j],
+ "ptrs[%u] == ptrs[%u]", i, j);
+ }
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_bulk_zero_flag(void)
+{
+ enum { N = 8, SIZE = 128 };
+ void *ptrs[N];
+ int rc;
+
+ rc = rte_fastmem_alloc_bulk(ptrs, N, SIZE, 0, RTE_FASTMEM_F_ZERO);
+ TEST_ASSERT_EQUAL(rc, 0, "alloc_bulk failed: %d", rc);
+
+ for (unsigned int i = 0; i < N; i++) {
+ uint8_t *p = ptrs[i];
+
+ for (unsigned int b = 0; b < SIZE; b++)
+ TEST_ASSERT_EQUAL(p[b], 0,
+ "ptrs[%u][%u] != 0", i, b);
+ }
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_bulk_exceeds_cache(void)
+{
+ /* Allocate more than cache capacity (64) in one bulk call. */
+ enum { N = 128 };
+ void *ptrs[N];
+ int rc;
+
+ rc = rte_fastmem_alloc_bulk(ptrs, N, 64, 0, 0);
+ TEST_ASSERT_EQUAL(rc, 0, "alloc_bulk(%u) failed: %d", N, rc);
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_alloc_bulk_socket(void)
+{
+ enum { N = 16 };
+ void *ptrs[N];
+ int socket_id;
+ int rc;
+
+ socket_id = rte_socket_id_by_idx(0);
+ TEST_ASSERT(socket_id >= 0, "no sockets");
+
+ rc = rte_fastmem_alloc_bulk_socket(ptrs, N, 64, 0, 0, socket_id);
+ TEST_ASSERT_EQUAL(rc, 0, "alloc_bulk_socket failed: %d", rc);
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ /* SOCKET_ID_ANY */
+ rc = rte_fastmem_alloc_bulk_socket(ptrs, N, 64, 0, 0, SOCKET_ID_ANY);
+ TEST_ASSERT_EQUAL(rc, 0, "alloc_bulk_socket(ANY) failed: %d", rc);
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_free_bulk(void)
+{
+ enum { N = 64 };
+ void *ptrs[N];
+ /* Allocate individually, free in bulk. */
+ for (unsigned int i = 0; i < N; i++) {
+ ptrs[i] = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "alloc[%u] failed", i);
+ }
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ /* Verify memory is reusable. */
+ for (unsigned int i = 0; i < N; i++) {
+ ptrs[i] = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "re-alloc[%u] failed", i);
+ }
+
+ rte_fastmem_free_bulk(ptrs, N);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_classes(void)
+{
+ size_t sizes[32];
+ unsigned int n;
+
+ n = rte_fastmem_classes(NULL);
+ TEST_ASSERT_EQUAL(n, 18u, "expected 18 classes, got %u", n);
+
+ n = rte_fastmem_classes(sizes);
+ TEST_ASSERT_EQUAL(n, 18u, "expected 18 classes, got %u", n);
+ TEST_ASSERT_EQUAL(sizes[0], (size_t)8, "class 0 != 8");
+ TEST_ASSERT_EQUAL(sizes[n - 1], (size_t)(1 << 20),
+ "last class != 1 MiB");
+
+ for (unsigned int i = 0; i < n; i++) {
+ TEST_ASSERT(sizes[i] != 0 && (sizes[i] & (sizes[i] - 1)) == 0,
+ "class %u size %zu not power of 2", i, sizes[i]);
+ if (i > 0)
+ TEST_ASSERT(sizes[i] > sizes[i - 1],
+ "classes not ascending at %u", i);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_stats_class(void)
+{
+ enum { N = 10 };
+ struct rte_fastmem_class_stats cs;
+ void *ptrs[N];
+ int rc;
+
+ for (unsigned int i = 0; i < N; i++) {
+ ptrs[i] = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "alloc[%u] failed", i);
+ }
+
+ rc = rte_fastmem_stats_class(64, &cs);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_class failed: %d", rc);
+ TEST_ASSERT_EQUAL(cs.class_size, (size_t)64, "wrong class_size");
+ TEST_ASSERT(cs.alloc_cache_hits + cs.alloc_cache_misses == N,
+ "alloc count != N: hits=%" PRIu64 " misses=%" PRIu64,
+ cs.alloc_cache_hits, cs.alloc_cache_misses);
+ TEST_ASSERT_EQUAL(cs.in_use, (uint64_t)N, "in_use != N");
+
+ for (unsigned int i = 0; i < N; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ rc = rte_fastmem_stats_class(64, &cs);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_class after free failed: %d", rc);
+ TEST_ASSERT_EQUAL(cs.in_use, (uint64_t)0, "in_use != 0 after free");
+
+ /* Invalid class size. */
+ rc = rte_fastmem_stats_class(13, &cs);
+ TEST_ASSERT_EQUAL(rc, -EINVAL, "expected -EINVAL for bad size");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_stats_lcore(void)
+{
+ struct rte_fastmem_lcore_stats ls;
+ void *ptr;
+ int rc;
+
+ ptr = rte_fastmem_alloc(128, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ rc = rte_fastmem_stats_lcore(rte_lcore_id(), &ls);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_lcore failed: %d", rc);
+ TEST_ASSERT(ls.alloc_cache_hits + ls.alloc_cache_misses > 0,
+ "no alloc activity on this lcore");
+
+ rte_fastmem_free(ptr);
+
+ rc = rte_fastmem_stats_lcore(rte_lcore_id(), &ls);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_lcore after free failed: %d", rc);
+ TEST_ASSERT(ls.free_cache_hits + ls.free_cache_misses > 0,
+ "no free activity on this lcore");
+
+ /* Invalid lcore. */
+ rc = rte_fastmem_stats_lcore(RTE_MAX_LCORE, &ls);
+ TEST_ASSERT_EQUAL(rc, -EINVAL, "expected -EINVAL for bad lcore");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_stats_lcore_class(void)
+{
+ struct rte_fastmem_lcore_class_stats lcs;
+ void *ptr;
+ int rc;
+
+ ptr = rte_fastmem_alloc(256, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ rc = rte_fastmem_stats_lcore_class(rte_lcore_id(), 256, &lcs);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_lcore_class failed: %d", rc);
+ TEST_ASSERT_EQUAL(lcs.class_size, (size_t)256, "wrong class_size");
+ TEST_ASSERT(lcs.alloc_cache_hits + lcs.alloc_cache_misses > 0,
+ "no alloc activity");
+
+ rte_fastmem_free(ptr);
+ return TEST_SUCCESS;
+}
+
+static int
+test_stats_reset(void)
+{
+ struct rte_fastmem_stats gs;
+ void *ptr;
+ int rc;
+
+ ptr = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+ rte_fastmem_free(ptr);
+
+ rte_fastmem_stats_reset();
+
+ rc = rte_fastmem_stats(&gs);
+ TEST_ASSERT_EQUAL(rc, 0, "stats failed: %d", rc);
+ TEST_ASSERT_EQUAL(gs.alloc_total, (uint64_t)0,
+ "alloc_total not zero after reset");
+ TEST_ASSERT_EQUAL(gs.free_total, (uint64_t)0,
+ "free_total not zero after reset");
+
+ return TEST_SUCCESS;
+}
+
+/*
+ * Counters are stored separately from the per-lcore caches, so a
+ * cache flush (which frees the cache structs) must not discard
+ * accumulated statistics.
+ */
+static int
+test_stats_survive_cache_flush(void)
+{
+ enum { N = 10 };
+ struct rte_fastmem_class_stats before, after;
+ struct rte_fastmem_lcore_stats lbefore, lafter;
+ void *ptrs[N];
+ unsigned int i;
+ int rc;
+
+ for (i = 0; i < N; i++) {
+ ptrs[i] = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "alloc[%u] failed", i);
+ }
+ for (i = 0; i < N; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ rc = rte_fastmem_stats_class(64, &before);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_class failed: %d", rc);
+ rc = rte_fastmem_stats_lcore(rte_lcore_id(), &lbefore);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_lcore failed: %d", rc);
+
+ TEST_ASSERT(before.alloc_cache_hits + before.alloc_cache_misses == N,
+ "expected %d allocs before flush", N);
+
+ rte_fastmem_cache_flush();
+
+ rc = rte_fastmem_stats_class(64, &after);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_class after flush failed: %d", rc);
+ rc = rte_fastmem_stats_lcore(rte_lcore_id(), &lafter);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_lcore after flush failed: %d", rc);
+
+ TEST_ASSERT_EQUAL(after.alloc_cache_hits, before.alloc_cache_hits,
+ "alloc_cache_hits lost across flush: %" PRIu64 " -> %" PRIu64,
+ before.alloc_cache_hits, after.alloc_cache_hits);
+ TEST_ASSERT_EQUAL(after.alloc_cache_misses, before.alloc_cache_misses,
+ "alloc_cache_misses lost across flush: %" PRIu64 " -> %" PRIu64,
+ before.alloc_cache_misses, after.alloc_cache_misses);
+ TEST_ASSERT_EQUAL(after.free_cache_hits, before.free_cache_hits,
+ "free_cache_hits lost across flush: %" PRIu64 " -> %" PRIu64,
+ before.free_cache_hits, after.free_cache_hits);
+ TEST_ASSERT_EQUAL(lafter.alloc_cache_hits + lafter.alloc_cache_misses,
+ lbefore.alloc_cache_hits + lbefore.alloc_cache_misses,
+ "per-lcore alloc counters lost across flush");
+
+ return TEST_SUCCESS;
+}
+
+/*
+ * Allocations made by a non-EAL thread cannot be attributed to an
+ * lcore, but must still be reflected in the global and per-class
+ * statistics.
+ */
+static uint32_t
+stats_non_eal_main(void *arg)
+{
+ struct non_eal_args *args = arg;
+ void *ptrs[8];
+ unsigned int i;
+
+ for (i = 0; i < RTE_DIM(ptrs); i++) {
+ ptrs[i] = rte_fastmem_alloc(64, 0, 0);
+ if (ptrs[i] == NULL)
+ return 1;
+ }
+ for (i = 0; i < RTE_DIM(ptrs); i++)
+ rte_fastmem_free(ptrs[i]);
+
+ args->ok = 1;
+ return 0;
+}
+
+static int
+test_stats_count_non_eal(void)
+{
+ enum { N = 8 };
+ struct rte_fastmem_stats before, after;
+ struct non_eal_args args = { 0 };
+ rte_thread_t thread_id;
+ int rc;
+
+ rte_fastmem_stats_reset();
+
+ rc = rte_fastmem_stats(&before);
+ TEST_ASSERT_EQUAL(rc, 0, "stats failed: %d", rc);
+
+ rc = rte_thread_create(&thread_id, NULL, stats_non_eal_main, &args);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_create() failed: %d", rc);
+ rc = rte_thread_join(thread_id, NULL);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_join() failed: %d", rc);
+ TEST_ASSERT_EQUAL(args.ok, 1, "non-EAL thread alloc/free failed");
+
+ rc = rte_fastmem_stats(&after);
+ TEST_ASSERT_EQUAL(rc, 0, "stats failed: %d", rc);
+
+ TEST_ASSERT_EQUAL(after.alloc_total - before.alloc_total, (uint64_t)N,
+ "non-EAL allocs not counted globally: delta=%" PRIu64,
+ after.alloc_total - before.alloc_total);
+ TEST_ASSERT_EQUAL(after.free_total - before.free_total, (uint64_t)N,
+ "non-EAL frees not counted globally: delta=%" PRIu64,
+ after.free_total - before.free_total);
+
+ return TEST_SUCCESS;
+}
+
+/*
+ * A non-EAL thread has no lcore id, so its traffic must land in the
+ * shared cache and be reported by rte_fastmem_stats_shared().
+ */
+static int
+test_stats_shared_non_eal(void)
+{
+ enum { N = 8 };
+ struct rte_fastmem_lcore_stats sh;
+ struct rte_fastmem_lcore_class_stats shc;
+ struct non_eal_args args = { 0 };
+ rte_thread_t thread_id;
+ int rc;
+
+ rte_fastmem_stats_reset();
+
+ rc = rte_thread_create(&thread_id, NULL, stats_non_eal_main, &args);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_create() failed: %d", rc);
+ rc = rte_thread_join(thread_id, NULL);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_join() failed: %d", rc);
+ TEST_ASSERT_EQUAL(args.ok, 1, "non-EAL thread alloc/free failed");
+
+ rc = rte_fastmem_stats_shared(&sh);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_shared failed: %d", rc);
+ TEST_ASSERT_EQUAL(sh.alloc_cache_hits + sh.alloc_cache_misses,
+ (uint64_t)N, "shared allocs not counted: %" PRIu64,
+ sh.alloc_cache_hits + sh.alloc_cache_misses);
+ TEST_ASSERT_EQUAL(sh.free_cache_hits + sh.free_cache_misses,
+ (uint64_t)N, "shared frees not counted: %" PRIu64,
+ sh.free_cache_hits + sh.free_cache_misses);
+
+ /* stats_non_eal_main allocates 64-byte objects. */
+ rc = rte_fastmem_stats_shared_class(64, &shc);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_shared_class failed: %d", rc);
+ TEST_ASSERT_EQUAL(shc.class_size, (size_t)64, "wrong class_size");
+ TEST_ASSERT_EQUAL(shc.alloc_cache_hits + shc.alloc_cache_misses,
+ (uint64_t)N, "shared class allocs not counted: %" PRIu64,
+ shc.alloc_cache_hits + shc.alloc_cache_misses);
+
+ /* The shared traffic must not be attributed to any lcore. */
+ struct rte_fastmem_lcore_stats ls;
+ rc = rte_fastmem_stats_lcore(rte_lcore_id(), &ls);
+ TEST_ASSERT_EQUAL(rc, 0, "stats_lcore failed: %d", rc);
+ TEST_ASSERT_EQUAL(ls.alloc_cache_hits + ls.alloc_cache_misses,
+ (uint64_t)0, "shared traffic leaked into lcore stats");
+
+ /* Error paths. */
+ rc = rte_fastmem_stats_shared(NULL);
+ TEST_ASSERT_EQUAL(rc, -EINVAL, "expected -EINVAL for NULL stats");
+ rc = rte_fastmem_stats_shared_class(13, &shc);
+ TEST_ASSERT_EQUAL(rc, -EINVAL, "expected -EINVAL for bad size");
+
+ return TEST_SUCCESS;
+}
+
+
+#define MIXED_LONG_LIVED_COUNT 25
+#define MIXED_SHORT_LIVED_ITERS 1000
+#define MIXED_MIN_LCORES 3
+
+static const size_t mixed_long_sizes[] = { 64, 256, 4096 };
+static const size_t mixed_short_sizes[] = { 8, 16, 32, 64, 128, 256, 512, 1024 };
+
+struct mixed_worker_args {
+ uint32_t seed;
+ int result;
+};
+
+static uint32_t
+xorshift32(uint32_t *state)
+{
+ uint32_t x = *state;
+
+ x ^= x << 13;
+ x ^= x >> 17;
+ x ^= x << 5;
+ *state = x;
+ return x;
+}
+
+static int
+mixed_worker(void *arg)
+{
+ struct mixed_worker_args *args = arg;
+ uint32_t seed = args->seed;
+ void *long_lived[MIXED_LONG_LIVED_COUNT];
+ size_t long_sizes[MIXED_LONG_LIVED_COUNT];
+ unsigned int i;
+
+ /* Allocate long-lived objects of mixed sizes. */
+ for (i = 0; i < MIXED_LONG_LIVED_COUNT; i++) {
+ long_sizes[i] = mixed_long_sizes[i % RTE_DIM(mixed_long_sizes)];
+ long_lived[i] = rte_fastmem_alloc(long_sizes[i], 0, 0);
+ if (long_lived[i] == NULL) {
+ args->result = TEST_FAILED;
+ return -1;
+ }
+ memset(long_lived[i], (int)(i + 1), long_sizes[i]);
+ }
+
+ /* Rapidly cycle short-lived objects. */
+ for (i = 0; i < MIXED_SHORT_LIVED_ITERS; i++) {
+ size_t sz = mixed_short_sizes[xorshift32(&seed) %
+ RTE_DIM(mixed_short_sizes)];
+ uint8_t pattern = (uint8_t)(i & 0xff);
+ uint8_t *p;
+
+ p = rte_fastmem_alloc(sz, 0, 0);
+ if (p == NULL) {
+ args->result = TEST_FAILED;
+ return -1;
+ }
+ memset(p, pattern, sz);
+
+ /* Verify before freeing. */
+ for (size_t j = 0; j < sz; j++) {
+ if (p[j] != pattern) {
+ args->result = TEST_FAILED;
+ return -1;
+ }
+ }
+ rte_fastmem_free(p);
+ }
+
+ /* Verify long-lived objects are still intact. */
+ for (i = 0; i < MIXED_LONG_LIVED_COUNT; i++) {
+ uint8_t *bytes = long_lived[i];
+ uint8_t expected = (uint8_t)(i + 1);
+
+ for (size_t j = 0; j < long_sizes[i]; j++) {
+ if (bytes[j] != expected) {
+ args->result = TEST_FAILED;
+ return -1;
+ }
+ }
+ rte_fastmem_free(long_lived[i]);
+ }
+
+ args->result = TEST_SUCCESS;
+ return 0;
+}
+
+static int
+test_mixed_lifetimes_multi_lcore(void)
+{
+ struct mixed_worker_args args[RTE_MAX_LCORE];
+ unsigned int lcore_id;
+ unsigned int count = 0;
+ struct rte_fastmem_stats stats;
+ int rc;
+
+ RTE_LCORE_FOREACH_WORKER(lcore_id)
+ count++;
+
+ if (count < MIXED_MIN_LCORES) {
+ printf("Not enough worker lcores (%u < %u), skipping\n",
+ count, MIXED_MIN_LCORES);
+ return TEST_SKIPPED;
+ }
+
+ /* Launch workers with distinct seeds. */
+ uint32_t seed = 0xdeadbeef;
+
+ RTE_LCORE_FOREACH_WORKER(lcore_id) {
+ args[lcore_id].seed = seed;
+ args[lcore_id].result = TEST_FAILED;
+ seed += 0x12345678;
+ rte_eal_remote_launch(mixed_worker, &args[lcore_id], lcore_id);
+ }
+
+ rte_eal_mp_wait_lcore();
+
+ /* Check all workers succeeded. */
+ RTE_LCORE_FOREACH_WORKER(lcore_id) {
+ TEST_ASSERT_EQUAL(args[lcore_id].result, TEST_SUCCESS,
+ "worker on lcore %u failed", lcore_id);
+ }
+
+ /* Verify no memory leak. */
+ rc = rte_fastmem_stats(&stats);
+ TEST_ASSERT_EQUAL(rc, 0, "stats failed: %d", rc);
+ TEST_ASSERT_EQUAL(stats.bytes_in_use, (uint64_t)0,
+ "bytes_in_use not zero after test: %" PRIu64,
+ stats.bytes_in_use);
+
+ return TEST_SUCCESS;
+}
+
+
+/*
+ * Memory limit tests.
+ *
+ * FASTMEM_MEMZONE_SIZE is 128 MiB. We use a limit of 128 MiB
+ * (one memzone) for most tests, and large objects (256 KiB) to
+ * exhaust slabs quickly.
+ */
+
+#define LIMIT_ONE_MZ ((size_t)128 << 20)
+#define LIMIT_OBJ_SIZE ((size_t)256 * 1024)
+
+static int
+test_memory_limit_basic(void)
+{
+ int rc;
+
+ rc = rte_fastmem_set_limit(SOCKET_ID_ANY, LIMIT_ONE_MZ);
+ TEST_ASSERT_EQUAL(rc, 0, "set_memory_limit failed: %d", rc);
+
+ const size_t got = rte_fastmem_get_limit(0);
+ TEST_ASSERT_EQUAL(got, LIMIT_ONE_MZ,
+ "get_memory_limit mismatch: %zu", got);
+
+ rc = rte_fastmem_reserve(LIMIT_ONE_MZ, SOCKET_ID_ANY);
+ TEST_ASSERT_EQUAL(rc, 0, "first reserve failed: %d", rc);
+
+ rc = rte_fastmem_reserve(LIMIT_ONE_MZ + 1, SOCKET_ID_ANY);
+ TEST_ASSERT(rc < 0, "second reserve should have failed");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_alloc_exhaustion(void)
+{
+ const unsigned int max_ptrs = 1024;
+ void *ptrs[max_ptrs];
+ unsigned int count = 0;
+ rte_fastmem_set_limit(SOCKET_ID_ANY, LIMIT_ONE_MZ);
+
+ for (count = 0; count < max_ptrs; count++) {
+ ptrs[count] = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ if (ptrs[count] == NULL)
+ break;
+ }
+
+ TEST_ASSERT(count > 0, "should have allocated at least one");
+ TEST_ASSERT(count < max_ptrs, "should have hit the limit");
+ TEST_ASSERT_EQUAL(rte_errno, ENOMEM, "expected ENOMEM, got %d", rte_errno);
+
+ rte_fastmem_free(ptrs[count - 1]);
+ void *p = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(p, "alloc after free should succeed");
+ rte_fastmem_free(p);
+
+ for (unsigned int i = 0; i < count - 1; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_zero_blocks_growth(void)
+{
+ int rc;
+
+ rte_fastmem_set_limit(SOCKET_ID_ANY, 0);
+
+ rc = rte_fastmem_reserve(1, SOCKET_ID_ANY);
+ TEST_ASSERT(rc < 0, "reserve with limit=0 should fail");
+
+ void *p = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NULL(p, "alloc with limit=0 should fail");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_below_current(void)
+{
+ int rc;
+
+ rc = rte_fastmem_reserve(LIMIT_ONE_MZ, SOCKET_ID_ANY);
+ TEST_ASSERT_EQUAL(rc, 0, "reserve failed: %d", rc);
+
+ rte_fastmem_set_limit(SOCKET_ID_ANY, 1);
+
+ void *p = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(p, "alloc from existing backing should work");
+ rte_fastmem_free(p);
+
+ rc = rte_fastmem_reserve(LIMIT_ONE_MZ * 2, SOCKET_ID_ANY);
+ TEST_ASSERT(rc < 0, "growth beyond limit should fail");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_socket_id_any(void)
+{
+ rte_fastmem_set_limit(SOCKET_ID_ANY, 42);
+
+ for (unsigned int i = 0; i < rte_socket_count(); i++) {
+ const int sid = rte_socket_id_by_idx(i);
+ const size_t lim = rte_fastmem_get_limit(sid);
+
+ TEST_ASSERT_EQUAL(lim, (size_t)42,
+ "socket %d limit mismatch: %zu", sid, lim);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_unlimited(void)
+{
+ int rc;
+
+ rte_fastmem_set_limit(SOCKET_ID_ANY, 0);
+ rte_fastmem_set_limit(SOCKET_ID_ANY, SIZE_MAX);
+
+ rc = rte_fastmem_reserve(LIMIT_ONE_MZ, SOCKET_ID_ANY);
+ TEST_ASSERT_EQUAL(rc, 0, "reserve after reset failed: %d", rc);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_alloc_integrity_under_oom(void)
+{
+ const unsigned int n = 128;
+ const size_t obj_size = 1024;
+ uint8_t *ptrs[n];
+ const unsigned int extra_max = 1024;
+ void *extra[extra_max];
+ unsigned int n_extra = 0;
+ unsigned int i;
+ rte_fastmem_set_limit(SOCKET_ID_ANY, LIMIT_ONE_MZ);
+
+ for (i = 0; i < n; i++) {
+ ptrs[i] = rte_fastmem_alloc(obj_size, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "alloc[%u] failed", i);
+ memset(ptrs[i], (int)(i & 0xff), obj_size);
+ }
+
+ /* Exhaust remaining backing with large objects. */
+ for (n_extra = 0; n_extra < extra_max; n_extra++) {
+ extra[n_extra] = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ if (extra[n_extra] == NULL)
+ break;
+ }
+
+ /* Verify original objects are intact. */
+ for (i = 0; i < n; i++) {
+ const uint8_t expected = (uint8_t)(i & 0xff);
+ for (unsigned int j = 0; j < obj_size; j++)
+ TEST_ASSERT_EQUAL(ptrs[i][j], expected,
+ "corruption at [%u][%u]", i, j);
+ }
+
+ for (i = 0; i < n; i++)
+ rte_fastmem_free(ptrs[i]);
+ for (i = 0; i < n_extra; i++)
+ rte_fastmem_free(extra[i]);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_bulk_alloc_oom(void)
+{
+ const unsigned int bulk_n = 64;
+ const unsigned int drain_max = 512;
+ void *ptrs[bulk_n];
+ void *drain[drain_max];
+ unsigned int drained = 0;
+ int rc;
+
+ rte_fastmem_set_limit(SOCKET_ID_ANY, LIMIT_ONE_MZ);
+
+ for (drained = 0; drained < drain_max; drained++) {
+ drain[drained] = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ if (drain[drained] == NULL)
+ break;
+ }
+
+ /* Free a few — enough for some but not bulk_n objects. */
+ const unsigned int freed = RTE_MIN(drained, 4u);
+ for (unsigned int i = 0; i < freed; i++)
+ rte_fastmem_free(drain[--drained]);
+
+ rc = rte_fastmem_alloc_bulk(ptrs, bulk_n, LIMIT_OBJ_SIZE, 0, 0);
+ TEST_ASSERT(rc < 0, "bulk alloc should fail");
+
+ for (unsigned int i = 0; i < drained; i++)
+ rte_fastmem_free(drain[i]);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_memory_limit_recovery_after_free(void)
+{
+ const unsigned int max_ptrs = 512;
+ void *ptrs[max_ptrs];
+ unsigned int count = 0;
+ rte_fastmem_set_limit(SOCKET_ID_ANY, LIMIT_ONE_MZ);
+
+ for (count = 0; count < max_ptrs; count++) {
+ ptrs[count] = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ if (ptrs[count] == NULL)
+ break;
+ }
+ TEST_ASSERT(count > 0 && count < max_ptrs,
+ "expected partial fill, got %u", count);
+
+ const unsigned int half = count / 2;
+ for (unsigned int i = 0; i < half; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ for (unsigned int i = 0; i < half; i++) {
+ ptrs[i] = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptrs[i], "recovery alloc[%u] failed", i);
+ }
+
+ for (unsigned int i = 0; i < count; i++)
+ rte_fastmem_free(ptrs[i]);
+
+ return TEST_SUCCESS;
+}
+
+struct limit_worker_args {
+ unsigned int alloc_count;
+ int result;
+};
+
+static int
+limit_worker(void *arg)
+{
+ struct limit_worker_args *args = arg;
+ const unsigned int max_ptrs = 128;
+ void *ptrs[max_ptrs];
+ unsigned int i;
+
+ args->alloc_count = 0;
+
+ for (i = 0; i < max_ptrs; i++) {
+ ptrs[i] = rte_fastmem_alloc(LIMIT_OBJ_SIZE, 0, 0);
+ if (ptrs[i] == NULL)
+ break;
+ memset(ptrs[i], 0xab, LIMIT_OBJ_SIZE);
+ args->alloc_count++;
+ }
+
+ for (unsigned int j = 0; j < args->alloc_count; j++) {
+ uint8_t *bytes = ptrs[j];
+ for (size_t k = 0; k < LIMIT_OBJ_SIZE; k++) {
+ if (bytes[k] != 0xab) {
+ args->result = TEST_FAILED;
+ return -1;
+ }
+ }
+ rte_fastmem_free(ptrs[j]);
+ }
+
+ args->result = TEST_SUCCESS;
+ return 0;
+}
+
+static int
+test_memory_limit_multi_lcore_oom(void)
+{
+ struct limit_worker_args args[RTE_MAX_LCORE];
+ unsigned int lcore_id;
+ unsigned int worker_count = 0;
+ RTE_LCORE_FOREACH_WORKER(lcore_id)
+ worker_count++;
+
+ if (worker_count < 2) {
+ printf("Not enough workers (%u < 2), skipping\n", worker_count);
+ return TEST_SKIPPED;
+ }
+
+ rte_fastmem_set_limit(SOCKET_ID_ANY, LIMIT_ONE_MZ);
+
+ RTE_LCORE_FOREACH_WORKER(lcore_id) {
+ args[lcore_id].result = TEST_FAILED;
+ rte_eal_remote_launch(limit_worker, &args[lcore_id], lcore_id);
+ }
+
+ rte_eal_mp_wait_lcore();
+
+ RTE_LCORE_FOREACH_WORKER(lcore_id) {
+ TEST_ASSERT_EQUAL(args[lcore_id].result, TEST_SUCCESS,
+ "worker on lcore %u failed", lcore_id);
+ }
+
+ struct rte_fastmem_stats stats;
+ rte_fastmem_stats(&stats);
+ TEST_ASSERT_EQUAL(stats.bytes_in_use, (uint64_t)0,
+ "bytes_in_use not zero: %" PRIu64, stats.bytes_in_use);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_same_class(void)
+{
+ void *ptr = rte_fastmem_alloc(32, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ /* Realloc to a smaller size within the same class (64 B class). */
+ void *ptr2 = rte_fastmem_realloc(ptr, 33, 0);
+ TEST_ASSERT_NOT_NULL(ptr2, "realloc failed");
+ TEST_ASSERT_EQUAL(ptr, ptr2,
+ "realloc returned different pointer for same class");
+
+ /* Realloc to exact class boundary — still same class. */
+ void *ptr3 = rte_fastmem_realloc(ptr2, 64, 0);
+ TEST_ASSERT_NOT_NULL(ptr3, "realloc failed");
+ TEST_ASSERT_EQUAL(ptr2, ptr3,
+ "realloc returned different pointer for same class");
+
+ rte_fastmem_free(ptr3);
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_grow(void)
+{
+ const uint8_t pattern = 0xab;
+ void *ptr = rte_fastmem_alloc(16, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ memset(ptr, pattern, 16);
+
+ /* Grow beyond current class. */
+ void *ptr2 = rte_fastmem_realloc(ptr, 128, 0);
+ TEST_ASSERT_NOT_NULL(ptr2, "realloc grow failed");
+
+ /* Verify contents preserved. */
+ uint8_t *bytes = ptr2;
+ for (unsigned int i = 0; i < 16; i++)
+ TEST_ASSERT_EQUAL(bytes[i], pattern,
+ "content corrupted at byte %u", i);
+
+ rte_fastmem_free(ptr2);
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_shrink(void)
+{
+ const uint8_t pattern = 0xcd;
+ void *ptr = rte_fastmem_alloc(256, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ memset(ptr, pattern, 256);
+
+ /* Shrink to a smaller class. */
+ void *ptr2 = rte_fastmem_realloc(ptr, 16, 0);
+ TEST_ASSERT_NOT_NULL(ptr2, "realloc shrink failed");
+
+ /* Verify contents preserved up to new size. */
+ uint8_t *bytes = ptr2;
+ for (unsigned int i = 0; i < 16; i++)
+ TEST_ASSERT_EQUAL(bytes[i], pattern,
+ "content corrupted at byte %u", i);
+
+ rte_fastmem_free(ptr2);
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_null_ptr(void)
+{
+ /* NULL ptr should behave like alloc. */
+ void *ptr = rte_fastmem_realloc(NULL, 64, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "realloc(NULL) failed");
+
+ rte_fastmem_free(ptr);
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_zero_size(void)
+{
+ void *ptr = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ /* size 0 should free and return NULL. */
+ void *ptr2 = rte_fastmem_realloc(ptr, 0, 0);
+ TEST_ASSERT_NULL(ptr2, "realloc(size=0) should return NULL");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_too_big(void)
+{
+ void *ptr = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ void *ptr2 = rte_fastmem_realloc(ptr, rte_fastmem_max_size() + 1, 0);
+ TEST_ASSERT_NULL(ptr2, "realloc should fail for oversized request");
+ TEST_ASSERT_EQUAL(rte_errno, E2BIG, "expected E2BIG");
+
+ /* Original pointer should still be valid. */
+ rte_fastmem_free(ptr);
+ return TEST_SUCCESS;
+}
+
+static int
+test_realloc_invalid_align(void)
+{
+ void *ptr = rte_fastmem_alloc(64, 0, 0);
+ TEST_ASSERT_NOT_NULL(ptr, "alloc failed");
+
+ void *ptr2 = rte_fastmem_realloc(ptr, 64, 3);
+ TEST_ASSERT_NULL(ptr2, "realloc should fail for non-power-of-2 align");
+ TEST_ASSERT_EQUAL(rte_errno, EINVAL, "expected EINVAL");
+
+ rte_fastmem_free(ptr);
+ return TEST_SUCCESS;
+}
+
+/*
+ * Handle-based allocation API.
+ */
+
+static int
+test_halloc_basic(void)
+{
+ rte_fastmem_handle_t handle;
+ void *ptrs[16];
+ void *p;
+ int rc;
+ unsigned int i;
+
+ rc = rte_fastmem_hlookup(64, 0, rte_socket_id_by_idx(0), &handle);
+ TEST_ASSERT_EQUAL(rc, 0, "hlookup failed: %d", rc);
+
+ p = rte_fastmem_halloc(handle, RTE_FASTMEM_F_ZERO);
+ TEST_ASSERT_NOT_NULL(p, "halloc failed: rte_errno=%d", rte_errno);
+ memset(p, 0x5a, 64);
+ rte_fastmem_hfree(handle, p);
+
+ /* NULL pointer free is a no-op. */
+ rte_fastmem_hfree(handle, NULL);
+
+ rc = rte_fastmem_halloc_bulk(handle, ptrs, RTE_DIM(ptrs), 0);
+ TEST_ASSERT_EQUAL(rc, 0, "halloc_bulk failed: %d", rc);
+ for (i = 0; i < RTE_DIM(ptrs); i++)
+ TEST_ASSERT_NOT_NULL(ptrs[i], "halloc_bulk[%u] NULL", i);
+ rte_fastmem_hfree_bulk(handle, ptrs, RTE_DIM(ptrs));
+
+ return TEST_SUCCESS;
+}
+
+struct halloc_worker_args {
+ rte_fastmem_handle_t handle;
+ int result;
+};
+
+/*
+ * Allocate and free using a handle that was looked up on a
+ * different lcore. The worker lcore has no pre-existing cache for
+ * the handle's size class, so this exercises the path where
+ * halloc/hfree must lazily create (or bypass) the per-lcore cache.
+ */
+static int
+halloc_worker(void *arg)
+{
+ struct halloc_worker_args *args = arg;
+ void *ptrs[8];
+ uint8_t *p;
+ unsigned int i;
+
+ args->result = TEST_FAILED;
+
+ p = rte_fastmem_halloc(args->handle, 0);
+ if (p == NULL)
+ return -1;
+ memset(p, 0x3c, 64);
+ rte_fastmem_hfree(args->handle, p);
+
+ if (rte_fastmem_halloc_bulk(args->handle, ptrs, RTE_DIM(ptrs), 0) < 0)
+ return -1;
+ for (i = 0; i < RTE_DIM(ptrs); i++) {
+ if (ptrs[i] == NULL)
+ return -1;
+ }
+ rte_fastmem_hfree_bulk(args->handle, ptrs, RTE_DIM(ptrs));
+
+ args->result = TEST_SUCCESS;
+ return 0;
+}
+
+static int
+test_halloc_other_lcore(void)
+{
+ struct halloc_worker_args args;
+ rte_fastmem_handle_t handle;
+ unsigned int lcore_id;
+ int rc;
+
+ lcore_id = rte_get_next_lcore(-1, 1, 0);
+ if (lcore_id == RTE_MAX_LCORE)
+ return TEST_SKIPPED;
+
+ /* Look up the handle on the main lcore only. */
+ rc = rte_fastmem_hlookup(64, 0, rte_socket_id_by_idx(0), &handle);
+ TEST_ASSERT_EQUAL(rc, 0, "hlookup failed: %d", rc);
+
+ args.handle = handle;
+ args.result = TEST_FAILED;
+
+ rte_eal_remote_launch(halloc_worker, &args, lcore_id);
+ rc = rte_eal_wait_lcore(lcore_id);
+ TEST_ASSERT_EQUAL(rc, 0, "worker returned %d", rc);
+ TEST_ASSERT_EQUAL(args.result, TEST_SUCCESS,
+ "halloc/hfree failed on a lcore that did not call hlookup");
+
+ return TEST_SUCCESS;
+}
+
+static uint32_t
+halloc_non_eal_main(void *arg)
+{
+ struct halloc_worker_args *args = arg;
+
+ return halloc_worker(args) == 0 ? 0 : 1;
+}
+
+static int
+test_halloc_non_eal_thread(void)
+{
+ struct halloc_worker_args args;
+ rte_fastmem_handle_t handle;
+ rte_thread_t thread_id;
+ int rc;
+
+ rc = rte_fastmem_hlookup(64, 0, rte_socket_id_by_idx(0), &handle);
+ TEST_ASSERT_EQUAL(rc, 0, "hlookup failed: %d", rc);
+
+ args.handle = handle;
+ args.result = TEST_FAILED;
+
+ rc = rte_thread_create(&thread_id, NULL, halloc_non_eal_main, &args);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_create() failed: %d", rc);
+ rc = rte_thread_join(thread_id, NULL);
+ TEST_ASSERT_EQUAL(rc, 0, "rte_thread_join() failed: %d", rc);
+
+ TEST_ASSERT_EQUAL(args.result, TEST_SUCCESS,
+ "halloc/hfree failed on a non-EAL thread");
+
+ return TEST_SUCCESS;
+}
+
+static int
+fastmem_setup(void)
+{
+ return rte_fastmem_init();
+}
+
+static void
+fastmem_teardown(void)
+{
+ rte_fastmem_deinit();
+}
+
+static struct unit_test_suite fastmem_testsuite = {
+ .suite_name = "fastmem tests",
+ .setup = NULL,
+ .teardown = NULL,
+ .unit_test_cases = {
+ TEST_CASE(test_init_deinit),
+ TEST_CASE(test_init_is_not_idempotent),
+ TEST_CASE(test_deinit_without_init),
+ TEST_CASE(test_max_size),
+ TEST_CASE(test_reserve_without_init),
+ TEST_CASE(test_cache_flush_without_init),
+ TEST_CASE(test_classes),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_reserve_small),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_reserve_multiple_memzones),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_reserve_cumulative),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_reserve_invalid_socket),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_reserve_any_socket),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_too_big),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_invalid_align),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_free_small),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_free_various_sizes),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_alignment),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_zero_flag),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_reuse),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_many_in_class),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_socket),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_block_repurposing),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_block_repurposing_no_growth),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_free_null),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_content_integrity),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_align_too_big),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_align_one),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_socket_numa_placement),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_cross_socket_deinit),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_cache_flush),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_cache_exceeds_capacity),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_non_eal_thread),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_cache_flush_returns_memory),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_bulk_basic),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_bulk_zero_flag),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_bulk_exceeds_cache),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_alloc_bulk_socket),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_free_bulk),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_class),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_lcore),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_lcore_class),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_reset),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_survive_cache_flush),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_count_non_eal),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_stats_shared_non_eal),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_mixed_lifetimes_multi_lcore),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_basic),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_alloc_exhaustion),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_zero_blocks_growth),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_below_current),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_socket_id_any),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_unlimited),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_alloc_integrity_under_oom),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_bulk_alloc_oom),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_recovery_after_free),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_memory_limit_multi_lcore_oom),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_same_class),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_grow),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_shrink),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_null_ptr),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_zero_size),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_too_big),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_realloc_invalid_align),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_halloc_basic),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_halloc_other_lcore),
+ TEST_CASE_ST(fastmem_setup, fastmem_teardown,
+ test_halloc_non_eal_thread),
+ TEST_CASES_END()
+ }
+};
+
+static int
+test_fastmem(void)
+{
+ return unit_test_suite_runner(&fastmem_testsuite);
+}
+
+REGISTER_FAST_TEST(fastmem_autotest, NOHUGE_SKIP, ASAN_OK, test_fastmem);
diff --git a/app/test/test_fastmem_perf.c b/app/test/test_fastmem_perf.c
new file mode 100644
index 0000000000..73c0a4c6ce
--- /dev/null
+++ b/app/test/test_fastmem_perf.c
@@ -0,0 +1,1040 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 Ericsson AB
+ */
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_launch.h>
+#include <rte_lcore.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_stdatomic.h>
+
+#include <rte_fastmem.h>
+
+#include "test.h"
+
+#define TEST_LOG(...) printf(__VA_ARGS__)
+
+static const size_t SIZES[] = { 8, 64, 256, 1024, 4096 };
+#define N_SIZES RTE_DIM(SIZES)
+
+/* Number of ops for warmup and measurement. */
+#define WARMUP_OPS 20000u
+#define MEASURE_OPS 2000000u
+
+/* Buffer for scenarios that allocate N then free N. */
+#define BATCH_N 256
+
+/*
+ * Allocator vtable: a thin adapter exposing alloc / free /
+ * per-allocator setup/teardown. Each scenario calls these
+ * indirectly so the same timing loop serves all allocators.
+ */
+struct allocator {
+ const char *name;
+ int (*setup)(size_t size, unsigned int n_max);
+ void (*teardown)(void);
+ void *(*alloc)(void);
+ void (*free_obj)(void *ptr);
+ int (*alloc_bulk)(void **ptrs, unsigned int n);
+ void (*free_bulk)(void **ptrs, unsigned int n);
+};
+
+/* Fastmem adapter -------------------------------------------------- */
+
+static size_t fastmem_size;
+
+static int
+fastmem_setup(size_t size, unsigned int n_max __rte_unused)
+{
+ fastmem_size = size;
+ return 0;
+}
+
+static void
+fastmem_teardown(void)
+{
+ rte_fastmem_cache_flush();
+}
+
+static void * __rte_noinline
+fastmem_alloc(void)
+{
+ return rte_fastmem_alloc(fastmem_size, 0, 0);
+}
+
+static void __rte_noinline
+fastmem_free(void *ptr)
+{
+ rte_fastmem_free(ptr);
+}
+
+/* Mempool adapter -------------------------------------------------- */
+
+static struct rte_mempool *mempool_pool;
+
+static int
+mempool_setup(size_t size, unsigned int n_max)
+{
+ char name[RTE_MEMPOOL_NAMESIZE];
+ unsigned int cache_size;
+
+ /*
+ * Pool size must accommodate the full batch burst plus
+ * per-lcore cache capacity. Use mempool's default cache
+ * size so we're measuring its standard hot path.
+ */
+ cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE;
+
+ snprintf(name, sizeof(name), "fmperf_mp_%zu", size);
+ mempool_pool = rte_mempool_create(name, n_max + cache_size * 2,
+ size, cache_size, 0, NULL, NULL, NULL, NULL,
+ SOCKET_ID_ANY, 0);
+ if (mempool_pool == NULL) {
+ TEST_LOG("mempool_create(%zu) failed\n", size);
+ return -1;
+ }
+
+ return 0;
+}
+
+static void
+mempool_teardown(void)
+{
+ rte_mempool_free(mempool_pool);
+ mempool_pool = NULL;
+}
+
+static void * __rte_noinline
+mempool_alloc_one(void)
+{
+ void *obj = NULL;
+
+ if (rte_mempool_get(mempool_pool, &obj) < 0)
+ return NULL;
+ return obj;
+}
+
+static void __rte_noinline
+mempool_free_one(void *ptr)
+{
+ rte_mempool_put(mempool_pool, ptr);
+}
+
+/* rte_malloc adapter ----------------------------------------------- */
+
+static size_t malloc_size;
+
+static int
+malloc_setup(size_t size, unsigned int n_max __rte_unused)
+{
+ malloc_size = size;
+ return 0;
+}
+
+static void
+malloc_teardown(void)
+{
+}
+
+static void * __rte_noinline
+malloc_alloc(void)
+{
+ return rte_malloc(NULL, malloc_size, 0);
+}
+
+static void __rte_noinline
+malloc_free(void *ptr)
+{
+ rte_free(ptr);
+}
+
+/* libc (glibc) malloc adapter -------------------------------------- */
+
+static size_t libc_size;
+
+static int
+libc_setup(size_t size, unsigned int n_max __rte_unused)
+{
+ /*
+ * Round up to cache-line alignment to match the other
+ * allocators' default alignment guarantees and keep the
+ * comparison honest. aligned_alloc() requires size to be
+ * a multiple of the alignment.
+ */
+ libc_size = RTE_ALIGN_CEIL(size, RTE_CACHE_LINE_SIZE);
+ return 0;
+}
+
+static void
+libc_teardown(void)
+{
+}
+
+static void * __rte_noinline
+libc_alloc(void)
+{
+ return aligned_alloc(RTE_CACHE_LINE_SIZE, libc_size);
+}
+
+static void __rte_noinline
+libc_free(void *ptr)
+{
+ free(ptr);
+}
+
+/* Bulk adapters ---------------------------------------------------- */
+
+static int __rte_noinline
+fastmem_alloc_bulk(void **ptrs, unsigned int n)
+{
+ return rte_fastmem_alloc_bulk(ptrs, n, fastmem_size, 0, 0);
+}
+
+static void __rte_noinline
+fastmem_free_bulk(void **ptrs, unsigned int n)
+{
+ rte_fastmem_free_bulk(ptrs, n);
+}
+
+/* Fastmem handle adapter ------------------------------------------- */
+
+static rte_fastmem_handle_t fastmem_handle;
+
+static int
+fastmem_h_setup(size_t size, unsigned int n_max __rte_unused)
+{
+ return rte_fastmem_hlookup(size, 0, rte_socket_id(), &fastmem_handle);
+}
+
+static void
+fastmem_h_teardown(void)
+{
+ rte_fastmem_cache_flush();
+}
+
+static void * __rte_noinline
+fastmem_h_alloc(void)
+{
+ return rte_fastmem_halloc(fastmem_handle, 0);
+}
+
+static void __rte_noinline
+fastmem_h_free(void *ptr)
+{
+ rte_fastmem_hfree(fastmem_handle, ptr);
+}
+
+static int __rte_noinline
+fastmem_h_alloc_bulk(void **ptrs, unsigned int n)
+{
+ return rte_fastmem_halloc_bulk(fastmem_handle, ptrs, n, 0);
+}
+
+static void __rte_noinline
+fastmem_h_free_bulk(void **ptrs, unsigned int n)
+{
+ rte_fastmem_hfree_bulk(fastmem_handle, ptrs, n);
+}
+
+/* Mempool adapter -------------------------------------------------- */
+
+static int __rte_noinline
+mempool_alloc_bulk(void **ptrs, unsigned int n)
+{
+ return rte_mempool_get_bulk(mempool_pool, ptrs, n);
+}
+
+static void __rte_noinline
+mempool_free_bulk(void **ptrs, unsigned int n)
+{
+ rte_mempool_put_bulk(mempool_pool, ptrs, n);
+}
+
+static int __rte_noinline
+generic_alloc_bulk(void **ptrs, unsigned int n, void *(*alloc_fn)(void))
+{
+ unsigned int i;
+
+ for (i = 0; i < n; i++) {
+ ptrs[i] = alloc_fn();
+ if (ptrs[i] == NULL)
+ return -1;
+ }
+ return 0;
+}
+
+static int __rte_noinline
+malloc_alloc_bulk(void **ptrs, unsigned int n)
+{
+ return generic_alloc_bulk(ptrs, n, malloc_alloc);
+}
+
+static void __rte_noinline
+malloc_free_bulk(void **ptrs, unsigned int n)
+{
+ unsigned int i;
+
+ for (i = 0; i < n; i++)
+ malloc_free(ptrs[i]);
+}
+
+static int __rte_noinline
+libc_alloc_bulk(void **ptrs, unsigned int n)
+{
+ return generic_alloc_bulk(ptrs, n, libc_alloc);
+}
+
+static void __rte_noinline
+libc_free_bulk(void **ptrs, unsigned int n)
+{
+ unsigned int i;
+
+ for (i = 0; i < n; i++)
+ libc_free(ptrs[i]);
+}
+
+/* Adapter table ---------------------------------------------------- */
+
+static const struct allocator allocators[] = {
+ { "fastmem", fastmem_setup, fastmem_teardown, fastmem_alloc, fastmem_free, fastmem_alloc_bulk, fastmem_free_bulk },
+ { "fastmem_h", fastmem_h_setup, fastmem_h_teardown, fastmem_h_alloc, fastmem_h_free, fastmem_h_alloc_bulk, fastmem_h_free_bulk },
+ { "mempool", mempool_setup, mempool_teardown, mempool_alloc_one, mempool_free_one, mempool_alloc_bulk, mempool_free_bulk },
+ { "rte_malloc", malloc_setup, malloc_teardown, malloc_alloc, malloc_free, malloc_alloc_bulk, malloc_free_bulk },
+ { "libc", libc_setup, libc_teardown, libc_alloc, libc_free, libc_alloc_bulk, libc_free_bulk },
+};
+#define N_ALLOCATORS RTE_DIM(allocators)
+
+/*
+ * Scenario 1: tight alloc+free loop. A single object is cycled
+ * repeatedly. The LIFO path keeps the same pointer hot, giving
+ * a best-case measurement.
+ */
+static double
+run_tight(const struct allocator *alloc, size_t size)
+{
+ void *p;
+ uint64_t tsc;
+ unsigned int i;
+
+ if (alloc->setup(size, 1) < 0)
+ return -1.0;
+
+ /* Warmup. */
+ for (i = 0; i < WARMUP_OPS; i++) {
+ p = alloc->alloc();
+ if (p == NULL)
+ goto err;
+ alloc->free_obj(p);
+ }
+
+ tsc = rte_rdtsc_precise();
+ for (i = 0; i < MEASURE_OPS; i++) {
+ p = alloc->alloc();
+ if (p == NULL)
+ goto err;
+ alloc->free_obj(p);
+ }
+ tsc = rte_rdtsc_precise() - tsc;
+
+ alloc->teardown();
+
+ return (double)tsc / MEASURE_OPS;
+err:
+ alloc->teardown();
+ return -1.0;
+}
+
+/*
+ * Scenario 2: allocate N, free N (FIFO free order). Exercises
+ * cache refill and drain paths when N exceeds cache capacity.
+ */
+static void
+run_batch(const struct allocator *alloc, size_t size,
+ double *cycles_alloc, double *cycles_free)
+{
+ void *ptrs[BATCH_N];
+ uint64_t tsc_alloc = 0, tsc_free = 0;
+ unsigned int iter, i;
+ unsigned int iters;
+
+ *cycles_alloc = -1.0;
+ *cycles_free = -1.0;
+
+ if (alloc->setup(size, BATCH_N) < 0)
+ return;
+
+ /* Pick iteration count so total ops ~= MEASURE_OPS. */
+ iters = MEASURE_OPS / BATCH_N;
+
+ /* Warmup. */
+ for (iter = 0; iter < WARMUP_OPS / BATCH_N; iter++) {
+ for (i = 0; i < BATCH_N; i++) {
+ ptrs[i] = alloc->alloc();
+ if (ptrs[i] == NULL)
+ goto err;
+ }
+ for (i = 0; i < BATCH_N; i++)
+ alloc->free_obj(ptrs[i]);
+ }
+
+ for (iter = 0; iter < iters; iter++) {
+ uint64_t t0;
+
+ t0 = rte_rdtsc_precise();
+ for (i = 0; i < BATCH_N; i++) {
+ ptrs[i] = alloc->alloc();
+ if (ptrs[i] == NULL)
+ goto err;
+ }
+ tsc_alloc += rte_rdtsc_precise() - t0;
+
+ t0 = rte_rdtsc_precise();
+ for (i = 0; i < BATCH_N; i++)
+ alloc->free_obj(ptrs[i]);
+ tsc_free += rte_rdtsc_precise() - t0;
+ }
+
+ alloc->teardown();
+
+ *cycles_alloc = (double)tsc_alloc / (iters * BATCH_N);
+ *cycles_free = (double)tsc_free / (iters * BATCH_N);
+ return;
+err:
+ alloc->teardown();
+}
+
+/*
+ * Scenario 3: allocate N, free N in reverse order.
+ */
+static void
+run_batch_reverse(const struct allocator *alloc, size_t size,
+ double *cycles_alloc, double *cycles_free)
+{
+ void *ptrs[BATCH_N];
+ uint64_t tsc_alloc = 0, tsc_free = 0;
+ unsigned int iter, i;
+ unsigned int iters;
+
+ *cycles_alloc = -1.0;
+ *cycles_free = -1.0;
+
+ if (alloc->setup(size, BATCH_N) < 0)
+ return;
+
+ iters = MEASURE_OPS / BATCH_N;
+
+ for (iter = 0; iter < WARMUP_OPS / BATCH_N; iter++) {
+ for (i = 0; i < BATCH_N; i++) {
+ ptrs[i] = alloc->alloc();
+ if (ptrs[i] == NULL)
+ goto err;
+ }
+ for (i = BATCH_N; i > 0; i--)
+ alloc->free_obj(ptrs[i - 1]);
+ }
+
+ for (iter = 0; iter < iters; iter++) {
+ uint64_t t0;
+
+ t0 = rte_rdtsc_precise();
+ for (i = 0; i < BATCH_N; i++) {
+ ptrs[i] = alloc->alloc();
+ if (ptrs[i] == NULL)
+ goto err;
+ }
+ tsc_alloc += rte_rdtsc_precise() - t0;
+
+ t0 = rte_rdtsc_precise();
+ for (i = BATCH_N; i > 0; i--)
+ alloc->free_obj(ptrs[i - 1]);
+ tsc_free += rte_rdtsc_precise() - t0;
+ }
+
+ alloc->teardown();
+
+ *cycles_alloc = (double)tsc_alloc / (iters * BATCH_N);
+ *cycles_free = (double)tsc_free / (iters * BATCH_N);
+ return;
+err:
+ alloc->teardown();
+}
+
+/*
+ * Scenario 4: multi-lcore alloc/work/free with a dummy-work
+ * baseline. Each worker runs a tight alloc → touch → free loop
+ * on its own lcore. A second run with the same dummy work but
+ * no allocator traffic establishes a baseline; the per-op
+ * allocator cost is reported as (alloc_run - baseline_run).
+ *
+ * Fixed size class and a fixed amount of dummy work per op —
+ * this scenario sweeps lcore count rather than size.
+ */
+#define MULTI_SIZE 256u
+#define MULTI_WORK_BYTES 64u
+#define MULTI_WORK_PASSES 8u /* RMW passes over the work region. */
+#define MULTI_OPS 200000u
+#define MULTI_WARMUP 2000u
+#define MAX_MULTI_LCORES 32u
+
+/*
+ * Per-worker volatile sink. Each worker writes to its own
+ * slot, preventing dead-code elimination of touch_buffer() and
+ * avoiding cross-lcore cache-line sharing on the hot path.
+ * Padded to cache-line stride to prevent false sharing between
+ * neighboring workers' slots.
+ */
+struct worker_sink {
+ volatile uint64_t value;
+} __rte_cache_aligned;
+
+static struct worker_sink worker_sinks[RTE_MAX_LCORE];
+
+/*
+ * Out-of-line dummy workload: run MULTI_WORK_PASSES
+ * read-modify-write passes over the first 'bytes' of the
+ * buffer. Each pass reads what the previous pass wrote, so the
+ * compiler cannot unroll or parallelize across passes — the
+ * work scales linearly with MULTI_WORK_PASSES. Returns an
+ * accumulator so the caller can feed it into a volatile sink;
+ * without that, the compiler could elide the whole function.
+ *
+ * __rte_noinline so it looks identical to the compiler in both
+ * the baseline (pre-allocated scratch buffer) and alloc-path
+ * runs, making the cycle-delta subtraction valid.
+ *
+ * The purpose of this being tunably expensive is to keep
+ * worker-per-iteration cost high relative to the allocator's
+ * critical section, so that even serialized allocators like
+ * rte_malloc spend most of their time outside the lock and the
+ * measured per-op allocator cost reflects its own work rather
+ * than its contention queue.
+ */
+static uint64_t __rte_noinline
+touch_buffer(void *buf, size_t bytes)
+{
+ uint64_t *p = buf;
+ size_t n = bytes / sizeof(uint64_t);
+ uint64_t acc = 0;
+ unsigned int pass;
+ size_t i;
+
+ /* Prime the buffer with a known pattern. */
+ for (i = 0; i < n; i++)
+ p[i] = i * 0x9E3779B97F4A7C15ULL;
+
+ /*
+ * Dependent RMW passes: each pass reads p[i] written by
+ * the previous pass, mixes the pass index in, and writes
+ * back. The XOR into acc keeps the chain live.
+ */
+ for (pass = 0; pass < MULTI_WORK_PASSES; pass++) {
+ for (i = 0; i < n; i++) {
+ uint64_t v = p[i];
+
+ v = v * 0xC2B2AE3D27D4EB4FULL + pass;
+ v ^= v >> 33;
+ p[i] = v;
+ acc ^= v;
+ }
+ }
+
+ return acc;
+}
+
+struct worker_args {
+ const struct allocator *alloc;
+ void *scratch; /* baseline only; NULL => alloc path */
+ unsigned int iters;
+ unsigned int warmup;
+ unsigned int bulk_n; /* 0 = single-object, >0 = bulk */
+ RTE_ATOMIC(bool) start_flag; /* barrier at worker entry */
+ uint64_t cycles; /* out */
+ unsigned int ops; /* out */
+ int err; /* out */
+};
+
+static int
+worker_run(void *arg)
+{
+ struct worker_args *wa = arg;
+ unsigned int lcore = rte_lcore_id();
+ uint64_t acc = 0;
+ uint64_t t0;
+ unsigned int i;
+
+ wa->err = 0;
+ wa->ops = 0;
+ wa->cycles = 0;
+
+ /* Wait for start flag (spin-barrier set by main). */
+ while (!rte_atomic_load_explicit(&wa->start_flag,
+ rte_memory_order_acquire))
+ rte_pause();
+
+ /* Warmup. */
+ for (i = 0; i < wa->warmup; i++) {
+ void *p;
+
+ if (wa->scratch != NULL)
+ p = wa->scratch;
+ else {
+ p = wa->alloc->alloc();
+ if (p == NULL) {
+ wa->err = -1;
+ return -1;
+ }
+ }
+ acc ^= touch_buffer(p, MULTI_WORK_BYTES);
+ if (wa->scratch == NULL)
+ wa->alloc->free_obj(p);
+ }
+
+ /* Measured loop. */
+ t0 = rte_rdtsc_precise();
+ for (i = 0; i < wa->iters; i++) {
+ void *p;
+
+ if (wa->scratch != NULL)
+ p = wa->scratch;
+ else {
+ p = wa->alloc->alloc();
+ if (p == NULL) {
+ wa->err = -1;
+ break;
+ }
+ }
+ acc ^= touch_buffer(p, MULTI_WORK_BYTES);
+ if (wa->scratch == NULL)
+ wa->alloc->free_obj(p);
+ }
+ wa->cycles = rte_rdtsc_precise() - t0;
+ wa->ops = i;
+
+ /* Publish accumulator to defeat dead-code elimination. */
+ worker_sinks[lcore].value ^= acc;
+
+ return 0;
+}
+
+static int
+worker_run_bulk(void *arg)
+{
+ struct worker_args *wa = arg;
+ unsigned int lcore = rte_lcore_id();
+ void *ptrs[BATCH_N];
+ uint64_t acc = 0;
+ uint64_t t0;
+ unsigned int i, j;
+ unsigned int bulk_n = wa->bulk_n;
+
+ wa->err = 0;
+ wa->ops = 0;
+ wa->cycles = 0;
+
+ while (!rte_atomic_load_explicit(&wa->start_flag,
+ rte_memory_order_acquire))
+ rte_pause();
+
+ /* Warmup. */
+ for (i = 0; i < wa->warmup; i++) {
+ if (wa->alloc->alloc_bulk(ptrs, bulk_n) < 0) {
+ wa->err = -1;
+ return -1;
+ }
+ for (j = 0; j < bulk_n; j++)
+ acc ^= touch_buffer(ptrs[j], MULTI_WORK_BYTES);
+ wa->alloc->free_bulk(ptrs, bulk_n);
+ }
+
+ t0 = rte_rdtsc_precise();
+ for (i = 0; i < wa->iters; i++) {
+ if (wa->alloc->alloc_bulk(ptrs, bulk_n) < 0) {
+ wa->err = -1;
+ break;
+ }
+ for (j = 0; j < bulk_n; j++)
+ acc ^= touch_buffer(ptrs[j], MULTI_WORK_BYTES);
+ wa->alloc->free_bulk(ptrs, bulk_n);
+ }
+ wa->cycles = rte_rdtsc_precise() - t0;
+ wa->ops = i * bulk_n;
+
+ worker_sinks[lcore].value ^= acc;
+
+ return 0;
+}
+
+/*
+ * Launch workers on the first 'n_workers' worker lcores, run
+ * either the baseline (scratch != NULL) or the alloc path
+ * (scratch == NULL), and return the mean per-op cycle cost
+ * averaged across participating workers.
+ *
+ * On any worker error, returns -1.0.
+ */
+static double
+run_multi_workers(const struct allocator *alloc, unsigned int n_workers,
+ void *const *scratches, unsigned int bulk_n)
+{
+ struct worker_args wargs[RTE_MAX_LCORE];
+ unsigned int worker_lcores[MAX_MULTI_LCORES];
+ unsigned int n = 0;
+ unsigned int lcore_id;
+ unsigned int i;
+ lcore_function_t *fn = bulk_n > 0 ? worker_run_bulk : worker_run;
+
+ /* Collect the first n_workers worker lcores. */
+ RTE_LCORE_FOREACH_WORKER(lcore_id) {
+ if (n >= n_workers)
+ break;
+ worker_lcores[n++] = lcore_id;
+ }
+ if (n < n_workers)
+ return -1.0;
+
+ /* Prepare per-worker args. */
+ for (i = 0; i < n_workers; i++) {
+ struct worker_args *wa = &wargs[worker_lcores[i]];
+
+ wa->alloc = alloc;
+ wa->scratch = scratches != NULL ? scratches[i] : NULL;
+ wa->iters = MULTI_OPS;
+ wa->warmup = MULTI_WARMUP;
+ wa->bulk_n = bulk_n;
+ rte_atomic_store_explicit(&wa->start_flag, false,
+ rte_memory_order_relaxed);
+ }
+
+ /* Launch workers. They spin on start_flag until released. */
+ for (i = 0; i < n_workers; i++)
+ rte_eal_remote_launch(fn, &wargs[worker_lcores[i]],
+ worker_lcores[i]);
+
+ /* Release all workers roughly simultaneously. */
+ for (i = 0; i < n_workers; i++)
+ rte_atomic_store_explicit(
+ &wargs[worker_lcores[i]].start_flag, true,
+ rte_memory_order_release);
+
+ /* Wait for completion. */
+ for (i = 0; i < n_workers; i++)
+ rte_eal_wait_lcore(worker_lcores[i]);
+
+ /* Aggregate: mean cycles per op across workers. */
+ {
+ double sum_cycles_per_op = 0.0;
+ unsigned int n_ok = 0;
+
+ for (i = 0; i < n_workers; i++) {
+ struct worker_args *wa = &wargs[worker_lcores[i]];
+
+ if (wa->err != 0 || wa->ops == 0)
+ return -1.0;
+ sum_cycles_per_op +=
+ (double)wa->cycles / (double)wa->ops;
+ n_ok++;
+ }
+ return sum_cycles_per_op / n_ok;
+ }
+}
+
+/*
+ * One sub-run of Scenario 4: given an allocator and a worker
+ * count, return (baseline, alloc_path) mean cycles per op.
+ */
+static void
+run_multi_lcore(const struct allocator *alloc, unsigned int n_workers,
+ unsigned int bulk_n, double *baseline, double *alloc_path)
+{
+ void *scratches[MAX_MULTI_LCORES] = {0};
+ unsigned int n_alloced = 0;
+ unsigned int i;
+
+ *baseline = -1.0;
+ *alloc_path = -1.0;
+
+ if (alloc->setup(MULTI_SIZE, n_workers * 64) < 0)
+ return;
+
+ /* Baseline: pre-allocate one scratch per worker. */
+ for (i = 0; i < n_workers; i++) {
+ scratches[i] = alloc->alloc();
+ if (scratches[i] == NULL)
+ goto err;
+ n_alloced++;
+ }
+
+ *baseline = run_multi_workers(alloc, n_workers, scratches, 0);
+
+ for (i = 0; i < n_alloced; i++)
+ alloc->free_obj(scratches[i]);
+ n_alloced = 0;
+
+ /* Alloc path: workers alloc+free each iter. */
+ *alloc_path = run_multi_workers(alloc, n_workers, NULL, bulk_n);
+
+ alloc->teardown();
+ return;
+err:
+ for (i = 0; i < n_alloced; i++)
+ alloc->free_obj(scratches[i]);
+ alloc->teardown();
+}
+
+/* Reporting -------------------------------------------------------- */
+
+static void
+print_header(const char *title)
+{
+ size_t i;
+
+ TEST_LOG("\n=== %s ===\n", title);
+ TEST_LOG("%-12s", "allocator");
+ for (i = 0; i < N_SIZES; i++)
+ TEST_LOG(" %10zu B", SIZES[i]);
+ TEST_LOG("\n");
+}
+
+static void
+print_row(const char *name, const double *values)
+{
+ size_t i;
+
+ TEST_LOG("%-12s", name);
+ for (i = 0; i < N_SIZES; i++) {
+ if (values[i] < 0)
+ TEST_LOG(" %12s", "--");
+ else
+ TEST_LOG(" %12.1f", values[i]);
+ }
+ TEST_LOG("\n");
+}
+
+static void
+print_multi_header(const char *title, const unsigned int *lcore_counts,
+ unsigned int n_counts)
+{
+ unsigned int i;
+
+ TEST_LOG("\n=== %s ===\n", title);
+ TEST_LOG("%-12s", "allocator");
+ for (i = 0; i < n_counts; i++)
+ TEST_LOG(" %8u lcore%c", lcore_counts[i],
+ lcore_counts[i] == 1 ? ' ' : 's');
+ TEST_LOG("\n");
+}
+
+static void
+print_multi_row(const char *name, const double *values, unsigned int n_counts)
+{
+ unsigned int i;
+
+ TEST_LOG("%-12s", name);
+ for (i = 0; i < n_counts; i++) {
+ if (values[i] < 0)
+ TEST_LOG(" %14s", "--");
+ else
+ TEST_LOG(" %14.1f", values[i]);
+ }
+ TEST_LOG("\n");
+}
+
+/* Driver ----------------------------------------------------------- */
+
+static int
+test_fastmem_perf(void)
+{
+ size_t i;
+ size_t a;
+ int rc;
+
+ rc = rte_fastmem_init();
+ if (rc < 0) {
+ TEST_LOG("rte_fastmem_init() failed: %d\n", rc);
+ return -1;
+ }
+
+ rc = rte_fastmem_reserve(128 * 1024 * 1024, SOCKET_ID_ANY);
+ if (rc < 0) {
+ TEST_LOG("rte_fastmem_reserve() failed: %d\n", rc);
+ rte_fastmem_deinit();
+ return -1;
+ }
+
+ TEST_LOG("\nfastmem performance — single-lcore, fixed-size\n");
+ TEST_LOG("All numbers are TSC cycles.\n");
+
+ /* Scenario 1: tight alloc+free. */
+ print_header("Scenario 1: Single-object hot path — cycles per (alloc + free)");
+ for (a = 0; a < N_ALLOCATORS; a++) {
+ double vals[N_SIZES];
+
+ for (i = 0; i < N_SIZES; i++)
+ vals[i] = run_tight(&allocators[a], SIZES[i]);
+ print_row(allocators[a].name, vals);
+ }
+
+ /* Scenario 2: batched, FIFO free. */
+ print_header("Scenario 2: Batch alloc, FIFO free — cycles per alloc");
+ for (a = 0; a < N_ALLOCATORS; a++) {
+ double vals_alloc[N_SIZES], vals_free[N_SIZES];
+
+ for (i = 0; i < N_SIZES; i++)
+ run_batch(&allocators[a], SIZES[i],
+ &vals_alloc[i], &vals_free[i]);
+ print_row(allocators[a].name, vals_alloc);
+ }
+ print_header("Scenario 2: Batch alloc, FIFO free — cycles per free");
+ for (a = 0; a < N_ALLOCATORS; a++) {
+ double vals_alloc[N_SIZES], vals_free[N_SIZES];
+
+ for (i = 0; i < N_SIZES; i++)
+ run_batch(&allocators[a], SIZES[i],
+ &vals_alloc[i], &vals_free[i]);
+ print_row(allocators[a].name, vals_free);
+ }
+
+ /* Scenario 3: batched, reverse free. */
+ print_header("Scenario 3: Batch alloc, LIFO free — cycles per alloc");
+ for (a = 0; a < N_ALLOCATORS; a++) {
+ double vals_alloc[N_SIZES], vals_free[N_SIZES];
+
+ for (i = 0; i < N_SIZES; i++)
+ run_batch_reverse(&allocators[a], SIZES[i],
+ &vals_alloc[i], &vals_free[i]);
+ print_row(allocators[a].name, vals_alloc);
+ }
+ print_header("Scenario 3: Batch alloc, LIFO free — cycles per free");
+ for (a = 0; a < N_ALLOCATORS; a++) {
+ double vals_alloc[N_SIZES], vals_free[N_SIZES];
+
+ for (i = 0; i < N_SIZES; i++)
+ run_batch_reverse(&allocators[a], SIZES[i],
+ &vals_alloc[i], &vals_free[i]);
+ print_row(allocators[a].name, vals_free);
+ }
+
+ /* Scenario 4: multi-lcore alloc/work/free with baseline. */
+ {
+ unsigned int max_workers = rte_lcore_count() - 1;
+ unsigned int lcore_counts[8];
+ unsigned int n_counts = 0;
+ unsigned int w;
+ double base_vals[N_ALLOCATORS][8];
+ double alloc_vals[N_ALLOCATORS][8];
+ double delta_vals[N_ALLOCATORS][8];
+
+ if (max_workers > MAX_MULTI_LCORES)
+ max_workers = MAX_MULTI_LCORES;
+
+ /* Sweep lcore counts: 1, 2, 4, 8, ... up to max_workers. */
+ for (w = 1; w <= max_workers && n_counts < RTE_DIM(lcore_counts); w *= 2)
+ lcore_counts[n_counts++] = w;
+ /* Ensure max_workers is the final column if not power of two. */
+ if (n_counts > 0 && lcore_counts[n_counts - 1] != max_workers &&
+ n_counts < RTE_DIM(lcore_counts) && max_workers >= 1)
+ lcore_counts[n_counts++] = max_workers;
+
+ if (n_counts == 0) {
+ TEST_LOG("\nScenario 4 (Multi-lcore contention) skipped: no worker lcores available.\n");
+ } else {
+ TEST_LOG("\nScenario 4 parameters: size=%u B\n",
+ MULTI_SIZE);
+
+ for (a = 0; a < N_ALLOCATORS; a++) {
+ unsigned int c;
+
+ for (c = 0; c < n_counts; c++)
+ run_multi_lcore(&allocators[a], lcore_counts[c],
+ 0, &base_vals[a][c],
+ &alloc_vals[a][c]);
+ for (c = 0; c < n_counts; c++) {
+ if (base_vals[a][c] < 0 || alloc_vals[a][c] < 0)
+ delta_vals[a][c] = -1.0;
+ else
+ delta_vals[a][c] = alloc_vals[a][c] -
+ base_vals[a][c];
+ }
+ }
+
+ TEST_LOG("Baseline (domain logic only): %.1f cycles/op\n",
+ base_vals[0][0]);
+
+ print_multi_header("Scenario 4: Multi-lcore contention — allocator overhead (cycles/op)",
+ lcore_counts, n_counts);
+ for (a = 0; a < N_ALLOCATORS; a++)
+ print_multi_row(allocators[a].name,
+ delta_vals[a], n_counts);
+ }
+ }
+
+ /* Scenario 5: multi-lcore bulk alloc/work/free. */
+ {
+ unsigned int max_workers = rte_lcore_count() - 1;
+ unsigned int lcore_counts[8];
+ unsigned int n_counts = 0;
+ unsigned int w;
+ double base_vals[N_ALLOCATORS][8];
+ double alloc_vals[N_ALLOCATORS][8];
+ double delta_vals[N_ALLOCATORS][8];
+ unsigned int bulk_n = 8;
+
+ if (max_workers > MAX_MULTI_LCORES)
+ max_workers = MAX_MULTI_LCORES;
+
+ for (w = 1; w <= max_workers && n_counts < RTE_DIM(lcore_counts); w *= 2)
+ lcore_counts[n_counts++] = w;
+ if (n_counts > 0 && lcore_counts[n_counts - 1] != max_workers &&
+ n_counts < RTE_DIM(lcore_counts) && max_workers >= 1)
+ lcore_counts[n_counts++] = max_workers;
+
+ if (n_counts == 0) {
+ TEST_LOG("\nScenario 5 (Multi-lcore bulk contention) skipped: no worker lcores available.\n");
+ } else {
+ TEST_LOG("\nScenario 5 parameters: size=%u B, "
+ "bulk=%u\n",
+ MULTI_SIZE, bulk_n);
+
+ for (size_t a = 0; a < N_ALLOCATORS; a++) {
+ unsigned int c;
+
+ for (c = 0; c < n_counts; c++)
+ run_multi_lcore(&allocators[a],
+ lcore_counts[c], bulk_n,
+ &base_vals[a][c],
+ &alloc_vals[a][c]);
+ for (c = 0; c < n_counts; c++) {
+ if (base_vals[a][c] < 0 || alloc_vals[a][c] < 0)
+ delta_vals[a][c] = -1.0;
+ else
+ delta_vals[a][c] = alloc_vals[a][c] -
+ base_vals[a][c];
+ }
+ }
+
+ TEST_LOG("Baseline (domain logic only): %.1f cycles/op\n",
+ base_vals[0][0]);
+
+ print_multi_header("Scenario 5: Multi-lcore bulk contention — allocator overhead (cycles/op)",
+ lcore_counts, n_counts);
+ for (size_t a = 0; a < N_ALLOCATORS; a++)
+ print_multi_row(allocators[a].name,
+ delta_vals[a], n_counts);
+ }
+ }
+
+ TEST_LOG("\n");
+ rte_fastmem_deinit();
+ return 0;
+}
+
+REGISTER_PERF_TEST(fastmem_perf_autotest, test_fastmem_perf);
diff --git a/app/test/test_fastmem_profile.c b/app/test/test_fastmem_profile.c
new file mode 100644
index 0000000000..9a5dc94018
--- /dev/null
+++ b/app/test/test_fastmem_profile.c
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 Ericsson AB
+ */
+
+/*
+ * A minimal fastmem workload intended for use with perf record /
+ * perf report. Runs a tight alloc/free loop for a fixed duration
+ * so that sampling profilers can attribute cycles to individual
+ * functions and instructions within the fastmem hot path.
+ *
+ * Usage:
+ * perf record -g -- dpdk-test --no-huge --no-pci -m 8192 \
+ * -l 0 <<< fastmem_profile_autotest
+ * perf report
+ */
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_lcore.h>
+#include <rte_memory.h>
+
+#include <rte_fastmem.h>
+
+#include "test.h"
+
+/* Duration of each sub-test in TSC cycles (~3 seconds at 3 GHz). */
+#define PROFILE_DURATION_CYCLES (3ULL * rte_get_tsc_hz())
+
+/* Allocation size for the profiling workload. */
+#define PROFILE_SIZE 256u
+
+/*
+ * Sub-test 1: tight alloc+free, exercises only the per-lcore
+ * cache (no bin interaction after warmup).
+ */
+static int
+profile_cache_hit(void)
+{
+ uint64_t deadline;
+ uint64_t ops = 0;
+
+ deadline = rte_rdtsc() + PROFILE_DURATION_CYCLES;
+
+ while (rte_rdtsc() < deadline) {
+ void *p = rte_fastmem_alloc(PROFILE_SIZE, 0, 0);
+
+ if (p == NULL)
+ return -1;
+ rte_fastmem_free(p);
+ ops++;
+ }
+
+ printf(" cache_hit: %" PRIu64 " ops\n", ops);
+ return 0;
+}
+
+/*
+ * Sub-test 2: alloc N then free N, where N exceeds the cache
+ * capacity. This forces repeated cache refills and drains,
+ * exercising the bin lock and slab free-list traversal.
+ */
+#define PROFILE_BATCH 256u
+
+static int
+profile_cache_miss(void)
+{
+ void *ptrs[PROFILE_BATCH];
+ uint64_t deadline;
+ uint64_t ops = 0;
+ unsigned int i;
+
+ deadline = rte_rdtsc() + PROFILE_DURATION_CYCLES;
+
+ while (rte_rdtsc() < deadline) {
+ for (i = 0; i < PROFILE_BATCH; i++) {
+ ptrs[i] = rte_fastmem_alloc(PROFILE_SIZE, 0, 0);
+ if (ptrs[i] == NULL)
+ return -1;
+ }
+ for (i = 0; i < PROFILE_BATCH; i++)
+ rte_fastmem_free(ptrs[i]);
+ ops += PROFILE_BATCH;
+ }
+
+ printf(" cache_miss: %" PRIu64 " ops\n", ops);
+ return 0;
+}
+
+static int
+test_fastmem_profile_cache_hit(void)
+{
+ int rc;
+
+ rc = rte_fastmem_init();
+ if (rc < 0) {
+ printf("rte_fastmem_init() failed: %d\n", rc);
+ return -1;
+ }
+
+ rc = rte_fastmem_reserve(128 * 1024 * 1024, SOCKET_ID_ANY);
+ if (rc < 0) {
+ printf("rte_fastmem_reserve() failed: %d\n", rc);
+ rte_fastmem_deinit();
+ return -1;
+ }
+
+ printf("fastmem profile: cache-hit workload (size=%u, ~%u s)\n",
+ PROFILE_SIZE, 3);
+
+ if (profile_cache_hit() < 0) {
+ rte_fastmem_deinit();
+ return -1;
+ }
+
+ rte_fastmem_deinit();
+ return 0;
+}
+
+static int
+test_fastmem_profile_cache_miss(void)
+{
+ int rc;
+
+ rc = rte_fastmem_init();
+ if (rc < 0) {
+ printf("rte_fastmem_init() failed: %d\n", rc);
+ return -1;
+ }
+
+ rc = rte_fastmem_reserve(128 * 1024 * 1024, SOCKET_ID_ANY);
+ if (rc < 0) {
+ printf("rte_fastmem_reserve() failed: %d\n", rc);
+ rte_fastmem_deinit();
+ return -1;
+ }
+
+ printf("fastmem profile: cache-miss workload (size=%u, ~%u s)\n",
+ PROFILE_SIZE, 3);
+
+ if (profile_cache_miss() < 0) {
+ rte_fastmem_deinit();
+ return -1;
+ }
+
+ rte_fastmem_deinit();
+ return 0;
+}
+
+REGISTER_PERF_TEST(fastmem_profile_cache_hit_autotest,
+ test_fastmem_profile_cache_hit);
+REGISTER_PERF_TEST(fastmem_profile_cache_miss_autotest,
+ test_fastmem_profile_cache_miss);
--
2.43.0
^ permalink raw reply related
* [RFC v4 2/3] lib: add fastmem library
From: Mattias Rönnblom @ 2026-05-30 9:26 UTC (permalink / raw)
To: dev
Cc: Morten Brørup, Konstantin Ananyev, Mattias Rönnblom,
Yogaraj Baskaravel, Stephen Hemminger, Bruce Richardson,
Mattias Rönnblom
In-Reply-To: <20260530092634.46218-1-hofors@lysator.liu.se>
Introduce fastmem, a fast general-purpose small-object allocator
for DPDK applications. It allows an application to replace its
many per-type mempools with a single allocator that handles
arbitrary sizes, grows on demand, and offers mempool-level
performance on the hot path.
Applications that manage many object types (connections, sessions,
work items, timers) currently maintain a separate mempool for each,
requiring upfront sizing and wasting memory on over-provisioned
pools. Fastmem removes both constraints.
Key properties:
* Huge-page-backed, NUMA-aware, DMA-usable.
* Per-lcore caches for lock-free alloc/free on EAL threads.
* Bulk alloc and free APIs.
* Power-of-two size classes from 8 B to 1 MiB.
* Backing memory grows lazily; rte_fastmem_reserve() allows
upfront reservation to avoid latency spikes.
* Always-on per-lcore and per-class statistics.
Bounded to small objects; requests above rte_fastmem_max_size()
are rejected. Replacing rte_malloc is currently not a goal.
--
RFC v4:
* Fix crash in halloc/hfree on lcores without hlookup: fall
back to shared bin on NULL cache.
* Keep per-lcore statistics across rte_fastmem_cache_flush():
retain the cache struct so counters survive.
* Guard free and IOVA paths against uninitialized state.
* Lazy-attach stats readers in secondary processes; distinguish
-ENODEV from -EINVAL.
* Add likely() hint to cache-present branch in
account_alloc_nomem().
* Protect bin statistics with the bin lock.
* Trim verbose comments.
* Add shared cache for callers without a private cache (non-EAL
threads, secondary processes). Add rte_fastmem_stats_shared()
and rte_fastmem_stats_shared_class().
* Document rte_fastmem_stats_reset() quiescence requirement.
RFC v3:
* Add rte_fastmem_realloc().
* Add __rte_malloc/__rte_dealloc attributes; remove incorrect
__rte_alloc_size/__rte_alloc_align.
* Extract normalize_align() helper.
* Remove inline directives from static functions.
RFC v2:
* Fix use-after-free in rte_fastmem_deinit() with cross-socket
caches: restructure into three-phase teardown.
* Add secondary process support (lazy attach, safe deinit).
* Add handle-based allocation API (rte_fastmem_hlookup,
rte_fastmem_halloc, rte_fastmem_halloc_bulk).
* Fix clang -Wthread-safety-analysis warnings.
Signed-off-by: Mattias Rönnblom <hofors@lysator.liu.se>
---
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf.in | 1 +
lib/fastmem/meson.build | 6 +
lib/fastmem/rfc-cover-letter.txt | 128 ++
lib/fastmem/rte_fastmem.c | 2123 ++++++++++++++++++++++++++++++
lib/fastmem/rte_fastmem.h | 908 +++++++++++++
lib/meson.build | 1 +
7 files changed, 3168 insertions(+)
create mode 100644 lib/fastmem/meson.build
create mode 100644 lib/fastmem/rfc-cover-letter.txt
create mode 100644 lib/fastmem/rte_fastmem.c
create mode 100644 lib/fastmem/rte_fastmem.h
diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 9296042119..7ebf1201ce 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -70,6 +70,7 @@ The public API headers are grouped by topics:
[memzone](@ref rte_memzone.h),
[mempool](@ref rte_mempool.h),
[malloc](@ref rte_malloc.h),
+ [fastmem](@ref rte_fastmem.h),
[memcpy](@ref rte_memcpy.h)
- **timers**:
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index bedd944681..4355e9fb2d 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -43,6 +43,7 @@ INPUT = @TOPDIR@/doc/api/doxy-api-index.md \
@TOPDIR@/lib/efd \
@TOPDIR@/lib/ethdev \
@TOPDIR@/lib/eventdev \
+ @TOPDIR@/lib/fastmem \
@TOPDIR@/lib/fib \
@TOPDIR@/lib/gpudev \
@TOPDIR@/lib/graph \
diff --git a/lib/fastmem/meson.build b/lib/fastmem/meson.build
new file mode 100644
index 0000000000..6c7834608f
--- /dev/null
+++ b/lib/fastmem/meson.build
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 Ericsson AB
+
+sources = files('rte_fastmem.c')
+headers = files('rte_fastmem.h')
+deps += ['eal']
diff --git a/lib/fastmem/rfc-cover-letter.txt b/lib/fastmem/rfc-cover-letter.txt
new file mode 100644
index 0000000000..53752c7e8b
--- /dev/null
+++ b/lib/fastmem/rfc-cover-letter.txt
@@ -0,0 +1,128 @@
+Subject: [RFC] lib/fastmem: fast small-object allocator
+
+This RFC introduces fastmem, a general-purpose small-object allocator
+for DPDK. It is intended to replace per-type mempools with a single
+allocator that handles arbitrary sizes, grows on demand, and matches
+mempool-level performance on the hot path.
+
+Motivation
+----------
+
+DPDK applications commonly maintain many mempools — one per object
+type (connections, sessions, timers, work items). Each must be sized
+up front, wastes memory when over-provisioned, and cannot serve
+objects of a different size. Fastmem eliminates this by accepting
+arbitrary sizes at runtime, backed by a slab allocator that
+repurposes memory across size classes as demand shifts.
+
+Design
+------
+
+Three-layer architecture:
+
+1. Backing memory: 128 MiB IOVA-contiguous memzones from EAL,
+ reserved lazily (or pre-reserved for deterministic latency).
+
+2. Slabs: 2 MiB, 2 MiB-aligned regions carved from memzones.
+ The alignment enables O(1) slab lookup from any object pointer
+ via bitmask — no radix tree or index structure. Slabs move
+ freely between 18 power-of-2 size classes (8 B to 1 MiB).
+
+3. Per-lcore caches: bounded LIFO stacks (no locks on the hot
+ path). Cache misses trigger bulk transfers to/from the shared
+ bin under a spinlock.
+
+Key properties:
+
+- Zero per-object metadata in the production build.
+- NUMA-aware, with per-socket bins and free-slab pools.
+- DMA-usable memory with O(1) virt-to-IOVA translation.
+- Bulk alloc/free with all-or-nothing semantics.
+- Backing memory never returned during lifetime (slabs recycled).
+- Non-EAL threads supported (bypass cache, take bin lock).
+- Secondary process support (lazy attach, no per-lcore caches).
+
+API surface
+-----------
+
+ rte_fastmem_init / deinit
+ rte_fastmem_reserve
+ rte_fastmem_set_limit / get_limit
+ rte_fastmem_alloc / alloc_socket
+ rte_fastmem_realloc
+ rte_fastmem_alloc_bulk / alloc_bulk_socket
+ rte_fastmem_free / free_bulk
+ rte_fastmem_hlookup / halloc / halloc_bulk / hfree / hfree_bulk
+ rte_fastmem_virt2iova
+ rte_fastmem_cache_flush
+ rte_fastmem_max_size / classes
+ rte_fastmem_stats / stats_class / stats_lcore / stats_lcore_class
+ rte_fastmem_stats_reset
+
+All APIs are marked __rte_experimental.
+
+Performance
+-----------
+
+The single-object hot path is roughly 2–3× the cost of mempool
+and an order of magnitude faster than rte_malloc. Under
+multi-lcore contention, fastmem scales similarly to mempool,
+while rte_malloc collapses.
+
+Limitations
+-----------
+
+- Maximum allocation: 1 MiB. Larger requests should use rte_malloc.
+- Power-of-2 classes only; worst-case internal fragmentation ~50%.
+- Backing memory not reclaimable short of deinit.
+
+Future work
+-----------
+
+- Lcore-affine allocations (false-sharing-free by construction).
+- Mempool ops driver for transparent drop-in use.
+- Debug mode (cookies, double-free detection, poison-on-free).
+- Telemetry integration.
+- EAL integration, allowing EAL-internal subsystems to use
+ fastmem for their small-object allocations.
+
+Changes in RFC v4:
+- Fix crash in halloc/hfree on lcores without hlookup: fall back
+ to shared bin on NULL cache.
+- Keep per-lcore statistics across rte_fastmem_cache_flush().
+- Guard free and IOVA paths against uninitialized state.
+- Lazy-attach stats readers in secondary processes; distinguish
+ -ENODEV from -EINVAL.
+- Protect bin statistics with the bin lock.
+- Trim verbose comments.
+- Add shared cache for callers without a private cache (non-EAL
+ threads, secondary processes). Add rte_fastmem_stats_shared()
+ and rte_fastmem_stats_shared_class().
+- Document rte_fastmem_stats_reset() quiescence requirement.
+- Add tests for handle alloc/free from uncached lcores, stats
+ survival across flush, and shared-cache statistics.
+- Update programming guide (shared cache, stats sections).
+
+Changes in RFC v3:
+- Add rte_fastmem_realloc() with full test coverage.
+- Add __rte_malloc/__rte_dealloc compiler attributes; remove
+ incorrect __rte_alloc_size/__rte_alloc_align.
+- Extract normalize_align() helper; remove redundant inline
+ directives.
+- Merge lifecycle and functional test suites.
+- Add realloc subsection to programming guide.
+
+Changes in RFC v2:
+- Fix cross-socket deinit use-after-free.
+- Add secondary process support.
+- Add handle-based allocation API.
+- Fix clang warnings; misc cleanup.
+
+Mattias Rönnblom (3):
+ doc: add fastmem programming guide
+ lib: add fastmem library
+ app/test: add fastmem test suite
+
+ doc/guides/prog_guide/fastmem_lib.rst | ...
+ lib/fastmem/ | ...
+ app/test/test_fastmem*.c | ...
diff --git a/lib/fastmem/rte_fastmem.c b/lib/fastmem/rte_fastmem.c
new file mode 100644
index 0000000000..4add00ce80
--- /dev/null
+++ b/lib/fastmem/rte_fastmem.c
@@ -0,0 +1,2123 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 Ericsson AB
+ */
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/queue.h>
+
+#include <eal_export.h>
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_errno.h>
+#include <rte_lcore.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_spinlock.h>
+
+#include <rte_fastmem.h>
+
+RTE_LOG_REGISTER_DEFAULT(fastmem_logtype, NOTICE);
+
+#define RTE_LOGTYPE_FASTMEM fastmem_logtype
+
+#define FASTMEM_LOG(level, ...) \
+ RTE_LOG_LINE(level, FASTMEM, "" __VA_ARGS__)
+
+#define FASTMEM_MEMZONE_SIZE_LOG2 27 /* 128 MiB */
+#define FASTMEM_MEMZONE_SIZE ((size_t)1 << FASTMEM_MEMZONE_SIZE_LOG2)
+
+#define FASTMEM_SLAB_SIZE_LOG2 21 /* 2 MiB */
+#define FASTMEM_SLAB_SIZE ((size_t)1 << FASTMEM_SLAB_SIZE_LOG2)
+#define FASTMEM_SLAB_MASK (FASTMEM_SLAB_SIZE - 1)
+
+#define FASTMEM_SLABS_PER_MEMZONE (FASTMEM_MEMZONE_SIZE / FASTMEM_SLAB_SIZE)
+
+#define FASTMEM_MAX_MEMZONES_PER_SOCKET 64
+
+#define FASTMEM_MIN_CLASS_LOG2 3 /* 8 B */
+#define FASTMEM_MAX_CLASS_LOG2 20 /* 1 MiB */
+#define FASTMEM_N_CLASSES (FASTMEM_MAX_CLASS_LOG2 - FASTMEM_MIN_CLASS_LOG2 + 1)
+
+#define FASTMEM_MIN_SIZE ((size_t)1 << FASTMEM_MIN_CLASS_LOG2)
+#define FASTMEM_MAX_ALLOC_SIZE ((size_t)1 << FASTMEM_MAX_CLASS_LOG2)
+
+#define FASTMEM_SLAB_HEADER_SIZE RTE_CACHE_LINE_SIZE
+
+#define FASTMEM_CACHE_BASE_CAPACITY 64
+#define FASTMEM_CACHE_FLOOR_CAPACITY 4
+#define FASTMEM_CACHE_BASE_CLASS_LOG2 12 /* 4 KiB */
+
+struct fastmem_bin;
+
+/*
+ * Slab header at offset 0 of each 2 MiB slab. Either free (linked
+ * via next_free) or assigned to a bin (linked via list).
+ */
+struct fastmem_slab {
+ struct fastmem_bin *bin;
+ void *free_head;
+ uint32_t free_count;
+ uint32_t n_slots;
+ struct fastmem_slab *next_free;
+ TAILQ_ENTRY(fastmem_slab) list;
+ rte_iova_t iova_base;
+} __rte_aligned(FASTMEM_SLAB_HEADER_SIZE);
+
+TAILQ_HEAD(fastmem_slab_list, fastmem_slab);
+
+struct fastmem_bin {
+ rte_spinlock_t lock;
+ uint32_t slot_size;
+ uint32_t slots_per_slab;
+ uint32_t class_idx;
+ struct fastmem_slab_list partial;
+ struct fastmem_slab_list full;
+ int socket_id;
+ uint64_t slab_acquires;
+ uint64_t slab_releases;
+ uint32_t slabs_partial;
+ uint32_t slabs_full;
+ /*
+ * Traffic served straight from the bin, with no cache of any kind
+ * backing it. Reached only on the fallback where a caller has no
+ * private per-lcore cache and the shared cache could not be created
+ * either (cache-struct allocation failed, e.g. under a memory limit
+ * or in an under-provisioned secondary). The normal cache-less path
+ * goes through the shared cache and is counted there, not here.
+ * Written under bin->lock, read locklessly by the stats functions.
+ * Not attributable to an lcore, so it appears only in the global and
+ * per-class statistics.
+ */
+ uint64_t nocache_allocs;
+ uint64_t nocache_frees;
+ uint64_t nocache_nomem;
+};
+
+/*
+ * Bounded LIFO of free object pointers, holding statistics counters
+ * alongside the hot-path fields so alloc and free stay on one cache line.
+ *
+ * Used in two ways: as a private per-(lcore, class, socket) cache for
+ * lcore-id-equipped primary threads (written only by its owning lcore, so
+ * lock-free), and as a per-(class, socket) cache shared by all other
+ * callers (serialized by the socket's shared_cache_lock).
+ *
+ * Never freed once created (rte_fastmem_cache_flush() drains the objects
+ * but keeps the struct), so the counters survive a flush and stats readers
+ * may touch it safely.
+ */
+struct fastmem_cache {
+ uint32_t count;
+ uint32_t capacity;
+ uint32_t target;
+ uint64_t alloc_cache_hits;
+ uint64_t alloc_cache_misses;
+ uint64_t alloc_nomem;
+ uint64_t free_cache_hits;
+ uint64_t free_cache_misses;
+ void *objs[];
+} __rte_cache_aligned;
+
+struct fastmem_socket_state {
+ rte_spinlock_t lock;
+ struct fastmem_slab *free_head;
+ size_t reserved_bytes;
+ size_t memory_limit;
+ unsigned int n_memzones;
+ unsigned int memzone_seq;
+ const struct rte_memzone *memzones[FASTMEM_MAX_MEMZONES_PER_SOCKET];
+ struct fastmem_bin bins[FASTMEM_N_CLASSES];
+ struct fastmem_cache *caches[RTE_MAX_LCORE][FASTMEM_N_CLASSES];
+ /*
+ * Cache shared by all callers lacking a private per-lcore cache
+ * (lcore-less primary threads and every secondary-process thread),
+ * guarded by one spinlock for the whole socket.
+ */
+ rte_spinlock_t shared_cache_lock;
+ struct fastmem_cache *shared_caches[FASTMEM_N_CLASSES];
+};
+
+struct fastmem {
+ struct fastmem_socket_state sockets[RTE_MAX_NUMA_NODES];
+};
+
+static struct fastmem *fastmem;
+static const struct rte_memzone *fastmem_mz;
+static bool fastmem_is_primary; /* cached; avoids function call on hot path */
+
+/*
+ * Ensure the global fastmem state is available to this process,
+ * lazily attaching a secondary to the shared memzone on first use.
+ * Returns false (rte_errno = ENODEV) if the primary has not
+ * initialized the library.
+ */
+static bool
+fastmem_assure(void)
+{
+ const struct rte_memzone *mz;
+
+ if (likely(fastmem != NULL))
+ return true;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ rte_errno = ENODEV;
+ return false;
+ }
+
+ mz = rte_memzone_lookup("fastmem_state");
+ if (mz == NULL) {
+ rte_errno = ENODEV;
+ return false;
+ }
+
+ fastmem_mz = mz;
+ fastmem = mz->addr;
+ return true;
+}
+
+static unsigned int
+size_to_class(size_t size, size_t align)
+{
+ size_t effective;
+ unsigned int log2;
+
+ effective = size < FASTMEM_MIN_SIZE ? FASTMEM_MIN_SIZE : size;
+ if (align > effective)
+ effective = align;
+
+ log2 = 64u - rte_clz64(effective - 1);
+
+ if (log2 < FASTMEM_MIN_CLASS_LOG2)
+ log2 = FASTMEM_MIN_CLASS_LOG2;
+ if (log2 > FASTMEM_MAX_CLASS_LOG2)
+ return FASTMEM_N_CLASSES;
+
+ return log2 - FASTMEM_MIN_CLASS_LOG2;
+}
+
+static size_t
+class_size(unsigned int class_idx)
+{
+ return (size_t)1 << (class_idx + FASTMEM_MIN_CLASS_LOG2);
+}
+
+/**
+ * Normalize and validate the alignment argument.
+ * Returns true on success (align updated in place), false on invalid input.
+ */
+static bool
+normalize_align(size_t *align)
+{
+ if (*align == 0) {
+ *align = RTE_CACHE_LINE_SIZE;
+ return true;
+ }
+ return rte_is_power_of_2(*align);
+}
+
+static_assert(sizeof(struct fastmem_slab) == FASTMEM_SLAB_HEADER_SIZE,
+ "fastmem slab header must fit in exactly one cache line");
+static_assert(sizeof(struct fastmem_slab) <= FASTMEM_SLAB_SIZE,
+ "slab header larger than a slab makes no sense");
+
+static struct fastmem_slab *
+slab_of(void *obj)
+{
+ return (struct fastmem_slab *)
+ ((uintptr_t)obj & ~(uintptr_t)FASTMEM_SLAB_MASK);
+}
+
+static size_t
+slab_slot0_offset(size_t class_size)
+{
+ return class_size < FASTMEM_SLAB_HEADER_SIZE ?
+ FASTMEM_SLAB_HEADER_SIZE : class_size;
+}
+
+static uint32_t
+slab_slot_count(size_t class_size)
+{
+ size_t offset = slab_slot0_offset(class_size);
+
+ return (uint32_t)((FASTMEM_SLAB_SIZE - offset) / class_size);
+}
+
+/* Must be called with bin->lock held. */
+static void
+slab_init(struct fastmem_bin *bin, struct fastmem_slab *slab)
+{
+ size_t slot_size = bin->slot_size;
+ size_t offset = slab_slot0_offset(slot_size);
+ uint32_t n = bin->slots_per_slab;
+ void *prev = NULL;
+ uint32_t i;
+
+ slab->bin = bin;
+ slab->n_slots = n;
+ slab->free_count = n;
+
+ /* Build in reverse so pops yield sequential addresses. */
+ for (i = 0; i < n; i++) {
+ void *slot = RTE_PTR_ADD(slab, offset + i * slot_size);
+ *(void **)slot = prev;
+ prev = slot;
+ }
+ slab->free_head = prev;
+}
+
+static int
+grow_socket(struct fastmem_socket_state *socket, int socket_id)
+{
+ char name[RTE_MEMZONE_NAMESIZE];
+ const struct rte_memzone *mz;
+ unsigned int i;
+
+ if (socket->reserved_bytes + FASTMEM_MEMZONE_SIZE > socket->memory_limit) {
+ FASTMEM_LOG(ERR,
+ "reserve would exceed memory_limit (%zu) on socket %d",
+ socket->memory_limit, socket_id);
+ return -ENOMEM;
+ }
+
+ if (socket->n_memzones == FASTMEM_MAX_MEMZONES_PER_SOCKET) {
+ FASTMEM_LOG(ERR,
+ "reached per-socket memzone cap (%u) on socket %d",
+ FASTMEM_MAX_MEMZONES_PER_SOCKET, socket_id);
+ return -ENOMEM;
+ }
+
+ snprintf(name, sizeof(name), "fastmem_%d_%u", socket_id,
+ socket->memzone_seq++);
+
+ mz = rte_memzone_reserve_aligned(name, FASTMEM_MEMZONE_SIZE,
+ socket_id, RTE_MEMZONE_IOVA_CONTIG,
+ FASTMEM_SLAB_SIZE);
+ if (mz == NULL) {
+ FASTMEM_LOG(ERR,
+ "failed to reserve %zu-byte memzone '%s' on socket %d: %s",
+ (size_t)FASTMEM_MEMZONE_SIZE, name, socket_id,
+ rte_strerror(rte_errno));
+ return -ENOMEM;
+ }
+
+ socket->memzones[socket->n_memzones++] = mz;
+ socket->reserved_bytes += FASTMEM_MEMZONE_SIZE;
+
+ for (i = 0; i < FASTMEM_SLABS_PER_MEMZONE; i++) {
+ struct fastmem_slab *slab = RTE_PTR_ADD(mz->addr,
+ i * FASTMEM_SLAB_SIZE);
+
+ slab->iova_base = mz->iova + i * FASTMEM_SLAB_SIZE;
+ slab->next_free = socket->free_head;
+ socket->free_head = slab;
+ }
+
+ FASTMEM_LOG(DEBUG,
+ "reserved memzone '%s' (%zu bytes) on socket %d; %zu slabs added",
+ name, (size_t)FASTMEM_MEMZONE_SIZE, socket_id,
+ (size_t)FASTMEM_SLABS_PER_MEMZONE);
+
+ return 0;
+}
+
+static struct fastmem_slab *
+slab_acquire(struct fastmem_socket_state *socket, int socket_id)
+{
+ struct fastmem_slab *slab;
+
+ rte_spinlock_lock(&socket->lock);
+
+ if (socket->free_head == NULL) {
+ int rc = grow_socket(socket, socket_id);
+
+ if (rc < 0) {
+ rte_spinlock_unlock(&socket->lock);
+ return NULL;
+ }
+ }
+
+ slab = socket->free_head;
+ socket->free_head = slab->next_free;
+ slab->next_free = NULL;
+
+ rte_spinlock_unlock(&socket->lock);
+
+ return slab;
+}
+
+static void
+slab_release(struct fastmem_socket_state *socket,
+ struct fastmem_slab *slab)
+{
+ rte_spinlock_lock(&socket->lock);
+
+ slab->next_free = socket->free_head;
+ socket->free_head = slab;
+
+ rte_spinlock_unlock(&socket->lock);
+}
+
+static void
+bin_init(struct fastmem_bin *bin, unsigned int class_idx, int socket_id)
+{
+ size_t slot_size = class_size(class_idx);
+
+ rte_spinlock_init(&bin->lock);
+ bin->slot_size = (uint32_t)slot_size;
+ bin->slots_per_slab = slab_slot_count(slot_size);
+ bin->class_idx = class_idx;
+ TAILQ_INIT(&bin->partial);
+ TAILQ_INIT(&bin->full);
+ bin->socket_id = socket_id;
+ bin->slab_acquires = 0;
+ bin->slab_releases = 0;
+ bin->slabs_partial = 0;
+ bin->slabs_full = 0;
+}
+
+static void
+bin_release(struct fastmem_bin *bin, struct fastmem_socket_state *socket)
+{
+ struct fastmem_slab *slab;
+
+ while ((slab = TAILQ_FIRST(&bin->partial)) != NULL) {
+ TAILQ_REMOVE(&bin->partial, slab, list);
+ slab_release(socket, slab);
+ }
+ while ((slab = TAILQ_FIRST(&bin->full)) != NULL) {
+ TAILQ_REMOVE(&bin->full, slab, list);
+ slab_release(socket, slab);
+ }
+}
+
+static unsigned int
+bin_pop_locked(struct fastmem_bin *bin, void **objs, unsigned int n)
+{
+ unsigned int got = 0;
+
+ while (got < n) {
+ struct fastmem_slab *slab = TAILQ_FIRST(&bin->partial);
+ void *obj;
+
+ if (slab == NULL)
+ break;
+
+ obj = slab->free_head;
+ slab->free_head = *(void **)obj;
+ slab->free_count--;
+ objs[got++] = obj;
+
+ if (slab->free_count == 0) {
+ TAILQ_REMOVE(&bin->partial, slab, list);
+ TAILQ_INSERT_HEAD(&bin->full, slab, list);
+ bin->slabs_partial--;
+ bin->slabs_full++;
+ }
+ }
+
+ return got;
+}
+
+/*
+ * Fully-drained slabs are accumulated in @p to_release for the
+ * caller to return after dropping the lock.
+ */
+static unsigned int
+bin_push_locked(struct fastmem_bin *bin, void **objs, unsigned int n,
+ struct fastmem_slab **to_release)
+{
+ unsigned int n_release = 0;
+ unsigned int i;
+
+ for (i = 0; i < n; i++) {
+ void *obj = objs[i];
+ struct fastmem_slab *slab = (struct fastmem_slab *)
+ ((uintptr_t)obj & ~(uintptr_t)FASTMEM_SLAB_MASK);
+ bool was_full = slab->free_count == 0;
+
+ *(void **)obj = slab->free_head;
+ slab->free_head = obj;
+ slab->free_count++;
+
+ if (was_full) {
+ TAILQ_REMOVE(&bin->full, slab, list);
+ TAILQ_INSERT_HEAD(&bin->partial, slab, list);
+ bin->slabs_full--;
+ bin->slabs_partial++;
+ }
+
+ if (slab->free_count == slab->n_slots) {
+ TAILQ_REMOVE(&bin->partial, slab, list);
+ bin->slabs_partial--;
+ bin->slab_releases++;
+ to_release[n_release++] = slab;
+ }
+ }
+
+ return n_release;
+}
+
+/*
+ * Allocate a single object from the bin. Pass @p nocache true only on the
+ * no-cache fallback (a user allocation that has neither a private nor a
+ * shared cache); it counts the alloc against the bin's no-cache statistics.
+ * Internal cache machinery (refills) passes false.
+ */
+static void *
+bin_alloc_one(struct fastmem_bin *bin, bool nocache)
+{
+ struct fastmem_socket_state *socket = &fastmem->sockets[bin->socket_id];
+ void *obj;
+
+ rte_spinlock_lock(&bin->lock);
+
+ while (bin_pop_locked(bin, &obj, 1) == 0) {
+ struct fastmem_slab *slab;
+
+ if (TAILQ_FIRST(&bin->partial) != NULL)
+ continue;
+
+ rte_spinlock_unlock(&bin->lock);
+
+ slab = slab_acquire(socket, bin->socket_id);
+ if (slab == NULL) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ rte_spinlock_lock(&bin->lock);
+
+ if (unlikely(TAILQ_FIRST(&bin->partial) != NULL)) {
+ /* Release surplus slab without holding bin->lock. */
+ rte_spinlock_unlock(&bin->lock);
+ slab_release(socket, slab);
+ rte_spinlock_lock(&bin->lock);
+ } else {
+ slab_init(bin, slab);
+ TAILQ_INSERT_HEAD(&bin->partial, slab, list);
+ bin->slabs_partial++;
+ bin->slab_acquires++;
+ }
+ }
+
+ if (nocache)
+ bin->nocache_allocs++;
+
+ rte_spinlock_unlock(&bin->lock);
+
+ return obj;
+}
+
+/*
+ * Allocate up to @p n objects from the bin. Pass @p nocache true only on the
+ * no-cache fallback (a user allocation that has neither a private nor a
+ * shared cache); it counts the allocs against the bin's no-cache statistics.
+ * Internal cache machinery (e.g. a cache refill) passes false.
+ */
+static unsigned int
+bin_alloc_bulk(struct fastmem_bin *bin, void **objs, unsigned int n,
+ bool nocache)
+{
+ struct fastmem_socket_state *socket = &fastmem->sockets[bin->socket_id];
+ unsigned int got = 0;
+
+ rte_spinlock_lock(&bin->lock);
+
+ while (got < n) {
+ struct fastmem_slab *slab;
+
+ got += bin_pop_locked(bin, objs + got, n - got);
+ if (got == n)
+ break;
+
+ if (TAILQ_FIRST(&bin->partial) != NULL)
+ continue;
+
+ rte_spinlock_unlock(&bin->lock);
+
+ slab = slab_acquire(socket, bin->socket_id);
+ if (slab == NULL) {
+ rte_spinlock_lock(&bin->lock);
+ break;
+ }
+
+ rte_spinlock_lock(&bin->lock);
+
+ if (unlikely(TAILQ_FIRST(&bin->partial) != NULL)) {
+ /* Release surplus slab without holding bin->lock. */
+ rte_spinlock_unlock(&bin->lock);
+ slab_release(socket, slab);
+ rte_spinlock_lock(&bin->lock);
+ } else {
+ slab_init(bin, slab);
+ TAILQ_INSERT_HEAD(&bin->partial, slab, list);
+ bin->slabs_partial++;
+ bin->slab_acquires++;
+ }
+ }
+
+ if (nocache)
+ bin->nocache_allocs += got;
+
+ rte_spinlock_unlock(&bin->lock);
+
+ return got;
+}
+
+/*
+ * Free a single object to the bin. Pass @p nocache true only on the no-cache
+ * fallback (a user free that has neither a private nor a shared cache); it
+ * counts the free against the bin's no-cache statistics. Internal cache
+ * machinery (drain, teardown, flush) passes false.
+ */
+static void
+bin_free_one(struct fastmem_bin *bin, void *obj, bool nocache)
+{
+ unsigned int n_release;
+ struct fastmem_slab *slab_to_release = NULL;
+ struct fastmem_socket_state *socket;
+
+ rte_spinlock_lock(&bin->lock);
+ n_release = bin_push_locked(bin, &obj, 1, &slab_to_release);
+ if (nocache)
+ bin->nocache_frees++;
+ rte_spinlock_unlock(&bin->lock);
+
+ if (n_release > 0) {
+ socket = &fastmem->sockets[bin->socket_id];
+ slab_release(socket, slab_to_release);
+ }
+}
+
+/*
+ * Free a batch of objects to the bin. Always internal cache machinery
+ * (drain, teardown, flush), never a no-cache user free, so unlike
+ * bin_free_one() it has no nocache flag and is never counted against the
+ * bin's no-cache statistics.
+ */
+static void
+bin_free_bulk(struct fastmem_bin *bin, void **objs, unsigned int n)
+{
+ struct fastmem_socket_state *socket = &fastmem->sockets[bin->socket_id];
+ struct fastmem_slab *to_release[FASTMEM_CACHE_BASE_CAPACITY];
+ unsigned int n_release;
+ unsigned int i;
+
+ RTE_VERIFY(n <= RTE_DIM(to_release));
+
+ rte_spinlock_lock(&bin->lock);
+ n_release = bin_push_locked(bin, objs, n, to_release);
+ rte_spinlock_unlock(&bin->lock);
+
+ for (i = 0; i < n_release; i++)
+ slab_release(socket, to_release[i]);
+}
+
+static unsigned int
+cache_capacity(unsigned int class_idx)
+{
+ unsigned int class_log2 = class_idx + FASTMEM_MIN_CLASS_LOG2;
+ unsigned int shift;
+ unsigned int cap;
+
+ if (class_log2 <= FASTMEM_CACHE_BASE_CLASS_LOG2)
+ return FASTMEM_CACHE_BASE_CAPACITY;
+
+ shift = class_log2 - FASTMEM_CACHE_BASE_CLASS_LOG2;
+ cap = FASTMEM_CACHE_BASE_CAPACITY >> shift;
+
+ return cap < FASTMEM_CACHE_FLOOR_CAPACITY ?
+ FASTMEM_CACHE_FLOOR_CAPACITY : cap;
+}
+
+static struct fastmem_cache **
+cache_slot(struct fastmem_socket_state *socket, unsigned int class_idx,
+ unsigned int lcore_id)
+{
+ if (lcore_id >= RTE_MAX_LCORE)
+ return NULL;
+ return &socket->caches[lcore_id][class_idx];
+}
+
+/*
+ * Allocate and initialize a cache struct, itself drawn from fastmem on the
+ * calling lcore's socket, bypassing the cache layer to avoid recursion.
+ */
+static struct fastmem_cache *
+cache_alloc(struct fastmem_socket_state *socket, unsigned int class_idx)
+{
+ struct fastmem_cache *cache;
+ unsigned int capacity = cache_capacity(class_idx);
+ size_t cache_size = sizeof(*cache) + capacity * sizeof(void *);
+ unsigned int cache_class = size_to_class(cache_size, RTE_CACHE_LINE_SIZE);
+ unsigned int own_socket = rte_socket_id();
+ struct fastmem_socket_state *alloc_socket;
+
+ if (cache_class >= FASTMEM_N_CLASSES) {
+ FASTMEM_LOG(ERR,
+ "cache size %zu exceeds max size class",
+ cache_size);
+ return NULL;
+ }
+
+ if (own_socket >= RTE_MAX_NUMA_NODES)
+ own_socket = (unsigned int)socket->bins[0].socket_id;
+
+ alloc_socket = &fastmem->sockets[own_socket];
+
+ cache = bin_alloc_one(&alloc_socket->bins[cache_class], false);
+ if (cache == NULL) {
+ FASTMEM_LOG(ERR,
+ "failed to allocate cache for class %u on socket %u",
+ class_idx, own_socket);
+ return NULL;
+ }
+
+ cache->count = 0;
+ cache->capacity = capacity;
+ cache->target = capacity / 2;
+ cache->alloc_cache_hits = 0;
+ cache->alloc_cache_misses = 0;
+ cache->alloc_nomem = 0;
+ cache->free_cache_hits = 0;
+ cache->free_cache_misses = 0;
+
+ return cache;
+}
+
+static struct fastmem_cache *
+cache_create(struct fastmem_socket_state *socket,
+ unsigned int class_idx, unsigned int lcore_id)
+{
+ struct fastmem_cache **slot = cache_slot(socket, class_idx, lcore_id);
+ struct fastmem_cache *cache;
+
+ if (slot == NULL)
+ return NULL;
+
+ cache = *slot;
+ if (cache != NULL)
+ return cache;
+
+ cache = cache_alloc(socket, class_idx);
+ if (cache == NULL)
+ return NULL;
+
+ *slot = cache;
+
+ return cache;
+}
+
+/*
+ * Get-or-create the private per-lcore cache. Returns NULL for callers that
+ * have no private cache (secondary process, or no lcore id), which then use
+ * the shared cache instead.
+ */
+static struct fastmem_cache *
+cache_get(struct fastmem_socket_state *socket, unsigned int class_idx,
+ unsigned int lcore_id)
+{
+ struct fastmem_cache **slot;
+ struct fastmem_cache *cache;
+
+ if (unlikely(!fastmem_is_primary))
+ return NULL;
+
+ slot = cache_slot(socket, class_idx, lcore_id);
+
+ if (slot == NULL)
+ return NULL;
+
+ cache = *slot;
+ if (cache != NULL)
+ return cache;
+
+ return cache_create(socket, class_idx, lcore_id);
+}
+
+static void *
+cache_pop(struct fastmem_cache *cache, struct fastmem_bin *bin)
+{
+ if (cache->count > 0) {
+ cache->alloc_cache_hits++;
+ return cache->objs[--cache->count];
+ }
+
+ cache->count = bin_alloc_bulk(bin, cache->objs, cache->target, false);
+ if (cache->count == 0)
+ return NULL;
+
+ cache->alloc_cache_misses++;
+ return cache->objs[--cache->count];
+}
+
+static void
+cache_push(struct fastmem_cache *cache, struct fastmem_bin *bin, void *obj)
+{
+ unsigned int drain;
+
+ if (cache->count < cache->capacity) {
+ cache->free_cache_hits++;
+ cache->objs[cache->count++] = obj;
+ return;
+ }
+
+ cache->free_cache_misses++;
+
+ /*
+ * Drain the oldest (bottom) half to the bin, keep the newest
+ * (top) half for temporal reuse.
+ */
+ drain = cache->count - cache->target;
+ bin_free_bulk(bin, cache->objs, drain);
+ memmove(cache->objs, cache->objs + drain,
+ cache->target * sizeof(cache->objs[0]));
+ cache->count = cache->target;
+
+ cache->objs[cache->count++] = obj;
+}
+
+/* Get-or-create the shared cache; call with shared_cache_lock held. */
+static struct fastmem_cache *
+shared_cache_get(struct fastmem_socket_state *socket, unsigned int class_idx)
+{
+ struct fastmem_cache *cache = socket->shared_caches[class_idx];
+
+ if (cache != NULL)
+ return cache;
+
+ cache = cache_alloc(socket, class_idx);
+ if (cache == NULL)
+ return NULL;
+
+ socket->shared_caches[class_idx] = cache;
+
+ return cache;
+}
+
+/* Allocate one object via the shared cache, or straight from the bin if the
+ * cache cannot be created. */
+static void *
+shared_alloc_one(struct fastmem_socket_state *socket, unsigned int class_idx)
+{
+ struct fastmem_bin *bin = &socket->bins[class_idx];
+ struct fastmem_cache *cache;
+ void *obj;
+
+ rte_spinlock_lock(&socket->shared_cache_lock);
+
+ cache = shared_cache_get(socket, class_idx);
+ if (likely(cache != NULL)) {
+ obj = cache_pop(cache, bin);
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+ return obj;
+ }
+
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+
+ return bin_alloc_one(bin, true);
+}
+
+/* Allocate up to @p n objects via the shared cache; returns the count got. */
+static unsigned int
+shared_alloc_bulk(struct fastmem_socket_state *socket, unsigned int class_idx,
+ void **objs, unsigned int n)
+{
+ struct fastmem_bin *bin = &socket->bins[class_idx];
+ struct fastmem_cache *cache;
+ unsigned int got = 0;
+
+ rte_spinlock_lock(&socket->shared_cache_lock);
+
+ cache = shared_cache_get(socket, class_idx);
+ if (likely(cache != NULL)) {
+ while (got < n) {
+ void *obj = cache_pop(cache, bin);
+
+ if (obj == NULL)
+ break;
+ objs[got++] = obj;
+ }
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+ return got;
+ }
+
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+
+ return bin_alloc_bulk(bin, objs, n, true);
+}
+
+/* Free one object via the shared cache. */
+static void
+shared_free_one(struct fastmem_socket_state *socket, unsigned int class_idx,
+ void *obj)
+{
+ struct fastmem_bin *bin = &socket->bins[class_idx];
+ struct fastmem_cache *cache;
+
+ rte_spinlock_lock(&socket->shared_cache_lock);
+
+ cache = shared_cache_get(socket, class_idx);
+ if (likely(cache != NULL)) {
+ cache_push(cache, bin, obj);
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+ return;
+ }
+
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+
+ bin_free_one(bin, obj, true);
+}
+
+/* Record an alloc failure against the per-lcore cache, the shared cache, or
+ * the bin's no-cache counter, in that order of preference. */
+static void
+account_alloc_nomem(struct fastmem_socket_state *socket,
+ unsigned int class_idx, unsigned int lcore_id)
+{
+ struct fastmem_cache *cache = cache_get(socket, class_idx, lcore_id);
+
+ if (likely(cache != NULL)) {
+ cache->alloc_nomem++;
+ return;
+ }
+
+ rte_spinlock_lock(&socket->shared_cache_lock);
+ cache = shared_cache_get(socket, class_idx);
+ if (likely(cache != NULL)) {
+ cache->alloc_nomem++;
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+ return;
+ }
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+
+ struct fastmem_bin *bin = &socket->bins[class_idx];
+
+ rte_spinlock_lock(&bin->lock);
+ bin->nocache_nomem++;
+ rte_spinlock_unlock(&bin->lock);
+}
+
+static void
+socket_release_caches(struct fastmem_socket_state *socket)
+{
+ unsigned int lcore;
+ unsigned int c;
+
+ for (lcore = 0; lcore < RTE_MAX_LCORE; lcore++) {
+ for (c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_cache *cache = socket->caches[lcore][c];
+ struct fastmem_slab *cache_slab;
+
+ if (cache == NULL)
+ continue;
+
+ if (cache->count > 0) {
+ bin_free_bulk(&socket->bins[c],
+ cache->objs, cache->count);
+ cache->count = 0;
+ }
+
+ cache_slab = slab_of(cache);
+ bin_free_one(cache_slab->bin, cache, false);
+
+ socket->caches[lcore][c] = NULL;
+ }
+ }
+
+ for (c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_cache *cache = socket->shared_caches[c];
+ struct fastmem_slab *cache_slab;
+
+ if (cache == NULL)
+ continue;
+
+ if (cache->count > 0) {
+ bin_free_bulk(&socket->bins[c],
+ cache->objs, cache->count);
+ cache->count = 0;
+ }
+
+ cache_slab = slab_of(cache);
+ bin_free_one(cache_slab->bin, cache, false);
+
+ socket->shared_caches[c] = NULL;
+ }
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_init, 24.11)
+int
+rte_fastmem_init(void)
+{
+ unsigned int s, c;
+
+ if (fastmem != NULL)
+ return -EBUSY;
+
+ fastmem_mz = rte_memzone_reserve_aligned("fastmem_state",
+ sizeof(*fastmem), SOCKET_ID_ANY, 0,
+ RTE_CACHE_LINE_SIZE);
+ if (fastmem_mz == NULL)
+ return -ENOMEM;
+
+ fastmem = fastmem_mz->addr;
+ fastmem_is_primary = true;
+ memset(fastmem, 0, sizeof(*fastmem));
+
+ for (s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+
+ rte_spinlock_init(&socket->lock);
+ rte_spinlock_init(&socket->shared_cache_lock);
+ socket->memory_limit = SIZE_MAX;
+
+ for (c = 0; c < FASTMEM_N_CLASSES; c++)
+ bin_init(&socket->bins[c], c, (int)s);
+ }
+
+ return 0;
+}
+
+static void
+release_socket_caches(struct fastmem_socket_state *socket)
+{
+ socket_release_caches(socket);
+}
+
+static void
+release_socket_bins(struct fastmem_socket_state *socket)
+{
+ unsigned int c;
+
+ for (c = 0; c < FASTMEM_N_CLASSES; c++)
+ bin_release(&socket->bins[c], socket);
+}
+
+static void
+release_socket_memzones(struct fastmem_socket_state *socket)
+{
+ unsigned int i;
+
+ for (i = 0; i < socket->n_memzones; i++)
+ rte_memzone_free(socket->memzones[i]);
+
+ socket->free_head = NULL;
+ socket->reserved_bytes = 0;
+ socket->n_memzones = 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_deinit, 24.11)
+void
+rte_fastmem_deinit(void)
+{
+ unsigned int i;
+
+ if (fastmem == NULL)
+ return;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ fastmem = NULL;
+ fastmem_mz = NULL;
+ return;
+ }
+
+ for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+ release_socket_caches(&fastmem->sockets[i]);
+
+ for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+ release_socket_bins(&fastmem->sockets[i]);
+
+ for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+ release_socket_memzones(&fastmem->sockets[i]);
+
+ rte_memzone_free(fastmem_mz);
+ fastmem_mz = NULL;
+ fastmem = NULL;
+}
+
+/* Same resolution order as rte_malloc's malloc_get_numa_socket(). */
+static unsigned int
+local_socket_id(void)
+{
+ int sid = (int)rte_socket_id();
+
+ if (likely(sid >= 0 && sid < RTE_MAX_NUMA_NODES))
+ return sid;
+
+ sid = (int)rte_lcore_to_socket_id(rte_get_main_lcore());
+ if (likely(sid >= 0 && sid < RTE_MAX_NUMA_NODES))
+ return sid;
+
+ sid = rte_socket_id_by_idx(0);
+ if (likely(sid >= 0 && sid < RTE_MAX_NUMA_NODES))
+ return sid;
+
+ return 0;
+}
+
+static int
+reserve_on_socket(int sid, size_t size)
+{
+ struct fastmem_socket_state *socket = &fastmem->sockets[sid];
+ int rc = 0;
+
+ rte_spinlock_lock(&socket->lock);
+
+ while (socket->reserved_bytes < size) {
+ rc = grow_socket(socket, sid);
+ if (rc < 0)
+ break;
+ }
+
+ rte_spinlock_unlock(&socket->lock);
+
+ return rc;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_reserve, 24.11)
+int
+rte_fastmem_reserve(size_t size, int socket_id)
+{
+ unsigned int i;
+ int rc;
+
+ if (fastmem == NULL)
+ return -EINVAL;
+
+ if (socket_id != SOCKET_ID_ANY) {
+ if (socket_id < 0 || socket_id >= RTE_MAX_NUMA_NODES)
+ return -EINVAL;
+ return reserve_on_socket(socket_id, size);
+ }
+
+ rc = reserve_on_socket(local_socket_id(), size);
+ if (rc == 0)
+ return 0;
+
+ for (i = 0; i < rte_socket_count(); i++) {
+ int sid = rte_socket_id_by_idx(i);
+
+ if (sid < 0 || (unsigned int)sid == local_socket_id())
+ continue;
+
+ rc = reserve_on_socket(sid, size);
+ if (rc == 0)
+ return 0;
+ }
+
+ return rc;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_set_limit, 24.11)
+int
+rte_fastmem_set_limit(int socket_id, size_t max_bytes)
+{
+ if (fastmem == NULL)
+ return -EINVAL;
+
+ if (socket_id == SOCKET_ID_ANY) {
+ for (unsigned int i = 0; i < RTE_MAX_NUMA_NODES; i++)
+ fastmem->sockets[i].memory_limit = max_bytes;
+ return 0;
+ }
+
+ if (socket_id < 0 || socket_id >= RTE_MAX_NUMA_NODES)
+ return -EINVAL;
+
+ fastmem->sockets[socket_id].memory_limit = max_bytes;
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_get_limit, 24.11)
+size_t
+rte_fastmem_get_limit(int socket_id)
+{
+ if (fastmem == NULL || socket_id < 0 || socket_id >= RTE_MAX_NUMA_NODES)
+ return 0;
+
+ return fastmem->sockets[socket_id].memory_limit;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_max_size, 24.11)
+size_t
+rte_fastmem_max_size(void)
+{
+ return FASTMEM_MAX_ALLOC_SIZE;
+}
+
+static void *
+alloc_from_socket(struct fastmem_socket_state *socket,
+ unsigned int class_idx, unsigned int lcore_id)
+{
+ struct fastmem_cache *cache;
+ struct fastmem_bin *bin = &socket->bins[class_idx];
+
+ cache = cache_get(socket, class_idx, lcore_id);
+ if (likely(cache != NULL))
+ return cache_pop(cache, bin);
+
+ return shared_alloc_one(socket, class_idx);
+}
+
+static void
+do_free(void *ptr)
+{
+ struct fastmem_slab *slab;
+ struct fastmem_bin *bin;
+ struct fastmem_socket_state *socket;
+ unsigned int lcore_id;
+ struct fastmem_cache *cache;
+
+ if (unlikely(!fastmem_assure()))
+ return;
+
+ slab = slab_of(ptr);
+ bin = slab->bin;
+ socket = &fastmem->sockets[bin->socket_id];
+
+ lcore_id = rte_lcore_id();
+ cache = cache_get(socket, bin->class_idx, lcore_id);
+ if (likely(cache != NULL))
+ cache_push(cache, bin, ptr);
+ else
+ shared_free_one(socket, bin->class_idx, ptr);
+}
+
+static int
+do_alloc_bulk(void **ptrs, unsigned int n, size_t size, size_t align,
+ unsigned int flags, unsigned int lcore_id,
+ int socket_id, bool fallback)
+{
+ unsigned int class_idx;
+ struct fastmem_socket_state *socket;
+ struct fastmem_cache *cache;
+ unsigned int got = 0;
+
+ if (unlikely(!fastmem_assure()))
+ return -rte_errno;
+
+ if (unlikely(!normalize_align(&align))) {
+ rte_errno = EINVAL;
+ return -EINVAL;
+ }
+
+ class_idx = size_to_class(size, align);
+ if (unlikely(class_idx >= FASTMEM_N_CLASSES)) {
+ rte_errno = E2BIG;
+ return -E2BIG;
+ }
+
+ socket = &fastmem->sockets[socket_id];
+ cache = cache_get(socket, class_idx, lcore_id);
+
+ if (likely(cache != NULL)) {
+ /* Drain from cache. */
+ unsigned int avail = RTE_MIN(cache->count, n);
+
+ cache->count -= avail;
+ memcpy(ptrs, &cache->objs[cache->count],
+ avail * sizeof(void *));
+ got = avail;
+ cache->alloc_cache_hits += avail;
+
+ if (got < n) {
+ unsigned int need = n - got;
+ unsigned int want = RTE_MAX(need, cache->target);
+ unsigned int filled;
+
+ if (want <= cache->capacity) {
+ /* Refill into cache, give caller their share. */
+ filled = bin_alloc_bulk(
+ &socket->bins[class_idx],
+ cache->objs, want, false);
+ if (filled > 0)
+ cache->alloc_cache_misses += RTE_MIN(filled, need);
+ if (filled >= need) {
+ memcpy(ptrs + got,
+ cache->objs + filled - need,
+ need * sizeof(void *));
+ cache->count = filled - need;
+ got = n;
+ } else {
+ memcpy(ptrs + got, cache->objs,
+ filled * sizeof(void *));
+ got += filled;
+ cache->count = 0;
+ }
+ } else {
+ /*
+ * n exceeds cache capacity; pull directly,
+ * but count as cache misses since the caller
+ * has a cache.
+ */
+ unsigned int pulled = bin_alloc_bulk(
+ &socket->bins[class_idx],
+ ptrs + got, need, false);
+ if (pulled > 0)
+ cache->alloc_cache_misses += pulled;
+ got += pulled;
+ }
+ }
+ } else {
+ got = shared_alloc_bulk(socket, class_idx, ptrs, n);
+ }
+
+ if (unlikely(got < n) && fallback) {
+ unsigned int i;
+
+ for (i = 0; i < rte_socket_count() && got < n; i++) {
+ int sid = rte_socket_id_by_idx(i);
+
+ if (sid < 0 || sid == socket_id)
+ continue;
+
+ socket = &fastmem->sockets[sid];
+ cache = cache_get(socket, class_idx, lcore_id);
+ if (likely(cache != NULL)) {
+ unsigned int avail =
+ RTE_MIN(cache->count, n - got);
+ cache->count -= avail;
+ memcpy(ptrs + got,
+ &cache->objs[cache->count],
+ avail * sizeof(void *));
+ cache->alloc_cache_hits += avail;
+ got += avail;
+ }
+ if (got < n) {
+ if (cache != NULL) {
+ unsigned int pulled = bin_alloc_bulk(
+ &socket->bins[class_idx],
+ ptrs + got, n - got, false);
+ if (pulled > 0)
+ cache->alloc_cache_misses += pulled;
+ got += pulled;
+ } else {
+ got += shared_alloc_bulk(socket,
+ class_idx, ptrs + got, n - got);
+ }
+ }
+ }
+ }
+
+ if (unlikely(got < n)) {
+ /* All-or-nothing: return what we got. */
+ unsigned int i;
+
+ for (i = 0; i < got; i++)
+ do_free(ptrs[i]);
+
+ account_alloc_nomem(&fastmem->sockets[socket_id], class_idx,
+ lcore_id);
+ rte_errno = ENOMEM;
+ return -ENOMEM;
+ }
+
+ if (flags & RTE_FASTMEM_F_ZERO) {
+ size_t cs = class_size(class_idx);
+ unsigned int i;
+
+ for (i = 0; i < n; i++)
+ memset(ptrs[i], 0, cs);
+ }
+
+ return 0;
+}
+
+static void *
+do_alloc(size_t size, size_t align, unsigned int flags,
+ unsigned int lcore_id, int socket_id, bool fallback)
+{
+ unsigned int class_idx;
+ void *obj;
+
+ if (unlikely(!fastmem_assure()))
+ return NULL;
+
+ if (unlikely(!normalize_align(&align))) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ class_idx = size_to_class(size, align);
+ if (unlikely(class_idx >= FASTMEM_N_CLASSES)) {
+ rte_errno = E2BIG;
+ return NULL;
+ }
+
+ obj = alloc_from_socket(&fastmem->sockets[socket_id],
+ class_idx, lcore_id);
+
+ if (likely(obj != NULL))
+ goto out;
+
+ if (fallback) {
+ unsigned int i;
+
+ for (i = 0; i < rte_socket_count(); i++) {
+ int sid = rte_socket_id_by_idx(i);
+
+ if (sid < 0 || sid == socket_id)
+ continue;
+
+ obj = alloc_from_socket(&fastmem->sockets[sid],
+ class_idx, lcore_id);
+ if (obj != NULL)
+ goto out;
+ }
+ }
+
+ account_alloc_nomem(&fastmem->sockets[socket_id], class_idx, lcore_id);
+ rte_errno = ENOMEM;
+ return NULL;
+
+out:
+ if (flags & RTE_FASTMEM_F_ZERO)
+ memset(obj, 0, class_size(class_idx));
+
+ return obj;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_alloc, 24.11)
+void *
+rte_fastmem_alloc(size_t size, size_t align, unsigned int flags)
+{
+ return do_alloc(size, align, flags, rte_lcore_id(),
+ local_socket_id(), false);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_alloc_socket, 24.11)
+void *
+rte_fastmem_alloc_socket(size_t size, size_t align, unsigned int flags,
+ int socket_id)
+{
+ if (socket_id == SOCKET_ID_ANY)
+ return do_alloc(size, align, flags, rte_lcore_id(),
+ local_socket_id(), true);
+
+ if (unlikely(socket_id < 0 || socket_id >= RTE_MAX_NUMA_NODES)) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ return do_alloc(size, align, flags, rte_lcore_id(), socket_id, false);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_free, 24.11)
+void
+rte_fastmem_free(void *ptr)
+{
+ if (unlikely(ptr == NULL))
+ return;
+
+ do_free(ptr);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_realloc, 24.11)
+void *
+rte_fastmem_realloc(void *ptr, size_t size, size_t align)
+{
+ struct fastmem_slab *slab;
+ unsigned int old_class, new_class;
+ size_t old_size;
+ void *new_ptr;
+
+ if (ptr == NULL)
+ return rte_fastmem_alloc(size, align, 0);
+
+ if (size == 0) {
+ rte_fastmem_free(ptr);
+ return NULL;
+ }
+
+ if (unlikely(!normalize_align(&align))) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ new_class = size_to_class(size, align);
+ if (unlikely(new_class >= FASTMEM_N_CLASSES)) {
+ rte_errno = E2BIG;
+ return NULL;
+ }
+
+ slab = slab_of(ptr);
+ old_class = slab->bin->class_idx;
+
+ if (new_class == old_class)
+ return ptr;
+
+ new_ptr = rte_fastmem_alloc(size, align, 0);
+ if (unlikely(new_ptr == NULL))
+ return NULL;
+
+ old_size = class_size(old_class);
+ memcpy(new_ptr, ptr, RTE_MIN(old_size, size));
+ rte_fastmem_free(ptr);
+
+ return new_ptr;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_alloc_bulk, 24.11)
+int
+rte_fastmem_alloc_bulk(void **ptrs, unsigned int n, size_t size, size_t align,
+ unsigned int flags)
+{
+ return do_alloc_bulk(ptrs, n, size, align, flags,
+ rte_lcore_id(), local_socket_id(), false);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_alloc_bulk_socket, 24.11)
+int
+rte_fastmem_alloc_bulk_socket(void **ptrs, unsigned int n, size_t size,
+ size_t align, unsigned int flags, int socket_id)
+{
+ if (socket_id == SOCKET_ID_ANY)
+ return do_alloc_bulk(ptrs, n, size, align, flags,
+ rte_lcore_id(), local_socket_id(), true);
+
+ if (unlikely(socket_id < 0 || socket_id >= RTE_MAX_NUMA_NODES)) {
+ rte_errno = EINVAL;
+ return -EINVAL;
+ }
+
+ return do_alloc_bulk(ptrs, n, size, align, flags,
+ rte_lcore_id(), socket_id, false);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_free_bulk, 24.11)
+void
+rte_fastmem_free_bulk(void **ptrs, unsigned int n)
+{
+ unsigned int lcore_id;
+ struct fastmem_slab *slab;
+ struct fastmem_bin *bin;
+ struct fastmem_socket_state *socket;
+ struct fastmem_cache *cache;
+ unsigned int space;
+ unsigned int i;
+
+ if (unlikely(n == 0))
+ return;
+
+ if (unlikely(!fastmem_assure()))
+ return;
+
+ lcore_id = rte_lcore_id();
+
+ /* Fast path: check if first object gives us the bin. */
+ slab = slab_of(ptrs[0]);
+ bin = slab->bin;
+ socket = &fastmem->sockets[bin->socket_id];
+ cache = cache_get(socket, bin->class_idx, lcore_id);
+
+ if (unlikely(cache == NULL)) {
+ for (i = 0; i < n; i++)
+ do_free(ptrs[i]);
+ return;
+ }
+
+ /*
+ * Try to push all objects into the cache in one memcpy.
+ * If any object belongs to a different bin, fall back to
+ * per-object free for the remainder.
+ */
+ space = cache->capacity - cache->count;
+ if (likely(n <= space)) {
+ /* Verify all same bin (common case). */
+ for (i = 1; i < n; i++)
+ if (slab_of(ptrs[i])->bin != bin)
+ goto slow;
+ cache->free_cache_hits += n;
+ memcpy(&cache->objs[cache->count], ptrs,
+ n * sizeof(void *));
+ cache->count += n;
+ return;
+ }
+
+ /* Would overflow cache — drain first, then push. */
+ if (n <= cache->capacity) {
+ unsigned int drain;
+
+ for (i = 1; i < n; i++)
+ if (slab_of(ptrs[i])->bin != bin)
+ goto slow;
+
+ cache->free_cache_misses += n;
+ drain = cache->count - cache->target + n;
+ if (drain > cache->count)
+ drain = cache->count;
+ if (drain > 0) {
+ bin_free_bulk(bin, cache->objs, drain);
+ cache->count -= drain;
+ memmove(cache->objs, cache->objs + drain,
+ cache->count * sizeof(cache->objs[0]));
+ }
+ memcpy(&cache->objs[cache->count], ptrs,
+ n * sizeof(void *));
+ cache->count += n;
+ return;
+ }
+
+slow:
+ for (i = 0; i < n; i++)
+ do_free(ptrs[i]);
+}
+
+#define fastmem_handle_class_BITS 8
+
+static rte_fastmem_handle_t
+fastmem_handle_pack(unsigned int class_idx, int socket_id)
+{
+ return (uint32_t)class_idx |
+ ((uint32_t)socket_id << fastmem_handle_class_BITS);
+}
+
+static unsigned int
+fastmem_handle_class(rte_fastmem_handle_t h)
+{
+ return h & ((1U << fastmem_handle_class_BITS) - 1);
+}
+
+static int
+fastmem_handle_socket(rte_fastmem_handle_t h)
+{
+ return (int)(h >> fastmem_handle_class_BITS);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_hlookup, 24.11)
+int
+rte_fastmem_hlookup(size_t size, size_t align, int socket_id,
+ rte_fastmem_handle_t *handle)
+{
+ unsigned int class_idx;
+ struct fastmem_socket_state *socket;
+
+ if (handle == NULL)
+ return -EINVAL;
+
+ if (!normalize_align(&align))
+ return -EINVAL;
+
+ if (socket_id < 0 || socket_id >= RTE_MAX_NUMA_NODES)
+ return -EINVAL;
+
+ class_idx = size_to_class(size, align);
+ if (class_idx >= FASTMEM_N_CLASSES)
+ return -E2BIG;
+
+ /* Pre-create the cache for the calling lcore. */
+ socket = &fastmem->sockets[socket_id];
+ cache_create(socket, class_idx, rte_lcore_id());
+
+ *handle = fastmem_handle_pack(class_idx, socket_id);
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_halloc, 24.11)
+void *
+rte_fastmem_halloc(rte_fastmem_handle_t handle, unsigned int flags)
+{
+ unsigned int class_idx = fastmem_handle_class(handle);
+ int socket_id = fastmem_handle_socket(handle);
+ unsigned int lcore_id = rte_lcore_id();
+ struct fastmem_socket_state *socket;
+ struct fastmem_bin *bin;
+ struct fastmem_cache *cache;
+ void *obj;
+
+ if (unlikely(!fastmem_assure()))
+ return NULL;
+
+ socket = &fastmem->sockets[socket_id];
+ bin = &socket->bins[class_idx];
+
+ cache = cache_get(socket, class_idx, lcore_id);
+ if (likely(cache != NULL))
+ obj = cache_pop(cache, bin);
+ else
+ obj = shared_alloc_one(socket, class_idx);
+
+ if (unlikely(obj == NULL)) {
+ account_alloc_nomem(socket, class_idx, lcore_id);
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ if (flags & RTE_FASTMEM_F_ZERO)
+ memset(obj, 0, class_size(class_idx));
+
+ return obj;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_halloc_bulk, 24.11)
+int
+rte_fastmem_halloc_bulk(rte_fastmem_handle_t handle,
+ void **ptrs, unsigned int n, unsigned int flags)
+{
+ unsigned int class_idx = fastmem_handle_class(handle);
+ int socket_id = fastmem_handle_socket(handle);
+
+ return do_alloc_bulk(ptrs, n, class_size(class_idx),
+ RTE_CACHE_LINE_SIZE, flags, rte_lcore_id(),
+ socket_id, false);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_hfree, 24.11)
+void
+rte_fastmem_hfree(rte_fastmem_handle_t handle, void *ptr)
+{
+ unsigned int class_idx = fastmem_handle_class(handle);
+ int socket_id = fastmem_handle_socket(handle);
+ unsigned int lcore_id = rte_lcore_id();
+ struct fastmem_socket_state *socket;
+ struct fastmem_bin *bin;
+ struct fastmem_cache *cache;
+
+ if (unlikely(ptr == NULL))
+ return;
+
+ if (unlikely(!fastmem_assure()))
+ return;
+
+ socket = &fastmem->sockets[socket_id];
+ bin = &socket->bins[class_idx];
+
+ cache = cache_get(socket, class_idx, lcore_id);
+ if (likely(cache != NULL))
+ cache_push(cache, bin, ptr);
+ else
+ shared_free_one(socket, class_idx, ptr);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_hfree_bulk, 24.11)
+void
+rte_fastmem_hfree_bulk(rte_fastmem_handle_t handle,
+ void **ptrs, unsigned int n)
+{
+ unsigned int class_idx = fastmem_handle_class(handle);
+ int socket_id = fastmem_handle_socket(handle);
+ struct fastmem_socket_state *socket;
+ struct fastmem_bin *bin;
+ unsigned int lcore_id;
+ struct fastmem_cache *cache;
+ unsigned int i;
+
+ if (unlikely(n == 0))
+ return;
+
+ if (unlikely(!fastmem_assure()))
+ return;
+
+ socket = &fastmem->sockets[socket_id];
+ bin = &socket->bins[class_idx];
+
+ lcore_id = rte_lcore_id();
+ cache = cache_get(socket, class_idx, lcore_id);
+
+ if (likely(cache != NULL)) {
+ for (i = 0; i < n; i++)
+ cache_push(cache, bin, ptrs[i]);
+ } else {
+ for (i = 0; i < n; i++)
+ shared_free_one(socket, class_idx, ptrs[i]);
+ }
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_virt2iova, 24.11)
+rte_iova_t
+rte_fastmem_virt2iova(const void *ptr)
+{
+ struct fastmem_slab *slab;
+
+ if (unlikely(!fastmem_assure()))
+ return RTE_BAD_IOVA;
+
+ slab = slab_of((void *)(uintptr_t)ptr);
+
+ return slab->iova_base + ((uintptr_t)ptr - (uintptr_t)slab);
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_cache_flush, 24.11)
+void
+rte_fastmem_cache_flush(void)
+{
+ unsigned int lcore_id;
+ unsigned int s, c;
+
+ if (fastmem == NULL)
+ return;
+
+ lcore_id = rte_lcore_id();
+ if (lcore_id >= RTE_MAX_LCORE)
+ return;
+
+ for (s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+
+ for (c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_cache *cache =
+ socket->caches[lcore_id][c];
+
+ if (cache == NULL)
+ continue;
+
+ /*
+ * Drain the objects back to the bin, but keep the
+ * cache struct: it holds the lcore's statistics,
+ * which must survive the flush.
+ */
+ if (cache->count > 0) {
+ bin_free_bulk(&socket->bins[c],
+ cache->objs, cache->count);
+ cache->count = 0;
+ }
+ }
+ }
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats, 24.11)
+int
+rte_fastmem_stats(struct rte_fastmem_stats *stats)
+{
+ if (stats == NULL)
+ return -EINVAL;
+ if (!fastmem_assure())
+ return -ENODEV;
+
+ *stats = (struct rte_fastmem_stats){0};
+ stats->n_classes = FASTMEM_N_CLASSES;
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+
+ stats->bytes_backing += socket->reserved_bytes;
+
+ for (unsigned int c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_bin *bin = &socket->bins[c];
+ uint64_t class_allocs, class_frees;
+
+ class_allocs = bin->nocache_allocs;
+ class_frees = bin->nocache_frees;
+ stats->alloc_nomem += bin->nocache_nomem;
+
+ for (unsigned int l = 0; l < RTE_MAX_LCORE; l++) {
+ struct fastmem_cache *cache =
+ socket->caches[l][c];
+
+ if (cache == NULL)
+ continue;
+
+ class_allocs += cache->alloc_cache_hits +
+ cache->alloc_cache_misses;
+ class_frees += cache->free_cache_hits +
+ cache->free_cache_misses;
+ stats->alloc_nomem += cache->alloc_nomem;
+ }
+
+ struct fastmem_cache *shared = socket->shared_caches[c];
+
+ if (shared != NULL) {
+ class_allocs += shared->alloc_cache_hits +
+ shared->alloc_cache_misses;
+ class_frees += shared->free_cache_hits +
+ shared->free_cache_misses;
+ stats->alloc_nomem += shared->alloc_nomem;
+ }
+
+ stats->alloc_total += class_allocs;
+ stats->free_total += class_frees;
+ if (class_allocs > class_frees)
+ stats->bytes_in_use += class_size(c) *
+ (class_allocs - class_frees);
+ }
+ }
+
+ return 0;
+}
+
+static unsigned int
+exact_class_idx(size_t sz)
+{
+ unsigned int log2;
+
+ if (sz < FASTMEM_MIN_SIZE || sz > FASTMEM_MAX_ALLOC_SIZE)
+ return FASTMEM_N_CLASSES;
+ if ((sz & (sz - 1)) != 0)
+ return FASTMEM_N_CLASSES;
+
+ log2 = (unsigned int)rte_ctz64(sz);
+ if (log2 < FASTMEM_MIN_CLASS_LOG2 || log2 > FASTMEM_MAX_CLASS_LOG2)
+ return FASTMEM_N_CLASSES;
+
+ return log2 - FASTMEM_MIN_CLASS_LOG2;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats_class, 24.11)
+int
+rte_fastmem_stats_class(size_t class_size_arg,
+ struct rte_fastmem_class_stats *stats)
+{
+ unsigned int c;
+ uint64_t allocs, frees;
+
+ if (stats == NULL)
+ return -EINVAL;
+ if (!fastmem_assure())
+ return -ENODEV;
+
+ c = exact_class_idx(class_size_arg);
+ if (c >= FASTMEM_N_CLASSES)
+ return -EINVAL;
+
+ *stats = (struct rte_fastmem_class_stats){0};
+ stats->class_size = class_size(c);
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+ struct fastmem_bin *bin = &socket->bins[c];
+
+ for (unsigned int l = 0; l < RTE_MAX_LCORE; l++) {
+ struct fastmem_cache *cache = socket->caches[l][c];
+
+ if (cache == NULL)
+ continue;
+
+ stats->alloc_cache_hits += cache->alloc_cache_hits;
+ stats->alloc_cache_misses += cache->alloc_cache_misses;
+ stats->alloc_nomem += cache->alloc_nomem;
+ stats->free_cache_hits += cache->free_cache_hits;
+ stats->free_cache_misses += cache->free_cache_misses;
+ }
+
+ struct fastmem_cache *shared = socket->shared_caches[c];
+
+ if (shared != NULL) {
+ stats->alloc_cache_hits += shared->alloc_cache_hits;
+ stats->alloc_cache_misses += shared->alloc_cache_misses;
+ stats->alloc_nomem += shared->alloc_nomem;
+ stats->free_cache_hits += shared->free_cache_hits;
+ stats->free_cache_misses += shared->free_cache_misses;
+ }
+
+ /* No-cache fallback traffic; fold into the miss counters. */
+ stats->alloc_cache_misses += bin->nocache_allocs;
+ stats->free_cache_misses += bin->nocache_frees;
+ stats->alloc_nomem += bin->nocache_nomem;
+
+ stats->slab_acquires += bin->slab_acquires;
+ stats->slab_releases += bin->slab_releases;
+ stats->slabs_partial += bin->slabs_partial;
+ stats->slabs_full += bin->slabs_full;
+ }
+
+ allocs = stats->alloc_cache_hits + stats->alloc_cache_misses;
+ frees = stats->free_cache_hits + stats->free_cache_misses;
+ if (allocs > frees)
+ stats->in_use = allocs - frees;
+
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats_lcore, 24.11)
+int
+rte_fastmem_stats_lcore(unsigned int lcore_id,
+ struct rte_fastmem_lcore_stats *stats)
+{
+ if (stats == NULL)
+ return -EINVAL;
+ if (!fastmem_assure())
+ return -ENODEV;
+ if (lcore_id >= RTE_MAX_LCORE)
+ return -EINVAL;
+
+ *stats = (struct rte_fastmem_lcore_stats){0};
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+
+ for (unsigned int c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_cache *cache =
+ socket->caches[lcore_id][c];
+
+ if (cache == NULL)
+ continue;
+
+ stats->alloc_cache_hits += cache->alloc_cache_hits;
+ stats->alloc_cache_misses += cache->alloc_cache_misses;
+ stats->alloc_nomem += cache->alloc_nomem;
+ stats->free_cache_hits += cache->free_cache_hits;
+ stats->free_cache_misses += cache->free_cache_misses;
+ }
+ }
+
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats_lcore_class, 24.11)
+int
+rte_fastmem_stats_lcore_class(unsigned int lcore_id, size_t class_size_arg,
+ struct rte_fastmem_lcore_class_stats *stats)
+{
+ unsigned int c;
+
+ if (stats == NULL)
+ return -EINVAL;
+ if (!fastmem_assure())
+ return -ENODEV;
+ if (lcore_id >= RTE_MAX_LCORE)
+ return -EINVAL;
+
+ c = exact_class_idx(class_size_arg);
+ if (c >= FASTMEM_N_CLASSES)
+ return -EINVAL;
+
+ *stats = (struct rte_fastmem_lcore_class_stats){0};
+ stats->class_size = class_size(c);
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_cache *cache =
+ fastmem->sockets[s].caches[lcore_id][c];
+
+ if (cache == NULL)
+ continue;
+
+ stats->alloc_cache_hits += cache->alloc_cache_hits;
+ stats->alloc_cache_misses += cache->alloc_cache_misses;
+ stats->alloc_nomem += cache->alloc_nomem;
+ stats->free_cache_hits += cache->free_cache_hits;
+ stats->free_cache_misses += cache->free_cache_misses;
+ }
+
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats_shared, 24.11)
+int
+rte_fastmem_stats_shared(struct rte_fastmem_lcore_stats *stats)
+{
+ if (stats == NULL)
+ return -EINVAL;
+ if (!fastmem_assure())
+ return -ENODEV;
+
+ *stats = (struct rte_fastmem_lcore_stats){0};
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+
+ for (unsigned int c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_cache *cache = socket->shared_caches[c];
+
+ if (cache == NULL)
+ continue;
+
+ stats->alloc_cache_hits += cache->alloc_cache_hits;
+ stats->alloc_cache_misses += cache->alloc_cache_misses;
+ stats->alloc_nomem += cache->alloc_nomem;
+ stats->free_cache_hits += cache->free_cache_hits;
+ stats->free_cache_misses += cache->free_cache_misses;
+ }
+ }
+
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats_shared_class, 24.11)
+int
+rte_fastmem_stats_shared_class(size_t class_size_arg,
+ struct rte_fastmem_lcore_class_stats *stats)
+{
+ unsigned int c;
+
+ if (stats == NULL)
+ return -EINVAL;
+ if (!fastmem_assure())
+ return -ENODEV;
+
+ c = exact_class_idx(class_size_arg);
+ if (c >= FASTMEM_N_CLASSES)
+ return -EINVAL;
+
+ *stats = (struct rte_fastmem_lcore_class_stats){0};
+ stats->class_size = class_size(c);
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_cache *cache =
+ fastmem->sockets[s].shared_caches[c];
+
+ if (cache == NULL)
+ continue;
+
+ stats->alloc_cache_hits += cache->alloc_cache_hits;
+ stats->alloc_cache_misses += cache->alloc_cache_misses;
+ stats->alloc_nomem += cache->alloc_nomem;
+ stats->free_cache_hits += cache->free_cache_hits;
+ stats->free_cache_misses += cache->free_cache_misses;
+ }
+
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_stats_reset, 24.11)
+void
+rte_fastmem_stats_reset(void)
+{
+ if (fastmem == NULL)
+ return;
+
+ for (unsigned int s = 0; s < RTE_MAX_NUMA_NODES; s++) {
+ struct fastmem_socket_state *socket = &fastmem->sockets[s];
+
+ for (unsigned int c = 0; c < FASTMEM_N_CLASSES; c++) {
+ struct fastmem_bin *bin = &socket->bins[c];
+
+ rte_spinlock_lock(&bin->lock);
+ bin->slab_acquires = 0;
+ bin->slab_releases = 0;
+ bin->nocache_allocs = 0;
+ bin->nocache_frees = 0;
+ bin->nocache_nomem = 0;
+ rte_spinlock_unlock(&bin->lock);
+
+ for (unsigned int l = 0; l < RTE_MAX_LCORE; l++) {
+ struct fastmem_cache *cache =
+ socket->caches[l][c];
+ if (cache == NULL)
+ continue;
+ cache->alloc_cache_hits = 0;
+ cache->alloc_cache_misses = 0;
+ cache->alloc_nomem = 0;
+ cache->free_cache_hits = 0;
+ cache->free_cache_misses = 0;
+ }
+
+ rte_spinlock_lock(&socket->shared_cache_lock);
+ struct fastmem_cache *shared = socket->shared_caches[c];
+ if (shared != NULL) {
+ shared->alloc_cache_hits = 0;
+ shared->alloc_cache_misses = 0;
+ shared->alloc_nomem = 0;
+ shared->free_cache_hits = 0;
+ shared->free_cache_misses = 0;
+ }
+ rte_spinlock_unlock(&socket->shared_cache_lock);
+ }
+ }
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_fastmem_classes, 24.11)
+unsigned int
+rte_fastmem_classes(size_t *sizes)
+{
+ if (sizes != NULL)
+ for (unsigned int i = 0; i < FASTMEM_N_CLASSES; i++)
+ sizes[i] = class_size(i);
+ return FASTMEM_N_CLASSES;
+}
diff --git a/lib/fastmem/rte_fastmem.h b/lib/fastmem/rte_fastmem.h
new file mode 100644
index 0000000000..8526d2a001
--- /dev/null
+++ b/lib/fastmem/rte_fastmem.h
@@ -0,0 +1,908 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 Ericsson AB
+ */
+
+#ifndef _RTE_FASTMEM_H_
+#define _RTE_FASTMEM_H_
+
+/**
+ * @file
+ *
+ * RTE Fastmem
+ *
+ * @warning
+ * @b EXPERIMENTAL:
+ * All functions in this file may be changed or removed without prior notice.
+ *
+ * The fastmem library is a fast, general-purpose small-object
+ * allocator for DPDK applications. It is intended to allow an
+ * application to replace its many per-type mempools — each sized
+ * for a single object type (a connection, a session, a work item,
+ * a timer, etc.) — with a single allocator that handles arbitrary
+ * object sizes, grows on demand, and offers mempool-level
+ * performance for the common allocation and free paths.
+ *
+ * Like mempool, fastmem is backed by huge pages, is NUMA-aware,
+ * supports bulk operations, and uses per-lcore caches to reduce
+ * shared-state contention. Unlike mempool, it does not require the
+ * caller to declare object sizes or counts up front.
+ *
+ * There is a single, global fastmem instance per process. The
+ * instance is brought up with rte_fastmem_init() and torn down with
+ * rte_fastmem_deinit(). Allocations are made with
+ * rte_fastmem_alloc() and freed with rte_fastmem_free().
+ *
+ * The allocator is bounded to small-object allocations. Requests
+ * larger than rte_fastmem_max_size() are rejected; callers with
+ * such needs should use rte_malloc() directly.
+ *
+ * Backing memory is reserved from DPDK memzones. Once reserved,
+ * backing memory is not returned to the system during the
+ * allocator's lifetime. Callers that need predictable latency may
+ * pre-reserve backing memory up front using rte_fastmem_reserve(),
+ * avoiding memzone-reservation overhead during steady-state
+ * operation.
+ *
+ * Alignment argument, @c align:
+ * If non-zero, @c align specifies an exact minimum alignment and
+ * must be a power of 2. If zero, the default alignment is
+ * @c RTE_CACHE_LINE_SIZE, so that objects obtained from distinct
+ * calls cannot false-share a cache line.
+ *
+ * Threads and caches:
+ * Only threads with an lcore id running in the primary process
+ * get a private per-lcore cache, which makes their common path
+ * lock-free. Every other caller — unregistered non-EAL threads
+ * (which have no lcore id), and all threads in a secondary
+ * process — instead shares a single spinlock-protected cache per
+ * (size class, socket). These callers still benefit from caching,
+ * but pay for the shared lock and so cost more per call than a
+ * private-cache thread.
+ *
+ * Non-preemptible caller:
+ * Callers should not be preemptible while inside a fastmem call.
+ * Fastmem uses internal spinlocks; if a caller is preempted
+ * while holding one, any other thread that subsequently needs
+ * the same lock stalls until the preempted caller resumes.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <rte_bitops.h>
+#include <rte_common.h>
+#include <rte_compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Flag for rte_fastmem_alloc() and its variants: initialize the
+ * returned memory to zero before returning it to the caller.
+ */
+#define RTE_FASTMEM_F_ZERO RTE_BIT32(0)
+
+/**
+ * Initialize the fastmem allocator.
+ *
+ * Sets up the library's internal state. Must be called before any
+ * allocation call. Typically called once per process, after
+ * rte_eal_init() and before the application's worker threads begin
+ * making allocations.
+ *
+ * Initialization does not pre-reserve any backing memory; memzones
+ * are reserved lazily as allocations require. An application that
+ * wants to avoid memzone-reservation latency on the allocation
+ * path should follow rte_fastmem_init() with one or more calls to
+ * rte_fastmem_reserve().
+ *
+ * This function is not thread-safe and must not be called
+ * concurrently with any other fastmem function.
+ *
+ * @return
+ * - 0: Success.
+ * - -EBUSY: The allocator is already initialized.
+ * - -ENOMEM: Unable to allocate internal state.
+ */
+__rte_experimental
+int
+rte_fastmem_init(void);
+
+/**
+ * Tear down the fastmem allocator.
+ *
+ * Releases the library's internal state and frees all backing
+ * memzones. After this call, no fastmem allocations or frees may
+ * be made until rte_fastmem_init() is called again.
+ *
+ * The caller is responsible for ensuring that no fastmem-allocated
+ * objects remain in use. Outstanding allocations at deinit time
+ * result in undefined behavior.
+ *
+ * This function is not thread-safe and must not be called
+ * concurrently with any other fastmem function.
+ */
+__rte_experimental
+void
+rte_fastmem_deinit(void);
+
+/**
+ * Pre-reserve backing memory.
+ *
+ * Ensures that at least @p size bytes of memzone-backed memory are
+ * available to the allocator on @p socket_id, reserving additional
+ * memzones from EAL as needed to reach that total. Subsequent
+ * allocations served from the pre-reserved memory do not incur
+ * memzone-reservation cost.
+ *
+ * The reservation is cumulative: repeated calls to
+ * rte_fastmem_reserve() with the same @p socket_id grow the
+ * reservation monotonically. Reserved memory is never returned to
+ * the system during the allocator's lifetime.
+ *
+ * A typical use is to call rte_fastmem_reserve() once at
+ * application startup, with a size chosen to cover the expected
+ * steady-state working set. Allocations and frees during
+ * steady-state operation then avoid memzone reservations entirely.
+ *
+ * @param size
+ * The minimum amount of backing memory, in bytes, to make
+ * available on @p socket_id. The allocator may reserve more than
+ * the requested amount due to internal rounding (e.g., to memzone
+ * or block granularity).
+ *
+ * @param socket_id
+ * The NUMA socket on which to reserve memory, or SOCKET_ID_ANY
+ * to leave the choice to the allocator. With SOCKET_ID_ANY, the
+ * allocator starts on the calling lcore's socket (or the first
+ * configured socket if the caller is not bound to one) and falls
+ * back to other sockets if the preferred socket cannot satisfy
+ * the reservation.
+ *
+ * @return
+ * - 0: Success.
+ * - -ENOMEM: Insufficient huge-page memory to satisfy the request.
+ * - -EINVAL: Invalid @p socket_id.
+ */
+__rte_experimental
+int
+rte_fastmem_reserve(size_t size, int socket_id);
+
+/**
+ * Set the maximum backing memory that may be reserved on a socket.
+ *
+ * Once the limit is reached, allocations that would require new
+ * backing memory on the constrained socket fail with ENOMEM.
+ * Already-reserved memory is not released.
+ *
+ * Setting a limit below the current reserved amount is allowed and
+ * prevents further growth.
+ *
+ * @param socket_id
+ * The NUMA socket to constrain, or SOCKET_ID_ANY to apply the
+ * limit to all sockets.
+ * @param max_bytes
+ * Maximum backing memory in bytes, or SIZE_MAX for unlimited (the default).
+ * @return
+ * - 0: Success.
+ * - -EINVAL: Fastmem not initialized, or invalid @p socket_id.
+ */
+__rte_experimental
+int
+rte_fastmem_set_limit(int socket_id, size_t max_bytes);
+
+/**
+ * Get the maximum backing memory limit for a socket.
+ *
+ * @param socket_id
+ * The NUMA socket to query.
+ * @return
+ * The limit in bytes, or SIZE_MAX if unlimited.
+ */
+__rte_experimental
+size_t
+rte_fastmem_get_limit(int socket_id);
+
+/**
+ * Retrieve the largest allocation size the allocator supports.
+ *
+ * Requests larger than this size are rejected by the allocation
+ * functions. The returned value is a property of the allocator
+ * implementation and does not change across the lifetime of the
+ * process.
+ *
+ * @return
+ * The largest supported allocation size, in bytes.
+ */
+__rte_experimental
+size_t
+rte_fastmem_max_size(void);
+
+/* Forward declaration for __rte_dealloc attribute. */
+void rte_fastmem_free(void *ptr);
+
+/**
+ * Allocate an object from the fastmem allocator.
+ *
+ * Allocates at least @p size bytes, aligned to at least @p align
+ * bytes. The returned memory is backed by huge pages and is
+ * DMA-usable; its IOVA can be obtained via rte_fastmem_virt2iova().
+ *
+ * On NUMA systems, the memory is allocated on the socket of the
+ * calling lcore. Use rte_fastmem_alloc_socket() to target a
+ * specific socket.
+ *
+ * The allocated memory must be freed with rte_fastmem_free(). An
+ * allocation may be freed from any lcore, not only the lcore that
+ * made the allocation.
+ *
+ * This function is MT-safe.
+ *
+ * @param size
+ * Requested allocation size, in bytes. Must not exceed
+ * rte_fastmem_max_size().
+ *
+ * @param align
+ * If 0, the returned pointer will be aligned to at least
+ * @c RTE_CACHE_LINE_SIZE. Otherwise, the returned pointer will
+ * be aligned on a multiple of @p align, which must be a power of
+ * 2.
+ *
+ * @param flags
+ * A bitwise OR of zero or more RTE_FASTMEM_F_* flags. Use
+ * RTE_FASTMEM_F_ZERO to obtain zero-initialized memory.
+ *
+ * @return
+ * - A pointer to the allocated object on success.
+ * - NULL on failure, with @c rte_errno set:
+ * - E2BIG: @p size exceeds rte_fastmem_max_size().
+ * - EINVAL: Invalid @p align (not a power of two).
+ * - ENOMEM: Allocation could not be served from existing
+ * backing memory and no additional memzone could be reserved.
+ */
+__rte_experimental
+void *
+rte_fastmem_alloc(size_t size, size_t align, unsigned int flags)
+ __rte_malloc __rte_dealloc(rte_fastmem_free, 1);
+
+/**
+ * Allocate an object on a specific NUMA socket.
+ *
+ * Like rte_fastmem_alloc(), but targets the specified NUMA socket
+ * rather than the socket of the calling lcore. Use this variant
+ * when the lifetime or access pattern of the allocation is not
+ * tied to the calling lcore's socket.
+ *
+ * This function is MT-safe.
+ *
+ * @param size
+ * Requested allocation size, in bytes. Must not exceed
+ * rte_fastmem_max_size().
+ *
+ * @param align
+ * If 0, the returned pointer will be aligned to at least
+ * @c RTE_CACHE_LINE_SIZE. Otherwise, the returned pointer will
+ * be aligned on a multiple of @p align, which must be a power of
+ * 2.
+ *
+ * @param flags
+ * A bitwise OR of zero or more RTE_FASTMEM_F_* flags.
+ *
+ * @param socket_id
+ * The NUMA socket on which to allocate, or SOCKET_ID_ANY to
+ * leave the choice to the allocator. With SOCKET_ID_ANY, the
+ * allocator starts on the calling lcore's socket (or the first
+ * configured socket if the caller is not bound to one) and falls
+ * back to other sockets if the preferred socket cannot satisfy
+ * the request.
+ *
+ * @return
+ * - A pointer to the allocated object on success.
+ * - NULL on failure, with @c rte_errno set (see rte_fastmem_alloc()).
+ */
+__rte_experimental
+void *
+rte_fastmem_alloc_socket(size_t size, size_t align, unsigned int flags,
+ int socket_id)
+ __rte_malloc __rte_dealloc(rte_fastmem_free, 1);
+
+/**
+ * Resize a fastmem allocation, preserving existing contents.
+ *
+ * If @p ptr is NULL, equivalent to rte_fastmem_alloc(size, align, 0).
+ * If @p size is 0, frees @p ptr and returns NULL.
+ *
+ * If the existing allocation can already satisfy the new size and
+ * alignment, the original pointer may be returned unchanged.
+ * Otherwise, a new allocation is made, the contents are copied
+ * (up to the minimum of old and new sizes), and the old allocation
+ * is freed.
+ *
+ * This function is MT-safe.
+ *
+ * @param ptr
+ * Pointer to an existing fastmem allocation, or NULL.
+ *
+ * @param size
+ * New requested size in bytes. If 0, the allocation is freed.
+ *
+ * @param align
+ * If 0, alignment is at least @c RTE_CACHE_LINE_SIZE. Otherwise,
+ * must be a power of 2.
+ *
+ * @return
+ * - A pointer to the resized allocation on success.
+ * - NULL on failure, with @c rte_errno set:
+ * - E2BIG: @p size exceeds rte_fastmem_max_size().
+ * - EINVAL: Invalid @p align.
+ * - ENOMEM: Allocation could not be served.
+ * On failure, the original allocation at @p ptr remains valid.
+ */
+__rte_experimental
+void *
+rte_fastmem_realloc(void *ptr, size_t size, size_t align)
+ __rte_dealloc(rte_fastmem_free, 1);
+
+/**
+ * Free an object previously allocated by the fastmem allocator.
+ *
+ * @p ptr must have been returned by a prior call to any fastmem
+ * allocation function, or be NULL. If @p ptr is NULL, no operation
+ * is performed.
+ *
+ * Free may be called from any lcore, regardless of which lcore
+ * made the original allocation.
+ *
+ * This function is MT-safe.
+ *
+ * @param ptr
+ * Pointer to an object previously allocated by fastmem, or NULL.
+ */
+__rte_experimental
+void
+rte_fastmem_free(void *ptr);
+
+/**
+ * Allocate multiple objects in bulk.
+ *
+ * Allocates @p n objects, each of size at least @p size and aligned
+ * to at least @p align bytes, and stores the resulting pointers
+ * into @p ptrs. All @p n objects have the same size and alignment.
+ *
+ * On NUMA systems, the memory is allocated on the socket of the
+ * calling lcore. Use rte_fastmem_alloc_bulk_socket() to target a
+ * specific socket.
+ *
+ * The bulk path amortizes per-object overhead and is typically
+ * faster than @p n individual calls to rte_fastmem_alloc().
+ *
+ * On failure no objects are allocated and @p ptrs is left
+ * untouched.
+ *
+ * This function is MT-safe.
+ *
+ * @param ptrs
+ * An array of at least @p n pointers into which the newly
+ * allocated object pointers are written.
+ *
+ * @param n
+ * The number of objects to allocate.
+ *
+ * @param size
+ * Requested size of each object, in bytes. Must not exceed
+ * rte_fastmem_max_size().
+ *
+ * @param align
+ * If 0, returned pointers will be aligned to at least
+ * @c RTE_CACHE_LINE_SIZE. Otherwise, returned pointers will be
+ * aligned on a multiple of @p align, which must be a power of 2.
+ *
+ * @param flags
+ * A bitwise OR of zero or more RTE_FASTMEM_F_* flags.
+ *
+ * @return
+ * - 0: All @p n objects were allocated and stored in @p ptrs.
+ * - -E2BIG: @p size exceeds rte_fastmem_max_size().
+ * - -EINVAL: Invalid @p align.
+ * - -ENOMEM: Not enough objects could be allocated to fill the
+ * request.
+ */
+__rte_experimental
+int
+rte_fastmem_alloc_bulk(void **ptrs, unsigned int n, size_t size, size_t align,
+ unsigned int flags);
+
+/**
+ * Allocate multiple objects in bulk on a specific NUMA socket.
+ *
+ * Like rte_fastmem_alloc_bulk(), but targets the specified NUMA
+ * socket rather than the socket of the calling lcore.
+ *
+ * This function is MT-safe.
+ *
+ * @param ptrs
+ * An array of at least @p n pointers into which the newly
+ * allocated object pointers are written.
+ *
+ * @param n
+ * The number of objects to allocate.
+ *
+ * @param size
+ * Requested size of each object, in bytes. Must not exceed
+ * rte_fastmem_max_size().
+ *
+ * @param align
+ * If 0, returned pointers will be aligned to at least
+ * @c RTE_CACHE_LINE_SIZE. Otherwise, returned pointers will be
+ * aligned on a multiple of @p align, which must be a power of 2.
+ *
+ * @param flags
+ * A bitwise OR of zero or more RTE_FASTMEM_F_* flags.
+ *
+ * @param socket_id
+ * The NUMA socket on which to allocate, or SOCKET_ID_ANY to
+ * leave the choice to the allocator. With SOCKET_ID_ANY, the
+ * allocator starts on the calling lcore's socket (or the first
+ * configured socket if the caller is not bound to one) and falls
+ * back to other sockets if the preferred socket cannot satisfy
+ * the request.
+ *
+ * @return
+ * - 0: All @p n objects were allocated and stored in @p ptrs.
+ * - Negative errno on failure (see rte_fastmem_alloc_bulk()).
+ */
+__rte_experimental
+int
+rte_fastmem_alloc_bulk_socket(void **ptrs, unsigned int n, size_t size,
+ size_t align, unsigned int flags, int socket_id);
+
+/**
+ * Free multiple objects in bulk.
+ *
+ * Frees the @p n objects pointed to by @p ptrs. Each pointer in
+ * the array must have been returned by a prior fastmem allocation
+ * call and must not have been freed. The objects need not have
+ * the same size, alignment, or socket.
+ *
+ * The bulk path amortizes per-object overhead and is typically
+ * faster than @p n individual calls to rte_fastmem_free().
+ *
+ * This function is MT-safe.
+ *
+ * @param ptrs
+ * An array of @p n pointers to fastmem-allocated objects.
+ *
+ * @param n
+ * The number of objects to free.
+ */
+__rte_experimental
+void
+rte_fastmem_free_bulk(void **ptrs, unsigned int n);
+
+/**
+ * Opaque handle encoding a (size class, NUMA socket) pair.
+ *
+ * Obtained via rte_fastmem_hlookup(). Passing a handle to
+ * rte_fastmem_halloc() avoids the per-call size-class
+ * lookup and socket resolution, improving allocation throughput
+ * for fixed-size objects.
+ */
+typedef uint32_t rte_fastmem_handle_t;
+
+/**
+ * Look up a handle for a given object size and NUMA socket.
+ *
+ * The returned handle encodes the size class and socket, and can
+ * be passed to rte_fastmem_halloc() to allocate objects
+ * without repeating the class lookup.
+ *
+ * @param size
+ * Object size in bytes. Must not exceed rte_fastmem_max_size().
+ *
+ * @param align
+ * Alignment requirement (power of two), or 0 for the default
+ * (RTE_CACHE_LINE_SIZE).
+ *
+ * @param socket_id
+ * NUMA socket to allocate from.
+ *
+ * @param[out] handle
+ * On success, set to the resolved handle.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: Invalid alignment or socket_id.
+ * - -E2BIG: @p size exceeds rte_fastmem_max_size().
+ */
+__rte_experimental
+int
+rte_fastmem_hlookup(size_t size, size_t align, int socket_id,
+ rte_fastmem_handle_t *handle);
+
+/**
+ * Allocate an object using a pre-resolved handle.
+ *
+ * Equivalent to rte_fastmem_alloc() but skips the size-class
+ * lookup and socket resolution, using the pre-resolved handle
+ * instead.
+ *
+ * A handle is not tied to the lcore that produced it: it may be
+ * shared across threads and used from any thread, including from
+ * lcores that never called rte_fastmem_hlookup() and from non-EAL
+ * threads. As with rte_fastmem_alloc(), callers without a private
+ * per-lcore cache use the shared cache instead.
+ *
+ * This function is MT-safe.
+ *
+ * @param handle
+ * A handle previously obtained from rte_fastmem_hlookup().
+ *
+ * @param flags
+ * Allocation flags (e.g., RTE_FASTMEM_F_ZERO).
+ *
+ * @return
+ * A pointer to the allocated object, or NULL on failure
+ * (rte_errno is set).
+ */
+__rte_experimental
+void *
+rte_fastmem_halloc(rte_fastmem_handle_t handle, unsigned int flags)
+ __rte_malloc __rte_dealloc(rte_fastmem_free, 1);
+
+/**
+ * Bulk-allocate objects using a pre-resolved handle.
+ *
+ * Equivalent to rte_fastmem_alloc_bulk() but uses a pre-resolved
+ * handle. All-or-nothing semantics apply.
+ *
+ * @param handle
+ * A handle previously obtained from rte_fastmem_hlookup().
+ *
+ * @param[out] ptrs
+ * Array to receive @p n allocated pointers.
+ *
+ * @param n
+ * Number of objects to allocate.
+ *
+ * @param flags
+ * Allocation flags (e.g., RTE_FASTMEM_F_ZERO).
+ *
+ * @return
+ * - 0: All @p n objects allocated successfully.
+ * - -ENOMEM: Allocation failed; no objects were allocated.
+ */
+__rte_experimental
+int
+rte_fastmem_halloc_bulk(rte_fastmem_handle_t handle,
+ void **ptrs, unsigned int n, unsigned int flags);
+
+/**
+ * Free an object using a pre-resolved handle.
+ *
+ * Equivalent to rte_fastmem_free() but skips the slab-header
+ * lookup by using the class and socket encoded in the handle.
+ *
+ * Like rte_fastmem_halloc(), this may be called from any thread,
+ * regardless of which thread produced the handle or the object.
+ *
+ * This function is MT-safe.
+ *
+ * @param handle
+ * A handle previously obtained from rte_fastmem_hlookup().
+ *
+ * @param ptr
+ * A pointer previously returned by a fastmem allocation function.
+ * Must belong to the same size class and socket as @p handle.
+ * NULL is permitted (no-op).
+ */
+__rte_experimental
+void
+rte_fastmem_hfree(rte_fastmem_handle_t handle, void *ptr);
+
+/**
+ * Bulk-free objects using a pre-resolved handle.
+ *
+ * Equivalent to rte_fastmem_free_bulk() but skips per-object
+ * slab-header lookups.
+ *
+ * All objects must belong to the same size class and socket as
+ * @p handle.
+ *
+ * @param handle
+ * A handle previously obtained from rte_fastmem_hlookup().
+ *
+ * @param ptrs
+ * An array of @p n pointers to fastmem-allocated objects.
+ *
+ * @param n
+ * The number of objects to free.
+ */
+__rte_experimental
+void
+rte_fastmem_hfree_bulk(rte_fastmem_handle_t handle,
+ void **ptrs, unsigned int n);
+
+/**
+ * Obtain the IOVA for a fastmem-allocated pointer.
+ *
+ * Translates a virtual address returned by a fastmem allocation
+ * function into the corresponding IOVA, suitable for use in device
+ * DMA descriptors.
+ *
+ * The returned IOVA is valid for the lifetime of the allocation.
+ *
+ * @p ptr must have been returned by a prior fastmem allocation
+ * function. Passing any other pointer results in undefined
+ * behavior.
+ *
+ * @param ptr
+ * A pointer previously returned by a fastmem allocation
+ * function.
+ *
+ * @return
+ * The IOVA corresponding to @p ptr, or ``RTE_BAD_IOVA`` if the
+ * library is not initialized (and could not be attached to).
+ */
+__rte_experimental
+rte_iova_t
+rte_fastmem_virt2iova(const void *ptr);
+
+/**
+ * Flush the calling lcore's per-lcore caches.
+ *
+ * Drains every cached object from the calling lcore's
+ * per-(size class, NUMA socket) caches back to their shared
+ * bins, and releases the cache state itself. A subsequent
+ * allocation or free on this lcore lazily recreates any caches
+ * it needs.
+ *
+ * This is useful in applications that have finished a bursty
+ * phase and want to release memory that would otherwise sit idle
+ * in caches. It is also useful in tests that want to observe
+ * bin-level state without per-lcore caching hiding activity.
+ *
+ * Only private per-lcore caches are flushed. The call has no
+ * effect when invoked from a thread that has no private cache (a
+ * lcore-less thread, or any thread in a secondary process); the
+ * shared cache is never flushed.
+ *
+ * This function is not thread-safe with respect to concurrent
+ * allocations or frees on the calling lcore; call it only when
+ * the calling lcore is not making other fastmem calls.
+ */
+__rte_experimental
+void
+rte_fastmem_cache_flush(void);
+
+/**
+ * Global summary statistics.
+ */
+struct rte_fastmem_stats {
+ uint64_t bytes_backing; /**< Bytes of backing memory (memzones) reserved from EAL. */
+ uint64_t bytes_in_use; /**< Approximate bytes in live objects. */
+ uint64_t alloc_total; /**< Total successful alloc operations (hits + misses). */
+ uint64_t free_total; /**< Total free operations (hits + misses). */
+ uint64_t alloc_nomem; /**< Alloc attempts that failed with ENOMEM. */
+ unsigned int n_classes; /**< Number of size classes. */
+};
+
+/**
+ * Per-size-class statistics (aggregated across all lcores).
+ *
+ * Allocation and free counters count individual objects, not
+ * operations. A bulk allocation of 32 objects that hits the cache
+ * increments alloc_cache_hits by 32.
+ */
+struct rte_fastmem_class_stats {
+ size_t class_size; /**< Usable size of this class (bytes). */
+ uint64_t in_use; /**< Objects currently live (allocs - frees). */
+ uint64_t alloc_cache_hits; /**< Allocs served from a per-lcore cache. */
+ uint64_t alloc_cache_misses; /**< Allocs that triggered a bin refill. */
+ uint64_t alloc_nomem; /**< Alloc attempts that failed with ENOMEM. */
+ uint64_t free_cache_hits; /**< Frees absorbed by a per-lcore cache. */
+ uint64_t free_cache_misses; /**< Frees that triggered a bin drain. */
+ uint64_t slab_acquires; /**< Slabs pulled from the free pool. */
+ uint64_t slab_releases; /**< Slabs returned to the free pool. */
+ uint32_t slabs_partial; /**< Current partial slab count. */
+ uint32_t slabs_full; /**< Current full slab count. */
+};
+
+/**
+ * Per-lcore statistics (aggregated across all classes).
+ *
+ * Covers activity served through the lcore's private per-lcore
+ * cache, which exists only for lcore-id-equipped threads in the
+ * primary process. Allocations and frees made without a private
+ * cache (lcore-less threads, and any thread in a secondary
+ * process) are not attributed to any lcore and so do not appear
+ * here; they are visible in the global and per-class statistics,
+ * and in the shared-cache statistics retrieved with
+ * rte_fastmem_stats_shared().
+ *
+ * This structure is also used to report the shared cache; see
+ * rte_fastmem_stats_shared().
+ */
+struct rte_fastmem_lcore_stats {
+ uint64_t alloc_cache_hits; /**< Allocs served from this lcore's caches. */
+ uint64_t alloc_cache_misses; /**< Allocs that missed this lcore's caches. */
+ uint64_t alloc_nomem; /**< Alloc attempts that failed with ENOMEM. */
+ uint64_t free_cache_hits; /**< Frees absorbed by this lcore's caches. */
+ uint64_t free_cache_misses; /**< Frees that bypassed this lcore's caches. */
+};
+
+/**
+ * Per-lcore, per-class statistics (no aggregation).
+ *
+ * Also used to report the shared cache for a single class; see
+ * rte_fastmem_stats_shared_class().
+ */
+struct rte_fastmem_lcore_class_stats {
+ size_t class_size; /**< Usable size of this class (bytes). */
+ uint64_t alloc_cache_hits; /**< Allocs served from cache. */
+ uint64_t alloc_cache_misses; /**< Allocs that triggered a bin refill. */
+ uint64_t alloc_nomem; /**< Alloc attempts that failed with ENOMEM. */
+ uint64_t free_cache_hits; /**< Frees absorbed by cache. */
+ uint64_t free_cache_misses; /**< Frees that triggered a bin drain. */
+};
+
+/**
+ * Get the number of size classes and optionally their sizes.
+ *
+ * @param[out] sizes
+ * If non-NULL, filled with the size (in bytes) of each class.
+ * The caller must provide space for at least the returned number
+ * of entries.
+ *
+ * @return
+ * The number of size classes.
+ */
+__rte_experimental
+unsigned int
+rte_fastmem_classes(size_t *sizes);
+
+/**
+ * Retrieve global summary statistics.
+ *
+ * @param[out] stats
+ * Structure to fill.
+ *
+ * May be called from a secondary process, which lazily attaches to
+ * the shared state on first use.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: @p stats is NULL.
+ * - -ENODEV: fastmem is not initialized.
+ */
+__rte_experimental
+int
+rte_fastmem_stats(struct rte_fastmem_stats *stats);
+
+/**
+ * Retrieve statistics for a single size class.
+ *
+ * @param class_size
+ * Exact size of the class to query (must match one of the values
+ * returned by rte_fastmem_classes()).
+ * @param[out] stats
+ * Structure to fill.
+ *
+ * May be called from a secondary process, which lazily attaches to
+ * the shared state on first use.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: @p stats is NULL, or @p class_size does not match any
+ * size class.
+ * - -ENODEV: fastmem is not initialized.
+ */
+__rte_experimental
+int
+rte_fastmem_stats_class(size_t class_size,
+ struct rte_fastmem_class_stats *stats);
+
+/**
+ * Retrieve per-lcore statistics (aggregated across all classes).
+ *
+ * @param lcore_id
+ * The lcore to query.
+ * @param[out] stats
+ * Structure to fill.
+ *
+ * May be called from a secondary process, which lazily attaches to
+ * the shared state on first use.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: @p stats is NULL, or @p lcore_id is invalid.
+ * - -ENODEV: fastmem is not initialized.
+ */
+__rte_experimental
+int
+rte_fastmem_stats_lcore(unsigned int lcore_id,
+ struct rte_fastmem_lcore_stats *stats);
+
+/**
+ * Retrieve per-lcore, per-class statistics.
+ *
+ * @param lcore_id
+ * The lcore to query.
+ * @param class_size
+ * Exact size of the class to query.
+ * @param[out] stats
+ * Structure to fill.
+ *
+ * May be called from a secondary process, which lazily attaches to
+ * the shared state on first use.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: @p stats is NULL, @p lcore_id is invalid, or
+ * @p class_size does not match any size class.
+ * - -ENODEV: fastmem is not initialized.
+ */
+__rte_experimental
+int
+rte_fastmem_stats_lcore_class(unsigned int lcore_id, size_t class_size,
+ struct rte_fastmem_lcore_class_stats *stats);
+
+/**
+ * Retrieve statistics for the shared cache (aggregated across all
+ * classes).
+ *
+ * The shared cache serves every caller without a private per-lcore
+ * cache: threads without an lcore id, and all threads in a
+ * secondary process (where private per-lcore caches are never
+ * used). Its activity is reported here rather than under any
+ * single lcore.
+ *
+ * @param[out] stats
+ * Structure to fill. The per-lcore stats layout is reused.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: @p stats is NULL.
+ * - -ENODEV: fastmem is not initialized.
+ */
+__rte_experimental
+int
+rte_fastmem_stats_shared(struct rte_fastmem_lcore_stats *stats);
+
+/**
+ * Retrieve shared-cache statistics for a single size class.
+ *
+ * @param class_size
+ * Exact size of the class to query.
+ * @param[out] stats
+ * Structure to fill. The per-lcore-per-class stats layout is reused.
+ *
+ * @return
+ * - 0: Success.
+ * - -EINVAL: @p stats is NULL, or @p class_size does not match any
+ * size class.
+ * - -ENODEV: fastmem is not initialized.
+ */
+__rte_experimental
+int
+rte_fastmem_stats_shared_class(size_t class_size,
+ struct rte_fastmem_lcore_class_stats *stats);
+
+/**
+ * Reset all statistics counters to zero.
+ *
+ * Zeroes the per-lcore, shared-cache, and per-bin counters. Does
+ * not affect the allocator's operational state.
+ *
+ * For an accurate reset, call when no other threads are
+ * actively allocating or freeing.
+ */
+__rte_experimental
+void
+rte_fastmem_stats_reset(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_FASTMEM_H_ */
diff --git a/lib/meson.build b/lib/meson.build
index 8f5cfd28a5..10906d4d53 100644
--- a/lib/meson.build
+++ b/lib/meson.build
@@ -38,6 +38,7 @@ libraries = [
'distributor',
'dmadev', # eventdev depends on this
'efd',
+ 'fastmem',
'eventdev',
'dispatcher', # dispatcher depends on eventdev
'gpudev',
--
2.43.0
^ permalink raw reply related
* [RFC v4 1/3] doc: add fastmem programming guide
From: Mattias Rönnblom @ 2026-05-30 9:26 UTC (permalink / raw)
To: dev
Cc: Morten Brørup, Konstantin Ananyev, Mattias Rönnblom,
Yogaraj Baskaravel, Stephen Hemminger, Bruce Richardson,
Mattias Rönnblom
In-Reply-To: <20260530092634.46218-1-hofors@lysator.liu.se>
Add a programming guide for the fastmem library covering usage,
API overview, design, and implementation details.
--
RFC v4:
* Document per-lcore statistics surviving cache flush and
bin-direct counters for non-cached traffic.
* Document shared cache for callers without a private cache
(non-EAL threads, secondary processes).
RFC v3:
* Add realloc subsection to Allocation and free section.
Signed-off-by: Mattias Rönnblom <hofors@lysator.liu.se>
---
doc/guides/prog_guide/fastmem_lib.rst | 351 ++++++++++++++++++++++++++
doc/guides/prog_guide/index.rst | 1 +
2 files changed, 352 insertions(+)
create mode 100644 doc/guides/prog_guide/fastmem_lib.rst
diff --git a/doc/guides/prog_guide/fastmem_lib.rst b/doc/guides/prog_guide/fastmem_lib.rst
new file mode 100644
index 0000000000..4d7d69770c
--- /dev/null
+++ b/doc/guides/prog_guide/fastmem_lib.rst
@@ -0,0 +1,351 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright(c) 2026 Ericsson AB
+
+Fastmem Library
+===============
+
+The fastmem library is a fast, general-purpose small-object
+allocator for DPDK applications. It lets an application replace
+its many per-type mempools — each sized for a single object type
+— with a single allocator that handles arbitrary object sizes,
+grows on demand, and offers mempool-level performance for the
+common allocation and free paths.
+
+Like mempool, fastmem is backed by huge pages, is NUMA-aware,
+supports bulk operations, and uses per-lcore caches to reduce
+shared-state contention. Unlike mempool, it does not require the
+caller to declare object sizes or counts up front.
+
+
+When to use fastmem
+-------------------
+
+Use fastmem when:
+
+* Small objects (up to 1 MiB) are allocated and freed on the
+ data path with low, predictable latency requirements.
+
+* Many object types of varying sizes exist and maintaining a
+ separate mempool for each is impractical.
+
+* DMA-usable memory with efficient virtual-to-IOVA translation
+ is needed.
+
+Do not use fastmem for allocations larger than 1 MiB. Use
+``rte_malloc()`` instead.
+
+
+Initialization and teardown
+----------------------------
+
+.. code-block:: c
+
+ /* At startup, after rte_eal_init(). */
+ rte_fastmem_init();
+
+ /* Optional: pre-reserve backing memory to avoid latency
+ * spikes from on-demand memzone reservation. */
+ rte_fastmem_reserve(64 * 1024 * 1024, SOCKET_ID_ANY);
+
+ /* ... application runs ... */
+
+ /* At shutdown, after all allocations have been freed. */
+ rte_fastmem_deinit();
+
+Neither ``rte_fastmem_init()`` nor ``rte_fastmem_deinit()`` is
+thread-safe; call them from the main lcore during startup and
+shutdown.
+
+
+Allocation and free
+-------------------
+
+.. code-block:: c
+
+ void *obj = rte_fastmem_alloc(128, 0, 0);
+ /* Use obj... */
+ rte_fastmem_free(obj);
+
+``rte_fastmem_alloc()`` allocates on the calling lcore's NUMA
+socket. Use ``rte_fastmem_alloc_socket()`` to target a specific
+socket or to enable cross-socket fallback with ``SOCKET_ID_ANY``.
+
+Realloc
+~~~~~~~
+
+.. code-block:: c
+
+ obj = rte_fastmem_realloc(obj, 256, 0);
+
+``rte_fastmem_realloc()`` resizes an allocation, preserving its
+contents. If the existing allocation already satisfies the new
+size, the original pointer may be returned unchanged. Otherwise a
+new allocation is made, contents are copied, and the old
+allocation is freed. On failure, the original allocation remains
+valid.
+
+Alignment
+~~~~~~~~~
+
+When ``align`` is 0, the returned pointer is aligned to at least
+``RTE_CACHE_LINE_SIZE``. A non-zero ``align`` must be a power of
+two. Specifying an alignment smaller than ``RTE_CACHE_LINE_SIZE``
+is permitted but the returned object may then share a cache line
+with an adjacent allocation, risking false sharing.
+
+Zeroing
+~~~~~~~
+
+Pass ``RTE_FASTMEM_F_ZERO`` to receive zero-initialized memory:
+
+.. code-block:: c
+
+ void *obj = rte_fastmem_alloc(256, 0, RTE_FASTMEM_F_ZERO);
+
+
+Bulk allocation and free
+-------------------------
+
+.. code-block:: c
+
+ void *ptrs[32];
+
+ if (rte_fastmem_alloc_bulk(ptrs, 32, 64, 0, 0) < 0)
+ /* handle error */;
+
+ /* Use objects... */
+
+ rte_fastmem_free_bulk(ptrs, 32);
+
+Bulk allocation has all-or-nothing semantics: either all
+requested objects are returned, or none are (and ``rte_errno``
+is set to ``ENOMEM``).
+
+Bulk free is most efficient when all objects belong to the same
+size class; in that case the objects are pushed into the
+caller's cache in a single operation.
+
+
+IOVA translation
+----------------
+
+Memory returned by fastmem is DMA-usable. To obtain the IOVA
+for use in device descriptors:
+
+.. code-block:: c
+
+ rte_iova_t iova = rte_fastmem_virt2iova(obj);
+
+The translation is O(1). The returned IOVA is valid for the
+lifetime of the allocation.
+
+
+NUMA awareness
+--------------
+
+``rte_fastmem_alloc()`` allocates on the calling lcore's socket.
+``rte_fastmem_alloc_socket()`` accepts an explicit socket ID or
+``SOCKET_ID_ANY``:
+
+* Explicit socket: allocate only from that socket; fail with
+ ``ENOMEM`` if exhausted.
+
+* ``SOCKET_ID_ANY``: try the caller's local socket first, then
+ fall back to other sockets.
+
+
+Caches
+------
+
+Only threads with an lcore id running in the **primary** process
+get a private cache per size class. The common allocation and free
+paths operate entirely within this private cache, avoiding locks.
+Cache misses (empty on alloc, full on free) trigger a bulk transfer
+to/from the shared bin under a lock.
+
+Every other caller — unregistered non-EAL threads (which have no
+lcore id), and all threads in a secondary process (which never use
+private caches) — shares a single **shared cache** per (size class,
+socket), protected by a per-socket spinlock. These callers still
+benefit from caching, but pay for the shared lock and so cost more
+per operation than a private-cache thread.
+
+``rte_fastmem_cache_flush()`` drains the calling lcore's private
+caches back to the shared bins. This is useful after bursty phases
+to release idle cached memory. It has no effect on a thread that
+has no private cache.
+
+
+Threading
+---------
+
+All allocation and free functions are thread-safe and may be
+called from any thread. An allocation made on one thread may be
+freed on any other.
+
+Fastmem uses internal spinlocks. A thread preempted while
+holding one delays other threads contending for the same lock
+(correctness is not affected, only latency).
+
+
+Pre-reserving memory
+--------------------
+
+By default, fastmem reserves backing memory lazily on first
+allocation. ``rte_fastmem_reserve(size, socket_id)`` forces
+reservation up front, ensuring subsequent allocations do not
+incur memzone-reservation latency:
+
+.. code-block:: c
+
+ /* Reserve 128 MiB on socket 0. */
+ rte_fastmem_reserve(128 * 1024 * 1024, 0);
+
+Once reserved, backing memory is never returned to the system
+during the allocator's lifetime.
+
+Memory limits
+~~~~~~~~~~~~~
+
+``rte_fastmem_set_limit(socket_id, max_bytes)`` caps how much
+backing memory may be reserved on a given socket. Once the limit is
+reached, allocations that would require new backing memory fail with
+``ENOMEM``. The default is ``SIZE_MAX`` (unlimited).
+``rte_fastmem_get_limit()`` returns the current limit for a socket.
+
+.. code-block:: c
+
+ /* Allow at most 256 MiB on socket 0. */
+ rte_fastmem_set_limit(0, 256 * 1024 * 1024);
+
+ /* Block all growth on socket 1. */
+ rte_fastmem_set_limit(1, 0);
+
+Pass ``SOCKET_ID_ANY`` to apply the same limit to all sockets.
+
+
+Size classes
+------------
+
+Fastmem uses power-of-two size classes from 8 bytes to 1 MiB
+(18 classes). A request for N bytes is served from the smallest
+class >= N. The maximum supported size is queryable via
+``rte_fastmem_max_size()``.
+
+With power-of-two classes, worst-case internal fragmentation is
+just under 50% (e.g., a 33-byte request occupies a 64-byte
+slot). Assuming a uniform distribution of request sizes, the
+average waste is 25%. In practice, DPDK workloads tend to
+cluster at or near powers of two, so typical waste is lower.
+
+Requests exceeding the maximum are rejected with ``E2BIG``.
+
+
+Implementation
+--------------
+
+Fastmem organizes memory in three layers: backing memzones, slabs,
+and caches.
+
+Backing memory and slabs
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Backing memory is obtained from EAL as 128 MiB IOVA-contiguous
+memzones, each aligned to 2 MiB. A memzone is partitioned into
+64 fixed-size, 2 MiB **slabs**. Slabs are the unit of memory
+that moves between size classes: a free slab can be assigned to
+any bin on demand, and an empty slab (all objects freed) returns
+to the free-slab pool for reuse by another size class.
+
+The 2 MiB slab alignment is the key structural property. Given
+any object pointer, the allocator recovers the owning slab by
+masking off the low 21 bits — no radix tree, hash table, or
+memzone lookup is needed. This makes the free path fast: a
+single pointer-mask load reaches the slab header, which
+identifies the size class and bin.
+
+Each slab reserves 64 bytes at offset 0 for its header. The
+remaining space is divided into fixed-size slots equal to the
+size class. Allocated objects carry no per-object metadata; the
+full slot is available to the caller.
+
+Three-level allocation hierarchy
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. **Cache** — a bounded LIFO stack of free object pointers.
+ Allocation pops; free pushes. Lcore-id-equipped primary threads
+ each get a private cache per (lcore, size class, socket), which
+ needs no lock because only the owning lcore touches it. All
+ other callers share one cache per (size class, socket), guarded
+ by a per-socket spinlock.
+
+2. **Bin** — one per (size class, socket). Owns the partial and
+ full slab lists. A spinlock serializes bulk transfers between
+ the bin and the caches. Most traffic is absorbed by the
+ caches, so bin-lock contention is low.
+
+3. **Free-slab pool** — one per socket. A spinlock protects slab
+ acquisition and release. These events are rare relative to
+ object-level operations (a single small-object slab serves
+ thousands of allocations).
+
+On a cache miss (empty on alloc, full on free), the cache
+exchanges objects with the bin in bulk, targeting half-full to
+maximize headroom in both directions.
+
+Cache sizing
+~~~~~~~~~~~~
+
+Cache capacity varies by size class to bound per-cache memory
+footprint:
+
+* Classes 8 B through 4 KiB: capacity 64.
+* Larger classes: capacity halves per class (32, 16, 8, 4),
+ flooring at 4.
+
+Even the largest classes remain cached. The capacity curve
+ensures that small, frequent allocations get the highest cache
+hit rate, while large allocations still avoid the bin lock on
+most operations. The shared cache uses the same capacities.
+
+
+Statistics
+----------
+
+Fastmem maintains always-on counters that track allocation and
+free activity. Statistics are queryable at several levels of
+granularity: global summary, per size class, per lcore, per lcore
+per class, and for the shared cache (with
+``rte_fastmem_stats_shared()`` and
+``rte_fastmem_stats_shared_class()``).
+
+Counters are stored independently of the caches, so they survive
+``rte_fastmem_cache_flush()`` and persist until an explicit
+``rte_fastmem_stats_reset()``.
+
+Allocations and frees made without a private per-lcore cache — by
+lcore-less threads and by all threads in a secondary process — go
+through the shared cache. They cannot be attributed to an lcore, so
+they do not appear in the per-lcore or per-lcore-per-class views,
+but they are counted in the global and per-class statistics and
+reported by the shared-cache statistics functions.
+
+``rte_fastmem_classes()`` returns the number of size classes and
+optionally fills an array with their sizes.
+
+See ``rte_fastmem.h`` for the full statistics API.
+
+
+Secondary Processes
+-------------------
+
+Fastmem works transparently in DPDK secondary processes. The shared
+state is discovered automatically on first allocation.
+
+Secondary processes do not use private per-lcore caches, even for
+their lcore-id-equipped threads; all of their traffic goes through
+the shared cache (the same one used by lcore-less primary threads).
+This is acceptable for control-plane secondaries with low allocation
+rates. The primary process should pre-reserve sufficient backing
+memory with ``rte_fastmem_reserve()`` since secondaries cannot grow
+the pool.
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index e6f24945b0..c85196c85e 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -28,6 +28,7 @@ Memory Management
mempool_lib
mbuf_lib
multi_proc_support
+ fastmem_lib
CPU Management
--
2.43.0
^ permalink raw reply related
* [RFC v4 0/3] lib/fastmem: fast small-object allocator
From: Mattias Rönnblom @ 2026-05-30 9:26 UTC (permalink / raw)
To: dev
Cc: Morten Brørup, Konstantin Ananyev, Mattias Rönnblom,
Yogaraj Baskaravel, Stephen Hemminger, Bruce Richardson,
Mattias Rönnblom
In-Reply-To: <20260527173042.93867-2-hofors@lysator.liu.se>
This RFC introduces fastmem, a general-purpose small-object allocator
for DPDK. It is intended to replace per-type mempools with a single
allocator that handles arbitrary sizes, grows on demand, and matches
mempool-level performance on the hot path.
Motivation
----------
DPDK applications commonly maintain many mempools — one per object
type (connections, sessions, timers, work items). Each must be sized
up front, wastes memory when over-provisioned, and cannot serve
objects of a different size. Fastmem eliminates this by accepting
arbitrary sizes at runtime, backed by a slab allocator that
repurposes memory across size classes as demand shifts.
Design
------
Three-layer architecture:
1. Backing memory: 128 MiB IOVA-contiguous memzones from EAL,
reserved lazily (or pre-reserved for deterministic latency).
2. Slabs: 2 MiB, 2 MiB-aligned regions carved from memzones.
The alignment enables O(1) slab lookup from any object pointer
via bitmask — no radix tree or index structure. Slabs move
freely between 18 power-of-2 size classes (8 B to 1 MiB).
3. Per-lcore caches: bounded LIFO stacks (no locks on the hot
path). Cache misses trigger bulk transfers to/from the shared
bin under a spinlock.
Key properties:
- Zero per-object metadata in the production build.
- NUMA-aware, with per-socket bins and free-slab pools.
- DMA-usable memory with O(1) virt-to-IOVA translation.
- Bulk alloc/free with all-or-nothing semantics.
- Backing memory never returned during lifetime (slabs recycled).
- Non-EAL threads supported (bypass cache, take bin lock).
- Secondary process support (lazy attach, no per-lcore caches).
API surface
-----------
rte_fastmem_init / deinit
rte_fastmem_reserve
rte_fastmem_set_limit / get_limit
rte_fastmem_alloc / alloc_socket
rte_fastmem_realloc
rte_fastmem_alloc_bulk / alloc_bulk_socket
rte_fastmem_free / free_bulk
rte_fastmem_hlookup / halloc / halloc_bulk / hfree / hfree_bulk
rte_fastmem_virt2iova
rte_fastmem_cache_flush
rte_fastmem_max_size / classes
rte_fastmem_stats / stats_class / stats_lcore / stats_lcore_class
rte_fastmem_stats_reset
All APIs are marked __rte_experimental.
Performance
-----------
The single-object hot path is roughly 2–3× the cost of mempool
and an order of magnitude faster than rte_malloc. Under
multi-lcore contention, fastmem scales similarly to mempool,
while rte_malloc collapses.
Limitations
-----------
- Maximum allocation: 1 MiB. Larger requests should use rte_malloc.
- Power-of-2 classes only; worst-case internal fragmentation ~50%.
- Backing memory not reclaimable short of deinit.
Future work
-----------
- Lcore-affine allocations (false-sharing-free by construction).
- Mempool ops driver for transparent drop-in use.
- Debug mode (cookies, double-free detection, poison-on-free).
- Telemetry integration.
- EAL integration, allowing EAL-internal subsystems to use
fastmem for their small-object allocations.
Changes in RFC v4:
- Fix crash in halloc/hfree on lcores without hlookup: fall back
to shared bin on NULL cache.
- Keep per-lcore statistics across rte_fastmem_cache_flush().
- Guard free and IOVA paths against uninitialized state.
- Lazy-attach stats readers in secondary processes; distinguish
-ENODEV from -EINVAL.
- Protect bin statistics with the bin lock.
- Trim verbose comments.
- Add shared cache for callers without a private cache (non-EAL
threads, secondary processes). Add rte_fastmem_stats_shared()
and rte_fastmem_stats_shared_class().
- Document rte_fastmem_stats_reset() quiescence requirement.
- Add tests for handle alloc/free from uncached lcores, stats
survival across flush, and shared-cache statistics.
- Update programming guide (shared cache, stats sections).
Changes in RFC v3:
- Add rte_fastmem_realloc() with full test coverage.
- Add __rte_malloc/__rte_dealloc compiler attributes; remove
incorrect __rte_alloc_size/__rte_alloc_align.
- Extract normalize_align() helper; remove redundant inline
directives.
- Merge lifecycle and functional test suites.
- Add realloc subsection to programming guide.
Changes in RFC v2:
- Fix cross-socket deinit use-after-free.
- Add secondary process support.
- Add handle-based allocation API.
- Fix clang warnings; misc cleanup.
Mattias Rönnblom (3):
doc: add fastmem programming guide
lib: add fastmem library
app/test: add fastmem test suite
doc/guides/prog_guide/fastmem_lib.rst | ...
lib/fastmem/ | ...
app/test/test_fastmem*.c | ...
Mattias Rönnblom (3):
doc: add fastmem programming guide
lib: add fastmem library
app/test: add fastmem test suite
app/test/meson.build | 3 +
app/test/test_fastmem.c | 2111 ++++++++++++++++++++++++
app/test/test_fastmem_perf.c | 1040 ++++++++++++
app/test/test_fastmem_profile.c | 157 ++
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf.in | 1 +
doc/guides/prog_guide/fastmem_lib.rst | 351 ++++
doc/guides/prog_guide/index.rst | 1 +
lib/fastmem/meson.build | 6 +
lib/fastmem/rfc-cover-letter.txt | 128 ++
lib/fastmem/rte_fastmem.c | 2123 +++++++++++++++++++++++++
lib/fastmem/rte_fastmem.h | 908 +++++++++++
lib/meson.build | 1 +
13 files changed, 6831 insertions(+)
create mode 100644 app/test/test_fastmem.c
create mode 100644 app/test/test_fastmem_perf.c
create mode 100644 app/test/test_fastmem_profile.c
create mode 100644 doc/guides/prog_guide/fastmem_lib.rst
create mode 100644 lib/fastmem/meson.build
create mode 100644 lib/fastmem/rfc-cover-letter.txt
create mode 100644 lib/fastmem/rte_fastmem.c
create mode 100644 lib/fastmem/rte_fastmem.h
--
2.43.0
^ permalink raw reply
* [PATCH v5 25/25] bus: add class device conversion macro
From: David Marchand @ 2026-05-30 7:51 UTC (permalink / raw)
To: dev
Cc: thomas, stephen, bruce.richardson, Nicolas Chautru, Parav Pandit,
Xueming Li, Nipun Gupta, Nikhil Agarwal, Chenbo Xia, Ashish Gupta,
Fan Zhang, Ankur Dwivedi, Anoob Joseph, Tejasree Kondoj,
Gagandeep Singh, Hemant Agrawal, Pavan Nikhilesh, Shijith Thotton,
Tirthendu Sarkar, Jerin Jacob, Shepard Siegel, Ed Czeck,
John Miller, Igor Russkikh, Steven Webster, Matt Peters,
Selwin Sebastian, Julien Aube, Kishore Padmanabha, Ajit Khaparde,
Nithin Dabilpuram, Kiran Kumar K, Sunil Kumar Kori, Satha Rao,
Harman Kalra, Potnuri Bharat Teja, Sachin Saxena, Shai Brandes,
Evgeny Schemeilin, Ron Beider, Amit Bernstein, Wajeeh Atrash,
Vanshika Shukla, John Daley, Hyong Youb Kim, Jeroen de Borst,
Joshua Washington, Xiaoyun Wang, Feifei Wang, Xingui Yang,
Chengwen Feng, Praveen Shetty, Vladimir Medvedkin,
Anatoly Burakov, Jingjing Wu, Rosen Xu, Dimon Zhao, Leon Yu,
Sam Chen, Long Li, Wei Hu, Chaoyong He, Jiawen Wu, Zaiyu Wang,
Vamsi Attunuru, Devendra Singh Rawat, Alok Prasad, Howard Wang,
Chunhao Lin, Xing Wang, Javen Xu, Wenbo Cao, Andrew Rybchenko,
Jie Liu, Maciej Czekaj, Maxime Coquelin, Jochen Behrens,
Renyong Wan, Na Na, Rong Qian, Xiaoxiong Zhang, Dongwei Xu,
Junlong Wang, Ming Ran
In-Reply-To: <20260530075201.869606-1-david.marchand@redhat.com>
Add a new helper macro RTE_CLASS_TO_BUS_DEVICE that provides a unified
way to convert from any device class (ethdev, cryptodev, eventdev, etc.)
to a bus-specific device type. This macro works with any device class
that has a 'device' field pointing to struct rte_device.
Remove the bus-specific ethdev convenience macros (RTE_ETH_DEV_TO_PCI,
RTE_ETH_DEV_TO_AUXILIARY, RTE_ETH_DEV_TO_VDEV) and replace all uses
with the generic RTE_CLASS_TO_BUS_DEVICE macro.
Convert all drivers to use RTE_CLASS_TO_BUS_DEVICE instead of
the pattern RTE_BUS_DEVICE(dev->device).
Simplify code that was using an intermediate struct rte_device pointer
by applying RTE_CLASS_TO_BUS_DEVICE directly on the device class pointer.
This reduces code duplication and provides a consistent interface that
can be used for all device classes.
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
Changes since v4:
- fixed conflict on net/ixgbe,
Changes since v3:
- updated sxe2 drivers,
---
drivers/baseband/acc/rte_acc100_pmd.c | 4 +-
drivers/baseband/acc/rte_vrb_pmd.c | 2 +-
.../fpga_5gnr_fec/rte_fpga_5gnr_fec.c | 4 +-
drivers/baseband/fpga_lte_fec/fpga_lte_fec.c | 2 +-
drivers/bus/auxiliary/bus_auxiliary_driver.h | 3 --
drivers/bus/cdx/bus_cdx_driver.h | 2 -
drivers/bus/pci/bus_pci_driver.h | 3 --
drivers/bus/vdev/bus_vdev_driver.h | 3 --
drivers/compress/octeontx/otx_zip.c | 2 +-
drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 2 +-
drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c | 3 +-
drivers/crypto/octeontx/otx_cryptodev_ops.c | 4 +-
drivers/event/cnxk/cnxk_eventdev.c | 2 +-
drivers/event/dlb2/pf/dlb2_pf.c | 2 +-
drivers/event/skeleton/skeleton_eventdev.c | 2 +-
drivers/net/ark/ark_ethdev.c | 2 +-
drivers/net/atlantic/atl_ethdev.c | 12 +++---
drivers/net/avp/avp_ethdev.c | 22 +++++-----
drivers/net/axgbe/axgbe_ethdev.c | 4 +-
drivers/net/bnx2x/bnx2x_ethdev.c | 2 +-
drivers/net/bnxt/bnxt_ethdev.c | 12 +++---
drivers/net/bnxt/tf_ulp/bnxt_ulp.c | 4 +-
drivers/net/cnxk/cnxk_ethdev.c | 2 +-
drivers/net/cnxk/cnxk_ethdev_ops.c | 2 +-
drivers/net/cxgbe/cxgbe_ethdev.c | 4 +-
drivers/net/cxgbe/cxgbevf_ethdev.c | 4 +-
drivers/net/dpaa/dpaa_ethdev.c | 16 +++-----
drivers/net/dpaa2/dpaa2_ethdev.c | 8 ++--
drivers/net/dpaa2/dpaa2_recycle.c | 6 +--
drivers/net/ena/ena_ethdev.c | 10 ++---
drivers/net/enetc/enetc4_ethdev.c | 4 +-
drivers/net/enetc/enetc4_vf.c | 4 +-
drivers/net/enetc/enetc_ethdev.c | 2 +-
drivers/net/enic/enic_ethdev.c | 4 +-
drivers/net/enic/enic_fm_flow.c | 6 +--
drivers/net/enic/enic_vf_representor.c | 2 +-
drivers/net/gve/gve_ethdev.c | 2 +-
drivers/net/hinic/hinic_pmd_ethdev.c | 8 ++--
drivers/net/hinic3/base/hinic3_hwdev.c | 7 ++--
drivers/net/hinic3/hinic3_ethdev.c | 16 ++++----
drivers/net/hns3/hns3_cmd.c | 2 +-
drivers/net/hns3/hns3_common.c | 8 ++--
drivers/net/hns3/hns3_ethdev.c | 6 +--
drivers/net/hns3/hns3_ethdev_vf.c | 6 +--
drivers/net/hns3/hns3_rxtx.c | 4 +-
drivers/net/intel/cpfl/cpfl_ethdev.c | 4 +-
drivers/net/intel/cpfl/cpfl_ethdev.h | 2 +-
drivers/net/intel/e1000/em_ethdev.c | 12 +++---
drivers/net/intel/e1000/em_rxtx.c | 2 +-
drivers/net/intel/e1000/igb_ethdev.c | 30 +++++++-------
drivers/net/intel/e1000/igb_pf.c | 2 +-
drivers/net/intel/e1000/igc_ethdev.c | 22 +++++-----
drivers/net/intel/fm10k/fm10k_ethdev.c | 16 ++++----
drivers/net/intel/i40e/i40e_ethdev.c | 28 ++++++-------
drivers/net/intel/i40e/i40e_ethdev.h | 2 +-
drivers/net/intel/iavf/iavf_ethdev.c | 8 ++--
drivers/net/intel/ice/ice_dcf.c | 6 +--
drivers/net/intel/ice/ice_ethdev.c | 6 +--
drivers/net/intel/ice/ice_ethdev.h | 2 +-
drivers/net/intel/idpf/idpf_ethdev.h | 2 +-
drivers/net/intel/ipn3ke/ipn3ke_ethdev.h | 3 --
drivers/net/intel/ipn3ke/ipn3ke_representor.c | 6 +--
drivers/net/intel/ixgbe/ixgbe_ethdev.c | 40 +++++++++----------
drivers/net/intel/ixgbe/ixgbe_pf.c | 2 +-
drivers/net/intel/ixgbe/ixgbe_tm.c | 2 +-
.../net/intel/ixgbe/ixgbe_vf_representor.c | 2 +-
drivers/net/intel/ixgbe/rte_pmd_ixgbe.c | 20 +++++-----
drivers/net/nbl/nbl_core.c | 2 +-
drivers/net/nbl/nbl_dev/nbl_dev.c | 6 +--
drivers/net/netvsc/hn_ethdev.c | 3 +-
drivers/net/nfp/nfp_ethdev.c | 8 ++--
drivers/net/nfp/nfp_ethdev_vf.c | 6 +--
drivers/net/nfp/nfp_net_common.c | 8 ++--
drivers/net/ngbe/ngbe_ethdev.c | 20 +++++-----
drivers/net/ngbe/ngbe_ethdev_vf.c | 16 ++++----
drivers/net/ngbe/ngbe_pf.c | 2 +-
drivers/net/octeon_ep/otx_ep_ethdev.c | 2 +-
drivers/net/octeon_ep/otx_ep_mbox.c | 6 +--
drivers/net/qede/qede_ethdev.c | 6 +--
drivers/net/r8169/r8169_ethdev.c | 6 +--
drivers/net/rnp/rnp_ethdev.c | 6 +--
drivers/net/sfc/sfc.c | 4 +-
drivers/net/sfc/sfc_ethdev.c | 2 +-
drivers/net/sfc/sfc_intr.c | 10 ++---
drivers/net/sfc/sfc_rx.c | 3 +-
drivers/net/sfc/sfc_sriov.c | 2 +-
drivers/net/sfc/sfc_tx.c | 3 +-
drivers/net/sxe2/sxe2_ethdev.c | 4 +-
drivers/net/thunderx/nicvf_ethdev.c | 4 +-
drivers/net/txgbe/txgbe_ethdev.c | 26 ++++++------
drivers/net/txgbe/txgbe_ethdev_vf.c | 16 ++++----
drivers/net/txgbe/txgbe_flow.c | 4 +-
drivers/net/txgbe/txgbe_pf.c | 2 +-
drivers/net/txgbe/txgbe_tm.c | 2 +-
drivers/net/virtio/virtio_pci_ethdev.c | 11 ++---
drivers/net/vmxnet3/vmxnet3_ethdev.c | 4 +-
drivers/net/xsc/xsc_ethdev.c | 2 +-
drivers/net/zxdh/zxdh_ethdev.c | 8 ++--
drivers/raw/ifpga/afu_pmd_n3000.c | 4 +-
lib/eal/include/bus_driver.h | 18 +++++++++
100 files changed, 336 insertions(+), 340 deletions(-)
diff --git a/drivers/baseband/acc/rte_acc100_pmd.c b/drivers/baseband/acc/rte_acc100_pmd.c
index 061f595a98..cbcacc7aa3 100644
--- a/drivers/baseband/acc/rte_acc100_pmd.c
+++ b/drivers/baseband/acc/rte_acc100_pmd.c
@@ -3993,7 +3993,7 @@ acc100_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
static void
acc100_bbdev_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
dev->dev_ops = &acc100_bbdev_ops;
dev->enqueue_enc_ops = acc100_enqueue_enc;
@@ -4646,7 +4646,7 @@ rte_acc_configure(const char *dev_name, struct rte_acc_conf *conf)
dev_name);
return -ENODEV;
}
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(bbdev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(bbdev, *pci_dev);
rte_bbdev_log(INFO, "Configure dev id %x", pci_dev->id.device_id);
if (pci_dev->id.device_id == ACC100_PF_DEVICE_ID)
return acc100_configure(dev_name, conf);
diff --git a/drivers/baseband/acc/rte_vrb_pmd.c b/drivers/baseband/acc/rte_vrb_pmd.c
index fe23c01b5c..1f85e33462 100644
--- a/drivers/baseband/acc/rte_vrb_pmd.c
+++ b/drivers/baseband/acc/rte_vrb_pmd.c
@@ -4353,7 +4353,7 @@ vrb2_dequeue_mldts(struct rte_bbdev_queue_data *q_data,
static void
vrb_bbdev_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct acc_device *d = dev->data->dev_private;
dev->dev_ops = &vrb_bbdev_ops;
diff --git a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c
index cb805a1732..45bd171ca7 100644
--- a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c
+++ b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c
@@ -2873,7 +2873,7 @@ fpga_5gnr_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
static void
fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
dev->dev_ops = &fpga_5gnr_ops;
dev->enqueue_ldpc_enc_ops = fpga_5gnr_enqueue_ldpc_enc;
@@ -3376,7 +3376,7 @@ int rte_fpga_5gnr_fec_configure(const char *dev_name, const struct rte_fpga_5gnr
dev_name);
return -ENODEV;
}
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(bbdev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(bbdev, *pci_dev);
rte_bbdev_log(INFO, "Configure dev id %x", pci_dev->id.device_id);
if (pci_dev->id.device_id == VC_5GNR_PF_DEVICE_ID)
return vc_5gnr_configure(dev_name, conf);
diff --git a/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c b/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c
index d27164c6f4..04ac445820 100644
--- a/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c
+++ b/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c
@@ -2316,7 +2316,7 @@ fpga_dequeue_dec(struct rte_bbdev_queue_data *q_data,
static void
fpga_lte_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
dev->dev_ops = &fpga_ops;
dev->enqueue_enc_ops = fpga_enqueue_enc;
diff --git a/drivers/bus/auxiliary/bus_auxiliary_driver.h b/drivers/bus/auxiliary/bus_auxiliary_driver.h
index cab5f86d03..65e1814ec0 100644
--- a/drivers/bus/auxiliary/bus_auxiliary_driver.h
+++ b/drivers/bus/auxiliary/bus_auxiliary_driver.h
@@ -128,9 +128,6 @@ struct rte_auxiliary_driver {
uint32_t drv_flags; /**< Flags RTE_AUXILIARY_DRV_*. */
};
-#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_auxiliary_device)
-
/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
diff --git a/drivers/bus/cdx/bus_cdx_driver.h b/drivers/bus/cdx/bus_cdx_driver.h
index d443178404..01684466ed 100644
--- a/drivers/bus/cdx/bus_cdx_driver.h
+++ b/drivers/bus/cdx/bus_cdx_driver.h
@@ -60,8 +60,6 @@ struct rte_cdx_device {
struct rte_intr_handle *intr_handle; /**< Interrupt handle */
};
-#define RTE_ETH_DEV_TO_CDX_DEV(eth_dev) RTE_BUS_DEVICE((eth_dev)->device, struct rte_cdx_device)
-
#ifdef __cplusplus
/** C++ macro used to help building up tables of device IDs. */
#define RTE_CDX_DEVICE(vend, dev) \
diff --git a/drivers/bus/pci/bus_pci_driver.h b/drivers/bus/pci/bus_pci_driver.h
index cb7039f8d6..c04ebddf59 100644
--- a/drivers/bus/pci/bus_pci_driver.h
+++ b/drivers/bus/pci/bus_pci_driver.h
@@ -47,9 +47,6 @@ struct rte_pci_device {
/**< Handler of VFIO request interrupt */
};
-#define RTE_ETH_DEV_TO_PCI(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_pci_device)
-
#ifdef __cplusplus
/** C++ macro used to help building up tables of device IDs */
#define RTE_PCI_DEVICE(vend, dev) \
diff --git a/drivers/bus/vdev/bus_vdev_driver.h b/drivers/bus/vdev/bus_vdev_driver.h
index 8d114e4b3b..ecfc5384fc 100644
--- a/drivers/bus/vdev/bus_vdev_driver.h
+++ b/drivers/bus/vdev/bus_vdev_driver.h
@@ -19,9 +19,6 @@ struct rte_vdev_device {
struct rte_device device; /**< Inherit core device */
};
-#define RTE_ETH_DEV_TO_VDEV(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_vdev_device)
-
static inline const char *
rte_vdev_device_name(const struct rte_vdev_device *dev)
{
diff --git a/drivers/compress/octeontx/otx_zip.c b/drivers/compress/octeontx/otx_zip.c
index 8673561a81..7cf3283680 100644
--- a/drivers/compress/octeontx/otx_zip.c
+++ b/drivers/compress/octeontx/otx_zip.c
@@ -142,7 +142,7 @@ zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *cmd)
int
zipvf_create(struct rte_compressdev *compressdev)
{
- struct rte_pci_device *pdev = RTE_BUS_DEVICE(compressdev->device, *pdev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(compressdev, *pdev);
struct zip_vf *zipvf = NULL;
char *dev_name = compressdev->data->name;
void *vbar0;
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index f437350539..d3cf1ddd57 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -481,7 +481,7 @@ cnxk_cpt_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
if (dev->data->queue_pairs[qp_id] != NULL)
cnxk_cpt_queue_pair_release(dev, qp_id);
- pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (pci_dev->mem_resource[2].addr == NULL) {
plt_err("Invalid PCI mem address");
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
index d7b53723e7..3d980d096f 100644
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
@@ -4383,7 +4383,6 @@ static int
dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
{
struct dpaa2_sec_dev_private *internals;
- struct rte_device *dev = cryptodev->device;
struct rte_dpaa2_device *dpaa2_dev;
struct rte_security_ctx *security_instance;
struct fsl_mc_io *dpseci;
@@ -4392,7 +4391,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
int retcode, hw_id;
PMD_INIT_FUNC_TRACE();
- dpaa2_dev = RTE_BUS_DEVICE(dev, *dpaa2_dev);
+ dpaa2_dev = RTE_CLASS_TO_BUS_DEVICE(cryptodev, *dpaa2_dev);
hw_id = dpaa2_dev->object_id;
cryptodev->driver_id = cryptodev_driver_id;
diff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.c b/drivers/crypto/octeontx/otx_cryptodev_ops.c
index a499c8d0bc..d6d1b2cea9 100644
--- a/drivers/crypto/octeontx/otx_cryptodev_ops.c
+++ b/drivers/crypto/octeontx/otx_cryptodev_ops.c
@@ -156,7 +156,7 @@ otx_cpt_que_pair_setup(struct rte_cryptodev *dev,
DEFAULT_CMD_QLEN);
}
- pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (pci_dev->mem_resource[0].addr == NULL) {
CPT_LOG_ERR("PCI mem address null");
@@ -1001,7 +1001,7 @@ static struct rte_cryptodev_ops cptvf_ops = {
int
otx_cpt_dev_create(struct rte_cryptodev *c_dev)
{
- struct rte_pci_device *pdev = RTE_BUS_DEVICE(c_dev->device, *pdev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(c_dev, *pdev);
struct cpt_vf *cptvf = NULL;
void *reg_base;
char dev_name[32];
diff --git a/drivers/event/cnxk/cnxk_eventdev.c b/drivers/event/cnxk/cnxk_eventdev.c
index 6f000ff49e..272ba235a4 100644
--- a/drivers/event/cnxk/cnxk_eventdev.c
+++ b/drivers/event/cnxk/cnxk_eventdev.c
@@ -654,7 +654,7 @@ cnxk_sso_init(struct rte_eventdev *event_dev)
return -ENOMEM;
}
- pci_dev = RTE_BUS_DEVICE(event_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(event_dev, *pci_dev);
dev->sso.pci_dev = pci_dev;
*(uint64_t *)mz->addr = (uint64_t)dev;
diff --git a/drivers/event/dlb2/pf/dlb2_pf.c b/drivers/event/dlb2/pf/dlb2_pf.c
index 82075bbf0b..c78783e59d 100644
--- a/drivers/event/dlb2/pf/dlb2_pf.c
+++ b/drivers/event/dlb2/pf/dlb2_pf.c
@@ -784,7 +784,7 @@ dlb2_eventdev_pci_init(struct rte_eventdev *eventdev)
dlb2_pf_iface_fn_ptrs_init();
- pci_dev = RTE_BUS_DEVICE(eventdev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eventdev, *pci_dev);
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
dlb2 = dlb2_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
index 4292644fde..c0e06e4aa0 100644
--- a/drivers/event/skeleton/skeleton_eventdev.c
+++ b/drivers/event/skeleton/skeleton_eventdev.c
@@ -332,7 +332,7 @@ skeleton_eventdev_init(struct rte_eventdev *eventdev)
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return 0;
- pci_dev = RTE_BUS_DEVICE(eventdev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eventdev, *pci_dev);
skel->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
if (!skel->reg_base) {
diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 8b25ed948f..d6e34021ce 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -315,7 +315,7 @@ ark_dev_init(struct rte_eth_dev *dev)
if (ret)
return ret;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
rte_eth_copy_pci_info(dev, pci_dev);
dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
diff --git a/drivers/net/atlantic/atl_ethdev.c b/drivers/net/atlantic/atl_ethdev.c
index f8744da221..0b2033c084 100644
--- a/drivers/net/atlantic/atl_ethdev.c
+++ b/drivers/net/atlantic/atl_ethdev.c
@@ -360,7 +360,7 @@ static int
eth_atl_dev_init(struct rte_eth_dev *eth_dev)
{
struct atl_adapter *adapter = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
int err = 0;
@@ -479,7 +479,7 @@ static int
atl_dev_start(struct rte_eth_dev *dev)
{
struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_vector = 0;
int status;
@@ -607,7 +607,7 @@ atl_dev_stop(struct rte_eth_dev *dev)
struct rte_eth_link link;
struct aq_hw_s *hw =
ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
PMD_INIT_FUNC_TRACE();
@@ -688,7 +688,7 @@ atl_dev_set_link_down(struct rte_eth_dev *dev)
static int
atl_dev_close(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct aq_hw_s *hw;
int ret;
@@ -1094,7 +1094,7 @@ atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
static int
atl_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
dev_info->max_rx_queues = AQ_HW_MAX_RX_QUEUES;
dev_info->max_tx_queues = AQ_HW_MAX_TX_QUEUES;
@@ -1345,7 +1345,7 @@ atl_dev_link_status_print(struct rte_eth_dev *dev)
#ifdef DEBUG
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
pci_dev->addr.domain,
diff --git a/drivers/net/avp/avp_ethdev.c b/drivers/net/avp/avp_ethdev.c
index 3bc5171336..8af6c45381 100644
--- a/drivers/net/avp/avp_ethdev.c
+++ b/drivers/net/avp/avp_ethdev.c
@@ -361,7 +361,7 @@ static void *
avp_dev_translate_address(struct rte_eth_dev *eth_dev,
rte_iova_t host_phys_addr)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_mem_resource *resource;
struct rte_avp_memmap_info *info;
struct rte_avp_memmap *map;
@@ -414,7 +414,7 @@ avp_dev_version_check(uint32_t version)
static int
avp_dev_check_regions(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_avp_memmap_info *memmap;
struct rte_avp_device_info *info;
struct rte_mem_resource *resource;
@@ -550,7 +550,7 @@ _avp_set_rx_queue_mappings(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
static void
_avp_set_queue_counts(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
struct rte_avp_device_info *host_info;
void *addr;
@@ -610,7 +610,7 @@ avp_dev_attach(struct rte_eth_dev *eth_dev)
* re-run the device create utility which will parse the new host info
* and setup the AVP device queue pointers.
*/
- ret = avp_dev_create(RTE_ETH_DEV_TO_PCI(eth_dev), eth_dev);
+ ret = avp_dev_create(RTE_CLASS_TO_BUS_DEVICE(eth_dev, struct rte_pci_device), eth_dev);
if (ret < 0) {
PMD_DRV_LOG_LINE(ERR, "Failed to re-create AVP device, ret=%d",
ret);
@@ -664,7 +664,7 @@ static void
avp_dev_interrupt_handler(void *data)
{
struct rte_eth_dev *eth_dev = data;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr;
uint32_t status, value;
int ret;
@@ -723,7 +723,7 @@ avp_dev_interrupt_handler(void *data)
static int
avp_dev_enable_interrupts(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr;
int ret;
@@ -748,7 +748,7 @@ avp_dev_enable_interrupts(struct rte_eth_dev *eth_dev)
static int
avp_dev_disable_interrupts(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr;
int ret;
@@ -773,7 +773,7 @@ avp_dev_disable_interrupts(struct rte_eth_dev *eth_dev)
static int
avp_dev_setup_interrupts(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int ret;
/* register a callback handler with UIO for interrupt notifications */
@@ -793,7 +793,7 @@ avp_dev_setup_interrupts(struct rte_eth_dev *eth_dev)
static int
avp_dev_migration_pending(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr;
uint32_t value;
@@ -954,7 +954,7 @@ eth_avp_dev_init(struct rte_eth_dev *eth_dev)
struct rte_pci_device *pci_dev;
int ret;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
eth_dev->dev_ops = &avp_eth_dev_ops;
eth_dev->rx_pkt_burst = &avp_recv_pkts;
eth_dev->tx_pkt_burst = &avp_xmit_pkts;
@@ -1977,7 +1977,7 @@ avp_dev_tx_queue_release_all(struct rte_eth_dev *eth_dev)
static int
avp_dev_configure(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
struct rte_avp_device_info *host_info;
struct rte_avp_device_config config;
diff --git a/drivers/net/axgbe/axgbe_ethdev.c b/drivers/net/axgbe/axgbe_ethdev.c
index e321959afd..61725d55ca 100644
--- a/drivers/net/axgbe/axgbe_ethdev.c
+++ b/drivers/net/axgbe/axgbe_ethdev.c
@@ -2231,7 +2231,7 @@ eth_axgbe_dev_init(struct rte_eth_dev *eth_dev)
rte_bit_relaxed_set32(AXGBE_STOPPED, &pdata->dev_state);
pdata->eth_dev = eth_dev;
- pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
pdata->pci_dev = pci_dev;
pdata->xgmac_regs =
@@ -2454,7 +2454,7 @@ axgbe_dev_close(struct rte_eth_dev *eth_dev)
return 0;
pdata = eth_dev->data->dev_private;
- pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
axgbe_dev_clear_queues(eth_dev);
/* disable uio intr before callback unregister */
diff --git a/drivers/net/bnx2x/bnx2x_ethdev.c b/drivers/net/bnx2x/bnx2x_ethdev.c
index 7b96e1acee..4f1f97a999 100644
--- a/drivers/net/bnx2x/bnx2x_ethdev.c
+++ b/drivers/net/bnx2x/bnx2x_ethdev.c
@@ -639,7 +639,7 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
/* Extract key data structures */
sc = eth_dev->data->dev_private;
- pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
pci_addr = pci_dev->addr;
snprintf(sc->devinfo.name, NAME_SIZE, PCI_SHORT_PRI_FMT ":dpdk-port-%u",
diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index ac61ffda80..1b8cf3a52a 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -862,7 +862,7 @@ static int bnxt_alloc_prev_ring_stats(struct bnxt *bp)
static int bnxt_start_nic(struct bnxt *bp)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(bp->eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_vector = 0;
uint32_t queue_id, base = BNXT_MISC_VEC_ID;
@@ -1167,7 +1167,7 @@ uint64_t bnxt_eth_rss_support(struct bnxt *bp)
static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pdev = RTE_BUS_DEVICE(eth_dev->device, *pdev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
struct bnxt *bp = eth_dev->data->dev_private;
uint16_t max_vnics, i, j, vpool, vrxq;
unsigned int max_rx_rings;
@@ -1719,7 +1719,7 @@ static int bnxt_ptp_start(struct bnxt *bp)
static int bnxt_dev_stop(struct rte_eth_dev *eth_dev)
{
struct bnxt *bp = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct rte_eth_link link;
uint16_t i;
@@ -5143,7 +5143,7 @@ bool bnxt_stratus_device(struct bnxt *bp)
static int bnxt_map_pci_bars(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct bnxt *bp = eth_dev->data->dev_private;
/* enable device (incl. PCI PM wakeup), and bus-mastering */
@@ -6600,7 +6600,7 @@ bnxt_parse_dev_args(struct bnxt *bp, struct rte_devargs *devargs)
*/
static int bnxt_drv_init(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct bnxt *bp = eth_dev->data->dev_private;
int rc = 0;
@@ -6684,7 +6684,7 @@ static int bnxt_drv_init(struct rte_eth_dev *eth_dev)
static int
bnxt_dev_init(struct rte_eth_dev *eth_dev, void *params __rte_unused)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
static int version_printed;
struct bnxt *bp;
int rc;
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
index df30dbfc0f..a75da8aa19 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
@@ -192,7 +192,7 @@ ulp_session_init(struct bnxt *bp,
if (!bp)
return NULL;
- pci_dev = RTE_BUS_DEVICE(bp->eth_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(bp->eth_dev, *pci_dev);
pci_addr = &pci_dev->addr;
pthread_mutex_lock(&bnxt_ulp_global_mutex);
@@ -556,7 +556,7 @@ bnxt_ulp_port_deinit(struct bnxt *bp)
bp->eth_dev->data->port_id);
/* Get the session details */
- pci_dev = RTE_BUS_DEVICE(bp->eth_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(bp->eth_dev, *pci_dev);
pci_addr = &pci_dev->addr;
pthread_mutex_lock(&bnxt_ulp_global_mutex);
session = ulp_get_session(bp, pci_addr);
diff --git a/drivers/net/cnxk/cnxk_ethdev.c b/drivers/net/cnxk/cnxk_ethdev.c
index 06d1c9b362..7ae16186c6 100644
--- a/drivers/net/cnxk/cnxk_ethdev.c
+++ b/drivers/net/cnxk/cnxk_ethdev.c
@@ -2177,7 +2177,7 @@ cnxk_eth_dev_init(struct rte_eth_dev *eth_dev)
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return 0;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
rte_eth_copy_pci_info(eth_dev, pci_dev);
/* Parse devargs string */
diff --git a/drivers/net/cnxk/cnxk_ethdev_ops.c b/drivers/net/cnxk/cnxk_ethdev_ops.c
index 49e77e49a6..460ffa32b6 100644
--- a/drivers/net/cnxk/cnxk_ethdev_ops.c
+++ b/drivers/net/cnxk/cnxk_ethdev_ops.c
@@ -7,7 +7,7 @@
int
cnxk_nix_info_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *devinfo)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
int max_rx_pktlen;
diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c
index 0c337a6cc8..82e67eeff1 100644
--- a/drivers/net/cxgbe/cxgbe_ethdev.c
+++ b/drivers/net/cxgbe/cxgbe_ethdev.c
@@ -1704,7 +1704,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
eth_dev->dev_ops = &cxgbe_eth_dev_ops;
eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;
eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* for secondary processes, we attach to ethdevs allocated by primary
* and do minimal initialization.
@@ -1767,7 +1767,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
static int eth_cxgbe_dev_uninit(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
uint16_t port_id;
int err = 0;
diff --git a/drivers/net/cxgbe/cxgbevf_ethdev.c b/drivers/net/cxgbe/cxgbevf_ethdev.c
index d8eba8afef..750dc7da4d 100644
--- a/drivers/net/cxgbe/cxgbevf_ethdev.c
+++ b/drivers/net/cxgbe/cxgbevf_ethdev.c
@@ -113,7 +113,7 @@ static int eth_cxgbevf_dev_init(struct rte_eth_dev *eth_dev)
eth_dev->dev_ops = &cxgbevf_eth_dev_ops;
eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;
eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* for secondary processes, we attach to ethdevs allocated by primary
* and do minimal initialization.
@@ -177,7 +177,7 @@ static int eth_cxgbevf_dev_init(struct rte_eth_dev *eth_dev)
static int eth_cxgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
uint16_t port_id;
int err = 0;
diff --git a/drivers/net/dpaa/dpaa_ethdev.c b/drivers/net/dpaa/dpaa_ethdev.c
index d4b4793f16..9f976d179b 100644
--- a/drivers/net/dpaa/dpaa_ethdev.c
+++ b/drivers/net/dpaa/dpaa_ethdev.c
@@ -217,7 +217,6 @@ dpaa_eth_dev_configure(struct rte_eth_dev *dev)
uint64_t rx_offloads = eth_conf->rxmode.offloads;
uint64_t tx_offloads = eth_conf->txmode.offloads;
struct dpaa_if *dpaa_intf = dev->data->dev_private;
- struct rte_device *rdev = dev->device;
struct rte_eth_link *link = &dev->data->dev_link;
struct rte_dpaa_device *dpaa_dev;
struct fman_if *fif = dev->process_private;
@@ -230,7 +229,7 @@ dpaa_eth_dev_configure(struct rte_eth_dev *dev)
PMD_INIT_FUNC_TRACE();
- dpaa_dev = RTE_BUS_DEVICE(rdev, *dpaa_dev);
+ dpaa_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *dpaa_dev);
intr_handle = dpaa_dev->intr_handle;
__fif = container_of(fif, struct __fman_if, __if);
@@ -426,13 +425,12 @@ dpaa_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements)
static void dpaa_interrupt_handler(void *param)
{
struct rte_eth_dev *dev = param;
- struct rte_device *rdev = dev->device;
struct rte_dpaa_device *dpaa_dev;
struct rte_intr_handle *intr_handle;
uint64_t buf;
int bytes_read;
- dpaa_dev = RTE_BUS_DEVICE(rdev, *dpaa_dev);
+ dpaa_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *dpaa_dev);
intr_handle = dpaa_dev->intr_handle;
if (rte_intr_fd_get(intr_handle) < 0)
@@ -502,7 +500,6 @@ static int dpaa_eth_dev_close(struct rte_eth_dev *dev)
{
struct fman_if *fif = dev->process_private;
struct __fman_if *__fif;
- struct rte_device *rdev = dev->device;
struct rte_dpaa_device *dpaa_dev;
struct rte_intr_handle *intr_handle;
struct rte_eth_link *link = &dev->data->dev_link;
@@ -530,7 +527,7 @@ static int dpaa_eth_dev_close(struct rte_eth_dev *dev)
}
}
- dpaa_dev = RTE_BUS_DEVICE(rdev, *dpaa_dev);
+ dpaa_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *dpaa_dev);
intr_handle = dpaa_dev->intr_handle;
__fif = container_of(fif, struct __fman_if, __if);
@@ -1267,9 +1264,8 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
/* Set up the device interrupt handler */
if (dev->intr_handle == NULL) {
struct rte_dpaa_device *dpaa_dev;
- struct rte_device *rdev = dev->device;
- dpaa_dev = RTE_BUS_DEVICE(rdev, *dpaa_dev);
+ dpaa_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *dpaa_dev);
dev->intr_handle = dpaa_dev->intr_handle;
if (rte_intr_vec_list_alloc(dev->intr_handle,
NULL, dpaa_push_queue_max_num())) {
@@ -2119,7 +2115,7 @@ dpaa_dev_init_secondary(struct rte_eth_dev *eth_dev)
PMD_INIT_FUNC_TRACE();
- dpaa_device = RTE_BUS_DEVICE(eth_dev->device, *dpaa_device);
+ dpaa_device = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *dpaa_device);
dev_id = dpaa_device->id.dev_id;
cfg = dpaa_get_eth_port_cfg(dev_id);
fman_intf = cfg->fman_if;
@@ -2236,7 +2232,7 @@ dpaa_dev_init(struct rte_eth_dev *eth_dev)
PMD_INIT_FUNC_TRACE();
- dpaa_device = RTE_BUS_DEVICE(eth_dev->device, *dpaa_device);
+ dpaa_device = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *dpaa_device);
dev_id = dpaa_device->id.dev_id;
dpaa_intf = eth_dev->data->dev_private;
cfg = dpaa_get_eth_port_cfg(dev_id);
diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c b/drivers/net/dpaa2/dpaa2_ethdev.c
index dc9ea700ac..803a8321e0 100644
--- a/drivers/net/dpaa2/dpaa2_ethdev.c
+++ b/drivers/net/dpaa2/dpaa2_ethdev.c
@@ -1339,7 +1339,6 @@ dpaa2_eth_setup_irqs(struct rte_eth_dev *dev, int enable)
static int
dpaa2_dev_start(struct rte_eth_dev *dev)
{
- struct rte_device *rdev = dev->device;
struct rte_dpaa2_device *dpaa2_dev;
struct rte_eth_dev_data *data = dev->data;
struct dpaa2_dev_priv *priv = data->dev_private;
@@ -1351,7 +1350,7 @@ dpaa2_dev_start(struct rte_eth_dev *dev)
int ret, i;
struct rte_intr_handle *intr_handle;
- dpaa2_dev = RTE_BUS_DEVICE(rdev, *dpaa2_dev);
+ dpaa2_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *dpaa2_dev);
intr_handle = dpaa2_dev->intr_handle;
PMD_INIT_FUNC_TRACE();
@@ -1458,12 +1457,11 @@ dpaa2_dev_stop(struct rte_eth_dev *dev)
struct fsl_mc_io *dpni = dev->process_private;
int ret;
struct rte_eth_link link;
- struct rte_device *rdev = dev->device;
struct rte_intr_handle *intr_handle;
struct rte_dpaa2_device *dpaa2_dev;
uint16_t i;
- dpaa2_dev = RTE_BUS_DEVICE(rdev, *dpaa2_dev);
+ dpaa2_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *dpaa2_dev);
intr_handle = dpaa2_dev->intr_handle;
PMD_INIT_FUNC_TRACE();
@@ -2918,7 +2916,7 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
- dpaa2_dev = RTE_BUS_DEVICE(dev, *dpaa2_dev);
+ dpaa2_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *dpaa2_dev);
hw_id = dpaa2_dev->object_id;
ret = dpni_open(dpni_dev, CMD_PRI_LOW, hw_id, &priv->token);
diff --git a/drivers/net/dpaa2/dpaa2_recycle.c b/drivers/net/dpaa2/dpaa2_recycle.c
index 14416c41d0..f78d12362e 100644
--- a/drivers/net/dpaa2/dpaa2_recycle.c
+++ b/drivers/net/dpaa2/dpaa2_recycle.c
@@ -607,9 +607,8 @@ lx_serdes_eth_lpbk(uint16_t mac_id, int en)
int
dpaa2_dev_recycle_config(struct rte_eth_dev *eth_dev)
{
- struct rte_device *dev = eth_dev->device;
struct dpaa2_dev_priv *priv = eth_dev->data->dev_private;
- struct rte_dpaa2_device *dpaa2_dev = RTE_BUS_DEVICE(dev, *dpaa2_dev);
+ struct rte_dpaa2_device *dpaa2_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *dpaa2_dev);
struct fsl_mc_io *dpni_dev = eth_dev->process_private;
struct dpni_port_cfg port_cfg;
int ret;
@@ -674,9 +673,8 @@ dpaa2_dev_recycle_config(struct rte_eth_dev *eth_dev)
int
dpaa2_dev_recycle_deconfig(struct rte_eth_dev *eth_dev)
{
- struct rte_device *dev = eth_dev->device;
struct dpaa2_dev_priv *priv = eth_dev->data->dev_private;
- struct rte_dpaa2_device *dpaa2_dev = RTE_BUS_DEVICE(dev, *dpaa2_dev);
+ struct rte_dpaa2_device *dpaa2_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *dpaa2_dev);
struct fsl_mc_io *dpni_dev = eth_dev->process_private;
struct dpni_port_cfg port_cfg;
int ret = 0;
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index ea4afbc75d..ad2ac6dbbf 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -924,7 +924,7 @@ static inline void ena_indirect_table_release(struct ena_adapter *adapter)
static int ena_close(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ena_adapter *adapter = dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
@@ -1457,7 +1457,7 @@ static int ena_stop(struct rte_eth_dev *dev)
{
struct ena_adapter *adapter = dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint16_t i;
int rc;
@@ -1503,7 +1503,7 @@ static int ena_create_io_queue(struct rte_eth_dev *dev, struct ena_ring *ring)
{
struct ena_adapter *adapter = ring->adapter;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ena_com_create_io_ctx ctx =
/* policy set to _HOST just to satisfy icc compiler */
@@ -2422,7 +2422,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
adapter->edev_data = eth_dev->data;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
PMD_INIT_LOG_LINE(INFO, "Initializing " PCI_PRI_FMT,
pci_dev->addr.domain,
@@ -3978,7 +3978,7 @@ static int ena_parse_devargs(struct ena_adapter *adapter, struct rte_devargs *de
static int ena_setup_rx_intr(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int rc;
uint16_t vectors_nb, i;
diff --git a/drivers/net/enetc/enetc4_ethdev.c b/drivers/net/enetc/enetc4_ethdev.c
index df9f007473..78eba70a08 100644
--- a/drivers/net/enetc/enetc4_ethdev.c
+++ b/drivers/net/enetc/enetc4_ethdev.c
@@ -956,7 +956,7 @@ enetc4_dev_hw_init(struct rte_eth_dev *eth_dev)
{
struct enetc_eth_hw *hw =
ENETC_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
eth_dev->rx_pkt_burst = &enetc_recv_pkts_nc;
eth_dev->tx_pkt_burst = &enetc_xmit_pkts_nc;
@@ -986,7 +986,7 @@ enetc4_dev_init(struct rte_eth_dev *eth_dev)
{
struct enetc_eth_hw *hw =
ENETC_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int error = 0;
uint32_t si_cap;
struct enetc_hw *enetc_hw = &hw->hw;
diff --git a/drivers/net/enetc/enetc4_vf.c b/drivers/net/enetc/enetc4_vf.c
index 3f257234a0..bec7128e41 100644
--- a/drivers/net/enetc/enetc4_vf.c
+++ b/drivers/net/enetc/enetc4_vf.c
@@ -1249,7 +1249,7 @@ enetc4_vf_dev_init(struct rte_eth_dev *eth_dev)
{
struct enetc_eth_hw *hw =
ENETC_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int error = 0;
uint32_t si_cap;
struct enetc_hw *enetc_hw = &hw->hw;
@@ -1297,7 +1297,7 @@ enetc4_vf_dev_intr(struct rte_eth_dev *eth_dev, bool enable)
struct enetc_eth_hw *hw =
ENETC_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
struct enetc_hw *enetc_hw = &hw->hw;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret = 0;
diff --git a/drivers/net/enetc/enetc_ethdev.c b/drivers/net/enetc/enetc_ethdev.c
index b2bbace16c..f41f3c1803 100644
--- a/drivers/net/enetc/enetc_ethdev.c
+++ b/drivers/net/enetc/enetc_ethdev.c
@@ -886,7 +886,7 @@ static int
enetc_dev_init(struct rte_eth_dev *eth_dev)
{
int error = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct enetc_eth_hw *hw =
ENETC_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
index a853a5047a..2e5cd186f9 100644
--- a/drivers/net/enic/enic_ethdev.c
+++ b/drivers/net/enic/enic_ethdev.c
@@ -454,7 +454,7 @@ static uint32_t speed_capa_from_pci_id(struct rte_eth_dev *eth_dev)
struct rte_pci_device *pdev;
uint16_t id;
- pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
id = pdev->id.subsystem_device_id;
for (m = vic_speed_capa_map; m->sub_devid != 0; m++) {
if (m->sub_devid == id)
@@ -1292,7 +1292,7 @@ static int eth_enic_dev_init(struct rte_eth_dev *eth_dev,
enic->rte_dev = eth_dev;
enic->dev_data = eth_dev->data;
- pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
rte_eth_copy_pci_info(eth_dev, pdev);
enic->pdev = pdev;
addr = &pdev->addr;
diff --git a/drivers/net/enic/enic_fm_flow.c b/drivers/net/enic/enic_fm_flow.c
index c2c3e55206..4b0a513977 100644
--- a/drivers/net/enic/enic_fm_flow.c
+++ b/drivers/net/enic/enic_fm_flow.c
@@ -3229,7 +3229,7 @@ enic_fm_init(struct enic *enic)
if (rte_eth_dev_is_repr(enic->rte_dev))
addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
else
- addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
+ addr = &RTE_CLASS_TO_BUS_DEVICE(enic->rte_dev, struct rte_pci_device)->addr;
rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
if (rc) {
ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
@@ -3361,7 +3361,7 @@ enic_fm_allocate_switch_domain(struct enic *pf)
if (rte_eth_dev_is_repr(pf->rte_dev))
return -EINVAL;
cur = pf;
- cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
+ cur_a = &RTE_CLASS_TO_BUS_DEVICE(cur->rte_dev, struct rte_pci_device)->addr;
/* Go through ports and find another PF that is on the same adapter */
RTE_ETH_FOREACH_DEV(pid) {
dev = &rte_eth_devices[pid];
@@ -3373,7 +3373,7 @@ enic_fm_allocate_switch_domain(struct enic *pf)
continue;
/* dev is another PF. Is it on the same adapter? */
prev = pmd_priv(dev);
- prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
+ prev_a = &RTE_CLASS_TO_BUS_DEVICE(dev, struct rte_pci_device)->addr;
if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
cur->rte_dev->data->port_id,
diff --git a/drivers/net/enic/enic_vf_representor.c b/drivers/net/enic/enic_vf_representor.c
index 05b2efedcb..fc836100b4 100644
--- a/drivers/net/enic/enic_vf_representor.c
+++ b/drivers/net/enic/enic_vf_representor.c
@@ -655,7 +655,7 @@ int enic_vf_representor_init(struct rte_eth_dev *eth_dev, void *init_params)
}
/* Check for non-existent VFs */
- pdev = RTE_ETH_DEV_TO_PCI(pf->rte_dev);
+ pdev = RTE_CLASS_TO_BUS_DEVICE(pf->rte_dev, *pdev);
if (vf->vf_id >= pdev->max_vfs) {
ENICPMD_LOG(ERR, "VF ID is invalid. vf_id %u max_vfs %u",
vf->vf_id, pdev->max_vfs);
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c
index ed25b82848..476b2c311f 100644
--- a/drivers/net/gve/gve_ethdev.c
+++ b/drivers/net/gve/gve_ethdev.c
@@ -1491,7 +1491,7 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
- pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
reg_bar = pci_dev->mem_resource[GVE_REG_BAR].addr;
if (!reg_bar) {
diff --git a/drivers/net/hinic/hinic_pmd_ethdev.c b/drivers/net/hinic/hinic_pmd_ethdev.c
index 75534c1ce2..91a4348fb6 100644
--- a/drivers/net/hinic/hinic_pmd_ethdev.c
+++ b/drivers/net/hinic/hinic_pmd_ethdev.c
@@ -1234,7 +1234,7 @@ static int hinic_dev_stop(struct rte_eth_dev *dev)
static void hinic_disable_interrupt(struct rte_eth_dev *dev)
{
struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
int ret, retries = 0;
rte_bit_relaxed_clear32(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
@@ -2745,7 +2745,7 @@ static int hinic_nic_dev_create(struct rte_eth_dev *eth_dev)
eth_dev->data->name);
return -ENOMEM;
}
- nic_dev->hwdev->pcidev_hdl = RTE_ETH_DEV_TO_PCI(eth_dev);
+ nic_dev->hwdev->pcidev_hdl = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *nic_dev->hwdev->pcidev_hdl);
/* init osdep*/
rc = hinic_osdep_init(nic_dev->hwdev);
@@ -3086,7 +3086,7 @@ static int hinic_func_init(struct rte_eth_dev *eth_dev)
u32 mac_size;
int rc;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* EAL is SECONDARY and eth_dev is already created */
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
@@ -3218,7 +3218,7 @@ static int hinic_dev_init(struct rte_eth_dev *eth_dev)
{
struct rte_pci_device *pci_dev;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
PMD_DRV_LOG(INFO, "Initializing pf hinic-" PCI_PRI_FMT " in %s process",
pci_dev->addr.domain, pci_dev->addr.bus,
diff --git a/drivers/net/hinic3/base/hinic3_hwdev.c b/drivers/net/hinic3/base/hinic3_hwdev.c
index 5d12cf7b5f..d09a8f7e7d 100644
--- a/drivers/net/hinic3/base/hinic3_hwdev.c
+++ b/drivers/net/hinic3/base/hinic3_hwdev.c
@@ -74,10 +74,11 @@ struct mgmt_event_handle {
};
bool
-hinic3_is_vfio_iommu_enable(const struct rte_eth_dev *rte_dev)
+hinic3_is_vfio_iommu_enable(const struct rte_eth_dev *eth_dev)
{
- return ((RTE_ETH_DEV_TO_PCI(rte_dev)->kdrv == RTE_PCI_KDRV_VFIO) &&
- (rte_vfio_noiommu_is_enabled() != 1));
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
+
+ return pci_dev->kdrv == RTE_PCI_KDRV_VFIO && rte_vfio_noiommu_is_enabled() != 1;
}
int
diff --git a/drivers/net/hinic3/hinic3_ethdev.c b/drivers/net/hinic3/hinic3_ethdev.c
index f4eb788686..361e52f7b9 100644
--- a/drivers/net/hinic3/hinic3_ethdev.c
+++ b/drivers/net/hinic3/hinic3_ethdev.c
@@ -1474,7 +1474,7 @@ hinic3_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t sq_id)
int
hinic3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = PCI_DEV_TO_INTR_HANDLE(pci_dev);
struct hinic3_nic_dev *nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
uint16_t msix_intr;
@@ -1493,7 +1493,7 @@ hinic3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
int
hinic3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = PCI_DEV_TO_INTR_HANDLE(pci_dev);
struct hinic3_nic_dev *nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
uint16_t msix_intr;
@@ -1695,7 +1695,7 @@ static void
hinic3_disable_interrupt(struct rte_eth_dev *dev)
{
struct hinic3_nic_dev *nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!hinic3_get_bit(HINIC3_DEV_INIT, &nic_dev->dev_status))
return;
@@ -1710,7 +1710,7 @@ static void
hinic3_enable_interrupt(struct rte_eth_dev *dev)
{
struct hinic3_nic_dev *nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!hinic3_get_bit(HINIC3_DEV_INIT, &nic_dev->dev_status))
return;
@@ -2080,7 +2080,7 @@ hinic3_dev_release(struct rte_eth_dev *eth_dev)
{
struct hinic3_nic_dev *nic_dev =
HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int qid;
/* Release io resource. */
@@ -3394,7 +3394,7 @@ hinic3_func_init(struct rte_eth_dev *eth_dev)
struct rte_pci_device *pci_dev = NULL;
int err;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* EAL is secondary and eth_dev is already created. */
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
@@ -3460,7 +3460,7 @@ hinic3_func_init(struct rte_eth_dev *eth_dev)
err = -ENOMEM;
goto alloc_hwdev_mem_fail;
}
- nic_dev->hwdev->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ nic_dev->hwdev->pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *nic_dev->hwdev->pci_dev);
nic_dev->hwdev->dev_handle = nic_dev;
nic_dev->hwdev->eth_dev = eth_dev;
nic_dev->hwdev->port_id = eth_dev->data->port_id;
@@ -3616,7 +3616,7 @@ hinic3_dev_init(struct rte_eth_dev *eth_dev)
{
struct rte_pci_device *pci_dev;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
PMD_DRV_LOG(INFO, "Initializing %.4x:%.2x:%.2x.%x in %s process",
pci_dev->addr.domain, pci_dev->addr.bus,
diff --git a/drivers/net/hns3/hns3_cmd.c b/drivers/net/hns3/hns3_cmd.c
index ad4ef9e189..34e12e7359 100644
--- a/drivers/net/hns3/hns3_cmd.c
+++ b/drivers/net/hns3/hns3_cmd.c
@@ -551,7 +551,7 @@ hns3_set_dcb_capability(struct hns3_hw *hw)
return;
eth_dev = &rte_eth_devices[hw->data->port_id];
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
device_id = pci_dev->id.device_id;
if (device_id == HNS3_DEV_ID_25GE_RDMA ||
diff --git a/drivers/net/hns3/hns3_common.c b/drivers/net/hns3/hns3_common.c
index 28d7e94ffb..29b51856d9 100644
--- a/drivers/net/hns3/hns3_common.c
+++ b/drivers/net/hns3/hns3_common.c
@@ -812,7 +812,7 @@ hns3_init_ring_with_vector(struct hns3_hw *hw)
int
hns3_map_rx_interrupt(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint16_t base = RTE_INTR_VEC_ZERO_OFFSET;
@@ -878,7 +878,7 @@ hns3_map_rx_interrupt(struct rte_eth_dev *dev)
void
hns3_unmap_rx_interrupt(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct hns3_adapter *hns = dev->data->dev_private;
struct hns3_hw *hw = &hns->hw;
@@ -912,7 +912,7 @@ int
hns3_restore_rx_interrupt(struct hns3_hw *hw)
{
struct rte_eth_dev *dev = &rte_eth_devices[hw->data->port_id];
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint16_t q_id;
int ret;
@@ -943,7 +943,7 @@ hns3_get_pci_revision_id(struct hns3_hw *hw, uint8_t *revision_id)
int ret;
eth_dev = &rte_eth_devices[hw->data->port_id];
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
ret = rte_pci_read_config(pci_dev, &revision, 1, RTE_PCI_REVISION_ID);
if (ret != 1) {
hns3_err(hw, "failed to read pci revision id, ret = %d", ret);
diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c
index a66fc5d81a..dbe26df77d 100644
--- a/drivers/net/hns3/hns3_ethdev.c
+++ b/drivers/net/hns3/hns3_ethdev.c
@@ -4526,8 +4526,7 @@ hns3_get_port_supported_speed(struct rte_eth_dev *eth_dev)
static int
hns3_init_pf(struct rte_eth_dev *eth_dev)
{
- struct rte_device *dev = eth_dev->device;
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct hns3_adapter *hns = eth_dev->data->dev_private;
struct hns3_hw *hw = &hns->hw;
int ret;
@@ -4656,8 +4655,7 @@ static void
hns3_uninit_pf(struct rte_eth_dev *eth_dev)
{
struct hns3_adapter *hns = eth_dev->data->dev_private;
- struct rte_device *dev = eth_dev->device;
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct hns3_hw *hw = &hns->hw;
PMD_INIT_FUNC_TRACE();
diff --git a/drivers/net/hns3/hns3_ethdev_vf.c b/drivers/net/hns3/hns3_ethdev_vf.c
index 59fb790240..84e733a0f5 100644
--- a/drivers/net/hns3/hns3_ethdev_vf.c
+++ b/drivers/net/hns3/hns3_ethdev_vf.c
@@ -1622,7 +1622,7 @@ hns3vf_clear_vport_list(struct hns3_hw *hw)
static int
hns3vf_init_vf(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct hns3_adapter *hns = eth_dev->data->dev_private;
struct hns3_hw *hw = &hns->hw;
int ret;
@@ -1739,7 +1739,7 @@ hns3vf_notify_uninit(struct hns3_hw *hw)
static void
hns3vf_uninit_vf(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct hns3_adapter *hns = eth_dev->data->dev_private;
struct hns3_hw *hw = &hns->hw;
@@ -2377,7 +2377,7 @@ static int
hns3vf_reinit_dev(struct hns3_adapter *hns)
{
struct rte_eth_dev *eth_dev = &rte_eth_devices[hns->hw.data->port_id];
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct hns3_hw *hw = &hns->hw;
int ret;
diff --git a/drivers/net/hns3/hns3_rxtx.c b/drivers/net/hns3/hns3_rxtx.c
index cd8b49999b..8060c37f23 100644
--- a/drivers/net/hns3/hns3_rxtx.c
+++ b/drivers/net/hns3/hns3_rxtx.c
@@ -1093,7 +1093,7 @@ hns3_dev_all_rx_queue_intr_enable(struct hns3_hw *hw, bool en)
int
hns3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -3066,7 +3066,7 @@ hns3_tx_push_get_queue_tail_reg(struct rte_eth_dev *dev, uint16_t queue_id)
#define HNS3_TX_PUSH_QUICK_DOORBELL_OFFSET 64
#define HNS3_TX_PUSH_PCI_BAR_INDEX 4
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint8_t bar_id = HNS3_TX_PUSH_PCI_BAR_INDEX;
/*
diff --git a/drivers/net/intel/cpfl/cpfl_ethdev.c b/drivers/net/intel/cpfl/cpfl_ethdev.c
index ec80a65dcd..7ac8797490 100644
--- a/drivers/net/intel/cpfl/cpfl_ethdev.c
+++ b/drivers/net/intel/cpfl/cpfl_ethdev.c
@@ -2764,7 +2764,7 @@ cpfl_dev_vport_init(struct rte_eth_dev *dev, void *init_params)
uint8_t p2p_q_vc_out_info[IDPF_DFLT_MBX_BUF_SIZE] = {0};
struct cpfl_vport_id vi;
struct cpchnl2_vport_id v_id;
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
int ret = 0;
dev->dev_ops = &cpfl_eth_dev_ops;
@@ -2836,7 +2836,7 @@ cpfl_dev_vport_init(struct rte_eth_dev *dev, void *init_params)
}
/* get the vport info */
if (adapter->base.hw.device_id == IXD_DEV_ID_VCPF) {
- pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
vi.func_type = VCPF_CPCHNL2_FTYPE_LAN_VF;
vi.pf_id = CPFL_HOST0_CPF_ID;
vi.vf_id = pci_dev->addr.function;
diff --git a/drivers/net/intel/cpfl/cpfl_ethdev.h b/drivers/net/intel/cpfl/cpfl_ethdev.h
index f8df5b6dfa..d41aa93191 100644
--- a/drivers/net/intel/cpfl/cpfl_ethdev.h
+++ b/drivers/net/intel/cpfl/cpfl_ethdev.h
@@ -298,7 +298,7 @@ int vcpf_add_queues(struct cpfl_adapter_ext *adapter);
int vcpf_del_queues(struct cpfl_adapter_ext *adapter);
#define CPFL_DEV_TO_PCI(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_pci_device)
+ RTE_CLASS_TO_BUS_DEVICE(eth_dev, struct rte_pci_device)
#define CPFL_ADAPTER_TO_EXT(p) \
container_of((p), struct cpfl_adapter_ext, base)
#define CPFL_DEV_TO_VPORT(dev) \
diff --git a/drivers/net/intel/e1000/em_ethdev.c b/drivers/net/intel/e1000/em_ethdev.c
index 9e15e882b9..62ab57268f 100644
--- a/drivers/net/intel/e1000/em_ethdev.c
+++ b/drivers/net/intel/e1000/em_ethdev.c
@@ -273,7 +273,7 @@ eth_em_dev_is_ich8(struct e1000_hw *hw)
static int
eth_em_dev_init(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct e1000_adapter *adapter =
E1000_DEV_PRIVATE(eth_dev->data->dev_private);
@@ -563,7 +563,7 @@ eth_em_start(struct rte_eth_dev *dev)
E1000_DEV_PRIVATE(dev->data->dev_private);
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret, mask;
uint32_t intr_vector = 0;
@@ -762,7 +762,7 @@ eth_em_stop(struct rte_eth_dev *dev)
{
struct rte_eth_link link;
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
/*
@@ -816,7 +816,7 @@ eth_em_close(struct rte_eth_dev *dev)
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct e1000_adapter *adapter =
E1000_DEV_PRIVATE(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret;
@@ -1062,7 +1062,7 @@ static int
eth_em_rx_queue_intr_enable(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id)
{
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
/* device interrupts are only subscribed to in primary processes */
@@ -1647,7 +1647,7 @@ static int
eth_em_interrupt_action(struct rte_eth_dev *dev,
struct rte_intr_handle *intr_handle)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct e1000_interrupt *intr =
diff --git a/drivers/net/intel/e1000/em_rxtx.c b/drivers/net/intel/e1000/em_rxtx.c
index 5879013a1d..f9665127df 100644
--- a/drivers/net/intel/e1000/em_rxtx.c
+++ b/drivers/net/intel/e1000/em_rxtx.c
@@ -2092,7 +2092,7 @@ em_flush_desc_rings(struct rte_eth_dev *dev)
{
uint32_t fextnvm11, tdlen;
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint16_t pci_cfg_status = 0;
int ret;
diff --git a/drivers/net/intel/e1000/igb_ethdev.c b/drivers/net/intel/e1000/igb_ethdev.c
index ef1599ac38..a4370fe32b 100644
--- a/drivers/net/intel/e1000/igb_ethdev.c
+++ b/drivers/net/intel/e1000/igb_ethdev.c
@@ -529,7 +529,7 @@ igb_intr_enable(struct rte_eth_dev *dev)
E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (rte_intr_allow_others(intr_handle) &&
@@ -546,7 +546,7 @@ igb_intr_disable(struct rte_eth_dev *dev)
{
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (rte_intr_allow_others(intr_handle) &&
@@ -783,7 +783,7 @@ static int
eth_igb_dev_init(struct rte_eth_dev *eth_dev)
{
int error = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
struct e1000_vfta * shadow_vfta =
@@ -1004,7 +1004,7 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
rte_eth_copy_pci_info(eth_dev, pci_dev);
hw->device_id = pci_dev->id.device_id;
@@ -1300,7 +1300,7 @@ eth_igb_start(struct rte_eth_dev *dev)
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct e1000_adapter *adapter =
E1000_DEV_PRIVATE(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret, mask;
uint32_t tqavctrl;
@@ -1537,7 +1537,7 @@ static int
eth_igb_stop(struct rte_eth_dev *dev)
{
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_eth_link link;
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct e1000_adapter *adapter =
@@ -1646,7 +1646,7 @@ eth_igb_close(struct rte_eth_dev *dev)
{
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct rte_eth_link link;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct e1000_filter_info *filter_info =
E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
@@ -2931,7 +2931,7 @@ static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
int ret;
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0;
struct rte_eth_dev_info dev_info;
@@ -3002,7 +3002,7 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev,
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct e1000_interrupt *intr =
E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_eth_link link;
int ret;
@@ -3496,7 +3496,7 @@ igbvf_dev_start(struct rte_eth_dev *dev)
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct e1000_adapter *adapter =
E1000_DEV_PRIVATE(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret;
uint32_t intr_vector = 0;
@@ -3560,7 +3560,7 @@ igbvf_dev_start(struct rte_eth_dev *dev)
static int
igbvf_dev_stop(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct e1000_adapter *adapter =
E1000_DEV_PRIVATE(dev->data->dev_private);
@@ -3608,7 +3608,7 @@ igbvf_dev_close(struct rte_eth_dev *dev)
{
struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct rte_ether_addr addr;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
int ret;
PMD_INIT_FUNC_TRACE();
@@ -5410,7 +5410,7 @@ eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = E1000_MISC_VEC_ID;
@@ -5434,7 +5434,7 @@ eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct e1000_hw *hw =
E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = E1000_MISC_VEC_ID;
@@ -5516,7 +5516,7 @@ eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
uint32_t vec = E1000_MISC_VEC_ID;
uint32_t base = E1000_MISC_VEC_ID;
uint32_t misc_shift = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
/* won't configure msix register if no mapping is done
diff --git a/drivers/net/intel/e1000/igb_pf.c b/drivers/net/intel/e1000/igb_pf.c
index a3327e0bf0..e7f77e69da 100644
--- a/drivers/net/intel/e1000/igb_pf.c
+++ b/drivers/net/intel/e1000/igb_pf.c
@@ -28,7 +28,7 @@
static inline uint16_t
dev_num_vf(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
return pci_dev->max_vfs;
}
diff --git a/drivers/net/intel/e1000/igc_ethdev.c b/drivers/net/intel/e1000/igc_ethdev.c
index 727ea36c2b..de35da2c36 100644
--- a/drivers/net/intel/e1000/igc_ethdev.c
+++ b/drivers/net/intel/e1000/igc_ethdev.c
@@ -440,7 +440,7 @@ static void
igc_intr_other_disable(struct rte_eth_dev *dev)
{
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (rte_intr_allow_others(intr_handle) &&
@@ -460,7 +460,7 @@ igc_intr_other_enable(struct rte_eth_dev *dev)
{
struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (rte_intr_allow_others(intr_handle) &&
@@ -576,7 +576,7 @@ static void
eth_igc_interrupt_action(struct rte_eth_dev *dev)
{
struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_eth_link link;
int ret;
@@ -679,7 +679,7 @@ eth_igc_stop(struct rte_eth_dev *dev)
{
struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct rte_eth_link link;
@@ -799,7 +799,7 @@ static void
igc_configure_msix_intr(struct rte_eth_dev *dev)
{
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_mask;
@@ -882,7 +882,7 @@ igc_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
uint32_t mask;
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0;
int nb_efd;
@@ -990,7 +990,7 @@ eth_igc_start(struct rte_eth_dev *dev)
{
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t nsec, sec, baset_l, baset_h, tqavctrl;
struct timespec system_time;
@@ -1307,7 +1307,7 @@ igc_dev_free_queues(struct rte_eth_dev *dev)
static int
eth_igc_close(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
@@ -1359,7 +1359,7 @@ igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
static int
eth_igc_dev_init(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
int i, error = 0;
@@ -2257,7 +2257,7 @@ static int
eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = IGC_MISC_VEC_ID;
@@ -2280,7 +2280,7 @@ static int
eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct e1000_hw *hw = IGC_DEV_PRIVATE_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = IGC_MISC_VEC_ID;
diff --git a/drivers/net/intel/fm10k/fm10k_ethdev.c b/drivers/net/intel/fm10k/fm10k_ethdev.c
index 97f61afec2..ca438d2d02 100644
--- a/drivers/net/intel/fm10k/fm10k_ethdev.c
+++ b/drivers/net/intel/fm10k/fm10k_ethdev.c
@@ -693,7 +693,7 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct fm10k_macvlan_filter_info *macvlan;
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
struct rte_intr_handle *intr_handle = pdev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
@@ -1161,7 +1161,7 @@ static int
fm10k_dev_stop(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
struct rte_intr_handle *intr_handle = pdev->intr_handle;
int i;
@@ -1371,7 +1371,7 @@ fm10k_dev_infos_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
PMD_INIT_FUNC_TRACE();
@@ -2364,7 +2364,7 @@ static int
fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
/* Enable ITR */
if (hw->mac.type == fm10k_mac_pf)
@@ -2381,7 +2381,7 @@ static int
fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
/* Disable ITR */
if (hw->mac.type == fm10k_mac_pf)
@@ -2397,7 +2397,7 @@ static int
fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
struct rte_intr_handle *intr_handle = pdev->intr_handle;
uint32_t intr_vector, vec;
uint16_t queue_id;
@@ -2794,7 +2794,7 @@ static int
fm10k_dev_close(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
struct rte_intr_handle *intr_handle = pdev->intr_handle;
int ret;
@@ -3060,7 +3060,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(dev, *pdev);
struct rte_intr_handle *intr_handle = pdev->intr_handle;
int diag, i, ret;
struct fm10k_macvlan_filter_info *macvlan;
diff --git a/drivers/net/intel/i40e/i40e_ethdev.c b/drivers/net/intel/i40e/i40e_ethdev.c
index e818b6fc7c..306ce5d9bc 100644
--- a/drivers/net/intel/i40e/i40e_ethdev.c
+++ b/drivers/net/intel/i40e/i40e_ethdev.c
@@ -980,7 +980,7 @@ is_floating_veb_supported(struct rte_devargs *devargs)
static void
config_floating_veb(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -1548,7 +1548,7 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
return 0;
}
i40e_set_default_ptype_table(dev);
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
intr_handle = pci_dev->intr_handle;
rte_eth_copy_pci_info(dev, pci_dev);
@@ -2040,7 +2040,7 @@ void
i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
{
struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vsi);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
uint16_t msix_vect = vsi->msix_intr;
@@ -2156,7 +2156,7 @@ int
i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t itr_idx)
{
struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vsi);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
uint16_t msix_vect = vsi->msix_intr;
@@ -2235,7 +2235,7 @@ void
i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
{
struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vsi);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
@@ -2262,7 +2262,7 @@ void
i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
{
struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vsi);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
@@ -2430,7 +2430,7 @@ i40e_dev_start(struct rte_eth_dev *dev)
struct i40e_adapter *ad =
I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
int ret, i;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_vector = 0;
struct i40e_vsi *vsi;
@@ -2611,7 +2611,7 @@ i40e_dev_stop(struct rte_eth_dev *dev)
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct i40e_vsi *main_vsi = pf->main_vsi;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int i;
@@ -2673,7 +2673,7 @@ i40e_dev_close(struct rte_eth_dev *dev)
{
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_filter_control_settings settings;
struct rte_flow *p_flow;
@@ -3830,7 +3830,7 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct i40e_vsi *vsi = pf->main_vsi;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
dev_info->max_rx_queues = vsi->nb_qps;
dev_info->max_tx_queues = vsi->nb_qps;
@@ -4883,7 +4883,7 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev)
{
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct i40e_hw *hw = I40E_PF_TO_HW(pf);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint16_t qp_count = 0, vsi_count = 0;
if (pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
@@ -10032,7 +10032,7 @@ i40e_dev_flow_ops_get(struct rte_eth_dev *dev,
static void
i40e_enable_extended_tag(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint32_t buf = 0;
int ret;
@@ -11218,7 +11218,7 @@ i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
static int
i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint16_t msix_intr;
@@ -11246,7 +11246,7 @@ i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
static int
i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint16_t msix_intr;
diff --git a/drivers/net/intel/i40e/i40e_ethdev.h b/drivers/net/intel/i40e/i40e_ethdev.h
index dcbdf65047..c39a5a8802 100644
--- a/drivers/net/intel/i40e/i40e_ethdev.h
+++ b/drivers/net/intel/i40e/i40e_ethdev.h
@@ -1467,7 +1467,7 @@ int i40e_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params);
int i40e_vf_representor_uninit(struct rte_eth_dev *ethdev);
#define I40E_DEV_TO_PCI(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_pci_device)
+ RTE_CLASS_TO_BUS_DEVICE(eth_dev, struct rte_pci_device)
/* I40E_DEV_PRIVATE_TO */
#define I40E_DEV_PRIVATE_TO_PF(adapter) \
diff --git a/drivers/net/intel/iavf/iavf_ethdev.c b/drivers/net/intel/iavf/iavf_ethdev.c
index bdf650b822..a8031e23a5 100644
--- a/drivers/net/intel/iavf/iavf_ethdev.c
+++ b/drivers/net/intel/iavf/iavf_ethdev.c
@@ -2031,7 +2031,7 @@ iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct iavf_adapter *adapter =
IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
uint16_t msix_intr;
@@ -2067,7 +2067,7 @@ iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
static int
iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint16_t msix_intr;
@@ -2986,7 +2986,7 @@ iavf_dev_init(struct rte_eth_dev *eth_dev)
IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int ret = 0;
PMD_INIT_FUNC_TRACE();
@@ -3149,7 +3149,7 @@ static int
iavf_dev_close(struct rte_eth_dev *dev)
{
struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct iavf_adapter *adapter =
IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
diff --git a/drivers/net/intel/ice/ice_dcf.c b/drivers/net/intel/ice/ice_dcf.c
index 98ae0f8980..3b635c0822 100644
--- a/drivers/net/intel/ice/ice_dcf.c
+++ b/drivers/net/intel/ice/ice_dcf.c
@@ -658,7 +658,7 @@ ice_dcf_send_aq_cmd(void *dcf_hw, struct ice_aq_desc *desc,
int
ice_dcf_handle_vsi_update_event(struct ice_dcf_hw *hw)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(hw->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(hw->eth_dev, *pci_dev);
int i = 0;
int err = -1;
@@ -738,7 +738,7 @@ dcf_get_vlan_offload_caps_v2(struct ice_dcf_hw *hw)
int
ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int ret, size;
hw->resetting = false;
@@ -873,7 +873,7 @@ ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw)
void
ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS)
diff --git a/drivers/net/intel/ice/ice_ethdev.c b/drivers/net/intel/ice/ice_ethdev.c
index 64cff6bcc2..b7cea3bfc1 100644
--- a/drivers/net/intel/ice/ice_ethdev.c
+++ b/drivers/net/intel/ice/ice_ethdev.c
@@ -2634,7 +2634,7 @@ ice_dev_init(struct rte_eth_dev *dev)
}
ice_set_default_ptype_table(dev);
- pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
intr_handle = pci_dev->intr_handle;
pf->adapter = ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
@@ -2967,7 +2967,7 @@ ice_dev_close(struct rte_eth_dev *dev)
{
struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ice_adapter *ad =
ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
@@ -4531,7 +4531,7 @@ ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct ice_vsi *vsi = pf->main_vsi;
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
bool is_safe_mode = pf->adapter->is_safe_mode;
u64 phy_type_low;
u64 phy_type_high;
diff --git a/drivers/net/intel/ice/ice_ethdev.h b/drivers/net/intel/ice/ice_ethdev.h
index 0e6790db35..20e8a13fe9 100644
--- a/drivers/net/intel/ice/ice_ethdev.h
+++ b/drivers/net/intel/ice/ice_ethdev.h
@@ -711,7 +711,7 @@ struct ice_vsi_vlan_pvid_info {
};
#define ICE_DEV_TO_PCI(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_pci_device)
+ RTE_CLASS_TO_BUS_DEVICE(eth_dev, struct rte_pci_device)
/* ICE_DEV_PRIVATE_TO */
#define ICE_DEV_PRIVATE_TO_PF(adapter) \
diff --git a/drivers/net/intel/idpf/idpf_ethdev.h b/drivers/net/intel/idpf/idpf_ethdev.h
index 5105eea1c5..99496c59da 100644
--- a/drivers/net/intel/idpf/idpf_ethdev.h
+++ b/drivers/net/intel/idpf/idpf_ethdev.h
@@ -85,7 +85,7 @@ struct idpf_adapter_ext {
TAILQ_HEAD(idpf_adapter_list, idpf_adapter_ext);
#define IDPF_DEV_TO_PCI(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_pci_device)
+ RTE_CLASS_TO_BUS_DEVICE(eth_dev, struct rte_pci_device)
#define IDPF_ADAPTER_TO_EXT(p) \
container_of((p), struct idpf_adapter_ext, base)
diff --git a/drivers/net/intel/ipn3ke/ipn3ke_ethdev.h b/drivers/net/intel/ipn3ke/ipn3ke_ethdev.h
index 9dc66017fb..3b8026114d 100644
--- a/drivers/net/intel/ipn3ke/ipn3ke_ethdev.h
+++ b/drivers/net/intel/ipn3ke/ipn3ke_ethdev.h
@@ -309,9 +309,6 @@ struct ipn3ke_hw {
uint8_t *hw_addr;
};
-#define RTE_ETH_DEV_TO_AFU(eth_dev) \
- RTE_BUS_DEVICE((eth_dev)->device, struct rte_afu_device)
-
/**
* PCIe MMIO Access
*/
diff --git a/drivers/net/intel/ipn3ke/ipn3ke_representor.c b/drivers/net/intel/ipn3ke/ipn3ke_representor.c
index 281c025820..05aad6cd15 100644
--- a/drivers/net/intel/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/intel/ipn3ke/ipn3ke_representor.c
@@ -2070,7 +2070,7 @@ ipn3ke_rpst_stats_reset(struct rte_eth_dev *ethdev)
return -EINVAL;
}
- afu_dev = RTE_ETH_DEV_TO_AFU(ethdev);
+ afu_dev = RTE_CLASS_TO_BUS_DEVICE(ethdev, *afu_dev);
if (!afu_dev) {
IPN3KE_AFU_PMD_ERR("afu device to reset is NULL!");
return -EINVAL;
@@ -2138,7 +2138,7 @@ ipn3ke_rpst_stats_get
return -EINVAL;
}
- afu_dev = RTE_ETH_DEV_TO_AFU(ethdev);
+ afu_dev = RTE_CLASS_TO_BUS_DEVICE(ethdev, *afu_dev);
if (!afu_dev) {
IPN3KE_AFU_PMD_ERR("afu device to get statistics is NULL!");
return -EINVAL;
@@ -2228,7 +2228,7 @@ ipn3ke_rpst_xstats_get
return -EINVAL;
}
- afu_dev = RTE_ETH_DEV_TO_AFU(ethdev);
+ afu_dev = RTE_CLASS_TO_BUS_DEVICE(ethdev, *afu_dev);
if (!afu_dev) {
IPN3KE_AFU_PMD_ERR("afu device to get statistics is NULL!");
return -EINVAL;
diff --git a/drivers/net/intel/ixgbe/ixgbe_ethdev.c b/drivers/net/intel/ixgbe/ixgbe_ethdev.c
index 9dc015dfff..f9de95e4fc 100644
--- a/drivers/net/intel/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/intel/ixgbe/ixgbe_ethdev.c
@@ -1085,7 +1085,7 @@ static int
eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
{
struct ixgbe_adapter *ad = IXGBE_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
@@ -1603,7 +1603,7 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
int diag;
uint32_t tc, tcs;
struct ixgbe_adapter *ad = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
@@ -2269,7 +2269,7 @@ ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
static int
ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
switch (nb_rx_q) {
case 1:
@@ -2511,7 +2511,7 @@ ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
struct rte_pci_device *pci_dev;
int ret;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
ret = rte_eth_link_get_nowait(dev->data->port_id, &link);
if (ret < 0)
return ret;
@@ -2621,7 +2621,7 @@ ixgbe_dev_start(struct rte_eth_dev *dev)
struct rte_eth_fdir_conf *fdir_conf = IXGBE_DEV_FDIR_CONF(dev);
struct ixgbe_vf_info *vfinfo =
*IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_vector = 0;
int err;
@@ -2930,7 +2930,7 @@ ixgbe_dev_stop(struct rte_eth_dev *dev)
IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct ixgbe_vf_info *vfinfo =
*IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int vf;
struct ixgbe_tm_conf *tm_conf =
@@ -3091,7 +3091,7 @@ ixgbe_dev_close(struct rte_eth_dev *dev)
{
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int retries = 0;
int ret;
@@ -3980,7 +3980,7 @@ ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
static int
ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
@@ -4110,7 +4110,7 @@ static int
ixgbevf_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
@@ -4678,7 +4678,7 @@ ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
static void
ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_eth_link link;
rte_eth_linkstatus_get(dev, &link);
@@ -4790,7 +4790,7 @@ static void
ixgbe_dev_interrupt_delayed_handler(void *param)
{
struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ixgbe_interrupt *intr =
IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
@@ -5354,7 +5354,7 @@ ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
static int
ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
/*
* This function calls into the base driver, which in turn will use
@@ -5518,7 +5518,7 @@ ixgbevf_dev_start(struct rte_eth_dev *dev)
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t intr_vector = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int err, mask = 0;
@@ -5631,7 +5631,7 @@ ixgbevf_dev_stop(struct rte_eth_dev *dev)
{
struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct ixgbe_adapter *adapter = dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
/*
@@ -5679,7 +5679,7 @@ static int
ixgbevf_dev_close(struct rte_eth_dev *dev)
{
struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret;
@@ -5985,7 +5985,7 @@ ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
static int
ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ixgbe_interrupt *intr =
IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
@@ -6015,7 +6015,7 @@ ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = IXGBE_MISC_VEC_ID;
@@ -6035,7 +6035,7 @@ ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
static int
ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t mask;
struct ixgbe_hw *hw =
@@ -6172,7 +6172,7 @@ ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
static void
ixgbevf_configure_msix(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -6223,7 +6223,7 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
static void
ixgbe_configure_msix(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ixgbe_hw *hw =
IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
diff --git a/drivers/net/intel/ixgbe/ixgbe_pf.c b/drivers/net/intel/ixgbe/ixgbe_pf.c
index d3db571918..939e7d1417 100644
--- a/drivers/net/intel/ixgbe/ixgbe_pf.c
+++ b/drivers/net/intel/ixgbe/ixgbe_pf.c
@@ -31,7 +31,7 @@
static inline uint16_t
dev_num_vf(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
return pci_dev->max_vfs;
}
diff --git a/drivers/net/intel/ixgbe/ixgbe_tm.c b/drivers/net/intel/ixgbe/ixgbe_tm.c
index e1d8364f46..984c157223 100644
--- a/drivers/net/intel/ixgbe/ixgbe_tm.c
+++ b/drivers/net/intel/ixgbe/ixgbe_tm.c
@@ -365,7 +365,7 @@ ixgbe_queue_base_nb_get(struct rte_eth_dev *dev, uint16_t tc_node_no,
uint16_t *base, uint16_t *nb)
{
uint8_t nb_tcs = ixgbe_tc_nb_get(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint16_t vf_num = pci_dev->max_vfs;
*base = 0;
diff --git a/drivers/net/intel/ixgbe/ixgbe_vf_representor.c b/drivers/net/intel/ixgbe/ixgbe_vf_representor.c
index 52b43530c0..c11983e307 100644
--- a/drivers/net/intel/ixgbe/ixgbe_vf_representor.c
+++ b/drivers/net/intel/ixgbe/ixgbe_vf_representor.c
@@ -223,7 +223,7 @@ ixgbe_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params)
if (pf_ethdev == NULL)
return -ENODEV;
- pci_dev = RTE_ETH_DEV_TO_PCI(pf_ethdev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(pf_ethdev, *pci_dev);
if (representor->vf_id >= pci_dev->max_vfs)
return -ENODEV;
diff --git a/drivers/net/intel/ixgbe/rte_pmd_ixgbe.c b/drivers/net/intel/ixgbe/rte_pmd_ixgbe.c
index 30dec57be8..f816fa3173 100644
--- a/drivers/net/intel/ixgbe/rte_pmd_ixgbe.c
+++ b/drivers/net/intel/ixgbe/rte_pmd_ixgbe.c
@@ -25,7 +25,7 @@ rte_pmd_ixgbe_set_vf_mac_addr(uint16_t port, uint16_t vf,
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -60,7 +60,7 @@ rte_pmd_ixgbe_ping_vf(uint16_t port, uint16_t vf)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -92,7 +92,7 @@ rte_pmd_ixgbe_set_vf_vlan_anti_spoof(uint16_t port, uint16_t vf, uint8_t on)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -123,7 +123,7 @@ rte_pmd_ixgbe_set_vf_mac_anti_spoof(uint16_t port, uint16_t vf, uint8_t on)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -153,7 +153,7 @@ rte_pmd_ixgbe_set_vf_vlan_insert(uint16_t port, uint16_t vf, uint16_t vlan_id)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -252,7 +252,7 @@ rte_pmd_ixgbe_set_vf_split_drop_en(uint16_t port, uint16_t vf, uint8_t on)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -289,7 +289,7 @@ rte_pmd_ixgbe_set_vf_vlan_stripq(uint16_t port, uint16_t vf, uint8_t on)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
if (!is_ixgbe_supported(dev))
@@ -338,7 +338,7 @@ rte_pmd_ixgbe_set_vf_rxmode(uint16_t port, uint16_t vf,
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -386,7 +386,7 @@ rte_pmd_ixgbe_set_vf_rx(uint16_t port, uint16_t vf, uint8_t on)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
@@ -438,7 +438,7 @@ rte_pmd_ixgbe_set_vf_tx(uint16_t port, uint16_t vf, uint8_t on)
RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
dev = &rte_eth_devices[port];
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!is_ixgbe_supported(dev))
return -ENOTSUP;
diff --git a/drivers/net/nbl/nbl_core.c b/drivers/net/nbl/nbl_core.c
index df8c0c76ed..6a823e9bfb 100644
--- a/drivers/net/nbl/nbl_core.c
+++ b/drivers/net/nbl/nbl_core.c
@@ -32,7 +32,7 @@ static void nbl_init_func_caps(const struct rte_pci_device *pci_dev, struct nbl_
int nbl_core_init(struct nbl_adapter *adapter, struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
const struct nbl_product_core_ops *product_base_ops = NULL;
struct nbl_common_info *common = NBL_ADAPTER_TO_COMMON(adapter);
int ret = 0;
diff --git a/drivers/net/nbl/nbl_dev/nbl_dev.c b/drivers/net/nbl/nbl_dev/nbl_dev.c
index 2b0413fb7c..35485ea691 100644
--- a/drivers/net/nbl/nbl_dev/nbl_dev.c
+++ b/drivers/net/nbl/nbl_dev/nbl_dev.c
@@ -868,7 +868,7 @@ static int nbl_dev_common_start(struct nbl_dev_mgt *dev_mgt)
struct nbl_dev_net_mgt *net_dev = NBL_DEV_MGT_TO_NET_DEV(dev_mgt);
struct nbl_common_info *common = NBL_DEV_MGT_TO_COMMON(dev_mgt);
struct nbl_board_port_info *board_info;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(net_dev->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(net_dev->eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
u8 *mac;
int ret;
@@ -991,7 +991,7 @@ static void nbl_dev_leonis_stop(void *p)
const struct nbl_common_info *common = NBL_DEV_MGT_TO_COMMON(dev_mgt);
const struct nbl_dispatch_ops *disp_ops = NBL_DEV_MGT_TO_DISP_OPS(dev_mgt);
const struct nbl_channel_ops *chan_ops = NBL_DEV_MGT_TO_CHAN_OPS(dev_mgt);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(net_dev->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(net_dev->eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
u8 *mac;
@@ -1105,7 +1105,7 @@ static int nbl_dev_setup_net_dev(struct nbl_dev_mgt *dev_mgt,
struct nbl_register_net_param register_param = { 0 };
struct nbl_register_net_result register_result = { 0 };
struct nbl_dev_ring_mgt *ring_mgt;
- const struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ const struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int ret = 0;
net_dev = rte_zmalloc("nbl_dev_net", sizeof(struct nbl_dev_net_mgt), 0);
diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index 7ec554cb83..9a193e4a7a 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -1627,7 +1627,6 @@ static int
eth_hn_dev_init(struct rte_eth_dev *eth_dev)
{
struct hn_data *hv = eth_dev->data->dev_private;
- struct rte_device *device = eth_dev->device;
struct rte_vmbus_device *vmbus;
uint32_t mtu;
unsigned int rxr_cnt;
@@ -1638,7 +1637,7 @@ eth_hn_dev_init(struct rte_eth_dev *eth_dev)
rte_spinlock_init(&hv->hotadd_lock);
LIST_INIT(&hv->hotadd_list);
- vmbus = RTE_BUS_DEVICE(device, *vmbus);
+ vmbus = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *vmbus);
eth_dev->dev_ops = &hn_eth_dev_ops;
eth_dev->rx_queue_count = hn_dev_rx_queue_count;
eth_dev->rx_descriptor_status = hn_dev_rx_queue_status;
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index cbd1deffb4..d2da18013c 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -363,7 +363,7 @@ nfp_net_start(struct rte_eth_dev *dev)
struct rte_eth_txmode *txmode;
struct nfp_net_hw_priv *hw_priv;
struct nfp_app_fw_nic *app_fw_nic;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
net_hw = dev->data->dev_private;
@@ -770,7 +770,7 @@ nfp_net_close(struct rte_eth_dev *dev)
hw = dev->data->dev_private;
pf_dev = hw_priv->pf_dev;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
app_fw_nic = NFP_PRIV_TO_APP_FW_NIC(pf_dev->app_fw_priv);
/*
@@ -1022,7 +1022,7 @@ nfp_net_init(struct rte_eth_dev *eth_dev,
struct nfp_net_hw_priv *hw_priv;
struct nfp_app_fw_nic *app_fw_nic;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
net_hw = eth_dev->data->dev_private;
hw_init = para;
@@ -2879,7 +2879,7 @@ nfp_pci_uninit(struct rte_eth_dev *eth_dev)
uint16_t port_id;
struct rte_pci_device *pci_dev;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* Free up all physical ports under PF */
RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
diff --git a/drivers/net/nfp/nfp_ethdev_vf.c b/drivers/net/nfp/nfp_ethdev_vf.c
index 23fa5b82ad..a86cc36592 100644
--- a/drivers/net/nfp/nfp_ethdev_vf.c
+++ b/drivers/net/nfp/nfp_ethdev_vf.c
@@ -30,7 +30,7 @@ nfp_netvf_start(struct rte_eth_dev *dev)
struct nfp_net_hw *net_hw;
struct rte_eth_conf *dev_conf;
struct rte_eth_rxmode *rxmode;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
/* Disabling queues just in case... */
@@ -169,7 +169,7 @@ nfp_netvf_close(struct rte_eth_dev *dev)
return 0;
net_hw = dev->data->dev_private;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
hw_priv = dev->process_private;
rte_free(net_hw->eth_xstats_base);
@@ -266,7 +266,7 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev)
const struct nfp_dev_info *dev_info;
port = eth_dev->data->port_id;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
dev_info = nfp_dev_info_get(pci_dev->id.device_id);
if (dev_info == NULL) {
diff --git a/drivers/net/nfp/nfp_net_common.c b/drivers/net/nfp/nfp_net_common.c
index d35eee970a..2d36311cfe 100644
--- a/drivers/net/nfp/nfp_net_common.c
+++ b/drivers/net/nfp/nfp_net_common.c
@@ -1568,7 +1568,7 @@ nfp_rx_queue_intr_enable(struct rte_eth_dev *dev,
struct nfp_net_hw *hw;
struct rte_pci_device *pci_dev;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (rte_intr_type_get(pci_dev->intr_handle) != RTE_INTR_HANDLE_UIO)
base = 1;
@@ -1589,7 +1589,7 @@ nfp_rx_queue_intr_disable(struct rte_eth_dev *dev,
struct nfp_net_hw *hw;
struct rte_pci_device *pci_dev;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (rte_intr_type_get(pci_dev->intr_handle) != RTE_INTR_HANDLE_UIO)
base = 1;
@@ -1606,7 +1606,7 @@ static void
nfp_net_dev_link_status_print(struct rte_eth_dev *dev)
{
struct rte_eth_link link;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
rte_eth_linkstatus_get(dev, &link);
if (link.link_status != 0)
@@ -1635,7 +1635,7 @@ nfp_net_irq_unmask(struct rte_eth_dev *dev)
struct rte_pci_device *pci_dev;
hw = nfp_net_get_hw(dev);
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
/* Make sure all updates are written before un-masking */
rte_wmb();
diff --git a/drivers/net/ngbe/ngbe_ethdev.c b/drivers/net/ngbe/ngbe_ethdev.c
index 8b9d6371fb..f7b4a8b159 100644
--- a/drivers/net/ngbe/ngbe_ethdev.c
+++ b/drivers/net/ngbe/ngbe_ethdev.c
@@ -321,7 +321,7 @@ ngbe_swfw_lock_reset(struct ngbe_hw *hw)
static int
eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct ngbe_hw *hw = ngbe_dev_hw(eth_dev);
struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(eth_dev);
struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(eth_dev);
@@ -958,7 +958,7 @@ ngbe_dev_start(struct rte_eth_dev *dev)
{
struct ngbe_hw *hw = ngbe_dev_hw(dev);
struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_vector = 0;
int err;
@@ -1160,7 +1160,7 @@ ngbe_dev_stop(struct rte_eth_dev *dev)
struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
struct ngbe_hw *hw = ngbe_dev_hw(dev);
struct ngbe_vf_info *vfinfo = *NGBE_DEV_VFDATA(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int vf;
@@ -1256,7 +1256,7 @@ static int
ngbe_dev_close(struct rte_eth_dev *dev)
{
struct ngbe_hw *hw = ngbe_dev_hw(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int retries = 0;
int ret;
@@ -1843,7 +1843,7 @@ ngbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
static int
ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct ngbe_hw *hw = ngbe_dev_hw(dev);
dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
@@ -2258,7 +2258,7 @@ ngbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
static void
ngbe_dev_link_status_print(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_eth_link link;
rte_eth_linkstatus_get(dev, &link);
@@ -2472,7 +2472,7 @@ static s32
ngbe_fc_hpbthresh_set(struct rte_eth_dev *dev)
{
struct ngbe_hw *hw = ngbe_dev_hw(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
u32 max_frame_size, tc, dv_id, rx_pb;
s32 kb, marker;
@@ -2653,7 +2653,7 @@ ngbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
static int
ngbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
ngbe_remove_rar(dev, 0);
ngbe_add_rar(dev, addr, 0, pci_dev->max_vfs);
@@ -2797,7 +2797,7 @@ ngbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
static int
ngbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ngbe_hw *hw = ngbe_dev_hw(dev);
uint32_t mask;
@@ -2867,7 +2867,7 @@ ngbe_set_ivar_map(struct ngbe_hw *hw, int8_t direction,
static void
ngbe_configure_msix(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ngbe_hw *hw = ngbe_dev_hw(dev);
uint32_t queue_id, base = NGBE_MISC_VEC_ID;
diff --git a/drivers/net/ngbe/ngbe_ethdev_vf.c b/drivers/net/ngbe/ngbe_ethdev_vf.c
index 6406df40d0..ea3a988df6 100644
--- a/drivers/net/ngbe/ngbe_ethdev_vf.c
+++ b/drivers/net/ngbe/ngbe_ethdev_vf.c
@@ -152,7 +152,7 @@ eth_ngbevf_dev_init(struct rte_eth_dev *eth_dev)
{
int err;
uint32_t tc, tcs;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ngbe_hw *hw = ngbe_dev_hw(eth_dev);
struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(eth_dev);
@@ -465,7 +465,7 @@ static int
ngbevf_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct ngbe_hw *hw = ngbe_dev_hw(dev);
dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
@@ -588,7 +588,7 @@ ngbevf_dev_start(struct rte_eth_dev *dev)
{
struct ngbe_hw *hw = ngbe_dev_hw(dev);
uint32_t intr_vector = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int err, mask = 0;
@@ -688,7 +688,7 @@ ngbevf_dev_stop(struct rte_eth_dev *dev)
{
struct ngbe_hw *hw = ngbe_dev_hw(dev);
struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (hw->adapter_stopped)
@@ -725,7 +725,7 @@ static int
ngbevf_dev_close(struct rte_eth_dev *dev)
{
struct ngbe_hw *hw = ngbe_dev_hw(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret;
@@ -898,7 +898,7 @@ ngbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
static int
ngbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
struct ngbe_hw *hw = ngbe_dev_hw(dev);
@@ -920,7 +920,7 @@ ngbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
struct ngbe_hw *hw = ngbe_dev_hw(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = NGBE_MISC_VEC_ID;
@@ -960,7 +960,7 @@ ngbevf_set_ivar_map(struct ngbe_hw *hw, int8_t direction,
static void
ngbevf_configure_msix(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct ngbe_hw *hw = ngbe_dev_hw(dev);
uint32_t q_idx;
diff --git a/drivers/net/ngbe/ngbe_pf.c b/drivers/net/ngbe/ngbe_pf.c
index bb62e2fbb7..db2384e28c 100644
--- a/drivers/net/ngbe/ngbe_pf.c
+++ b/drivers/net/ngbe/ngbe_pf.c
@@ -18,7 +18,7 @@
static inline uint16_t
dev_num_vf(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* EM only support 7 VFs. */
return pci_dev->max_vfs;
diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 99be30523a..876d2f9d7d 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -777,7 +777,7 @@ static int otx_ep_eth_dev_query_set_vf_mac(struct rte_eth_dev *eth_dev,
static int
otx_ep_eth_dev_init(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev);
struct rte_ether_addr vf_mac_addr;
int ret = 0;
diff --git a/drivers/net/octeon_ep/otx_ep_mbox.c b/drivers/net/octeon_ep/otx_ep_mbox.c
index 3e94c66677..5e6be29a96 100644
--- a/drivers/net/octeon_ep/otx_ep_mbox.c
+++ b/drivers/net/octeon_ep/otx_ep_mbox.c
@@ -346,7 +346,7 @@ otx_ep_mbox_intr_handler(void *param)
{
struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param;
struct otx_ep_device *otx_ep = (struct otx_ep_device *)eth_dev->data->dev_private;
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
union otx_ep_mbox_word mbox_cmd;
if (otx2_read64(otx_ep->hw_addr + CNXK_EP_R_MBOX_PF_VF_INT(0)) & CNXK_EP_MBOX_INTR) {
@@ -369,7 +369,7 @@ int
otx_ep_mbox_init(struct rte_eth_dev *eth_dev)
{
struct otx_ep_device *otx_ep = (struct otx_ep_device *)eth_dev->data->dev_private;
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
uint64_t reg_val;
int rc;
@@ -402,7 +402,7 @@ void
otx_ep_mbox_uninit(struct rte_eth_dev *eth_dev)
{
struct otx_ep_device *otx_ep = (struct otx_ep_device *)eth_dev->data->dev_private;
- struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pdev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pdev);
otx2_write64(0, otx_ep->hw_addr + CNXK_EP_R_MBOX_PF_VF_INT(0));
diff --git a/drivers/net/qede/qede_ethdev.c b/drivers/net/qede/qede_ethdev.c
index c676c6fa75..4efc2dd349 100644
--- a/drivers/net/qede/qede_ethdev.c
+++ b/drivers/net/qede/qede_ethdev.c
@@ -1231,7 +1231,7 @@ static int qede_args_check(const char *key, const char *val, void *opaque)
static int qede_args(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_kvargs *kvlist;
struct rte_devargs *devargs;
int ret;
@@ -1540,7 +1540,7 @@ static void qede_poll_sp_sb_cb(void *param)
static int qede_dev_close(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
int ret = 0;
@@ -2529,7 +2529,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
adapter = eth_dev->data->dev_private;
adapter->ethdev = eth_dev;
edev = &adapter->edev;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
pci_addr = pci_dev->addr;
PMD_INIT_FUNC_TRACE(edev);
diff --git a/drivers/net/r8169/r8169_ethdev.c b/drivers/net/r8169/r8169_ethdev.c
index b2b1882aa5..2ac0189b61 100644
--- a/drivers/net/r8169/r8169_ethdev.c
+++ b/drivers/net/r8169/r8169_ethdev.c
@@ -301,7 +301,7 @@ rtl_dev_start(struct rte_eth_dev *dev)
{
struct rtl_adapter *adapter = RTL_DEV_PRIVATE(dev);
struct rtl_hw *hw = &adapter->hw;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int err;
@@ -684,7 +684,7 @@ rtl_dev_interrupt_handler(void *param)
static int
rtl_dev_close(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct rtl_adapter *adapter = RTL_DEV_PRIVATE(dev);
struct rtl_hw *hw = &adapter->hw;
@@ -908,7 +908,7 @@ rtl_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf
static int
rtl_dev_init(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct rtl_adapter *adapter = RTL_DEV_PRIVATE(dev);
struct rtl_hw *hw = &adapter->hw;
diff --git a/drivers/net/rnp/rnp_ethdev.c b/drivers/net/rnp/rnp_ethdev.c
index 3420842823..e48ad0e317 100644
--- a/drivers/net/rnp/rnp_ethdev.c
+++ b/drivers/net/rnp/rnp_ethdev.c
@@ -728,7 +728,7 @@ static int rnp_dev_close(struct rte_eth_dev *eth_dev)
if (adapter->intr_registered && adapter->eth_dev == eth_dev)
rnp_change_manage_port(adapter);
if (adapter->closed_ports == adapter->inited_ports) {
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
if (adapter->intr_registered) {
/* disable uio irq before callback unregister */
rte_intr_disable(pci_dev->intr_handle);
@@ -1667,7 +1667,7 @@ rnp_rx_reset_pool_setup(struct rnp_eth_adapter *adapter)
static int
rnp_eth_dev_init(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct rnp_eth_port *port = RNP_DEV_TO_PORT(eth_dev);
char name[RTE_ETH_NAME_MAX_LEN] = " ";
@@ -1798,7 +1798,7 @@ rnp_eth_dev_init(struct rte_eth_dev *eth_dev)
static int
rnp_eth_dev_uninit(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(eth_dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
uint16_t port_id;
int err = 0;
diff --git a/drivers/net/sfc/sfc.c b/drivers/net/sfc/sfc.c
index 69747e49ae..39cd8d519a 100644
--- a/drivers/net/sfc/sfc.c
+++ b/drivers/net/sfc/sfc.c
@@ -781,7 +781,7 @@ static int
sfc_mem_bar_init(struct sfc_adapter *sa, const efx_bar_region_t *mem_ebrp)
{
struct rte_eth_dev *eth_dev = sa->eth_dev;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
efsys_bar_t *ebp = &sa->mem_bar;
struct rte_mem_resource *res =
&pci_dev->mem_resource[mem_ebrp->ebr_index];
@@ -1283,7 +1283,7 @@ sfc_probe(struct sfc_adapter *sa)
{
efx_bar_region_t mem_ebrp;
struct rte_eth_dev *eth_dev = sa->eth_dev;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
efx_nic_t *enp;
int rc;
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index 6be98c49d0..6be91789cf 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -3309,7 +3309,7 @@ static int
sfc_eth_dev_init(struct rte_eth_dev *dev, void *init_params)
{
struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct sfc_ethdev_init_data *init_data = init_params;
uint32_t logtype_main;
struct sfc_adapter *sa;
diff --git a/drivers/net/sfc/sfc_intr.c b/drivers/net/sfc/sfc_intr.c
index ddddefad7b..6a09da9f67 100644
--- a/drivers/net/sfc/sfc_intr.c
+++ b/drivers/net/sfc/sfc_intr.c
@@ -56,7 +56,7 @@ sfc_intr_line_handler(void *cb_arg)
boolean_t fatal;
uint32_t qmask;
unsigned int lsc_seq = sa->port.lsc_seq;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
sfc_log_init(sa, "entry");
@@ -102,7 +102,7 @@ sfc_intr_message_handler(void *cb_arg)
efx_nic_t *enp = sa->nic;
boolean_t fatal;
unsigned int lsc_seq = sa->port.lsc_seq;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
sfc_log_init(sa, "entry");
@@ -158,7 +158,7 @@ sfc_intr_start(struct sfc_adapter *sa)
if (rc != 0)
goto fail_intr_init;
- pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
intr_handle = pci_dev->intr_handle;
if (intr->handler != NULL) {
@@ -240,7 +240,7 @@ void
sfc_intr_stop(struct sfc_adapter *sa)
{
struct sfc_intr *intr = &sa->intr;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
sfc_log_init(sa, "entry");
@@ -318,7 +318,7 @@ int
sfc_intr_attach(struct sfc_adapter *sa)
{
struct sfc_intr *intr = &sa->intr;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
sfc_log_init(sa, "entry");
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index a193229265..305c680944 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -1277,8 +1277,9 @@ sfc_rx_qinit(struct sfc_adapter *sa, sfc_sw_index_t sw_index,
info.nic_dma_info = &sas->nic_dma_info;
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
rc = sa->priv.dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index,
- &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr,
+ &pci_dev->addr,
socket_id, &info, &rxq_info->dp);
if (rc != 0)
goto fail_dp_rx_qcreate;
diff --git a/drivers/net/sfc/sfc_sriov.c b/drivers/net/sfc/sfc_sriov.c
index 009b884d8d..f41d1b1719 100644
--- a/drivers/net/sfc/sfc_sriov.c
+++ b/drivers/net/sfc/sfc_sriov.c
@@ -44,7 +44,7 @@ sriov_mac_addr_assigned(const efx_vport_config_t *vport_config,
int
sfc_sriov_attach(struct sfc_adapter *sa)
{
- const struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+ const struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
struct sfc_sriov *sriov = &sa->sriov;
efx_vport_config_t *vport_config;
unsigned int i;
diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c
index ebc0a8235b..fac56cb27c 100644
--- a/drivers/net/sfc/sfc_tx.c
+++ b/drivers/net/sfc/sfc_tx.c
@@ -230,8 +230,9 @@ sfc_tx_qinit(struct sfc_adapter *sa, sfc_sw_index_t sw_index,
info.max_pdu = encp->enc_mac_pdu_max;
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(sa->eth_dev, *pci_dev);
rc = sa->priv.dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index,
- &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr,
+ &pci_dev->addr,
socket_id, &info, &txq_info->dp);
if (rc != 0)
goto fail_dp_tx_qinit;
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 6b82209f62..b6cc8703a7 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -561,7 +561,7 @@ static int32_t sxe2_dev_info_init(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter =
SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct sxe2_dev_info *dev_info = &adapter->dev_info;
struct sxe2_drv_dev_info_resp dev_info_resp = {0};
struct sxe2_drv_dev_fw_info_resp dev_fw_info_resp = {0};
@@ -600,7 +600,7 @@ static int32_t sxe2_dev_info_init(struct rte_eth_dev *dev)
int32_t sxe2_dev_pci_map_init(struct rte_eth_dev *dev)
{
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
- struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev->device, *pci_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct sxe2_pci_map_context *map_ctxt = &adapter->map_ctxt;
struct sxe2_pci_map_bar_info *bar_info = NULL;
struct sxe2_pci_map_segment_info *seg_info = NULL;
diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
index 76ed76a045..6e34da7c3c 100644
--- a/drivers/net/thunderx/nicvf_ethdev.c
+++ b/drivers/net/thunderx/nicvf_ethdev.c
@@ -1471,7 +1471,7 @@ static int
nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
{
struct nicvf *nic = nicvf_pmd_priv(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
PMD_INIT_FUNC_TRACE();
@@ -2234,7 +2234,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
}
}
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
rte_eth_copy_pci_info(eth_dev, pci_dev);
eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
diff --git a/drivers/net/txgbe/txgbe_ethdev.c b/drivers/net/txgbe/txgbe_ethdev.c
index 5d360f8305..0f484dfe91 100644
--- a/drivers/net/txgbe/txgbe_ethdev.c
+++ b/drivers/net/txgbe/txgbe_ethdev.c
@@ -525,7 +525,7 @@ static void
txgbe_parse_devargs(struct rte_eth_dev *dev)
{
struct rte_eth_fdir_conf *fdir_conf = TXGBE_DEV_FDIR_CONF(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_devargs *devargs = pci_dev->device.devargs;
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
struct rte_kvargs *kvlist;
@@ -601,7 +601,7 @@ static int
eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
{
struct txgbe_adapter *ad = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(eth_dev);
struct txgbe_vfta *shadow_vfta = TXGBE_DEV_VFTA(eth_dev);
struct txgbe_hwstrip *hwstrip = TXGBE_DEV_HWSTRIP(eth_dev);
@@ -1397,7 +1397,7 @@ txgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
static int
txgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
switch (nb_rx_q) {
case 1:
@@ -1664,7 +1664,7 @@ txgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
struct rte_pci_device *pci_dev;
int ret;
- pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
ret = rte_eth_link_get_nowait(dev->data->port_id, &link);
if (ret < 0)
return ret;
@@ -1736,7 +1736,7 @@ txgbe_dev_start(struct rte_eth_dev *dev)
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
struct txgbe_hw_stats *hw_stats = TXGBE_DEV_STATS(dev);
struct txgbe_vf_info *vfinfo = *TXGBE_DEV_VFDATA(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t intr_vector = 0;
int err;
@@ -2034,7 +2034,7 @@ txgbe_dev_stop(struct rte_eth_dev *dev)
struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
struct txgbe_vf_info *vfinfo = *TXGBE_DEV_VFDATA(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int vf;
struct txgbe_tm_conf *tm_conf = TXGBE_DEV_TM_CONF(dev);
@@ -2163,7 +2163,7 @@ static int
txgbe_dev_close(struct rte_eth_dev *dev)
{
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int retries = 0;
int ret;
@@ -2822,7 +2822,7 @@ txgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
static int
txgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
@@ -3500,7 +3500,7 @@ txgbe_dev_interrupt_get_status(struct rte_eth_dev *dev,
static void
txgbe_dev_link_status_print(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_eth_link link;
rte_eth_linkstatus_get(dev, &link);
@@ -3631,7 +3631,7 @@ static void
txgbe_dev_interrupt_delayed_handler(void *param)
{
struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
@@ -3978,7 +3978,7 @@ txgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
static int
txgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
txgbe_remove_rar(dev, 0);
txgbe_add_rar(dev, addr, 0, pci_dev->max_vfs);
@@ -4149,7 +4149,7 @@ txgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
static int
txgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t mask;
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
@@ -4231,7 +4231,7 @@ txgbe_set_ivar_map(struct txgbe_hw *hw, int8_t direction,
static void
txgbe_configure_msix(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
uint32_t queue_id, base = TXGBE_MISC_VEC_ID;
diff --git a/drivers/net/txgbe/txgbe_ethdev_vf.c b/drivers/net/txgbe/txgbe_ethdev_vf.c
index 39a5fff65c..7a50c7a855 100644
--- a/drivers/net/txgbe/txgbe_ethdev_vf.c
+++ b/drivers/net/txgbe/txgbe_ethdev_vf.c
@@ -232,7 +232,7 @@ eth_txgbevf_dev_init(struct rte_eth_dev *eth_dev)
int err;
uint32_t tc, tcs;
struct txgbe_adapter *ad = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct txgbe_hw *hw = TXGBE_DEV_HW(eth_dev);
struct txgbe_vfta *shadow_vfta = TXGBE_DEV_VFTA(eth_dev);
@@ -561,7 +561,7 @@ static int
txgbevf_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
@@ -696,7 +696,7 @@ txgbevf_dev_start(struct rte_eth_dev *dev)
{
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
uint32_t intr_vector = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int err, mask = 0;
@@ -801,7 +801,7 @@ txgbevf_dev_stop(struct rte_eth_dev *dev)
{
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
if (hw->adapter_stopped)
@@ -841,7 +841,7 @@ static int
txgbevf_dev_close(struct rte_eth_dev *dev)
{
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
int ret;
@@ -1023,7 +1023,7 @@ txgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
static int
txgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
@@ -1045,7 +1045,7 @@ txgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
{
struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
uint32_t vec = TXGBE_MISC_VEC_ID;
@@ -1085,7 +1085,7 @@ txgbevf_set_ivar_map(struct txgbe_hw *hw, int8_t direction,
static void
txgbevf_configure_msix(struct rte_eth_dev *dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
uint32_t q_idx;
diff --git a/drivers/net/txgbe/txgbe_flow.c b/drivers/net/txgbe/txgbe_flow.c
index a97588e57a..1bb0d3978c 100644
--- a/drivers/net/txgbe/txgbe_flow.c
+++ b/drivers/net/txgbe/txgbe_flow.c
@@ -1171,7 +1171,7 @@ cons_parse_l2_tn_filter(struct rte_eth_dev *dev,
const struct rte_flow_item_e_tag *e_tag_mask;
const struct rte_flow_action *act;
const struct rte_flow_action_vf *act_vf;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
if (!pattern) {
rte_flow_error_set(error, EINVAL,
@@ -1328,7 +1328,7 @@ txgbe_parse_l2_tn_filter(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
int ret = 0;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint16_t vf_num;
if (!txgbe_is_pf(TXGBE_DEV_HW(dev))) {
diff --git a/drivers/net/txgbe/txgbe_pf.c b/drivers/net/txgbe/txgbe_pf.c
index 700632bd88..91f73521fe 100644
--- a/drivers/net/txgbe/txgbe_pf.c
+++ b/drivers/net/txgbe/txgbe_pf.c
@@ -33,7 +33,7 @@
static inline uint16_t
dev_num_vf(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
return pci_dev->max_vfs;
}
diff --git a/drivers/net/txgbe/txgbe_tm.c b/drivers/net/txgbe/txgbe_tm.c
index b62bcf54aa..29c7c4adfb 100644
--- a/drivers/net/txgbe/txgbe_tm.c
+++ b/drivers/net/txgbe/txgbe_tm.c
@@ -354,7 +354,7 @@ txgbe_queue_base_nb_get(struct rte_eth_dev *dev, uint16_t tc_node_no,
uint16_t *base, uint16_t *nb)
{
uint8_t nb_tcs = txgbe_tc_nb_get(dev);
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(dev, *pci_dev);
uint16_t vf_num = pci_dev->max_vfs;
*base = 0;
diff --git a/drivers/net/virtio/virtio_pci_ethdev.c b/drivers/net/virtio/virtio_pci_ethdev.c
index fcda002297..d4f4bb0920 100644
--- a/drivers/net/virtio/virtio_pci_ethdev.c
+++ b/drivers/net/virtio/virtio_pci_ethdev.c
@@ -73,13 +73,13 @@ eth_virtio_pci_init(struct rte_eth_dev *eth_dev)
{
struct virtio_pci_dev *dev = eth_dev->data->dev_private;
struct virtio_hw *hw = &dev->hw;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int ret;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
hw->port_id = eth_dev->data->port_id;
VTPCI_DEV(hw) = pci_dev;
- ret = vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev), dev);
+ ret = vtpci_init(pci_dev, dev);
if (ret) {
PMD_INIT_LOG(ERR, "Failed to init PCI device");
return -1;
@@ -91,7 +91,7 @@ eth_virtio_pci_init(struct rte_eth_dev *eth_dev)
else
VIRTIO_OPS(hw) = &virtio_legacy_ops;
- ret = virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev), dev);
+ ret = virtio_remap_pci(pci_dev, dev);
if (ret < 0) {
PMD_INIT_LOG(ERR, "Failed to remap PCI device");
return -1;
@@ -111,7 +111,7 @@ eth_virtio_pci_init(struct rte_eth_dev *eth_dev)
return 0;
err_unmap:
- rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev));
+ rte_pci_unmap_device(pci_dev);
if (!dev->modern)
vtpci_legacy_ioport_unmap(hw);
@@ -127,11 +127,12 @@ eth_virtio_pci_uninit(struct rte_eth_dev *eth_dev)
PMD_INIT_FUNC_TRACE();
if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
dev = eth_dev->data->dev_private;
hw = &dev->hw;
if (dev->modern)
- rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev));
+ rte_pci_unmap_device(pci_dev);
else
vtpci_legacy_ioport_unmap(hw);
return 0;
diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c
index da9af08207..b7cf217724 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c
@@ -308,7 +308,7 @@ eth_vmxnet3_setup_capabilities(struct vmxnet3_hw *hw,
struct rte_eth_dev *eth_dev)
{
uint32_t dcr, ptcr, value;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
VMXNET3_CMD_GET_MAX_CAPABILITIES);
@@ -381,7 +381,7 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev)
eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts;
eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts;
eth_dev->rx_queue_count = vmxnet3_dev_rx_queue_count;
- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
/* extra mbuf field is required to guess MSS */
vmxnet3_segs_dynfield_offset =
diff --git a/drivers/net/xsc/xsc_ethdev.c b/drivers/net/xsc/xsc_ethdev.c
index 07fc52ac7b..39a67ff8cd 100644
--- a/drivers/net/xsc/xsc_ethdev.c
+++ b/drivers/net/xsc/xsc_ethdev.c
@@ -1048,7 +1048,7 @@ xsc_ethdev_init(struct rte_eth_dev *eth_dev)
PMD_INIT_FUNC_TRACE();
priv->eth_dev = eth_dev;
- priv->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ priv->pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *priv->pci_dev);
ret = xsc_dev_init(priv->pci_dev, &priv->xdev);
if (ret) {
diff --git a/drivers/net/zxdh/zxdh_ethdev.c b/drivers/net/zxdh/zxdh_ethdev.c
index aeb01f4652..80ff19b3ea 100644
--- a/drivers/net/zxdh/zxdh_ethdev.c
+++ b/drivers/net/zxdh/zxdh_ethdev.c
@@ -111,7 +111,7 @@ zxdh_intr_unmask(struct rte_eth_dev *dev)
if (rte_intr_ack(dev->intr_handle) < 0)
return -1;
- hw->use_msix = zxdh_pci_msix_detect(RTE_ETH_DEV_TO_PCI(dev));
+ hw->use_msix = zxdh_pci_msix_detect(RTE_CLASS_TO_BUS_DEVICE(dev, struct rte_pci_device));
return 0;
}
@@ -1586,7 +1586,7 @@ static int32_t
zxdh_init_device(struct rte_eth_dev *eth_dev)
{
struct zxdh_hw *hw = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
int ret = 0;
ret = zxdh_read_pci_caps(pci_dev, hw);
@@ -1820,7 +1820,7 @@ zxdh_get_dev_shared_data_idx(uint32_t dev_serial_id)
static int zxdh_init_dev_share_data(struct rte_eth_dev *eth_dev)
{
struct zxdh_hw *hw = eth_dev->data->dev_private;
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
uint32_t serial_id = (pci_dev->addr.domain << 16) |
(pci_dev->addr.bus << 8) | pci_dev->addr.devid;
uint16_t slot_id = 0;
@@ -2201,7 +2201,7 @@ is_inic_pf(uint16_t device_id)
static int
zxdh_eth_dev_init(struct rte_eth_dev *eth_dev)
{
- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
struct zxdh_hw *hw = eth_dev->data->dev_private;
int ret = 0;
diff --git a/drivers/raw/ifpga/afu_pmd_n3000.c b/drivers/raw/ifpga/afu_pmd_n3000.c
index f092ee2dec..d5520a0d71 100644
--- a/drivers/raw/ifpga/afu_pmd_n3000.c
+++ b/drivers/raw/ifpga/afu_pmd_n3000.c
@@ -1467,11 +1467,11 @@ static struct rte_pci_device *n3000_afu_get_pci_dev(struct afu_rawdev *dev)
if (!dev || !dev->rawdev || !dev->rawdev->device)
return NULL;
- afudev = RTE_BUS_DEVICE(dev->rawdev->device, *afudev);
+ afudev = RTE_CLASS_TO_BUS_DEVICE(dev->rawdev, *afudev);
if (!afudev->rawdev || !afudev->rawdev->device)
return NULL;
- return RTE_BUS_DEVICE(afudev->rawdev->device, struct rte_pci_device);
+ return RTE_CLASS_TO_BUS_DEVICE(afudev->rawdev, struct rte_pci_device);
}
static int dma_afu_set_irqs(struct afu_rawdev *dev, uint32_t vec_start,
diff --git a/lib/eal/include/bus_driver.h b/lib/eal/include/bus_driver.h
index 3d04efbd3f..0a7e23d98d 100644
--- a/lib/eal/include/bus_driver.h
+++ b/lib/eal/include/bus_driver.h
@@ -461,6 +461,24 @@ void rte_bus_unregister(struct rte_bus *bus);
#define RTE_BUS_DRIVER(drv, bus_drv_type) \
container_of(drv, typeof(bus_drv_type), driver)
+/**
+ * Helper macro to convert a device class pointer to a bus-specific device type.
+ * Works with any device class (ethdev, cryptodev, eventdev, bbdev, etc.) that has
+ * a 'device' field pointing to struct rte_device.
+ *
+ * Example: RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev) or
+ * RTE_CLASS_TO_BUS_DEVICE(eth_dev, struct rte_pci_device)
+ *
+ * @param class_dev
+ * Pointer to device class structure (e.g., struct rte_eth_dev *)
+ * @param bus_dev_type
+ * Bus device type for type inference (e.g., *pci_dev or struct rte_pci_device)
+ * @return
+ * Pointer to the bus-specific device structure
+ */
+#define RTE_CLASS_TO_BUS_DEVICE(class_dev, bus_dev_type) \
+ RTE_BUS_DEVICE((class_dev)->device, bus_dev_type)
+
/**
* Helper macro to iterate over all devices on a bus.
*
--
2.53.0
^ permalink raw reply related
* [PATCH v5 24/25] eventdev: rename dev field to device
From: David Marchand @ 2026-05-30 7:51 UTC (permalink / raw)
To: dev
Cc: thomas, stephen, bruce.richardson, Pavan Nikhilesh,
Shijith Thotton, Tirthendu Sarkar, Jerin Jacob
In-Reply-To: <20260530075201.869606-1-david.marchand@redhat.com>
Rename the rte_eventdev structure field from 'dev' to 'device' to align
with the naming convention used by all other device classes in DPDK
(ethdev, cryptodev, bbdev, compressdev, rawdev, regexdev, dmadev, gpudev,
and mldev).
This change provides consistency across all device classes: each device
class structure now contains a 'struct rte_device *device' field
pointing to the backing device.
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
drivers/event/cnxk/cn10k_eventdev.c | 8 ++++----
drivers/event/cnxk/cn20k_eventdev.c | 8 ++++----
drivers/event/cnxk/cn9k_eventdev.c | 6 +++---
drivers/event/cnxk/cnxk_eventdev.c | 2 +-
drivers/event/dlb2/pf/dlb2_pf.c | 2 +-
drivers/event/skeleton/skeleton_eventdev.c | 2 +-
lib/eventdev/eventdev_pmd.h | 2 +-
lib/eventdev/eventdev_pmd_pci.h | 4 ++--
lib/eventdev/eventdev_pmd_vdev.h | 2 +-
lib/eventdev/rte_eventdev.c | 14 +++++++-------
10 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c
index 2e4b8aab92..8289fc44d6 100644
--- a/drivers/event/cnxk/cn10k_eventdev.c
+++ b/drivers/event/cnxk/cn10k_eventdev.c
@@ -921,7 +921,7 @@ static int
cn10k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev, uint32_t *caps)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", ENOTSUP);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn10k", ENOTSUP);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", ENOTSUP);
*caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD |
@@ -939,7 +939,7 @@ cn10k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
{
int ret;
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn10k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", EINVAL);
cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
@@ -954,7 +954,7 @@ static int
cn10k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn10k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", EINVAL);
return cnxk_crypto_adapter_qp_del(cdev, queue_pair_id);
@@ -973,7 +973,7 @@ cn10k_crypto_adapter_vec_limits(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev,
struct rte_event_crypto_adapter_vector_limits *limits)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn10k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", EINVAL);
limits->log2_sz = false;
diff --git a/drivers/event/cnxk/cn20k_eventdev.c b/drivers/event/cnxk/cn20k_eventdev.c
index ff3aaac16a..9d34168c32 100644
--- a/drivers/event/cnxk/cn20k_eventdev.c
+++ b/drivers/event/cnxk/cn20k_eventdev.c
@@ -1125,7 +1125,7 @@ static int
cn20k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev, uint32_t *caps)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn20k", ENOTSUP);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn20k", ENOTSUP);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn20k", ENOTSUP);
*caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD |
@@ -1142,7 +1142,7 @@ cn20k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev, const struct r
{
int ret;
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn20k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn20k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn20k", EINVAL);
cn20k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
@@ -1157,7 +1157,7 @@ static int
cn20k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn20k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn20k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn20k", EINVAL);
return cnxk_crypto_adapter_qp_del(cdev, queue_pair_id);
@@ -1175,7 +1175,7 @@ cn20k_crypto_adapter_vec_limits(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev,
struct rte_event_crypto_adapter_vector_limits *limits)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn20k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn20k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn20k", EINVAL);
limits->log2_sz = false;
diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c
index 5f24366770..313dcbb384 100644
--- a/drivers/event/cnxk/cn9k_eventdev.c
+++ b/drivers/event/cnxk/cn9k_eventdev.c
@@ -1038,7 +1038,7 @@ static int
cn9k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
uint32_t *caps)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k", ENOTSUP);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn9k", ENOTSUP);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k", ENOTSUP);
*caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD |
@@ -1055,7 +1055,7 @@ cn9k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
{
int ret;
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn9k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k", EINVAL);
cn9k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
@@ -1070,7 +1070,7 @@ static int
cn9k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
{
- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k", EINVAL);
+ CNXK_VALID_DEV_OR_ERR_RET(event_dev->device, "event_cn9k", EINVAL);
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k", EINVAL);
return cnxk_crypto_adapter_qp_del(cdev, queue_pair_id);
diff --git a/drivers/event/cnxk/cnxk_eventdev.c b/drivers/event/cnxk/cnxk_eventdev.c
index 8eff2ba8e0..6f000ff49e 100644
--- a/drivers/event/cnxk/cnxk_eventdev.c
+++ b/drivers/event/cnxk/cnxk_eventdev.c
@@ -654,7 +654,7 @@ cnxk_sso_init(struct rte_eventdev *event_dev)
return -ENOMEM;
}
- pci_dev = RTE_BUS_DEVICE(event_dev->dev, *pci_dev);
+ pci_dev = RTE_BUS_DEVICE(event_dev->device, *pci_dev);
dev->sso.pci_dev = pci_dev;
*(uint64_t *)mz->addr = (uint64_t)dev;
diff --git a/drivers/event/dlb2/pf/dlb2_pf.c b/drivers/event/dlb2/pf/dlb2_pf.c
index a498ba8c41..82075bbf0b 100644
--- a/drivers/event/dlb2/pf/dlb2_pf.c
+++ b/drivers/event/dlb2/pf/dlb2_pf.c
@@ -784,7 +784,7 @@ dlb2_eventdev_pci_init(struct rte_eventdev *eventdev)
dlb2_pf_iface_fn_ptrs_init();
- pci_dev = RTE_BUS_DEVICE(eventdev->dev, *pci_dev);
+ pci_dev = RTE_BUS_DEVICE(eventdev->device, *pci_dev);
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
dlb2 = dlb2_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
index e07744d2f1..4292644fde 100644
--- a/drivers/event/skeleton/skeleton_eventdev.c
+++ b/drivers/event/skeleton/skeleton_eventdev.c
@@ -332,7 +332,7 @@ skeleton_eventdev_init(struct rte_eventdev *eventdev)
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return 0;
- pci_dev = RTE_BUS_DEVICE(eventdev->dev, *pci_dev);
+ pci_dev = RTE_BUS_DEVICE(eventdev->device, *pci_dev);
skel->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
if (!skel->reg_base) {
diff --git a/lib/eventdev/eventdev_pmd.h b/lib/eventdev/eventdev_pmd.h
index d13cc433a7..9309dce5e1 100644
--- a/lib/eventdev/eventdev_pmd.h
+++ b/lib/eventdev/eventdev_pmd.h
@@ -156,7 +156,7 @@ struct __rte_cache_aligned rte_eventdev {
/**< Pointer to device data */
struct eventdev_ops *dev_ops;
/**< Functions exported by PMD */
- struct rte_device *dev;
+ struct rte_device *device;
/**< Device info. supplied by probing */
uint8_t attached : 1;
diff --git a/lib/eventdev/eventdev_pmd_pci.h b/lib/eventdev/eventdev_pmd_pci.h
index 5cb5916a84..ebc7d12b1d 100644
--- a/lib/eventdev/eventdev_pmd_pci.h
+++ b/lib/eventdev/eventdev_pmd_pci.h
@@ -68,7 +68,7 @@ rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
"device data");
}
- eventdev->dev = &pci_dev->device;
+ eventdev->device = &pci_dev->device;
/* Invoke PMD device initialization function */
retval = devinit(eventdev);
@@ -150,7 +150,7 @@ rte_event_pmd_pci_remove(struct rte_pci_device *pci_dev,
/* Free event device */
rte_event_pmd_release(eventdev);
- eventdev->dev = NULL;
+ eventdev->device = NULL;
return 0;
}
diff --git a/lib/eventdev/eventdev_pmd_vdev.h b/lib/eventdev/eventdev_pmd_vdev.h
index 4eaefa0b0b..ae1c950bed 100644
--- a/lib/eventdev/eventdev_pmd_vdev.h
+++ b/lib/eventdev/eventdev_pmd_vdev.h
@@ -67,7 +67,7 @@ rte_event_pmd_vdev_init(const char *name, size_t dev_private_size,
rte_panic("Cannot allocate memzone for private device"
" data");
}
- eventdev->dev = &vdev->device;
+ eventdev->device = &vdev->device;
return eventdev;
}
diff --git a/lib/eventdev/rte_eventdev.c b/lib/eventdev/rte_eventdev.c
index b921142d7b..572cd5bd7d 100644
--- a/lib/eventdev/rte_eventdev.c
+++ b/lib/eventdev/rte_eventdev.c
@@ -68,8 +68,8 @@ rte_event_dev_get_dev_id(const char *name)
for (i = 0; i < eventdev_globals.nb_devs; i++) {
cmp = (strncmp(rte_event_devices[i].data->name, name,
RTE_EVENTDEV_NAME_MAX_LEN) == 0) ||
- (rte_event_devices[i].dev ? (strncmp(
- rte_event_devices[i].dev->driver->name, name,
+ (rte_event_devices[i].device ? (strncmp(
+ rte_event_devices[i].device->driver->name, name,
RTE_EVENTDEV_NAME_MAX_LEN) == 0) : 0);
if (cmp && (rte_event_devices[i].attached ==
RTE_EVENTDEV_ATTACHED)) {
@@ -114,9 +114,9 @@ rte_event_dev_info_get(uint8_t dev_id, struct rte_event_dev_info *dev_info)
dev_info->dequeue_timeout_ns = dev->data->dev_conf.dequeue_timeout_ns;
- dev_info->dev = dev->dev;
- if (dev->dev != NULL && dev->dev->driver != NULL)
- dev_info->driver_name = dev->dev->driver->name;
+ dev_info->dev = dev->device;
+ if (dev->device != NULL && dev->device->driver != NULL)
+ dev_info->driver_name = dev->device->driver->name;
rte_eventdev_trace_info_get(dev_id, dev_info, dev_info->dev);
@@ -1812,8 +1812,8 @@ handle_dev_info(const char *cmd __rte_unused,
rte_tel_data_start_dict(d);
rte_tel_data_add_dict_int(d, "dev_id", dev_id);
- rte_tel_data_add_dict_string(d, "dev_name", dev->dev->name);
- rte_tel_data_add_dict_string(d, "dev_driver", dev->dev->driver->name);
+ rte_tel_data_add_dict_string(d, "dev_name", dev->device->name);
+ rte_tel_data_add_dict_string(d, "dev_driver", dev->device->driver->name);
rte_tel_data_add_dict_string(d, "state",
dev->data->dev_started ? "started" : "stopped");
rte_tel_data_add_dict_int(d, "socket_id", dev->data->socket_id);
--
2.53.0
^ permalink raw reply related
* [PATCH v5 13/25] bus: factorize driver lookup
From: David Marchand @ 2026-05-30 7:51 UTC (permalink / raw)
To: dev
Cc: thomas, stephen, bruce.richardson, Chengwen Feng, Parav Pandit,
Xueming Li, Nipun Gupta, Nikhil Agarwal, Hemant Agrawal,
Sachin Saxena, Rosen Xu, Chenbo Xia, Tomasz Duszynski, Long Li,
Wei Hu, Kevin Laatz
In-Reply-To: <20260530075201.869606-1-david.marchand@redhat.com>
Introduce a new bus operation 'match' that checks whether a driver can
handle a given device. This separates the matching logic from iteration,
with buses providing match logic and EAL providing generic iteration
through rte_bus_find_driver().
The match operation returns true if a driver matches a device.
Matching logic is bus-specific (e.g., ID table matching for PCI/CDX,
device type matching for DPAA/FSLMC, UUID matching for IFPGA/VMBUS,
name matching for vdev/platform, API/algorithm matching for uacce).
A generic helper rte_bus_find_driver() iterates through all drivers
on a bus and returns the next matching driver, eliminating the need
for each bus to duplicate iteration logic.
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Chengwen Feng <fengchengwen@huawei.com>
---
drivers/bus/auxiliary/auxiliary_common.c | 14 ++++-----
drivers/bus/auxiliary/private.h | 6 ----
drivers/bus/cdx/cdx.c | 16 +++++-----
drivers/bus/dpaa/dpaa_bus.c | 21 ++++++-------
drivers/bus/fslmc/fslmc_bus.c | 23 +++++++-------
drivers/bus/ifpga/ifpga_bus.c | 14 +++++----
drivers/bus/pci/pci_common.c | 20 ++++++-------
drivers/bus/pci/private.h | 15 ----------
drivers/bus/platform/platform.c | 9 ++++--
drivers/bus/uacce/uacce.c | 13 ++++----
drivers/bus/vdev/vdev.c | 20 +++++++++++++
drivers/bus/vmbus/vmbus_common.c | 21 ++++---------
drivers/dma/idxd/idxd_bus.c | 35 ++++++++++++++--------
lib/eal/common/eal_common_bus.c | 19 ++++++++++++
lib/eal/include/bus_driver.h | 38 ++++++++++++++++++++++++
15 files changed, 172 insertions(+), 112 deletions(-)
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
index 05299db8fe..21b5bcb416 100644
--- a/drivers/bus/auxiliary/auxiliary_common.c
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -59,13 +59,12 @@ auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
aux_dev->device.devargs = rte_bus_find_devargs(&auxiliary_bus.bus, aux_dev->name);
}
-/*
- * Match the auxiliary driver and device using driver function.
- */
-bool
-auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
- const struct rte_auxiliary_device *aux_dev)
+static bool
+auxiliary_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_auxiliary_driver *aux_drv = RTE_BUS_DRIVER(drv, *aux_drv);
+ const struct rte_auxiliary_device *aux_dev = RTE_BUS_DEVICE(dev, *aux_dev);
+
if (aux_drv->match == NULL)
return false;
return aux_drv->match(aux_dev->name);
@@ -82,7 +81,7 @@ rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
int ret;
/* Check if driver supports it. */
- if (!auxiliary_match(drv, dev))
+ if (!auxiliary_bus_match(&drv->driver, &dev->device))
/* Match of device and driver failed */
return 1;
@@ -340,6 +339,7 @@ struct rte_auxiliary_bus auxiliary_bus = {
.probe = auxiliary_probe,
.cleanup = auxiliary_cleanup,
.find_device = rte_bus_generic_find_device,
+ .match = auxiliary_bus_match,
.plug = auxiliary_plug,
.unplug = auxiliary_unplug,
.parse = auxiliary_parse,
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
index 659d798cd6..116154eb56 100644
--- a/drivers/bus/auxiliary/private.h
+++ b/drivers/bus/auxiliary/private.h
@@ -44,10 +44,4 @@ int auxiliary_scan(void);
*/
void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
-/*
- * Match the auxiliary driver and device by driver function.
- */
-bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
- const struct rte_auxiliary_device *aux_dev);
-
#endif /* BUS_AUXILIARY_PRIVATE_H */
diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index d6f83e2e80..c898ce9271 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -279,13 +279,12 @@ cdx_unmap_resource(void *requested_addr, size_t size)
requested_addr, size, rte_strerror(rte_errno));
}
}
-/*
- * Match the CDX Driver and Device using device id and vendor id.
- */
+
static bool
-cdx_match(const struct rte_cdx_driver *cdx_drv,
- const struct rte_cdx_device *cdx_dev)
+cdx_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_cdx_driver *cdx_drv = RTE_BUS_DRIVER(drv, *cdx_drv);
+ const struct rte_cdx_device *cdx_dev = RTE_BUS_DEVICE(dev, *cdx_dev);
const struct rte_cdx_id *id_table;
for (id_table = cdx_drv->id_table; id_table->vendor_id != 0;
@@ -298,10 +297,10 @@ cdx_match(const struct rte_cdx_driver *cdx_drv,
id_table->device_id != RTE_CDX_ANY_ID)
continue;
- return 1;
+ return true;
}
- return 0;
+ return false;
}
/*
@@ -317,7 +316,7 @@ cdx_probe_one_driver(struct rte_cdx_driver *dr,
int ret;
/* The device is not blocked; Check if driver supports it */
- if (!cdx_match(dr, dev))
+ if (!cdx_bus_match(&dr->driver, &dev->device))
/* Match of device and driver failed */
return 1;
@@ -524,6 +523,7 @@ struct rte_cdx_bus rte_cdx_bus = {
.scan = cdx_scan,
.probe = cdx_probe,
.find_device = rte_bus_generic_find_device,
+ .match = cdx_bus_match,
.plug = cdx_plug,
.unplug = cdx_unplug,
.parse = cdx_parse,
diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index b3a754cbf4..ca80fff6ec 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -626,19 +626,16 @@ rte_dpaa_driver_unregister(struct rte_dpaa_driver *driver)
rte_bus_remove_driver(&rte_dpaa_bus.bus, &driver->driver);
}
-static int
-rte_dpaa_device_match(struct rte_dpaa_driver *drv,
- struct rte_dpaa_device *dev)
+static bool
+dpaa_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
- if (!drv || !dev) {
- DPAA_BUS_DEBUG("Invalid drv or dev received.");
- return -1;
- }
+ const struct rte_dpaa_driver *dpaa_drv = RTE_BUS_DRIVER(drv, *dpaa_drv);
+ const struct rte_dpaa_device *dpaa_dev = RTE_BUS_DEVICE(dev, *dpaa_dev);
- if (drv->drv_type == dev->device_type)
- return 0;
+ if (dpaa_drv->drv_type == dpaa_dev->device_type)
+ return true;
- return -1;
+ return false;
}
static int
@@ -793,8 +790,7 @@ rte_dpaa_bus_probe(void)
/* For each registered driver, and device, call the driver->probe */
RTE_BUS_FOREACH_DEV(dev, &rte_dpaa_bus.bus) {
RTE_BUS_FOREACH_DRV(drv, &rte_dpaa_bus.bus) {
- ret = rte_dpaa_device_match(drv, dev);
- if (ret)
+ if (!dpaa_bus_match(&drv->driver, &dev->device))
continue;
if (rte_dev_is_probed(&dev->device))
@@ -902,6 +898,7 @@ static struct rte_dpaa_bus rte_dpaa_bus = {
.dev_compare = dpaa_bus_dev_compare,
.find_device = rte_bus_generic_find_device,
.get_iommu_class = rte_dpaa_get_iommu_class,
+ .match = dpaa_bus_match,
.plug = dpaa_bus_plug,
.unplug = dpaa_bus_unplug,
.dev_iterate = rte_bus_generic_dev_iterate,
diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index 716f0178b5..8cd9b1eb88 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -381,14 +381,16 @@ rte_fslmc_scan(void)
return 0;
}
-static int
-rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
- struct rte_dpaa2_device *dpaa2_dev)
+static bool
+fslmc_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_dpaa2_driver *dpaa2_drv = RTE_BUS_DRIVER(drv, *dpaa2_drv);
+ const struct rte_dpaa2_device *dpaa2_dev = RTE_BUS_DEVICE(dev, *dpaa2_dev);
+
if (dpaa2_drv->drv_type == dpaa2_dev->dev_type)
- return 0;
+ return true;
- return 1;
+ return false;
}
static int
@@ -455,8 +457,7 @@ rte_fslmc_probe(void)
RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus.bus) {
RTE_BUS_FOREACH_DRV(drv, &rte_fslmc_bus.bus) {
- ret = rte_fslmc_match(drv, dev);
- if (ret)
+ if (!fslmc_bus_match(&drv->driver, &dev->device))
continue;
if (rte_dev_is_probed(&dev->device))
@@ -504,14 +505,12 @@ rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver)
static inline int
fslmc_all_device_support_iova(void)
{
- int ret = 0;
struct rte_dpaa2_device *dev;
struct rte_dpaa2_driver *drv;
RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus.bus) {
RTE_BUS_FOREACH_DRV(drv, &rte_fslmc_bus.bus) {
- ret = rte_fslmc_match(drv, dev);
- if (ret)
+ if (!fslmc_bus_match(&drv->driver, &dev->device))
continue;
/* if the driver is not supporting IOVA */
if (!(drv->drv_flags & RTE_DPAA2_DRV_IOVA_AS_VA))
@@ -548,8 +547,7 @@ fslmc_bus_plug(struct rte_device *rte_dev)
struct rte_dpaa2_driver *drv;
RTE_BUS_FOREACH_DRV(drv, &rte_fslmc_bus.bus) {
- ret = rte_fslmc_match(drv, dev);
- if (ret)
+ if (!fslmc_bus_match(&drv->driver, &dev->device))
continue;
if (rte_dev_is_probed(&dev->device))
@@ -602,6 +600,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
.dev_compare = fslmc_dev_compare,
.find_device = rte_bus_generic_find_device,
.get_iommu_class = rte_dpaa2_get_iommu_class,
+ .match = fslmc_bus_match,
.plug = fslmc_bus_plug,
.unplug = fslmc_bus_unplug,
.dev_iterate = rte_bus_generic_dev_iterate,
diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
index 7d3331fe7e..021171e955 100644
--- a/drivers/bus/ifpga/ifpga_bus.c
+++ b/drivers/bus/ifpga/ifpga_bus.c
@@ -245,10 +245,11 @@ ifpga_scan(void)
/*
* Match the AFU Driver and AFU Device using the ID Table
*/
-static int
-rte_afu_match(const struct rte_afu_driver *afu_drv,
- const struct rte_afu_device *afu_dev)
+static bool
+ifpga_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_afu_driver *afu_drv = RTE_BUS_DRIVER(drv, *afu_drv);
+ const struct rte_afu_device *afu_dev = RTE_BUS_DEVICE(dev, *afu_dev);
const struct rte_afu_uuid *id_table;
for (id_table = afu_drv->id_table;
@@ -260,10 +261,10 @@ rte_afu_match(const struct rte_afu_driver *afu_drv,
afu_dev->id.uuid.uuid_high)
continue;
- return 1;
+ return true;
}
- return 0;
+ return false;
}
static int
@@ -272,7 +273,7 @@ ifpga_probe_one_driver(struct rte_afu_driver *drv,
{
int ret;
- if (!rte_afu_match(drv, afu_dev))
+ if (!ifpga_bus_match(&drv->driver, &afu_dev->device))
/* Match of device and driver failed */
return 1;
@@ -452,6 +453,7 @@ static struct rte_bus rte_ifpga_bus = {
.probe = ifpga_probe,
.cleanup = ifpga_cleanup,
.find_device = rte_bus_generic_find_device,
+ .match = ifpga_bus_match,
.plug = ifpga_plug,
.unplug = ifpga_unplug,
.parse = ifpga_parse,
diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c
index 70ce63eac7..d7fda1752a 100644
--- a/drivers/bus/pci/pci_common.c
+++ b/drivers/bus/pci/pci_common.c
@@ -141,13 +141,12 @@ pci_unmap_resource(void *requested_addr, size_t size)
} else
PCI_LOG(DEBUG, " PCI memory unmapped at %p", requested_addr);
}
-/*
- * Match the PCI Driver and Device using the ID Table
- */
-int
-rte_pci_match(const struct rte_pci_driver *pci_drv,
- const struct rte_pci_device *pci_dev)
+
+static bool
+pci_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_pci_driver *pci_drv = RTE_BUS_DRIVER(drv, *pci_drv);
+ const struct rte_pci_device *pci_dev = RTE_BUS_DEVICE(dev, *pci_dev);
const struct rte_pci_id *id_table;
for (id_table = pci_drv->id_table; id_table->vendor_id != 0;
@@ -171,10 +170,10 @@ rte_pci_match(const struct rte_pci_driver *pci_drv,
id_table->class_id != RTE_CLASS_ANY_ID)
continue;
- return 1;
+ return true;
}
- return 0;
+ return false;
}
/*
@@ -195,7 +194,7 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
loc = &dev->addr;
/* The device is not blocked; Check if driver supports it */
- if (!rte_pci_match(dr, dev))
+ if (!pci_bus_match(&dr->driver, &dev->device))
/* Match of device and driver failed */
return 1;
@@ -680,7 +679,7 @@ rte_pci_get_iommu_class(void)
RTE_BUS_FOREACH_DRV(drv, &rte_pci_bus.bus) {
enum rte_iova_mode dev_iova_mode;
- if (!rte_pci_match(drv, dev))
+ if (!pci_bus_match(&drv->driver, &dev->device))
continue;
dev_iova_mode = pci_device_iova_mode(drv, dev);
@@ -861,6 +860,7 @@ struct rte_pci_bus rte_pci_bus = {
.probe = pci_probe,
.cleanup = pci_cleanup,
.find_device = rte_bus_generic_find_device,
+ .match = pci_bus_match,
.plug = pci_plug,
.unplug = pci_unplug,
.parse = pci_parse,
diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h
index 21637882f8..c54ea7b9d8 100644
--- a/drivers/bus/pci/private.h
+++ b/drivers/bus/pci/private.h
@@ -217,21 +217,6 @@ pci_uio_remap_resource(struct rte_pci_device *dev);
int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
struct mapped_pci_resource *uio_res, int map_idx);
-/*
- * Match the PCI Driver and Device using the ID Table
- *
- * @param pci_drv
- * PCI driver from which ID table would be extracted
- * @param pci_dev
- * PCI device to match against the driver
- * @return
- * 1 for successful match
- * 0 for unsuccessful match
- */
-int
-rte_pci_match(const struct rte_pci_driver *pci_drv,
- const struct rte_pci_device *pci_dev);
-
/**
* OS specific callbacks for rte_pci_get_iommu_class
*
diff --git a/drivers/bus/platform/platform.c b/drivers/bus/platform/platform.c
index 636f051049..3d6b6efe6e 100644
--- a/drivers/bus/platform/platform.c
+++ b/drivers/bus/platform/platform.c
@@ -371,8 +371,10 @@ driver_probe_device(struct rte_platform_driver *pdrv, struct rte_platform_device
}
static bool
-driver_match_device(struct rte_platform_driver *pdrv, struct rte_platform_device *pdev)
+platform_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_platform_driver *pdrv = RTE_BUS_DRIVER(drv, *pdrv);
+ const struct rte_platform_device *pdev = RTE_BUS_DEVICE(dev, *pdev);
bool match = false;
char *kdrv;
@@ -408,7 +410,7 @@ device_attach(struct rte_platform_device *pdev)
struct rte_platform_driver *pdrv;
RTE_BUS_FOREACH_DRV(pdrv, &platform_bus.bus) {
- if (driver_match_device(pdrv, pdev))
+ if (platform_bus_match(&pdrv->driver, &pdev->device))
break;
}
@@ -488,7 +490,7 @@ platform_bus_parse(const char *name, void *addr)
rte_strscpy(pdev.name, name, sizeof(pdev.name));
RTE_BUS_FOREACH_DRV(pdrv, &platform_bus.bus) {
- if (driver_match_device(pdrv, &pdev))
+ if (platform_bus_match(&pdrv->driver, &pdev.device))
break;
}
@@ -556,6 +558,7 @@ struct rte_platform_bus platform_bus = {
.scan = platform_bus_scan,
.probe = platform_bus_probe,
.find_device = rte_bus_generic_find_device,
+ .match = platform_bus_match,
.plug = platform_bus_plug,
.unplug = platform_bus_unplug,
.parse = platform_bus_parse,
diff --git a/drivers/bus/uacce/uacce.c b/drivers/bus/uacce/uacce.c
index af1ada0bd3..16767a3b88 100644
--- a/drivers/bus/uacce/uacce.c
+++ b/drivers/bus/uacce/uacce.c
@@ -315,15 +315,17 @@ uacce_match_api(const struct rte_uacce_device *dev, bool forward_compat,
}
static bool
-uacce_match(const struct rte_uacce_driver *dr, const struct rte_uacce_device *dev)
+uacce_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_uacce_driver *dr = RTE_BUS_DRIVER(drv, *dr);
+ const struct rte_uacce_device *uacce_dev = RTE_BUS_DEVICE(dev, *uacce_dev);
bool forward_compat = !!(dr->drv_flags & RTE_UACCE_DRV_FORWARD_COMPATIBILITY_DEV);
const struct rte_uacce_id *id_table;
const char *map;
uint32_t len;
for (id_table = dr->id_table; id_table->dev_api != NULL; id_table++) {
- if (!uacce_match_api(dev, forward_compat, id_table))
+ if (!uacce_match_api(uacce_dev, forward_compat, id_table))
continue;
if (id_table->dev_alg == NULL)
@@ -334,10 +336,10 @@ uacce_match(const struct rte_uacce_driver *dr, const struct rte_uacce_device *de
* algorithms: aaa, bbbb and cc.
* The id_table->dev_alg should be a single algrithm, e.g. bbbb.
*/
- map = strstr(dev->algs, id_table->dev_alg);
+ map = strstr(uacce_dev->algs, id_table->dev_alg);
if (map == NULL)
continue;
- if (map != dev->algs && map[-1] != '\n')
+ if (map != uacce_dev->algs && map[-1] != '\n')
continue;
len = strlen(id_table->dev_alg);
if (map[len] != '\0' && map[len] != '\n')
@@ -356,7 +358,7 @@ uacce_probe_one_driver(struct rte_uacce_driver *dr, struct rte_uacce_device *dev
bool already_probed;
int ret;
- if (!uacce_match(dr, dev))
+ if (!uacce_bus_match(&dr->driver, &dev->device))
/* Match of device and driver failed */
return 1;
@@ -623,6 +625,7 @@ static struct rte_uacce_bus uacce_bus = {
.scan = uacce_scan,
.probe = uacce_probe,
.cleanup = uacce_cleanup,
+ .match = uacce_bus_match,
.plug = uacce_plug,
.unplug = uacce_unplug,
.find_device = rte_bus_generic_find_device,
diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c
index 4003805315..0308be5cbe 100644
--- a/drivers/bus/vdev/vdev.c
+++ b/drivers/bus/vdev/vdev.c
@@ -162,6 +162,25 @@ vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
return 0;
}
+/*
+ * Check if a vdev driver matches a vdev device by name.
+ */
+static bool
+vdev_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
+{
+ const char *name = dev->name;
+
+ /* Check driver name match */
+ if (strncmp(drv->name, name, strlen(drv->name)) == 0)
+ return true;
+
+ /* Check driver alias match */
+ if (drv->alias && strncmp(drv->alias, name, strlen(drv->alias)) == 0)
+ return true;
+
+ return false;
+}
+
static int
vdev_probe_all_drivers(struct rte_vdev_device *dev)
{
@@ -631,6 +650,7 @@ static struct rte_bus rte_vdev_bus = {
.probe = vdev_probe,
.cleanup = vdev_cleanup,
.find_device = vdev_find_device,
+ .match = vdev_bus_match,
.plug = vdev_plug,
.unplug = vdev_unplug,
.parse = vdev_parse,
diff --git a/drivers/bus/vmbus/vmbus_common.c b/drivers/bus/vmbus/vmbus_common.c
index 3260bd5395..d811f1a229 100644
--- a/drivers/bus/vmbus/vmbus_common.c
+++ b/drivers/bus/vmbus/vmbus_common.c
@@ -65,25 +65,15 @@ vmbus_unmap_resource(void *requested_addr, size_t size)
}
}
-/**
- * Match the VMBUS driver and device using UUID table
- *
- * @param drv
- * VMBUS driver from which ID table would be extracted
- * @param pci_dev
- * VMBUS device to match against the driver
- * @return
- * true for successful match
- * false for unsuccessful match
- */
static bool
-vmbus_match(const struct rte_vmbus_driver *dr,
- const struct rte_vmbus_device *dev)
+vmbus_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
{
+ const struct rte_vmbus_driver *dr = RTE_BUS_DRIVER(drv, *dr);
+ const struct rte_vmbus_device *vmbus_dev = RTE_BUS_DEVICE(dev, *vmbus_dev);
const rte_uuid_t *id_table;
for (id_table = dr->id_table; !rte_uuid_is_null(*id_table); ++id_table) {
- if (rte_uuid_compare(*id_table, dev->class_id) == 0)
+ if (rte_uuid_compare(*id_table, vmbus_dev->class_id) == 0)
return true;
}
@@ -99,7 +89,7 @@ vmbus_probe_one_driver(struct rte_vmbus_driver *dr,
char guid[RTE_UUID_STRLEN];
int ret;
- if (!vmbus_match(dr, dev))
+ if (!vmbus_bus_match(&dr->driver, &dev->device))
return 1; /* not supported */
rte_uuid_unparse(dev->device_id, guid, sizeof(guid));
@@ -281,6 +271,7 @@ struct rte_vmbus_bus rte_vmbus_bus = {
.probe = rte_vmbus_probe,
.cleanup = rte_vmbus_cleanup,
.find_device = rte_bus_generic_find_device,
+ .match = vmbus_bus_match,
.parse = vmbus_parse,
.dev_compare = vmbus_dev_compare,
},
diff --git a/drivers/dma/idxd/idxd_bus.c b/drivers/dma/idxd/idxd_bus.c
index 269aac1946..cba26d0cdc 100644
--- a/drivers/dma/idxd/idxd_bus.c
+++ b/drivers/dma/idxd/idxd_bus.c
@@ -43,6 +43,7 @@ struct rte_dsa_device {
/* forward prototypes */
struct dsa_bus;
static int dsa_scan(void);
+static bool dsa_match(const struct rte_driver *drv, const struct rte_device *dev);
static int dsa_probe(void);
static enum rte_iova_mode dsa_get_iommu_class(void);
static int dsa_addr_parse(const char *name, void *addr);
@@ -58,6 +59,7 @@ struct dsa_bus {
struct dsa_bus dsa_bus = {
.bus = {
.scan = dsa_scan,
+ .match = dsa_match,
.probe = dsa_probe,
.find_device = rte_bus_generic_find_device,
.get_iommu_class = dsa_get_iommu_class,
@@ -126,7 +128,7 @@ idxd_bus_mmap_wq(struct rte_dsa_device *dev)
}
static int
-read_wq_string(struct rte_dsa_device *dev, const char *filename,
+read_wq_string(const struct rte_dsa_device *dev, const char *filename,
char *value, size_t valuelen)
{
char sysfs_node[PATH_MAX];
@@ -241,7 +243,7 @@ idxd_probe_dsa(struct rte_dsa_device *dev)
}
static int
-is_for_this_process_use(struct rte_dsa_device *dev, const char *name)
+is_for_this_process_use(const char *name)
{
char prefix[256];
int retval = 0;
@@ -256,9 +258,6 @@ is_for_this_process_use(struct rte_dsa_device *dev, const char *name)
if (strncmp(name, prefix, prefixlen) == 0 && name[prefixlen] == '_')
retval = 1;
- if (retval)
- retval = !rte_bus_device_is_ignored(&dsa_bus.bus, dev->device.name);
-
return retval;
}
@@ -268,14 +267,8 @@ dsa_probe(void)
struct rte_dsa_device *dev;
RTE_BUS_FOREACH_DEV(dev, &dsa_bus.bus) {
- char type[64], name[64];
-
- if (read_wq_string(dev, "type", type, sizeof(type)) < 0 ||
- read_wq_string(dev, "name", name, sizeof(name)) < 0)
- continue;
-
- if (strncmp(type, "user", 4) == 0 &&
- is_for_this_process_use(dev, name)) {
+ if (dsa_match(&dsa_bus.driver, &dev->device) &&
+ !rte_bus_device_is_ignored(&dsa_bus.bus, dev->device.name)) {
dev->device.driver = &dsa_bus.driver;
idxd_probe_dsa(dev);
continue;
@@ -286,6 +279,22 @@ dsa_probe(void)
return 0;
}
+static bool dsa_match(const struct rte_driver *drv, const struct rte_device *dev)
+{
+ const struct rte_dsa_device *dsa_dev = RTE_BUS_DEVICE(dev, *dsa_dev);
+
+ if (drv == &dsa_bus.driver) {
+ char type[64], name[64];
+
+ if (read_wq_string(dsa_dev, "type", type, sizeof(type)) >= 0 &&
+ read_wq_string(dsa_dev, "name", name, sizeof(name)) >= 0) {
+ return strncmp(type, "user", 4) == 0 && is_for_this_process_use(name);
+ }
+ }
+
+ return false;
+}
+
static int
dsa_scan(void)
{
diff --git a/lib/eal/common/eal_common_bus.c b/lib/eal/common/eal_common_bus.c
index 46a8e68532..4884cdfa50 100644
--- a/lib/eal/common/eal_common_bus.c
+++ b/lib/eal/common/eal_common_bus.c
@@ -473,3 +473,22 @@ rte_bus_generic_dev_iterate(const struct rte_bus *bus,
rte_kvargs_free(kvargs);
return dev;
}
+
+RTE_EXPORT_INTERNAL_SYMBOL(rte_bus_find_driver)
+struct rte_driver *
+rte_bus_find_driver(const struct rte_bus *bus, const struct rte_driver *start,
+ const struct rte_device *dev)
+{
+ struct rte_driver *drv;
+
+ if (start != NULL)
+ drv = TAILQ_NEXT(start, next);
+ else
+ drv = TAILQ_FIRST(&bus->driver_list);
+ while (drv != NULL) {
+ if (bus->match(drv, dev))
+ break;
+ drv = TAILQ_NEXT(drv, next);
+ }
+ return drv;
+}
diff --git a/lib/eal/include/bus_driver.h b/lib/eal/include/bus_driver.h
index 9568d820e5..c4d9ac2719 100644
--- a/lib/eal/include/bus_driver.h
+++ b/lib/eal/include/bus_driver.h
@@ -233,6 +233,24 @@ typedef int (*rte_bus_sigbus_handler_t)(const void *failure_addr);
*/
typedef int (*rte_bus_cleanup_t)(void);
+/**
+ * Check if a driver matches a device.
+ *
+ * This function checks whether a driver can handle a given device.
+ * Matching logic is bus-specific (e.g., PCI uses ID tables, vdev uses
+ * name matching, fslmc uses device type).
+ *
+ * @param drv
+ * Driver to check.
+ * @param dev
+ * Device to check against.
+ *
+ * @return
+ * true if the driver matches the device, false otherwise.
+ */
+typedef bool (*rte_bus_match_t)(const struct rte_driver *drv,
+ const struct rte_device *dev);
+
/**
* Bus scan policies
*/
@@ -297,6 +315,7 @@ struct rte_bus {
rte_bus_scan_t scan; /**< Scan for devices attached to bus */
rte_bus_probe_t probe; /**< Probe devices on bus */
rte_bus_find_device_t find_device; /**< Find a device on the bus */
+ rte_bus_match_t match; /**< Check if driver matches device */
rte_bus_plug_t plug; /**< Probe single device for drivers */
rte_bus_unplug_t unplug; /**< Remove single device from driver */
rte_bus_parse_t parse; /**< Parse a device name */
@@ -544,6 +563,25 @@ void rte_bus_add_driver(struct rte_bus *bus, struct rte_driver *driver);
__rte_internal
void rte_bus_remove_driver(struct rte_bus *bus, struct rte_driver *driver);
+/**
+ * Find the first driver that matches a device on a bus.
+ *
+ * Iterates through all registered drivers on the bus and returns the next
+ * one that matches the given device according to the bus's match operation.
+ *
+ * @param bus
+ * A pointer to a rte_bus structure.
+ * @param start
+ * Starting iteration context.
+ * @param dev
+ * A pointer to a rte_device structure.
+ * @return
+ * Pointer to the matching driver, or NULL if no match found.
+ */
+__rte_internal
+struct rte_driver *rte_bus_find_driver(const struct rte_bus *bus, const struct rte_driver *start,
+ const struct rte_device *dev);
+
#ifdef __cplusplus
}
#endif
--
2.53.0
^ permalink raw reply related
* [PATCH v5 12/25] bus: consolidate device iteration
From: David Marchand @ 2026-05-30 7:51 UTC (permalink / raw)
To: dev
Cc: thomas, stephen, bruce.richardson, Chengwen Feng, Parav Pandit,
Xueming Li, Nipun Gupta, Nikhil Agarwal, Hemant Agrawal,
Sachin Saxena, Chenbo Xia, Tomasz Duszynski, Andrew Rybchenko
In-Reply-To: <20260530075201.869606-1-david.marchand@redhat.com>
Many buses (auxiliary, cdx, dpaa, fslmc, platform, uacce, vdev...) had
nearly identical dev_iterate implementations using name-based matching:
- Parse kvargs with "name" parameter
- Match device name via strcmp
- Call rte_bus_find_device()
Extend bus device iterator callback and introduce
rte_bus_generic_dev_iterate() generic helper in EAL.
Only the PCI bus is left with its matching on PCI address criteria.
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Chengwen Feng <fengchengwen@huawei.com>
---
Changes since v2:
- renamed some input variables in the bus header for readability,
- fixed some doxygen comments,
---
drivers/bus/auxiliary/auxiliary_common.c | 2 +-
drivers/bus/auxiliary/auxiliary_params.c | 63 -----------------------
drivers/bus/auxiliary/meson.build | 5 +-
drivers/bus/auxiliary/private.h | 6 ---
drivers/bus/cdx/cdx.c | 52 +------------------
drivers/bus/dpaa/dpaa_bus.c | 46 +----------------
drivers/bus/fslmc/fslmc_bus.c | 46 +----------------
drivers/bus/pci/pci_params.c | 3 +-
drivers/bus/pci/private.h | 6 ++-
drivers/bus/platform/meson.build | 5 +-
drivers/bus/platform/platform.c | 2 +-
drivers/bus/platform/platform_params.c | 65 ------------------------
drivers/bus/platform/private.h | 7 ---
drivers/bus/uacce/uacce.c | 48 +----------------
drivers/bus/vdev/meson.build | 5 +-
drivers/bus/vdev/vdev.c | 18 +++----
drivers/bus/vdev/vdev_logs.h | 16 ------
drivers/bus/vdev/vdev_params.c | 64 -----------------------
drivers/bus/vdev/vdev_private.h | 28 ----------
lib/eal/common/eal_common_bus.c | 41 +++++++++++++++
lib/eal/common/eal_common_dev.c | 4 +-
lib/eal/include/bus_driver.h | 53 ++++++++++++++++++-
lib/ethdev/rte_ethdev.c | 2 +-
23 files changed, 121 insertions(+), 466 deletions(-)
delete mode 100644 drivers/bus/auxiliary/auxiliary_params.c
delete mode 100644 drivers/bus/platform/platform_params.c
delete mode 100644 drivers/bus/vdev/vdev_logs.h
delete mode 100644 drivers/bus/vdev/vdev_params.c
delete mode 100644 drivers/bus/vdev/vdev_private.h
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
index eb0a27cc11..05299db8fe 100644
--- a/drivers/bus/auxiliary/auxiliary_common.c
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -346,7 +346,7 @@ struct rte_auxiliary_bus auxiliary_bus = {
.dma_map = auxiliary_dma_map,
.dma_unmap = auxiliary_dma_unmap,
.get_iommu_class = auxiliary_get_iommu_class,
- .dev_iterate = auxiliary_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
},
};
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
deleted file mode 100644
index 1a76155c67..0000000000
--- a/drivers/bus/auxiliary/auxiliary_params.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2021 NVIDIA Corporation & Affiliates
- */
-
-#include <string.h>
-
-#include <bus_driver.h>
-#include <dev_driver.h>
-#include <rte_errno.h>
-#include <rte_kvargs.h>
-
-#include "private.h"
-
-enum auxiliary_params {
- RTE_AUXILIARY_PARAM_NAME,
-};
-
-static const char * const auxiliary_params_keys[] = {
- [RTE_AUXILIARY_PARAM_NAME] = "name",
- NULL,
-};
-
-static int
-auxiliary_dev_match(const struct rte_device *dev,
- const void *_kvlist)
-{
- const struct rte_kvargs *kvlist = _kvlist;
- const char *key = auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME];
- const char *name;
-
- /* no kvlist arg, all devices match */
- if (kvlist == NULL)
- return 0;
-
- /* if key is present in kvlist and does not match, filter device */
- name = rte_kvargs_get(kvlist, key);
- if (name != NULL && strcmp(name, dev->name))
- return -1;
-
- return 0;
-}
-
-void *
-auxiliary_dev_iterate(const void *start,
- const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- struct rte_kvargs *kvargs = NULL;
- struct rte_device *dev;
-
- if (str != NULL) {
- kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
- if (kvargs == NULL) {
- AUXILIARY_LOG(ERR, "cannot parse argument list %s",
- str);
- rte_errno = EINVAL;
- return NULL;
- }
- }
- dev = rte_bus_generic_find_device(&auxiliary_bus.bus, start, auxiliary_dev_match, kvargs);
- rte_kvargs_free(kvargs);
- return dev;
-}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
index 38d2f05d4b..846b714e2a 100644
--- a/drivers/bus/auxiliary/meson.build
+++ b/drivers/bus/auxiliary/meson.build
@@ -2,10 +2,7 @@
# Copyright (c) 2021 NVIDIA Corporation & Affiliates
driver_sdk_headers += files('bus_auxiliary_driver.h')
-sources = files(
- 'auxiliary_common.c',
- 'auxiliary_params.c',
-)
+sources = files('auxiliary_common.c')
if is_linux
cflags += '-DAUXILIARY_OS_SUPPORTED'
sources += files(
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
index 0b3d73a08d..659d798cd6 100644
--- a/drivers/bus/auxiliary/private.h
+++ b/drivers/bus/auxiliary/private.h
@@ -50,10 +50,4 @@ void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
const struct rte_auxiliary_device *aux_dev);
-/*
- * Iterate over devices, matching any device against the provided string.
- */
-void *auxiliary_dev_iterate(const void *start, const char *str,
- const struct rte_dev_iterator *it);
-
#endif /* BUS_AUXILIARY_PRIVATE_H */
diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index 45c6e8335d..d6f83e2e80 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -86,15 +86,6 @@
struct rte_cdx_bus rte_cdx_bus;
-enum cdx_params {
- RTE_CDX_PARAM_NAME,
-};
-
-static const char * const cdx_params_keys[] = {
- [RTE_CDX_PARAM_NAME] = "name",
- NULL,
-};
-
static int
cdx_get_kernel_driver_by_path(const char *filename, char *driver_name,
size_t len)
@@ -528,47 +519,6 @@ cdx_get_iommu_class(void)
return RTE_IOVA_VA;
}
-static int
-cdx_dev_match(const struct rte_device *dev,
- const void *_kvlist)
-{
- const struct rte_kvargs *kvlist = _kvlist;
- const char *key = cdx_params_keys[RTE_CDX_PARAM_NAME];
- const char *name;
-
- /* no kvlist arg, all devices match */
- if (kvlist == NULL)
- return 0;
-
- /* if key is present in kvlist and does not match, filter device */
- name = rte_kvargs_get(kvlist, key);
- if (name != NULL && strcmp(name, dev->name))
- return -1;
-
- return 0;
-}
-
-static void *
-cdx_dev_iterate(const void *start,
- const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- struct rte_kvargs *kvargs = NULL;
- struct rte_device *dev;
-
- if (str != NULL) {
- kvargs = rte_kvargs_parse(str, cdx_params_keys);
- if (kvargs == NULL) {
- CDX_BUS_ERR("cannot parse argument list %s", str);
- rte_errno = EINVAL;
- return NULL;
- }
- }
- dev = rte_bus_generic_find_device(&rte_cdx_bus.bus, start, cdx_dev_match, kvargs);
- rte_kvargs_free(kvargs);
- return dev;
-}
-
struct rte_cdx_bus rte_cdx_bus = {
.bus = {
.scan = cdx_scan,
@@ -580,7 +530,7 @@ struct rte_cdx_bus rte_cdx_bus = {
.dma_map = cdx_dma_map,
.dma_unmap = cdx_dma_unmap,
.get_iommu_class = cdx_get_iommu_class,
- .dev_iterate = cdx_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
},
};
diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index 0bacc0e9d5..b3a754cbf4 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -845,50 +845,6 @@ dpaa_bus_unplug(struct rte_device *dev __rte_unused)
return 0;
}
-static void *
-dpaa_bus_dev_iterate(const void *start, const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- char *dup, *dev_name = NULL;
- struct rte_device *dev;
-
- if (str == NULL) {
- DPAA_BUS_DEBUG("No device string");
- return NULL;
- }
-
- /* Expectation is that device would be name=device_name */
- if (strncmp(str, "name=", 5) != 0) {
- DPAA_BUS_DEBUG("Invalid device string (%s)", str);
- return NULL;
- }
-
- /* Now that name=device_name format is available, split */
- dup = strdup(str);
- if (dup == NULL) {
- DPAA_BUS_DEBUG("Dup string (%s) failed!", str);
- return NULL;
- }
- dev_name = dup + strlen("name=");
-
- if (start != NULL) {
- dev = TAILQ_NEXT((const struct rte_device *)start, next);
- } else {
- dev = TAILQ_FIRST(&rte_dpaa_bus.bus.device_list);
- }
-
- while (dev != NULL) {
- if (strcmp(dev->name, dev_name) == 0) {
- free(dup);
- return dev;
- }
- dev = TAILQ_NEXT(dev, next);
- }
-
- free(dup);
- return NULL;
-}
-
static int
dpaa_bus_cleanup(void)
{
@@ -948,7 +904,7 @@ static struct rte_dpaa_bus rte_dpaa_bus = {
.get_iommu_class = rte_dpaa_get_iommu_class,
.plug = dpaa_bus_plug,
.unplug = dpaa_bus_unplug,
- .dev_iterate = dpaa_bus_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
.cleanup = dpaa_bus_cleanup,
},
.max_push_rxq_num = DPAA_DEFAULT_PUSH_MODE_QUEUE,
diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index 692f7d26f2..716f0178b5 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -593,50 +593,6 @@ fslmc_bus_unplug(struct rte_device *rte_dev)
return -ENODEV;
}
-static void *
-fslmc_bus_dev_iterate(const void *start, const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- char *dup, *dev_name = NULL;
- struct rte_device *dev;
-
- if (str == NULL) {
- DPAA2_BUS_DEBUG("No device string");
- return NULL;
- }
-
- /* Expectation is that device would be name=device_name */
- if (strncmp(str, "name=", 5) != 0) {
- DPAA2_BUS_DEBUG("Invalid device string (%s)", str);
- return NULL;
- }
-
- /* Now that name=device_name format is available, split */
- dup = strdup(str);
- if (dup == NULL) {
- DPAA2_BUS_DEBUG("Dup string (%s) failed!", str);
- return NULL;
- }
- dev_name = dup + strlen("name=");
-
- if (start != NULL) {
- dev = TAILQ_NEXT((const struct rte_device *) start, next);
- } else {
- dev = TAILQ_FIRST(&rte_fslmc_bus.bus.device_list);
- }
-
- while (dev != NULL) {
- if (strcmp(dev->name, dev_name) == 0) {
- free(dup);
- return dev;
- }
- dev = TAILQ_NEXT(dev, next);
- }
-
- free(dup);
- return NULL;
-}
-
struct rte_fslmc_bus rte_fslmc_bus = {
.bus = {
.scan = rte_fslmc_scan,
@@ -648,7 +604,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
.get_iommu_class = rte_dpaa2_get_iommu_class,
.plug = fslmc_bus_plug,
.unplug = fslmc_bus_unplug,
- .dev_iterate = fslmc_bus_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
},
.device_count = {0},
};
diff --git a/drivers/bus/pci/pci_params.c b/drivers/bus/pci/pci_params.c
index d596c3bba8..e308c85ed2 100644
--- a/drivers/bus/pci/pci_params.c
+++ b/drivers/bus/pci/pci_params.c
@@ -59,7 +59,8 @@ pci_dev_match(const struct rte_device *dev,
}
void *
-rte_pci_dev_iterate(const void *start,
+rte_pci_dev_iterate(const struct rte_bus *bus __rte_unused,
+ const void *start,
const char *str,
const struct rte_dev_iterator *it __rte_unused)
{
diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h
index 52fa6b0f76..21637882f8 100644
--- a/drivers/bus/pci/private.h
+++ b/drivers/bus/pci/private.h
@@ -258,6 +258,9 @@ rte_pci_get_iommu_class(void);
* matching any device against the provided
* string.
*
+ * @param bus
+ * A pointer to the bus structure.
+ *
* @param start
* Iteration starting point.
*
@@ -272,7 +275,8 @@ rte_pci_get_iommu_class(void);
* NULL otherwise.
*/
void *
-rte_pci_dev_iterate(const void *start,
+rte_pci_dev_iterate(const struct rte_bus *bus,
+ const void *start,
const char *str,
const struct rte_dev_iterator *it);
diff --git a/drivers/bus/platform/meson.build b/drivers/bus/platform/meson.build
index 8633cc4e75..9b1f55c3bb 100644
--- a/drivers/bus/platform/meson.build
+++ b/drivers/bus/platform/meson.build
@@ -11,8 +11,5 @@ endif
require_iova_in_mbuf = false
deps += ['kvargs']
-sources = files(
- 'platform_params.c',
- 'platform.c',
-)
+sources = files('platform.c')
driver_sdk_headers += files('bus_platform_driver.h')
diff --git a/drivers/bus/platform/platform.c b/drivers/bus/platform/platform.c
index c795bd4b9c..636f051049 100644
--- a/drivers/bus/platform/platform.c
+++ b/drivers/bus/platform/platform.c
@@ -562,7 +562,7 @@ struct rte_platform_bus platform_bus = {
.dma_map = platform_bus_dma_map,
.dma_unmap = platform_bus_dma_unmap,
.get_iommu_class = platform_bus_get_iommu_class,
- .dev_iterate = platform_bus_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
.cleanup = platform_bus_cleanup,
},
};
diff --git a/drivers/bus/platform/platform_params.c b/drivers/bus/platform/platform_params.c
deleted file mode 100644
index f8538a1d84..0000000000
--- a/drivers/bus/platform/platform_params.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2023 Marvell.
- */
-
-#include <string.h>
-#include <errno.h>
-
-#include <rte_bus.h>
-#include <rte_common.h>
-#include <rte_dev.h>
-#include <rte_errno.h>
-#include <rte_kvargs.h>
-
-#include "bus_platform_driver.h"
-#include "private.h"
-
-enum platform_params {
- RTE_PLATFORM_PARAM_NAME,
-};
-
-static const char * const platform_params_keys[] = {
- [RTE_PLATFORM_PARAM_NAME] = "name",
- NULL
-};
-
-static int
-platform_dev_match(const struct rte_device *dev, const void *_kvlist)
-{
- const char *key = platform_params_keys[RTE_PLATFORM_PARAM_NAME];
- const struct rte_kvargs *kvlist = _kvlist;
- const char *name;
-
- /* no kvlist arg, all devices match */
- if (kvlist == NULL)
- return 0;
-
- /* if key is present in kvlist and does not match, filter device */
- name = rte_kvargs_get(kvlist, key);
- if (name != NULL && strcmp(name, dev->name))
- return -1;
-
- return 0;
-}
-
-void *
-platform_bus_dev_iterate(const void *start, const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- struct rte_kvargs *kvargs = NULL;
- struct rte_device *dev;
-
- if (str != NULL) {
- kvargs = rte_kvargs_parse(str, platform_params_keys);
- if (!kvargs) {
- PLATFORM_LOG_LINE(ERR, "cannot parse argument list %s", str);
- rte_errno = EINVAL;
- return NULL;
- }
- }
-
- dev = rte_bus_generic_find_device(&platform_bus.bus, start, platform_dev_match, kvargs);
- rte_kvargs_free(kvargs);
-
- return dev;
-}
diff --git a/drivers/bus/platform/private.h b/drivers/bus/platform/private.h
index 81a8984052..bf5d75df03 100644
--- a/drivers/bus/platform/private.h
+++ b/drivers/bus/platform/private.h
@@ -28,11 +28,4 @@ extern int platform_bus_logtype;
#define PLATFORM_LOG_LINE(level, ...) \
RTE_LOG_LINE(level, PLATFORM_BUS, __VA_ARGS__)
-/*
- * Iterate registered platform devices and find one that matches provided string.
- */
-void *
-platform_bus_dev_iterate(const void *start, const char *str,
- const struct rte_dev_iterator *it __rte_unused);
-
#endif /* PLATFORM_PRIVATE_H */
diff --git a/drivers/bus/uacce/uacce.c b/drivers/bus/uacce/uacce.c
index 24f3c05878..af1ada0bd3 100644
--- a/drivers/bus/uacce/uacce.c
+++ b/drivers/bus/uacce/uacce.c
@@ -44,14 +44,6 @@ struct rte_uacce_bus {
/* Forward declaration of UACCE bus. */
static struct rte_uacce_bus uacce_bus;
-enum uacce_params {
- RTE_UACCE_PARAM_NAME,
-};
-
-static const char *const uacce_params_keys[] = {
- [RTE_UACCE_PARAM_NAME] = "name",
- NULL,
-};
extern int uacce_bus_logtype;
#define RTE_LOGTYPE_UACCE_BUS uacce_bus_logtype
@@ -519,44 +511,6 @@ uacce_parse(const char *name, void *addr)
return ret;
}
-static int
-uacce_dev_match(const struct rte_device *dev, const void *_kvlist)
-{
- const char *key = uacce_params_keys[RTE_UACCE_PARAM_NAME];
- const struct rte_kvargs *kvlist = _kvlist;
- const char *name;
-
- /* no kvlist arg, all devices match. */
- if (kvlist == NULL)
- return 0;
-
- /* if key is present in kvlist and does not match, filter device. */
- name = rte_kvargs_get(kvlist, key);
- if (name != NULL && strcmp(name, dev->name))
- return -1;
-
- return 0;
-}
-
-static void *
-uacce_dev_iterate(const void *start, const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- struct rte_kvargs *kvargs = NULL;
- struct rte_device *dev;
-
- if (str != NULL) {
- kvargs = rte_kvargs_parse(str, uacce_params_keys);
- if (kvargs == NULL) {
- UACCE_BUS_ERR("cannot parse argument list %s", str);
- return NULL;
- }
- }
- dev = rte_bus_generic_find_device(&uacce_bus.bus, start, uacce_dev_match, kvargs);
- rte_kvargs_free(kvargs);
- return dev;
-}
-
RTE_EXPORT_INTERNAL_SYMBOL(rte_uacce_avail_queues)
int
rte_uacce_avail_queues(struct rte_uacce_device *dev)
@@ -673,7 +627,7 @@ static struct rte_uacce_bus uacce_bus = {
.unplug = uacce_unplug,
.find_device = rte_bus_generic_find_device,
.parse = uacce_parse,
- .dev_iterate = uacce_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
},
};
diff --git a/drivers/bus/vdev/meson.build b/drivers/bus/vdev/meson.build
index 50f0c8918d..6487b0d672 100644
--- a/drivers/bus/vdev/meson.build
+++ b/drivers/bus/vdev/meson.build
@@ -1,10 +1,7 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2017 Intel Corporation
-sources = files(
- 'vdev.c',
- 'vdev_params.c',
-)
+sources = files('vdev.c')
headers = files('rte_bus_vdev.h')
driver_sdk_headers = files('bus_vdev_driver.h')
diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c
index 65643b380d..4003805315 100644
--- a/drivers/bus/vdev/vdev.c
+++ b/drivers/bus/vdev/vdev.c
@@ -10,12 +10,14 @@
#include <stdbool.h>
#include <sys/queue.h>
+#include <rte_os_shim.h>
#include <eal_export.h>
#include <rte_eal.h>
#include <dev_driver.h>
#include <bus_driver.h>
#include <rte_common.h>
#include <rte_devargs.h>
+#include <rte_log.h>
#include <rte_memory.h>
#include <rte_tailq.h>
#include <rte_spinlock.h>
@@ -23,11 +25,15 @@
#include <rte_errno.h>
#include "bus_vdev_driver.h"
-#include "vdev_logs.h"
-#include "vdev_private.h"
#define VDEV_MP_KEY "bus_vdev_mp"
+int vdev_logtype_bus;
+#define RTE_LOGTYPE_VDEV_BUS vdev_logtype_bus
+
+#define VDEV_LOG(level, ...) \
+ RTE_LOG_LINE_PREFIX(level, VDEV_BUS, "%s(): ", __func__, __VA_ARGS__)
+
/* Forward declare to access virtual bus name */
static struct rte_bus rte_vdev_bus;
@@ -589,12 +595,6 @@ vdev_find_device(const struct rte_bus *bus, const struct rte_device *start,
return dev;
}
-struct rte_device *
-rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data)
-{
- return vdev_find_device(&rte_vdev_bus, start, cmp, data);
-}
-
static int
vdev_plug(struct rte_device *dev)
{
@@ -637,7 +637,7 @@ static struct rte_bus rte_vdev_bus = {
.dma_map = vdev_dma_map,
.dma_unmap = vdev_dma_unmap,
.get_iommu_class = vdev_get_iommu_class,
- .dev_iterate = rte_vdev_dev_iterate,
+ .dev_iterate = rte_bus_generic_dev_iterate,
};
RTE_REGISTER_BUS(vdev, rte_vdev_bus);
diff --git a/drivers/bus/vdev/vdev_logs.h b/drivers/bus/vdev/vdev_logs.h
deleted file mode 100644
index 38859ae4b7..0000000000
--- a/drivers/bus/vdev/vdev_logs.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017 Intel Corporation
- */
-
-#ifndef _VDEV_LOGS_H_
-#define _VDEV_LOGS_H_
-
-#include <rte_log.h>
-
-extern int vdev_logtype_bus;
-#define RTE_LOGTYPE_VDEV_BUS vdev_logtype_bus
-
-#define VDEV_LOG(level, ...) \
- RTE_LOG_LINE_PREFIX(level, VDEV_BUS, "%s(): ", __func__, __VA_ARGS__)
-
-#endif /* _VDEV_LOGS_H_ */
diff --git a/drivers/bus/vdev/vdev_params.c b/drivers/bus/vdev/vdev_params.c
deleted file mode 100644
index 68ae09e2e9..0000000000
--- a/drivers/bus/vdev/vdev_params.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2018 Gaëtan Rivet
- */
-
-#include <errno.h>
-#include <string.h>
-
-#include <dev_driver.h>
-#include <rte_kvargs.h>
-#include <rte_errno.h>
-
-#include "vdev_logs.h"
-#include "vdev_private.h"
-
-enum vdev_params {
- RTE_VDEV_PARAM_NAME,
- RTE_VDEV_PARAM_MAX,
-};
-
-static const char * const vdev_params_keys[] = {
- [RTE_VDEV_PARAM_NAME] = "name",
- [RTE_VDEV_PARAM_MAX] = NULL,
-};
-
-static int
-vdev_dev_match(const struct rte_device *dev,
- const void *_kvlist)
-{
- const struct rte_kvargs *kvlist = _kvlist;
- const char *key = vdev_params_keys[RTE_VDEV_PARAM_NAME];
- const char *name;
-
- /* no kvlist arg, all devices match */
- if (kvlist == NULL)
- return 0;
-
- /* if key is present in kvlist and does not match, filter device */
- name = rte_kvargs_get(kvlist, key);
- if (name != NULL && strcmp(name, dev->name))
- return -1;
-
- return 0;
-}
-
-void *
-rte_vdev_dev_iterate(const void *start,
- const char *str,
- const struct rte_dev_iterator *it __rte_unused)
-{
- struct rte_kvargs *kvargs = NULL;
- struct rte_device *dev;
-
- if (str != NULL) {
- kvargs = rte_kvargs_parse(str, vdev_params_keys);
- if (kvargs == NULL) {
- VDEV_LOG(ERR, "cannot parse argument list");
- rte_errno = EINVAL;
- return NULL;
- }
- }
- dev = rte_vdev_find_device(start, vdev_dev_match, kvargs);
- rte_kvargs_free(kvargs);
- return dev;
-}
diff --git a/drivers/bus/vdev/vdev_private.h b/drivers/bus/vdev/vdev_private.h
deleted file mode 100644
index e683f5f133..0000000000
--- a/drivers/bus/vdev/vdev_private.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2018 Gaëtan Rivet
- */
-
-#ifndef _VDEV_PRIVATE_H_
-#define _VDEV_PRIVATE_H_
-
-#include <rte_os_shim.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct rte_device *
-rte_vdev_find_device(const struct rte_device *start,
- rte_dev_cmp_t cmp,
- const void *data);
-
-void *
-rte_vdev_dev_iterate(const void *start,
- const char *str,
- const struct rte_dev_iterator *it);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _VDEV_PRIVATE_H_ */
diff --git a/lib/eal/common/eal_common_bus.c b/lib/eal/common/eal_common_bus.c
index f81d13e7d0..46a8e68532 100644
--- a/lib/eal/common/eal_common_bus.c
+++ b/lib/eal/common/eal_common_bus.c
@@ -9,6 +9,7 @@
#include <bus_driver.h>
#include <rte_debug.h>
#include <rte_devargs.h>
+#include <rte_kvargs.h>
#include <rte_string_fns.h>
#include <rte_errno.h>
@@ -432,3 +433,43 @@ rte_bus_remove_driver(struct rte_bus *bus, struct rte_driver *driver)
TAILQ_REMOVE(&bus->driver_list, driver, next);
driver->bus = NULL;
}
+
+static int
+bus_dev_match_by_name(const struct rte_device *dev, const void *_kvlist)
+{
+ const struct rte_kvargs *kvlist = _kvlist;
+ const char *name;
+
+ if (kvlist == NULL)
+ return 0;
+
+ name = rte_kvargs_get(kvlist, "name");
+ if (name != NULL && strcmp(name, dev->name))
+ return -1;
+
+ return 0;
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(rte_bus_generic_dev_iterate)
+void *
+rte_bus_generic_dev_iterate(const struct rte_bus *bus,
+ const void *start,
+ const char *str,
+ const struct rte_dev_iterator *it __rte_unused)
+{
+ static const char * const params_keys[] = { "name", NULL };
+ struct rte_kvargs *kvargs = NULL;
+ struct rte_device *dev;
+
+ if (str != NULL) {
+ kvargs = rte_kvargs_parse(str, params_keys);
+ if (kvargs == NULL) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ }
+
+ dev = rte_bus_generic_find_device(bus, start, bus_dev_match_by_name, kvargs);
+ rte_kvargs_free(kvargs);
+ return dev;
+}
diff --git a/lib/eal/common/eal_common_dev.c b/lib/eal/common/eal_common_dev.c
index e08a0f9dbc..17e8901546 100644
--- a/lib/eal/common/eal_common_dev.c
+++ b/lib/eal/common/eal_common_dev.c
@@ -756,13 +756,13 @@ bus_next_dev_cmp(const struct rte_bus *bus,
if (rte_errno != 0)
return -1;
if (it->cls_str == NULL) {
- dev = bus->dev_iterate(dev, bus_str, it);
+ dev = bus->dev_iterate(bus, dev, bus_str, it);
goto end;
}
/* cls_str != NULL */
if (dev == NULL) {
next_dev_on_bus:
- dev = bus->dev_iterate(dev, bus_str, it);
+ dev = bus->dev_iterate(bus, dev, bus_str, it);
it->device = dev;
}
if (dev == NULL)
diff --git a/lib/eal/include/bus_driver.h b/lib/eal/include/bus_driver.h
index 16e989c10c..9568d820e5 100644
--- a/lib/eal/include/bus_driver.h
+++ b/lib/eal/include/bus_driver.h
@@ -262,6 +262,32 @@ struct rte_bus_conf {
*/
typedef enum rte_iova_mode (*rte_bus_get_iommu_class_t)(void);
+/**
+ * Per bus, device iteration function.
+ *
+ * Similar to rte_dev_iterate_t but also pass along the bus pointer.
+ *
+ * @param bus
+ * A pointer to the bus structure.
+ *
+ * @param start
+ * Starting iteration context.
+ *
+ * @param devstr
+ * Device description string.
+ *
+ * @param it
+ * Device iterator.
+ *
+ * @return
+ * The address of the current element matching the device description
+ * string.
+ */
+typedef void *(*rte_bus_dev_iterate_t)(const struct rte_bus *bus,
+ const void *start,
+ const char *devstr,
+ const struct rte_dev_iterator *it);
+
/**
* A structure describing a generic bus.
*/
@@ -280,7 +306,7 @@ struct rte_bus {
rte_dev_dma_unmap_t dma_unmap; /**< DMA unmap for device in the bus */
struct rte_bus_conf conf; /**< Bus configuration */
rte_bus_get_iommu_class_t get_iommu_class; /**< Get iommu class */
- rte_dev_iterate_t dev_iterate; /**< Device iterator. */
+ rte_bus_dev_iterate_t dev_iterate; /**< Bus device iterator. */
rte_bus_hot_unplug_handler_t hot_unplug_handler;
/**< handle hot-unplug failure on the bus */
rte_bus_sigbus_handler_t sigbus_handler;
@@ -321,6 +347,31 @@ struct rte_devargs *rte_bus_find_devargs(const struct rte_bus *bus, const char *
__rte_internal
bool rte_bus_device_is_ignored(const struct rte_bus *bus, const char *dev_name);
+/**
+ * Generic device iterator for buses using name-based matching.
+ *
+ * This helper implements the standard name-based device iteration pattern
+ * using kvargs parsing. Buses that only support "name" parameter matching
+ * can use this instead of implementing their own dev_iterate function.
+ *
+ * @param bus
+ * A pointer to the bus structure.
+ * @param start
+ * The starting device (NULL to start from the beginning).
+ * @param devstr
+ * The device filter string (e.g., "name=eth0").
+ * @param it
+ * Device iterator.
+ *
+ * @return
+ * Pointer to the matching device, or NULL if not found.
+ */
+__rte_internal
+void *rte_bus_generic_dev_iterate(const struct rte_bus *bus,
+ const void *start,
+ const char *devstr,
+ const struct rte_dev_iterator *it);
+
/**
* Helper for Bus registration.
* The constructor has higher priority than PMD constructors.
diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c
index d0273e3f7b..ce0407b67f 100644
--- a/lib/ethdev/rte_ethdev.c
+++ b/lib/ethdev/rte_ethdev.c
@@ -313,7 +313,7 @@ rte_eth_iterator_next(struct rte_dev_iterator *iter)
iter->class_device == NULL) {
/* get next rte_device to try. */
iter->device = iter->bus->dev_iterate(
- iter->device, iter->bus_str, iter);
+ iter->bus, iter->device, iter->bus_str, iter);
if (iter->device == NULL)
break; /* no more rte_device candidate */
}
--
2.53.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox