* [PATCH v4 03/20] drivers: add supported packet types get callback
From: liujie5 @ 2026-06-02 3:16 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260602031707.578932-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Implement dev_supported_ptypes_get ethdev callback for sxe2 PMD.
This allows applications to query the packet types the driver
is capable of identifying, such as L2, L3 (IPv4/IPv6), and
L4 (TCP/UDP/SCTP) layers.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_ptype.h | 1793 ++++++++++++++++++++++++++++++
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 22 +
drivers/net/sxe2/sxe2_cmd_chnl.h | 2 +
drivers/net/sxe2/sxe2_drv_cmd.h | 6 +
drivers/net/sxe2/sxe2_ethdev.c | 169 ++-
drivers/net/sxe2/sxe2_ethdev.h | 19 +-
drivers/net/sxe2/sxe2_mac.c | 103 ++
drivers/net/sxe2/sxe2_mac.h | 50 +
9 files changed, 2126 insertions(+), 39 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_ptype.h
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
diff --git a/drivers/common/sxe2/sxe2_ptype.h b/drivers/common/sxe2/sxe2_ptype.h
new file mode 100644
index 0000000000..9c19570979
--- /dev/null
+++ b/drivers/common/sxe2/sxe2_ptype.h
@@ -0,0 +1,1793 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef _SXE2_PTYPE_H_
+#define _SXE2_PTYPE_H_
+#include <rte_ethdev.h>
+
+static inline const uint32_t *
+sxe2_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements)
+{
+ const uint32_t *ret = NULL;
+
+ static const uint32_t ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER_TIMESYNC,
+ RTE_PTYPE_L2_ETHER_LLDP,
+ RTE_PTYPE_L2_ETHER_ARP,
+ RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L4_FRAG,
+ RTE_PTYPE_L4_ICMP,
+ RTE_PTYPE_L4_NONFRAG,
+ RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT,
+ RTE_PTYPE_TUNNEL_IP,
+ RTE_PTYPE_INNER_L2_ETHER,
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_INNER_L4_FRAG,
+ RTE_PTYPE_INNER_L4_ICMP,
+ RTE_PTYPE_INNER_L4_NONFRAG,
+ RTE_PTYPE_INNER_L4_SCTP,
+ RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_UNKNOWN
+ };
+
+ if (dev->rx_pkt_burst != NULL) {
+ *no_of_elements = RTE_DIM(ptypes);
+ ret = ptypes;
+ } else {
+ ret = NULL;
+ }
+
+ return ret;
+}
+
+static inline void sxe2_init_ptype_list(uint32_t *ptype)
+{
+ /* ptype[0] reserved */
+ ptype[1] = RTE_PTYPE_L2_ETHER;
+ ptype[2] = RTE_PTYPE_L2_ETHER_TIMESYNC;
+ /* ptype[3] - ptype[5] reserved */
+ ptype[6] = RTE_PTYPE_L2_ETHER_LLDP;
+ /* ECP */
+ ptype[7] = RTE_PTYPE_UNKNOWN;
+ /* ptype[8] - ptype[9] reserved */
+ /* EAPol */
+ ptype[10] = RTE_PTYPE_UNKNOWN;
+ ptype[11] = RTE_PTYPE_L2_ETHER_ARP;
+ /* ptype[12] - ptype[21] reserved */
+
+ /* Non tunneled IPv4 */
+ ptype[22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_FRAG;
+ ptype[23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[25] reserved */
+ ptype[26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_SCTP;
+ ptype[28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_ICMP;
+
+ /* IPv4 --> IPv4 */
+ ptype[29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[32] reserved */
+ ptype[33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> IPv6 */
+ ptype[36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[39] reserved */
+ ptype[40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN */
+ ptype[43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> IPv4 */
+ ptype[44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[47] reserved */
+ ptype[48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> IPv6 */
+ ptype[51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[54] reserved */
+ ptype[55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC */
+ ptype[58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC --> IPv4 */
+ ptype[59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[62] reserved */
+ ptype[63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC --> IPv6 */
+ ptype[66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[69] reserved */
+ ptype[70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC/VLAN */
+ ptype[73] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv4 */
+ ptype[74] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[75] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[76] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[77] reserved */
+ ptype[78] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[79] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv6 */
+ ptype[81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_FRAG;
+ ptype[82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[64] reserved */
+ ptype[85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[87] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+ /* Non tunneled IPv6 */
+ ptype[88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_FRAG;
+ ptype[89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[91] reserved */
+ ptype[92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_SCTP;
+ ptype[94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_ICMP;
+
+ /* IPv6 --> IPv4 */
+ ptype[95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[98] reserved */
+ ptype[99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> IPv6 */
+ ptype[102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[105] reserved */
+ ptype[106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN */
+ ptype[109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> IPv4 */
+ ptype[110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[113] reserved */
+ ptype[114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> IPv6 */
+ ptype[117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[120] reserved */
+ ptype[121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC */
+ ptype[124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC --> IPv4 */
+ ptype[125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[128] reserved */
+ ptype[129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC --> IPv6 */
+ ptype[132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[135] reserved */
+ ptype[136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC/VLAN */
+ ptype[139] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv4 */
+ ptype[140] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[141] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[142] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[143] reserved */
+ ptype[144] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[145] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[146] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/GENEVE/VXLAN --> MAC/VLAN --> IPv6 */
+ ptype[147] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[148] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[149] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[150] reserved */
+ ptype[151] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP;
+ ptype[152] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP;
+ ptype[153] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_ICMP;
+ /* ptype[154] - ptype[159] reserved */
+ /* IPSec */
+ ptype[160] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_ESP;
+ ptype[161] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_ESP;
+ /* AH */
+ ptype[162] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[163] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* NAT-T-ESP */
+ ptype[164] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ ptype[165] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ /* SDN-ESP */
+ ptype[166] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ ptype[167] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_ESP;
+ /* ptype[168] - ptype[271] reserved */
+ /* IPV4 --> VRRP */
+ ptype[272] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPV4 --> OSPF */
+ ptype[273] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPV6 --> VRRP */
+ ptype[274] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPV6 --> VRRP */
+ ptype[275] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* ATAoE */
+ ptype[276] = RTE_PTYPE_UNKNOWN;
+ /* Control */
+ ptype[278] = RTE_PTYPE_UNKNOWN;
+ /* ptype[279] - ptype[324] reserved */
+ /* GTP */
+ ptype[325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPC;
+ ptype[329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU;
+ ptype[330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU;
+ ptype[331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ ptype[336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ ptype[341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ ptype[346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_GTPU |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* PFCP */
+ ptype[351] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[352] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[353] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[354] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[355] - ptype[359] reserved */
+ /* L2TPv3 */
+ ptype[360] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_L2TP;
+ ptype[361] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_L2TP;
+ /* ptype[362] - ptype[370] reserved */
+ /* eCPRI */
+ ptype[371] = RTE_PTYPE_UNKNOWN;
+ ptype[381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[396] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, IGMP */
+ ptype[397] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, EIGRP */
+ ptype[398] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, EIGRP */
+ ptype[399] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, PIM */
+ ptype[400] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, PIM */
+ ptype[401] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, IGMP */
+ ptype[402] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, IGMP */
+ ptype[403] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, EIGRP */
+ ptype[404] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, EIGRP */
+ ptype[405] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, PIM */
+ ptype[406] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, PIM */
+ ptype[407] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, VRRP */
+ ptype[408] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, VRRP */
+ ptype[409] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, VRRP */
+ ptype[410] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, VRRP */
+ ptype[411] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, OSPF */
+ ptype[412] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, OSPF */
+ ptype[413] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, OSPF */
+ ptype[414] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, OSPF */
+ ptype[415] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, L2_TP_V3 */
+ ptype[416] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, L2_TP_V3 */
+ ptype[417] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, L2_TP_V3 */
+ ptype[418] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, L2_TP_V3 */
+ ptype[419] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, AH */
+ ptype[420] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, AH */
+ ptype[421] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, AH */
+ ptype[422] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, AH */
+ ptype[423] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv4, ESP */
+ ptype[424] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv4, ESP */
+ ptype[425] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv4, UDP, TUN, MAC, IPv6, ESP */
+ ptype[426] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* MAC, IPv6, UDP, TUN, MAC, IPv6, ESP */
+ ptype[427] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ /* TP-TUN GTPU */
+ ptype[450] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[451] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[452] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[453] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[454] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[455] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[456] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[457] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[458] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[459] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[460] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[461] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[462] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[463] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[464] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[465] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[466] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[467] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[468] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[469] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[470] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[471] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[472] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[473] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[474] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[475] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[476] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[477] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[478] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[479] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[480] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[481] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[482] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[483] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[484] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[485] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[486] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[487] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[488] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[489] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[490] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[491] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[492] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[493] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[494] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[495] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[496] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[497] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_IP | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ /* ptype[498] - ptype[767] reserved */
+ /* L2(NETWORK CPU) */
+ /* ISIS */
+ ptype[768] = RTE_PTYPE_UNKNOWN;
+ /* SDF */
+ ptype[769] = RTE_PTYPE_UNKNOWN;
+ /* PPoE_NEGO */
+ ptype[770] = RTE_PTYPE_L2_ETHER_PPPOE;
+ /* PPoE_PROTOCOL */
+ ptype[771] = RTE_PTYPE_L2_ETHER_PPPOE;
+ ptype[772] = RTE_PTYPE_L2_ETHER_PPPOE;
+ /* LACP */
+ ptype[773] = RTE_PTYPE_UNKNOWN;
+ /* ptype[774] - ptype[775] reserved */
+ /* IPv4 L3(NETWORK CPU) */
+ ptype[776] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_IGMP;
+ /* EIGRP */
+ ptype[777] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* PIM */
+ ptype[778] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[779] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[780] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[781] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ /* ptype[782] - ptype[783] reserved */
+ /* IPv6 L3(NETWORK CPU) */
+ ptype[784] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_IGMP;
+ /* EIGRP */
+ ptype[785] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* PIM */
+ ptype[786] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[787] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[788] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ ptype[789] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_L2TP;
+ /* ptype[790] - ptype[791] reserved */
+ /* IPv4 L4(NETWORK CPU) */
+ ptype[792] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[793] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[794] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[795] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[796] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[797] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[798] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[799] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[800] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[801] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[802] - ptype[807] reserved */
+ /* IPv6 L4(NETWORK CPU) */
+ ptype[808] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[809] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[810] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[811] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[812] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[813] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[814] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[815] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[816] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[817] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ /* ptype[818] - ptype[819] reserved */
+ /* IPv6 -> MAC */
+ ptype[820] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC -> IPv4*/
+ ptype[821] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[822] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[823] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[824] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[825] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[826] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC -> IPv4*/
+ ptype[827] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[828] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[829] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[830] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[831] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[832] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* ptype[833] - ptype[834] reserved */
+ /* IPv6 -> MAC/VLAN */
+ ptype[835] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC/VLAN -> IPv4 */
+ ptype[836] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[837] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[838] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[839] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[840] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[841] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ /* IPv6 -> MAC/VLAN -> IPv6 */
+ ptype[842] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[843] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[844] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[845] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[846] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[847] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> PAY */
+ ptype[878] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> IPv4 */
+ ptype[877] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[876] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[879] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[880] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[875] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[874] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> IPv6 */
+ ptype[871] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[870] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[872] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[873] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[869] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[868] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> PAY */
+ ptype[891] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> IPv4 */
+ ptype[890] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[889] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[892] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[893] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[888] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[887] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> IPv6 */
+ ptype[884] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[883] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[885] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[886] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[882] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[881] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> GRE -> PAY */
+ ptype[904] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv6 -> UDP -> GRE -> IPv4 */
+ ptype[903] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[902] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[905] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[906] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[901] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[900] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv6 -> UDP -> GRE -> IPv6 */
+ ptype[897] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[896] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[898] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[899] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[895] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[894] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> GRE -> PAY */
+ ptype[917] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv4 -> UDP -> GRE -> IPv4 */
+ ptype[916] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[915] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[918] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[919] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[914] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[913] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> GRE -> IPv6 */
+ ptype[910] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[909] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[911] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[912] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[908] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[907] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MACVLAN -> PAY */
+ ptype[930] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv4 */
+ ptype[929] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[928] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[931] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[932] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[927] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[926] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv6 */
+ ptype[923] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[922] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[924] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[925] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[921] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[920] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MACVLAN -> PAY */
+ ptype[943] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv4 */
+ ptype[942] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[941] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[944] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[945] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[940] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[939] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MACVLAN -> IPv6 */
+ ptype[936] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[935] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[937] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[938] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[934] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[933] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 > UDP -> GRE -> MACVLAN -> PAY */
+ ptype[956] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv6 -> UDP -> GRE -> MACVLAN -> IPv4 */
+ ptype[955] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[954] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[957] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[958] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[953] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[952] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv6 -> UDP -> GRE -> MACVLAN -> IPv6 */
+ ptype[949] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[948] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[950] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[951] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[947] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[946] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> GRE -> MACVLAN -> PAY */
+ ptype[969] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN;
+ /* IPv4 -> UDP -> GRE -> MACVLAN -> IPv4 */
+ ptype[968] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[967] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[970] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[971] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[966] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[965] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> GRE -> MACVLAN -> IPv6 */
+ ptype[962] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[961] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[963] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[964] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[960] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[959] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER_VLAN |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MAC -> PAY */
+ ptype[982] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MAC -> IPv4 */
+ ptype[981] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[980] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[983] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[984] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[979] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[978] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 -> UDP -> VXLAN/GENEVE -> MAC -> IPv6 */
+ ptype[975] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[974] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[976] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[977] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[973] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[972] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MAC -> PAY */
+ ptype[995] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MAC -> IPv4 */
+ ptype[994] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[993] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[996] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[997] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[992] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[991] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> VXLAN/GENEVE -> MAC -> IPv6 */
+ ptype[988] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[987] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[989] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[990] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[986] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[985] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 > UDP -> GRE -> MAC -> PAY */
+ ptype[1008] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+ /* IPv6 -> UDP -> GRE -> MAC -> IPv4 */
+ ptype[1007] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1006] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1009] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1010] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[1005] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[1004] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv6 -> UDP -> GRE -> MAC -> IPv6 */
+ ptype[1001] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1000] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1002] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1003] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[999] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[998] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 -> UDP -> GRE -> MAC -> PAY */
+ ptype[1021] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+ /* IPv4 -> UDP -> GRE -> MAC -> IPv4 */
+ ptype[1020] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1019] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1022] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1023] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[1018] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[1017] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+ /* IPv4 -> UDP -> GRE -> MAC -> IPv6 */
+ ptype[1014] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[1013] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[1015] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[1016] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[1012] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ ptype[1011] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+}
+
+#endif /* _RTE_PTYPE_TUNNEL_GRENAT_H_ */
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index c225dd7cd8..b14b5120c1 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -60,4 +60,5 @@ sources += files(
'sxe2_txrx_poll.c',
'sxe2_txrx.c',
'sxe2_txrx_vec.c',
+ 'sxe2_mac.c',
)
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index d16b6528d0..07eeb7f38c 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -321,3 +321,25 @@ int32_t sxe2_drv_txq_switch(struct sxe2_adapter *adapter, struct sxe2_tx_queue *
return ret;
}
+
+int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter)
+{
+ int32_t ret = 0;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_drv_link_info_resp resp = {0};
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_LINK_STATUS_GET,
+ NULL, 0,
+ &resp, sizeof(resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "link status get failed, ret=%d", ret);
+ goto l_end;
+ }
+ adapter->link_ctxt.speed = resp.speed;
+ adapter->link_ctxt.link_up = resp.status;
+
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index ed5e842346..cda676ed97 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -34,4 +34,6 @@ int32_t sxe2_drv_txq_ctxt_cfg(struct sxe2_adapter *adapter,
struct sxe2_tx_queue *txq,
uint16_t txq_cnt);
+int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
+
#endif /* SXE2_CMD_CHNL_H */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index ccc9c20ef4..a0f08b5184 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -227,6 +227,12 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
struct sxe2_drv_msix_caps used_msix;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_link_info_resp {
+ uint32_t speed;
+ uint8_t status;
+ uint8_t rsv[3];
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index e0f7002138..01552a8202 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -27,7 +27,9 @@
#include "sxe2_tx.h"
#include "sxe2_rx.h"
#include "sxe2_txrx.h"
+#include "sxe2_mac.h"
#include "sxe2_common.h"
+#include "sxe2_ptype.h"
#include "sxe2_common_log.h"
#include "sxe2_host_regs.h"
#include "sxe2_ioctl_chnl_func.h"
@@ -78,6 +80,41 @@ static struct sxe2_pci_map_addr_info sxe2_net_map_addr_info_pf[SXE2_PCI_MAP_RES_
.reg_width = 10},
};
+static int32_t sxe2_dev_configure(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_start(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_stop(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_close(struct rte_eth_dev *dev);
+static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info);
+static const uint32_t *sxe2_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev
+ __rte_unused, size_t *no_of_elements __rte_unused);
+
+static const struct eth_dev_ops sxe2_eth_dev_ops = {
+ .dev_configure = sxe2_dev_configure,
+ .dev_start = sxe2_dev_start,
+ .dev_stop = sxe2_dev_stop,
+ .dev_close = sxe2_dev_close,
+ .dev_infos_get = sxe2_dev_infos_get,
+ .dev_supported_ptypes_get = sxe2_dev_supported_ptypes_get,
+ .link_update = sxe2_link_update,
+
+ .rx_queue_start = sxe2_rx_queue_start,
+ .rx_queue_stop = sxe2_rx_queue_stop,
+ .tx_queue_start = sxe2_tx_queue_start,
+ .tx_queue_stop = sxe2_tx_queue_stop,
+ .rx_queue_setup = sxe2_rx_queue_setup,
+ .rx_queue_release = sxe2_rx_queue_release,
+ .tx_queue_setup = sxe2_tx_queue_setup,
+ .tx_queue_release = sxe2_tx_queue_release,
+ .rxq_info_get = sxe2_rx_queue_info_get,
+ .txq_info_get = sxe2_tx_queue_info_get,
+ .rx_burst_mode_get = sxe2_rx_burst_mode_get,
+ .tx_burst_mode_get = sxe2_tx_burst_mode_get,
+ .tx_done_cleanup = sxe2_tx_done_cleanup,
+
+ .mtu_set = sxe2_mtu_set,
+ .buffer_split_supported_hdr_ptypes_get = sxe2_buffer_split_supported_hdr_ptypes_get,
+};
+
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
{
int32_t ret = 0;
@@ -122,6 +159,12 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
sxe2_rx_mode_func_set(dev);
sxe2_tx_mode_func_set(dev);
+ ret = sxe2_link_update_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
+ goto l_end;
+ }
+
ret = sxe2_queues_start(dev);
if (ret) {
PMD_LOG_ERR(INIT, "enable queues failed");
@@ -136,16 +179,6 @@ static int32_t sxe2_dev_start(struct rte_eth_dev *dev)
return ret;
}
-static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
-{
- (void)sxe2_dev_stop(dev);
- (void)sxe2_queues_release(dev);
- sxe2_vsi_uninit(dev);
- sxe2_dev_pci_map_uinit(dev);
-
- return 0;
-}
-
static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
@@ -270,28 +303,59 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
return 0;
}
-static const struct eth_dev_ops sxe2_eth_dev_ops = {
- .dev_configure = sxe2_dev_configure,
- .dev_start = sxe2_dev_start,
- .dev_stop = sxe2_dev_stop,
- .dev_close = sxe2_dev_close,
- .dev_infos_get = sxe2_dev_infos_get,
+static const uint32_t *
+sxe2_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev __rte_unused,
+ size_t *no_of_elements __rte_unused)
+{
+ static const uint32_t ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_SCTP,
+
+ RTE_PTYPE_TUNNEL_GRENAT,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP,
+
+ RTE_PTYPE_UNKNOWN
+ };
- .rx_queue_start = sxe2_rx_queue_start,
- .rx_queue_stop = sxe2_rx_queue_stop,
- .tx_queue_start = sxe2_tx_queue_start,
- .tx_queue_stop = sxe2_tx_queue_stop,
- .rx_queue_setup = sxe2_rx_queue_setup,
- .rx_queue_release = sxe2_rx_queue_release,
- .tx_queue_setup = sxe2_tx_queue_setup,
- .tx_queue_release = sxe2_tx_queue_release,
+ return ptypes;
+}
- .rxq_info_get = sxe2_rx_queue_info_get,
- .txq_info_get = sxe2_tx_queue_info_get,
- .rx_burst_mode_get = sxe2_rx_burst_mode_get,
- .tx_burst_mode_get = sxe2_tx_burst_mode_get,
- .tx_done_cleanup = sxe2_tx_done_cleanup,
-};
+static inline void sxe2_init_ptype_tbl(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ uint32_t *ptype = adapter->ptype_tbl;
+
+ PMD_INIT_FUNC_TRACE();
+
+ sxe2_init_ptype_list(ptype);
+}
struct sxe2_pci_map_bar_info *sxe2_dev_get_bar_info(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type)
@@ -360,6 +424,29 @@ void *sxe2_pci_map_addr_get(struct sxe2_adapter *adapter,
return addr;
}
+static int32_t sxe2_eth_init(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+
+ ret = sxe2_link_update_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize link update, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_mtu_set(dev, dev->data->mtu);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to set mtu, ret=%d", ret);
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static void sxe2_eth_uinit(struct rte_eth_dev *dev __rte_unused)
+{
+}
+
static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
struct sxe2_drv_dev_caps_resp *dev_caps)
{
@@ -765,6 +852,8 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto l_end;
}
+ sxe2_init_ptype_tbl(dev);
+
ret = sxe2_hw_init(dev);
if (ret) {
PMD_LOG_ERR(INIT, "Failed to initialize hw, ret=[%d]", ret);
@@ -789,18 +878,38 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_dev_info_err;
}
+ ret = sxe2_eth_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to initialize eth parameters, ret=%d", ret);
+ goto init_eth_err;
+ }
+
goto l_end;
+init_eth_err:
init_dev_info_err:
sxe2_vsi_uninit(dev);
init_vsi_err:
+ sxe2_dev_pci_map_uinit(dev);
l_end:
return ret;
}
+static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
+{
+ (void)sxe2_dev_stop(dev);
+ (void)sxe2_queues_release(dev);
+ sxe2_vsi_uninit(dev);
+ sxe2_dev_pci_map_uinit(dev);
+ sxe2_eth_uinit(dev);
+
+ return 0;
+}
+
static int32_t sxe2_dev_uninit(struct rte_eth_dev *dev)
{
int32_t ret = 0;
+
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
goto l_end;
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index ff0876cd4c..c188b41a0e 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -120,12 +120,6 @@ enum {
SXE2_FLAGS_NBITS
};
-struct sxe2_link_context {
- rte_spinlock_t link_lock;
- bool link_up;
- uint32_t speed;
-};
-
struct sxe2_devargs {
uint8_t flow_dup_pattern_mode;
uint8_t func_flow_direct_en;
@@ -267,6 +261,12 @@ struct sxe2_sched_hw_cap {
uint8_t adj_lvl;
};
+struct sxe2_link_context {
+ rte_spinlock_t link_lock;
+ bool link_up;
+ uint32_t speed;
+};
+
struct sxe2_adapter {
struct sxe2_common_device *cdev;
struct sxe2_dev_info dev_info;
@@ -276,9 +276,10 @@ struct sxe2_adapter {
struct sxe2_irq_context irq_ctxt;
struct sxe2_queue_context q_ctxt;
struct sxe2_vsi_context vsi_ctxt;
- struct sxe2_devargs devargs;
- uint16_t dev_port_id;
- uint64_t cap_flags;
+ struct sxe2_link_context link_ctxt;
+ struct sxe2_devargs devargs;
+ uint16_t dev_port_id;
+ uint64_t cap_flags;
enum sxe2_dev_type dev_type;
uint32_t ptype_tbl[SXE2_MAX_PTYPE_NUM];
struct rte_ether_addr mac_addr;
diff --git a/drivers/net/sxe2/sxe2_mac.c b/drivers/net/sxe2/sxe2_mac.c
new file mode 100644
index 0000000000..3c2f909002
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_mac.c
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_os.h>
+#include "sxe2_osal.h"
+#include "sxe2_mac.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+#include "sxe2_host_regs.h"
+
+int32_t sxe2_link_update_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret;
+
+ PMD_INIT_FUNC_TRACE();
+
+ rte_spinlock_init(&adapter->link_ctxt.link_lock);
+
+ ret = sxe2_drv_mac_link_status_get(adapter);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to get link status, ret=%d", ret);
+ goto l_end;
+ }
+
+ (void)sxe2_link_update(dev, 0);
+
+l_end:
+ return ret;
+}
+int32_t sxe2_link_update(struct rte_eth_dev *dev, __rte_unused int32_t wait_to_complete)
+{
+ struct rte_eth_link new_link;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ memset(&new_link, 0, sizeof(new_link));
+
+ switch (adapter->link_ctxt.speed) {
+ case 0:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_NONE;
+ break;
+ case 10:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_10M;
+ break;
+ case 100:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_100M;
+ break;
+ case 1000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_1G;
+ break;
+ case 10000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_10G;
+ break;
+ case 20000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_20G;
+ break;
+ case 25000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_25G;
+ break;
+ case 40000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_40G;
+ break;
+ case 50000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_50G;
+ break;
+ case 100000:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_100G;
+ break;
+ default:
+ new_link.link_speed = RTE_ETH_SPEED_NUM_NONE;
+ break;
+ }
+
+ new_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
+ new_link.link_status = adapter->link_ctxt.link_up ? RTE_ETH_LINK_UP :
+ RTE_ETH_LINK_DOWN;
+ new_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
+ RTE_ETH_LINK_SPEED_FIXED);
+
+ return rte_eth_linkstatus_set(dev, &new_link);
+}
+
+int32_t sxe2_mtu_set(struct rte_eth_dev *dev, uint16_t mtu __rte_unused)
+{
+ int32_t ret = -1;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (dev->data->dev_started != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "port %d must be stopped before configuration",
+ dev->data->port_id);
+ ret = -1;
+ goto l_end;
+ }
+
+ ret = 0;
+
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_mac.h b/drivers/net/sxe2/sxe2_mac.h
new file mode 100644
index 0000000000..f2f3edaeff
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_mac.h
@@ -0,0 +1,50 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_MAC_H__
+#define __SXE2_MAC_H__
+#include <ethdev_driver.h>
+#include "sxe2_host_regs.h"
+
+#define SXE2_NUM_MACADDR_MAX 64
+
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_8021Q SXE2_VSI_L2TAGSTXVALID_ID_OUT_VLAN1
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_8021AD SXE2_VSI_L2TAGSTXVALID_ID_STAG
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_QINQ1 SXE2_VSI_L2TAGSTXVALID_ID_OUT_VLAN2
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_VLAN SXE2_VSI_L2TAGSTXVALID_ID_VLAN
+
+#define SXE2_DPDK_OFFLOAD_OUTER_INSERT_ENABLE SXE2_VSI_L2TAGSTXVALID_L2TAG1_VALID
+
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021Q SXE2_VSI_TSR_ID_OUT_VLAN1
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021AD SXE2_VSI_TSR_ID_STAG
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_QINQ1 SXE2_VSI_TSR_ID_OUT_VLAN2
+
+#define SXE2_DPDK_OFFLOAD_INNER_INSERT_QINQ1 SXE2_VSI_L2TAGSTXVALID_ID_VLAN
+#define SXE2_DPDK_OFFLOAD_INNER_INSERT_ENABLE SXE2_VSI_L2TAGSTXVALID_L2TAG2_VALID
+
+#define SXE2_DPDK_OFFLOAD_INNER_STRIP_QINQ1 SXE2_VSI_TSR_ID_VLAN
+
+#define SXE2_DPDK_OFFLOAD_FIELD (0X0F)
+#define SXE2_DPDK_OFFLOAD_TAGID_FIELD (0X07)
+
+#define SXE2_DPDK_OFFLOAD_OUTER_STRIP_MASK (SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021Q | \
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_8021AD | \
+ SXE2_DPDK_OFFLOAD_OUTER_STRIP_QINQ1)
+#define SXE2_DPDK_OFFLOAD_STRIP_OFFSET SXE2_VSI_TSR_SHOW_TAG_S
+
+#define SXE2_DPDK_OFFLOAD_INSERT_ENABLE (RTE_BIT32(3))
+
+struct sxe2_mac_mc_list {
+ uint32_t count;
+ struct rte_ether_addr addr[SXE2_NUM_MACADDR_MAX];
+};
+
+int32_t sxe2_link_update_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_link_update(struct rte_eth_dev *dev, __rte_unused int32_t wait_to_complete);
+
+int32_t sxe2_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
+
+#endif /* __SXE2_MAC_H__ */
--
2.52.0
^ permalink raw reply related
* [PATCH v4 02/20] net/sxe2: add AVX2 vector data path for Rx and Tx
From: liujie5 @ 2026-06-02 3:16 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260602031707.578932-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Added AVX256 vectorized versions of Rx and Tx data path functions to
improve packet processing performance.
The vector path uses AVX2 SIMD instructions to process multiple
descriptors per loop, significantly reducing the per-packet overhead.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
net/sxe2: support AVX512 vectorized path for Rx and Tx
---
drivers/net/sxe2/meson.build | 9 +
drivers/net/sxe2/sxe2_txrx.c | 38 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 12 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 777 ++++++++++++++++++++++++++
4 files changed, 831 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 7bd0d8120c..c225dd7cd8 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -39,6 +39,15 @@ if arch_subdir == 'x86'
c_args: avx512_args)
objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
endif
+ sxe2_avx2_lib = static_library('sxe2_avx2_lib',
+ 'sxe2_txrx_vec_avx2.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev,
+ static_rte_bus_pci],
+ include_directories: includes,
+ c_args: [cflags, '-mavx2'])
+ objs += sxe2_avx2_lib.extract_objects('sxe2_txrx_vec_avx2.c')
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index aa1c474088..eaf95259a5 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -167,8 +167,14 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
#endif
}
- if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
- tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+ if (((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX2;
+
+ if ((0 == (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK)))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
@@ -197,6 +203,13 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
}
#endif
+ } else if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX2) {
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2_simple;
+ }
} else {
if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
@@ -231,6 +244,10 @@ static const struct {
{ sxe2_tx_pkts_vec_avx512_simple,
"Vector AVX512 Simple" },
#endif
+ { sxe2_tx_pkts_vec_avx2,
+ "Vector AVX2" },
+ { sxe2_tx_pkts_vec_avx2_simple,
+ "Vector AVX2 Simple" },
{ sxe2_tx_pkts_vec_sse,
"Vector SSE" },
{ sxe2_tx_pkts_vec_sse_simple,
@@ -330,7 +347,13 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
#endif
}
- if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX2;
+
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
#endif
@@ -354,6 +377,11 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
else
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
#endif
+ } else if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX2) {
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2;
} else {
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
}
@@ -381,6 +409,10 @@ static const struct {
{ sxe2_rx_pkts_scattered_vec_avx512_offload,
"Offload Vector AVX512 Scattered" },
#endif
+ { sxe2_rx_pkts_scattered_vec_avx2,
+ "Vector AVX2 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx2_offload,
+ "Offload Vector AVX2 Scattered" },
{ sxe2_rx_pkts_scattered_vec_sse_offload,
"Vector SSE Scattered" },
#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index af7c8d12b2..369777606f 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,19 +11,21 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
- SXE2_RX_MODE_VEC_AVX512)
+ SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
- SXE2_TX_MODE_VEC_AVX512)
+ SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -68,6 +70,12 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
new file mode 100644
index 0000000000..be9dc8f303
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
@@ -0,0 +1,777 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static inline void
+sxe2_tx_desc_fill_one_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m256i desc_group0;
+ __m256i desc_group1;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ if (((uint64_t)desc & 0x1F) != 0 && pkts_num != 0) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+
+ while (pkts_num > 3) {
+ desc3_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group1 = _mm256_set_epi64x(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]));
+
+ desc_group0 = _mm256_set_epi64x(desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc + 2), desc_group1);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc), desc_group0);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx2 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx2(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+ }
+
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num = sxe2_tx_pkts_vec_avx2_batch(txq,
+ tx_pkts + tx_done_num, tx_need_num, with_offloads);
+
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx2(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read), dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ?
+ (rxq->ring_depth - 1) : (rxq->realloc_start - 1);
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+l_end:
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx2(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m256i mbufs6_7, mbufs4_5, mbufs2_3, mbufs0_1;
+ uint32_t bit_num;
+ uint16_t done_num;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+ done_num = 0;
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx2(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m256i crc_adjust =
+ _mm256_set_epi16(0, 0, 0, -rxq->crc_len,
+ 0, -rxq->crc_len, 0,
+ 0, 0, 0, 0,
+ -rxq->crc_len, 0, -rxq->crc_len, 0, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+ const __m256i rvp_shuf_mask =
+ _mm256_set_epi8(7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 8, 0, 10, 2,
+ 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ if (split_rxe_flags) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs6_7 = _mm256_shuffle_epi8(descs6_7, rvp_shuf_mask);
+ mbufs4_5 = _mm256_shuffle_epi8(descs4_5, rvp_shuf_mask);
+
+ mbufs6_7 = _mm256_add_epi16(mbufs6_7, crc_adjust);
+ mbufs4_5 = _mm256_add_epi16(mbufs4_5, crc_adjust);
+
+ const __m256i ptype_mask = _mm256_set1_epi32(SXE2_RX_DESC_PTYPE_MASK);
+
+ const __m256i staterrs4_7 = _mm256_unpackhi_epi32(descs6_7, descs4_5);
+
+ __m256i ptypes4_7 = _mm256_and_si256(staterrs4_7, ptype_mask);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes4_7, 9);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes4_7, 1);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_7, 11);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_7, 3);
+
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype7], 4);
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype6], 0);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype5], 4);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype4], 0);
+
+ mbufs2_3 = _mm256_shuffle_epi8(descs2_3, rvp_shuf_mask);
+ mbufs0_1 = _mm256_shuffle_epi8(descs0_1, rvp_shuf_mask);
+
+ mbufs2_3 = _mm256_add_epi16(mbufs2_3, crc_adjust);
+ mbufs0_1 = _mm256_add_epi16(mbufs0_1, crc_adjust);
+
+ const __m256i staterrs0_3 = _mm256_unpackhi_epi32(descs2_3, descs0_1);
+
+ __m256i ptypes0_3 = _mm256_and_si256(staterrs0_3, ptype_mask);
+
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes0_3, 9);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes0_3, 1);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_3, 11);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_3, 3);
+
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype3], 4);
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype2], 0);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype1], 4);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype0], 0);
+
+ __m256i staterrs0_7 = _mm256_unpacklo_epi64(staterrs4_7, staterrs0_3);
+
+ __m256i stu_len0_7 = _mm256_unpackhi_epi64(staterrs4_7, staterrs0_3);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0x00001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+ const __m256i vlan_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 8), 4);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 4), 4);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 4);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(mbuf_flags, 4), 4);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] =
+ _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] =
+ _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags != NULL) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 = _mm256_and_si256(staterrs0_7,
+ umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+
+ __m128i umbcast_bits_128 = _mm_packs_epi32
+ (_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256
+ (umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags != NULL) {
+ const __m256i eop_rxe_mask = _mm256_set1_epi32
+ (SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m128i eop_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits = _mm_or_si128
+ (not_eop_bits,
+ _mm_srli_epi16
+ (_mm_and_si128(eop_rxe_bits, rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((1 == (rxq->processing_idx & 1)) && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ split_rxe_flags64[0] == 0 && split_rxe_flags64[1] == 0 &&
+ split_rxe_flags64[2] == 0 && split_rxe_flags64[3] == 0) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num =
+ sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM,
+ do_offload);
+ done_num += once_num;
+ nb_pkts -= once_num;
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num, nb_pkts, do_offload);
+l_end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
--
2.52.0
^ permalink raw reply related
* [PATCH v4 01/20] net/sxe2: support AVX512 vectorized path for Rx and Tx
From: liujie5 @ 2026-06-02 3:16 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260602031707.578932-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add AVX512 vector data path for Rx and Tx burst functions.
The decision to use AVX512 is based on:
1. CPU hardware flags (AVX512F, AVX512BW).
2. Compiler support (CC_AVX512_SUPPORT).
3. Max SIMD bitwidth configuration.
Performance shows approximately X% improvement in small packet
forwarding scenarios.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 24 +
drivers/net/sxe2/sxe2_drv_cmd.h | 80 +--
drivers/net/sxe2/sxe2_ethdev.c | 2 +-
drivers/net/sxe2/sxe2_txrx.c | 92 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 18 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 ++++++++++++++++++++++++
7 files changed, 1099 insertions(+), 60 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 6b2eb75b0e..7bd0d8120c 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -15,6 +15,30 @@ includes += include_directories('../../common/sxe2')
if arch_subdir == 'x86'
sources += files('sxe2_txrx_vec_sse.c')
+
+ sxe2_avx512_cpu_support =(
+ cc.get_define('__AVX512F__', args: machine_args) != '' and
+ cc.get_define('__AVX512BW__', args: machine_args) != '')
+
+ sxe2_avx512_cc_support = (
+ not machine_args.contains('-mno-avx512f') and
+ cc.has_argument('-mavx512f') and
+ cc.has_argument('-mavx512bw'))
+
+ if sxe2_avx512_cpu_support == true or sxe2_avx512_cc_support == true
+ cflags += ['-DCC_AVX512_SUPPORT']
+ avx512_args = [cflags, '-mavx512f', '-mavx512bw']
+ if cc.has_argument('-march=skylake-avx512')
+ avx512_args += '-march=skylake-avx512'
+ endif
+ sxe2_avx512_lib = static_library('sxe2_avx512_lib', 'sxe2_txrx_vec_avx512.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev, static_rte_bus_pci],
+ include_directories: includes,
+ c_args: avx512_args)
+ objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
+ endif
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index bba6476c2e..ccc9c20ef4 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -67,20 +67,20 @@ enum sxe2_dev_type {
SXE2_DEV_T_MAX,
};
-struct sxe2_drv_queue_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_caps {
uint16_t queues_cnt;
uint16_t base_idx_in_pf;
-};
+} __rte_packed_end;
-struct sxe2_drv_msix_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_msix_caps {
uint16_t msix_vectors_cnt;
uint16_t base_idx_in_func;
-};
+} __rte_packed_end;
-struct sxe2_drv_rss_hash_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rss_hash_caps {
uint16_t hash_key_size;
uint16_t lut_key_size;
-};
+} __rte_packed_end;
enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_BOTH = 0,
@@ -89,18 +89,18 @@ enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_MAX,
};
-struct sxe2_drv_vsi_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_caps {
uint16_t func_id;
uint16_t dpdk_vsi_id;
uint16_t kernel_vsi_id;
uint16_t vsi_type;
-};
+} __rte_packed_end;
-struct sxe2_drv_representor_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_representor_caps {
uint16_t cnt_repr_vf;
uint8_t rsv[2];
struct sxe2_drv_vsi_caps repr_vf_id[256];
-};
+} __rte_packed_end;
enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_NOTSET = 0,
@@ -111,25 +111,25 @@ enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_UNKNOWN,
};
-struct sxe2_switchdev_mode_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_mode_info {
uint8_t pf_id;
uint8_t is_switchdev;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_switchdev_cpvsi_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_cpvsi_info {
uint16_t cp_vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_txsch_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_txsch_caps {
uint8_t layer_cap;
uint8_t tm_mid_node_num;
uint8_t prio_num;
uint8_t rev;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_caps_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_caps_resp {
struct sxe2_drv_queue_caps queue_caps;
struct sxe2_drv_msix_caps msix_caps;
struct sxe2_drv_rss_hash_caps rss_hash_caps;
@@ -141,24 +141,24 @@ struct sxe2_drv_dev_caps_resp {
uint8_t dev_type;
uint8_t rev;
uint32_t cap_flags;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_info_resp {
uint64_t dsn;
uint16_t vsi_id;
uint8_t rsv[2];
uint8_t mac_addr[SXE2_ETH_ALEN];
uint8_t rsv2[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_fw_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_fw_info_resp {
uint8_t main_version_id;
uint8_t sub_version_id;
uint8_t fix_version_id;
uint8_t build_id;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_ctxt {
uint64_t dma_addr;
uint32_t max_lro_size;
uint32_t split_type_mask;
@@ -170,62 +170,62 @@ struct sxe2_drv_rxq_ctxt {
uint8_t keep_crc_en;
uint8_t split_en;
uint8_t desc_size;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
uint16_t max_frame_size;
uint8_t rsv[2];
struct sxe2_drv_rxq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_ctxt {
uint64_t dma_addr;
uint32_t sched_mode;
uint16_t queue_id;
uint16_t depth;
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
struct sxe2_drv_txq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_q_switch_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_q_switch_req {
uint16_t q_idx;
uint16_t vsi_id;
uint8_t is_enable;
uint8_t sched_mode;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_create_req_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_create_req_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_free_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_free_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 8d66e5d8c5..e0f7002138 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -891,7 +891,7 @@ static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
static int32_t sxe2_parse_eth_devargs(struct rte_device *dev,
struct rte_eth_devargs *eth_da)
{
- int ret = 0;
+ int32_t ret = 0;
if (dev->devargs == NULL)
return 0;
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 8d17535301..aa1c474088 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -157,6 +157,19 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
tx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
+#endif
+ }
+ if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
if (ret != 0)
@@ -172,14 +185,25 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
tx_mode_flags = adapter->q_ctxt.tx_mode_flags;
}
-#ifdef RTE_ARCH_X86
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
- if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
- dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ dev->tx_pkt_prepare = NULL;
+#ifdef RTE_ARCH_X86
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
+ }
+#endif
} else {
- dev->tx_pkt_prepare = NULL;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ }
}
} else {
#endif
@@ -201,8 +225,16 @@ static const struct {
} sxe2_tx_burst_infos[] = {
{ sxe2_tx_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
- { sxe2_tx_pkts_vec_sse, "Vector SSE" },
- { sxe2_tx_pkts_vec_sse_simple, "Vector SSE Simple" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_tx_pkts_vec_avx512,
+ "Vector AVX512" },
+ { sxe2_tx_pkts_vec_avx512_simple,
+ "Vector AVX512 Simple" },
+#endif
+ { sxe2_tx_pkts_vec_sse,
+ "Vector SSE" },
+ { sxe2_tx_pkts_vec_sse_simple,
+ "Vector SSE Simple" },
#endif
};
@@ -288,6 +320,20 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
rx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
+#endif
+ }
+ if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
+ rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
if (ret != 0)
@@ -301,7 +347,16 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
#ifdef RTE_ARCH_X86
if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
- dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
+#endif
+ } else {
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ }
return;
}
#endif
@@ -315,19 +370,30 @@ static const struct {
eth_rx_burst_t rx_burst;
const char *info;
} sxe2_rx_burst_infos[] = {
- { sxe2_rx_pkts_scattered, "Scalar Scattered" },
- { sxe2_rx_pkts_scattered_split, "Scalar Scattered split" },
+ { sxe2_rx_pkts_scattered,
+ "Scalar Scattered" },
+ { sxe2_rx_pkts_scattered_split,
+ "Scalar Scattered split" },
#ifdef RTE_ARCH_X86
- { sxe2_rx_pkts_scattered_vec_sse_offload, "Vector SSE Scattered" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_rx_pkts_scattered_vec_avx512,
+ "Vector AVX512 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx512_offload,
+ "Offload Vector AVX512 Scattered" },
+#endif
+ { sxe2_rx_pkts_scattered_vec_sse_offload,
+ "Vector SSE Scattered" },
#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
- __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode)
+ __rte_unused uint16_t queue_id,
+ struct rte_eth_burst_mode *mode)
{
eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
int32_t ret = -EINVAL;
uint32_t i, size;
+
size = RTE_DIM(sxe2_rx_burst_infos);
for (i = 0; i < size; ++i) {
if (pkt_burst == sxe2_rx_burst_infos[i].rx_burst) {
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index 8df4954d86..cf004f5eb2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -165,16 +165,54 @@ static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
return;
}
i = txq->next_dd - (txq->rs_thresh - 1);
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
+#ifdef CC_AVX512_SUPPORT
+ struct rte_eth_dev *dev;
+ struct sxe2_tx_buffer_vec *buffer_vec;
+
+ dev = &rte_eth_devices[txq->port_id];
+
+ if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
+ dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
+ buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for ( ; i < txq->next_use; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ } else {
+#endif
+ buffer = txq->buffer_ring;
+ buffer = txq->buffer_ring;
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer[i].mbuf);
+ buffer[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
buffer[i].mbuf = NULL;
}
}
- i = 0;
+#ifdef CC_AVX512_SUPPORT
}
+#endif
+
for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 04ff4d96a5..af7c8d12b2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,15 +11,19 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
- SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE)
+ SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
+ SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
- SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE)
+ SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
+ SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -54,6 +58,16 @@ uint16_t sxe2_tx_pkts_vec_sse(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_
uint16_t sxe2_tx_pkts_vec_sse_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_sse_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_ctx_offload(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
new file mode 100644
index 0000000000..2aec8037dd
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -0,0 +1,897 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_TEST
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_queue *txq)
+{
+ struct sxe2_tx_buffer_vec *buffer;
+ struct rte_mbuf *mbuf;
+ struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
+ struct rte_mempool *mp;
+ struct rte_mempool_cache *cache;
+ void **cache_objs;
+ uint32_t copied;
+ uint32_t i;
+ int32_t ret;
+ uint16_t rs_thresh;
+ uint16_t free_num;
+
+ if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
+ (txq->desc_ring[txq->next_dd].wb.dd &
+ rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
+ ret = 0;
+ goto l_end;
+ }
+
+ rs_thresh = txq->rs_thresh;
+
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += txq->next_dd - (rs_thresh - 1);
+
+ if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
+ (rs_thresh & 31) == 0) {
+ mp = buffer[0].mbuf->pool;
+ cache = rte_mempool_default_cache(mp, rte_lcore_id());
+
+ if (cache == NULL || cache->len)
+ goto normal;
+
+ if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer, rs_thresh);
+ goto done;
+ }
+ cache_objs = &cache->objs[cache->len];
+
+ copied = 0;
+ while (copied < rs_thresh) {
+ const __m512i objs0 = _mm512_loadu_si512(&buffer[copied]);
+ const __m512i objs1 = _mm512_loadu_si512(&buffer[copied + 8]);
+ const __m512i objs2 = _mm512_loadu_si512(&buffer[copied + 16]);
+ const __m512i objs3 = _mm512_loadu_si512(&buffer[copied + 24]);
+
+ _mm512_storeu_si512(&cache_objs[copied], objs0);
+ _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
+ _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
+ _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
+ copied += 32;
+ }
+ cache->len += rs_thresh;
+
+ if (cache->len >= cache->flushthresh) {
+ (void)rte_mempool_ops_enqueue_bulk(mp,
+ &cache->objs[cache->size], cache->len - cache->size);
+ cache->len = cache->size;
+ }
+ goto done;
+ }
+
+normal:
+ mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
+
+ if (likely(mbuf)) {
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+
+ if (likely(mbuf)) {
+ if (likely(mbuf->pool == mbuf_free_arr[0]->pool)) {
+ mbuf_free_arr[free_num] = mbuf;
+ free_num++;
+ } else {
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+ }
+ }
+ }
+
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+ } else {
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+ if (mbuf != NULL)
+ rte_mempool_put(mbuf->pool, mbuf);
+ }
+ }
+
+done:
+ txq->desc_free_num += txq->rs_thresh;
+ txq->next_dd += txq->rs_thresh;
+ if (txq->next_dd >= txq->ring_depth)
+ txq->next_dd = txq->rs_thresh - 1;
+ ret = rs_thresh;
+
+l_end:
+ return ret;
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline
+void sxe2_tx_desc_fill_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m512i desc_group;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ while (pkts_num > 3) {
+ desc3_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group =
+ _mm512_set_epi64(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]),
+ desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, desc), desc_group);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx512(desc, *pkts, desc_cmd, with_offloads);
+
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline void
+sxe2_tx_pkts_mbuf_fill_avx512(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ uint16_t i;
+
+ for (i = 0; i < nb_pkts; ++i)
+ buffer[i].mbuf = tx_pkts[i];
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer_vec *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec_avx512(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx512 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += next_use;
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx512(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = txq->desc_ring;
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ }
+
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num =
+ sxe2_tx_pkts_vec_avx512_batch(txq, tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx512(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read),
+ dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ? (rxq->ring_depth - 1) :
+ (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m512i mbufs4_7;
+ __m512i mbufs0_3;
+ __m256i mbufs6_7;
+ __m256i mbufs4_5;
+ __m256i mbufs2_3;
+ __m256i mbufs0_1;
+ uint32_t bit_num = 0;
+ uint16_t done_num = 0;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx512(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) & SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m512i crc_adjust =
+ _mm512_set4_epi32(0, -rxq->crc_len, -rxq->crc_len, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+
+ const __m512i rvp_shuf_mask =
+ _mm512_set4_epi32((7 << 24) | (6 << 16) | (5 << 8) | 4,
+ (3 << 24) | (2 << 16) | (13 << 8) | 12,
+ (0xFFU << 24) | (0xFF << 16) | (13 << 8) | 12,
+ 0xFFFFFFFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 8, 0, 10, 2, 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ const __m512i descs4_7 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs4_5), descs6_7, 1);
+ const __m512i descs0_3 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs0_1), descs2_3, 1);
+
+ if (split_rxe_flags != NULL) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs4_7 = _mm512_shuffle_epi8(descs4_7, rvp_shuf_mask);
+ mbufs0_3 = _mm512_shuffle_epi8(descs0_3, rvp_shuf_mask);
+
+ mbufs4_7 = _mm512_add_epi32(mbufs4_7, crc_adjust);
+ mbufs0_3 = _mm512_add_epi32(mbufs0_3, crc_adjust);
+
+ const __m512i ptype_mask = _mm512_set1_epi64(SXE2_RX_FLEX_DESC_PTYPE_M <<
+ SXE2_RX_FLEX_DESC_PTYPE_S);
+
+ __m512i ptypes4_7 = _mm512_and_si512(descs4_7, ptype_mask);
+ __m512i ptypes0_3 = _mm512_and_si512(descs0_3, ptype_mask);
+
+ const __m256i ptypes6_7 = _mm512_extracti64x4_epi64(ptypes4_7, 1);
+ const __m256i ptypes4_5 = _mm512_extracti64x4_epi64(ptypes4_7, 0);
+ const __m256i ptypes2_3 = _mm512_extracti64x4_epi64(ptypes0_3, 1);
+ const __m256i ptypes0_1 = _mm512_extracti64x4_epi64(ptypes0_3, 0);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 13);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 5);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 13);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 5);
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 13);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 5);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 13);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 5);
+
+ const __m512i ptype_mask4_7 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype7],
+ 0, 0, 0, ptype_tbl[ptype6],
+ 0, 0, 0, ptype_tbl[ptype5],
+ 0, 0, 0, ptype_tbl[ptype4]);
+ const __m512i ptype_mask0_3 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype3],
+ 0, 0, 0, ptype_tbl[ptype2],
+ 0, 0, 0, ptype_tbl[ptype1],
+ 0, 0, 0, ptype_tbl[ptype0]);
+
+ mbufs4_7 = _mm512_or_si512(mbufs4_7, ptype_mask4_7);
+ mbufs0_3 = _mm512_or_si512(mbufs0_3, ptype_mask0_3);
+
+ mbufs6_7 = _mm512_extracti64x4_epi64(mbufs4_7, 1);
+ mbufs4_5 = _mm512_extracti64x4_epi64(mbufs4_7, 0);
+ mbufs2_3 = _mm512_extracti64x4_epi64(mbufs0_3, 1);
+ mbufs0_1 = _mm512_extracti64x4_epi64(mbufs0_3, 0);
+
+ const __m512i staterr_per_mask =
+ _mm512_set_epi32(0x17, 0x1F, 0x07, 0x0F,
+ 0x13, 0x1B, 0x03, 0x0B,
+ 0x16, 0x1E, 0x06, 0x0E,
+ 0x12, 0x1A, 0x02, 0x0A);
+ __m512i qw1_0_7 = _mm512_permutex2var_epi32(descs4_7,
+ staterr_per_mask,
+ descs0_3);
+
+ __m256i staterrs0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 0);
+
+ __m256i stu_len0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 1);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0xC0001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0,
+ 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+
+ const __m256i vlan_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 8), 0x04);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 4), 0x04);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(mbuf_flags, 4), 0x04);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] = _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 =
+ _mm256_and_si256(staterrs0_7, umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+ __m128i umbcast_bits_128 =
+ _mm_packs_epi32(_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256(umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags) {
+ const __m256i eop_rxe_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+ const __m128i eop_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits =
+ _mm_or_si128(not_eop_bits,
+ _mm_srli_epi16(_mm_and_si128(eop_rxe_bits,
+ rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((rxq->processing_idx & 1) == 1 && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_rxe_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ umbcast_flags, do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ NULL, do_offload);
+ }
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_rxe_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ !split_rxe_flags64[0] && !split_rxe_flags64[1] &&
+ !split_rxe_flags64[2] && !split_rxe_flags64[3]) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0)
+ rx_pkt_done_num++;
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num = 0;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue, rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM, offload);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue,
+ rx_pkts + done_num, nb_pkts, offload);
+
+end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
+
+#endif
--
2.52.0
^ permalink raw reply related
* [PATCH v4 00/20] net/sxe2: added Linkdata sxe2 ethernet driver
From: liujie5 @ 2026-06-02 3:16 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260601084950.269887-21-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch set implements core functionality for the SXE PMD,
including basic driver framework, data path setup, and advanced
offload features (VLAN, RSS, DCB, PTP etc.).
V4:
- Addressed AI comments
Jie Liu (20):
net/sxe2: support AVX512 vectorized path for Rx and Tx
net/sxe2: add AVX2 vector data path for Rx and Tx
drivers: add supported packet types get callback
net/sxe2: support L2 filtering and MAC config
drivers: support RSS feature
net/sxe2: support TM hierarchy and shaping
net/sxe2: support IPsec inline protocol offload
net/sxe2: support statistics and multi-process
drivers: interrupt handling
net/sxe2: add NEON vec Rx/Tx burst functions
drivers: add support for VF representors
net/sxe2: add support for custom UDP tunnel ports
net/sxe2: support firmware version reading
net/sxe2: implement get monitor address
common/sxe2: add shared SFP module definitions
net/sxe2: support SFP module info and EEPROM access
net/sxe2: implement private dump info
net/sxe2: add mbuf validation in Tx debug mode
drivers: add testpmd commands for private features
net/sxe2: update sxe2 feature matrix docs
doc/guides/nics/features/sxe2.ini | 65 +
drivers/common/sxe2/sxe2_common.c | 156 ++
drivers/common/sxe2/sxe2_common.h | 4 +
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++
drivers/common/sxe2/sxe2_ioctl_chnl.c | 178 +-
drivers/common/sxe2/sxe2_ioctl_chnl_func.h | 18 +
drivers/common/sxe2/sxe2_msg.h | 118 ++
drivers/common/sxe2/sxe2_ptype.h | 1793 ++++++++++++++++++
drivers/net/sxe2/meson.build | 56 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 1587 +++++++++++++++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 139 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 521 +++++-
drivers/net/sxe2/sxe2_dump.c | 304 +++
drivers/net/sxe2/sxe2_dump.h | 12 +
drivers/net/sxe2/sxe2_ethdev.c | 1531 +++++++++++++++-
drivers/net/sxe2/sxe2_ethdev.h | 112 +-
drivers/net/sxe2/sxe2_ethdev_repr.c | 610 ++++++
drivers/net/sxe2/sxe2_ethdev_repr.h | 32 +
drivers/net/sxe2/sxe2_filter.c | 895 +++++++++
drivers/net/sxe2/sxe2_filter.h | 100 +
drivers/net/sxe2/sxe2_flow.c | 1391 ++++++++++++++
drivers/net/sxe2/sxe2_flow.h | 30 +
drivers/net/sxe2/sxe2_flow_define.h | 144 ++
drivers/net/sxe2/sxe2_flow_parse_action.c | 1182 ++++++++++++
drivers/net/sxe2/sxe2_flow_parse_action.h | 23 +
drivers/net/sxe2/sxe2_flow_parse_engine.c | 106 ++
drivers/net/sxe2/sxe2_flow_parse_engine.h | 13 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 1935 ++++++++++++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 46 +
drivers/net/sxe2/sxe2_ipsec.c | 1565 ++++++++++++++++
drivers/net/sxe2/sxe2_ipsec.h | 254 +++
drivers/net/sxe2/sxe2_irq.c | 1025 +++++++++++
drivers/net/sxe2/sxe2_irq.h | 25 +
drivers/net/sxe2/sxe2_mac.c | 535 ++++++
drivers/net/sxe2/sxe2_mac.h | 84 +
drivers/net/sxe2/sxe2_mp.c | 414 +++++
drivers/net/sxe2/sxe2_mp.h | 67 +
drivers/net/sxe2/sxe2_queue.c | 17 +-
drivers/net/sxe2/sxe2_rss.c | 584 ++++++
drivers/net/sxe2/sxe2_rss.h | 81 +
drivers/net/sxe2/sxe2_rx.c | 38 +
drivers/net/sxe2/sxe2_rx.h | 2 +
drivers/net/sxe2/sxe2_security.c | 335 ++++
drivers/net/sxe2/sxe2_security.h | 77 +
drivers/net/sxe2/sxe2_stats.c | 591 ++++++
drivers/net/sxe2/sxe2_stats.h | 39 +
drivers/net/sxe2/sxe2_switchdev.c | 332 ++++
drivers/net/sxe2/sxe2_switchdev.h | 33 +
drivers/net/sxe2/sxe2_testpmd.c | 733 ++++++++
drivers/net/sxe2/sxe2_testpmd_lib.c | 969 ++++++++++
drivers/net/sxe2/sxe2_testpmd_lib.h | 142 ++
drivers/net/sxe2/sxe2_tm.c | 1169 ++++++++++++
drivers/net/sxe2/sxe2_tm.h | 78 +
drivers/net/sxe2/sxe2_tx.c | 7 +
drivers/net/sxe2/sxe2_txrx.c | 176 +-
drivers/net/sxe2/sxe2_txrx.h | 4 +
drivers/net/sxe2/sxe2_txrx_check_mbuf.c | 595 ++++++
drivers/net/sxe2/sxe2_txrx_check_mbuf.h | 38 +
drivers/net/sxe2/sxe2_txrx_poll.c | 243 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 38 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 777 ++++++++
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 897 +++++++++
drivers/net/sxe2/sxe2_txrx_vec_common.h | 1 +
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 724 ++++++++
drivers/net/sxe2/sxe2_vsi.c | 146 ++
drivers/net/sxe2/sxe2_vsi.h | 12 +-
drivers/net/sxe2/sxe2vf_regs.h | 82 +
68 files changed, 26585 insertions(+), 124 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/common/sxe2/sxe2_msg.h
create mode 100644 drivers/common/sxe2/sxe2_ptype.h
create mode 100644 drivers/net/sxe2/sxe2_dump.c
create mode 100644 drivers/net/sxe2/sxe2_dump.h
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.c
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.h
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
create mode 100644 drivers/net/sxe2/sxe2_flow.c
create mode 100644 drivers/net/sxe2/sxe2_flow.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.h
create mode 100644 drivers/net/sxe2/sxe2_ipsec.c
create mode 100644 drivers/net/sxe2/sxe2_ipsec.h
create mode 100644 drivers/net/sxe2/sxe2_irq.c
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
create mode 100644 drivers/net/sxe2/sxe2_mp.c
create mode 100644 drivers/net/sxe2/sxe2_mp.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
create mode 100644 drivers/net/sxe2/sxe2_security.c
create mode 100644 drivers/net/sxe2/sxe2_security.h
create mode 100644 drivers/net/sxe2/sxe2_stats.c
create mode 100644 drivers/net/sxe2/sxe2_stats.h
create mode 100644 drivers/net/sxe2/sxe2_switchdev.c
create mode 100644 drivers/net/sxe2/sxe2_switchdev.h
create mode 100644 drivers/net/sxe2/sxe2_testpmd.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.c
create mode 100644 drivers/net/sxe2/sxe2_testpmd_lib.h
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
create mode 100644 drivers/net/sxe2/sxe2vf_regs.h
--
2.52.0
^ permalink raw reply
* Re: [PATCH] ring: avoid extra store at move head
From: Stephen Hemminger @ 2026-06-01 22:23 UTC (permalink / raw)
To: Konstantin Ananyev; +Cc: dev
In-Reply-To: <20260601181509.71007-1-konstantin.ananyev@huawei.com>
On Mon, 1 Jun 2026 19:15:09 +0100
Konstantin Ananyev <konstantin.ananyev@huawei.com> wrote:
> C11 __rte_ring_headtail_move_head_mt() uses output
> parameter: 'uint32_t *old_head' directly within CAS operation.
> In x86_64 that cause gcc to generate extra instructions to
> store return value of CAS (eax) within 'old_head' memory location,
> even when CAS was not successful and another attempt should be
> performed. In some cases, even extra branch can be observed.
> To be more specific the code like that is generated:
> // start of 'do { } while();' loop
> .L2
> ...
> lock cmpxchgl %r8d, (%rdi)
> jne .L17 //
> .L1: // <---- successful completion of CAS, finish
> movl %edx, %eax
> ret
> .L17: // <---- unsuccessful completion of CAS, repeat
> movl %eax, (%r9)
> jmp .L2
>
> In constrast, x86 specific version that uses
> __sync_bool_compare_and_swap() doesn't exibit such problem,
> as __sync_bool_compare_and_swap() doesn't update the 'old_head'
> with new value, and we have to re-read it explicitly on each iteration.
>
> Overcome that problem by using local variable 'head' inside the loop,
> and updaing '*old_head' value only at exit.
> With such change gcc manages to avoid extra store(/branch).
>
> Depends-on: series-38225 ("deprecate rte_atomicNN family")
>
> Signed-off-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
> ---
I used the standard ring perf tests and ran 10 times via:
! /bin/bash
if [ -z "$1" ]; then
echo "Usage $0 version"
exit 1
fi
VERSION=$1
for i in $(seq 1 10); do
sudo DPDK_TEST=ring_perf_autotest \
./build/app/dpdk-test -l 2-5 -n 4 --no-pci --file-prefix=run$i \
> ~/DPDK/ring_perf_results/${VERSION}_run${i}.log 2>&1
echo "${VERSION} run $i done"
done
Then had Claude compare results:
Key metric (two physical cores legacy MP/MC bulk n=128):
main: 5.380 cycles/elem
sync-bool: 5.377 cycles/elem (-0.07%)
avoid-store: 5.892 cycles/elem (+9.52%) ← regresses
Looking at the dissassembly of ring_enqueue_bulk:
The inner loop of main and sync-bool versions is:
mov 0x80(%rdi),%r11d ; load d->head via displacement
mov 0x104(%rdi),%ebx ; load s->tail
add %ecx,%ebx
sub %r11d,%ebx
cmp %ebx,%r12d
jae [exit]
lea (%r8,%r11,1),%r13d ; new_head = old_head + n
mov %r11d,%eax ; expected → eax
lock cmpxchg %r13d,0x80(%rdi) ; ← displacement addressing
jne [retry] ; ← direct jne, eax preserved
Using atomic_compare_exchange and your patch:
mov 0x38(%rdi),%r10d
mov 0x80(%rdi),%eax ; load d->head directly into %eax
lea 0x80(%rdi),%rcx ; ← MATERIALIZE &d->head into %rcx
lea -0x1(%r8),%r12d
mov 0x104(%rdi),%r11d
add %r10d,%r11d
sub %eax,%r11d
cmp %r11d,%r12d
jae [exit]
lea (%r8,%rax,1),%r13d ; new_head
lock cmpxchg %r13d,(%rcx) ; ← INDIRECT addressing via %rcx
mov %eax,%ebx ; ← EXTRA: save post-CAS %eax to %ebx
jne [retry]
Bottom line: good idea but still fighting with Gcc optimizer here.
^ permalink raw reply
* Re: [PATCH v4 03/27] ring: unify memory model on C11, remove atomic32
From: Stephen Hemminger @ 2026-06-01 22:07 UTC (permalink / raw)
To: dev; +Cc: Konstantin Ananyev, Wathsala Vithanage
In-Reply-To: <20260526232542.620966-4-stephen@networkplumber.org>
On Tue, 26 May 2026 16:23:53 -0700
Stephen Hemminger <stephen@networkplumber.org> wrote:
> Remove the RTE_USE_C11_MEM_MODEL build switch; C11 atomics are now
> the default for all platforms. Unifies __rte_ring_update_tail into
> the C11 form (atomic_store_release replaces the older rte_smp_wmb +
> plain store on the generic path) and renames rte_ring_generic_pvt.h
> to rte_ring_x86_pvt.h to reflect its new scope.
>
> Also splits the head-move helper into separate ST and MT variants,
> removing the runtime is_st branch from the MT retry loop.
> This gets small boost and scopes the following exception
> more tightly.
>
> Exception: on x86 with GCC, atomic_compare_exchange on the head CAS
> regresses MP/MC contended throughput by ~20% existing hand-written
> cmpxchg. As a workaround, GCC-on-x86 builds use the older
> __sync_bool_compare_and_swap builtin, which generates equivalent
> code to the original asm. Can be reverted if/when GCC gets
> fixed; similar issue was observed in Linux kernel.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
This is a big enough change, going to split it up as its own series.
^ permalink raw reply
* Re: [PATCH v4 03/27] ring: unify memory model on C11, remove atomic32
From: Stephen Hemminger @ 2026-06-01 21:18 UTC (permalink / raw)
To: Konstantin Ananyev; +Cc: dev@dpdk.org, Wathsala Vithanage
In-Reply-To: <b8523a4e0ee34e8b9194935574e86f7c@huawei.com>
On Mon, 1 Jun 2026 18:18:18 +0000
Konstantin Ananyev <konstantin.ananyev@huawei.com> wrote:
> > +
> > + /* check that we have enough room in ring */
> > + if (unlikely(n > *entries))
> > + n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
> > +
> > + if (n > 0) {
> > + *new_head = *old_head + n;
> > + d->head = *new_head;
>
> There is a bit of inconsistency with the 'load' operation above:
> If we use atomic_load(&d->head. ...) then it would be better to use
> atomic_store(&d->head,..., order_relaxed) here.
>
Will switch to atomic_store, with relaxed.
It generates same code on x86 and ARM as simple store.
And makes C11 checkers happy.
^ permalink raw reply
* Re: [PATCH v4 03/27] ring: unify memory model on C11, remove atomic32
From: Stephen Hemminger @ 2026-06-01 21:05 UTC (permalink / raw)
To: Konstantin Ananyev; +Cc: dev@dpdk.org, Wathsala Vithanage
In-Reply-To: <b8523a4e0ee34e8b9194935574e86f7c@huawei.com>
On Mon, 1 Jun 2026 18:18:18 +0000
Konstantin Ananyev <konstantin.ananyev@huawei.com> wrote:
> > + /* check that we have enough room in ring */
> > + if (unlikely(n > *entries))
> > + n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
> > +
> > + if (n > 0) {
> > + *new_head = *old_head + n;
> > + d->head = *new_head;
>
> There is a bit of inconsistency with the 'load' operation above:
> If we use atomic_load(&d->head. ...) then it would be better to use
> atomic_store(&d->head,..., order_relaxed) here.
This is single thread case, so not sure atomic_store is needed.
The old code didn't do it.
There is a little confusion in the ST path.
It is used in two different context SP and SC.
For move_head, it is SC that matters and consumer moves the head;
BUT if using multiple produces MP the tail is also examined to
determine space.
^ permalink raw reply
* Re: [PATCH v5 2/2] node: lookup with RISC-V vector extension
From: Thomas Monjalon @ 2026-06-01 21:01 UTC (permalink / raw)
To: Zijian
Cc: dev, Sun Yuechi, Nithin Dabilpuram, Pavan Nikhilesh,
Stanisław Kardach
In-Reply-To: <20260505062142.575105-3-sunyuechi@iscas.ac.cn>
Hello,
05/05/2026 08:21, Sun Yuechi:
> Implement ip4_lookup_node_process_vec function for RISC-V architecture
> using RISC-V Vector Extension instruction set.
>
> Signed-off-by: Sun Yuechi <sunyuechi@iscas.ac.cn>
> Signed-off-by: Zijian <zijian.oerv@isrc.iscas.ac.cn>
Signed-off confirms we can apply the DCO rule:
https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin
For this, we require the (quite) full name of the contributor.
Here I feel Zijian is not the complete name.
Please could you help by telling how we should identify you
with english alphabet?
Thank you
^ permalink raw reply
* Re: [PATCH v2] eal/riscv: add Z sub-extension detection via hwprobe syscall
From: Thomas Monjalon @ 2026-06-01 20:24 UTC (permalink / raw)
To: sunyuechi, dangshiwei; +Cc: dev, stanislaw.kardach
In-Reply-To: <20260323041714.4386-1-1138222970gg@gmail.com>
Hello,
We are waiting for a review of this patch.
The maintainer is Sun Yuechi.
23/03/2026 05:17, dangshiwei:
> The existing RISC-V rte_cpuflags implementation only detects
> single-letter ISA extensions (A, C, D, F, M, V, etc.) via
> AT_HWCAP. Z sub-extensions such as Zba, Zbb, Zbc, Zawrs and
> cryptography extensions (Zksed, Zksh, Zknd, Zkne) are not
> visible through AT_HWCAP and therefore cannot be used for
> runtime dispatch.
>
> Linux 6.4 introduced the riscv_hwprobe syscall (NR 258) which
> exposes Z sub-extensions via the RISCV_HWPROBE_KEY_IMA_EXT_0
> key as a 64-bit bitmask. This patch adds support for querying
> Z sub-extensions through this interface.
>
> Changes:
> - Add REG_HWPROBE_EXT0 source to cpu_register_t
> - Fix hwcap_registers_t type from uint32_t to uint64_t to
> avoid truncation of AT_HWCAP on 64-bit platforms
> - Call riscv_hwprobe at EAL init via RTE_INIT and cache the
> result; falls back silently on kernels older than 6.4
> - Add 56 new enum values for Z sub-extensions in rte_cpuflags.h
> - Extend cpuflags_autotest to cover all new flags
>
> Signed-off-by: dangshiwei <1138222970gg@gmail.com>
Please could you separate you first and family names?
Thank you
^ permalink raw reply
* Re: [PATCH v2] bus/uacce: support no-iommu mode
From: Thomas Monjalon @ 2026-06-01 20:21 UTC (permalink / raw)
To: Chengwen Feng; +Cc: dev, liuyonglong, qianweili
In-Reply-To: <20260518105857.23083-1-fengchengwen@huawei.com>
18/05/2026 12:58, Chengwen Feng:
> The uacce bus originally only supports devices with SVA capability.
> This patch extends the uacce bus to support no-iommu mode for devices
> without or not enabling IOMMU/SVA.
>
> For no-iommu mode UACCE devices:
> - The device api name is suffixed with _noiommu
> - The device flags bit1 was set (UACCE_DEV_FLAG_NOIOMMU)
>
> To support such devices, DPDK device drivers can mark
> RTE_UACCE_DRV_SUPPORT_NOIOMMU_MODE in drv_flags to declare capability
> of working with no-iommu devices.
>
> This commit also fixes typo UACCE_DEV_FLGA_SVA -> UACCE_DEV_FLAG_SVA.
>
> Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
Applied, thanks.
^ permalink raw reply
* Re: [PATCH 0/2] implement missing APIs on FreeBSD
From: Thomas Monjalon @ 2026-06-01 20:12 UTC (permalink / raw)
To: Bruce Richardson; +Cc: dev
In-Reply-To: <20260525171340.1701509-1-bruce.richardson@intel.com>
25/05/2026 19:13, Bruce Richardson:
> The APIs for converting a virtual address to an IOVA or a physical
> address were implemented only as stubs on FreeBSD, but unfortunately
> they are used by drivers such as "igc", part of e1000, causing the
> drivers to fail on BSD. A workaround would be to change the use of these
> functions with the rte_malloc equivalents (e.g. rte_malloc_virt2iova) -
> which do work on FreeBSD - but a better solution is just to replace the
> stubs with proper implementations in FreeBSD-specific EAL flags.
>
> Bruce Richardson (2):
> eal/freebsd: fix memseg addresses in IOVA_AS_VA mode
> eal/freebsd: implement virtual to IOVA translation fn
Applied, thanks.
^ permalink raw reply
* Re: [PATCH] ci: update versions of actions in GHA
From: Thomas Monjalon @ 2026-06-01 19:56 UTC (permalink / raw)
To: David Marchand; +Cc: dev, stable, Aaron Conole
In-Reply-To: <20260511161116.18652-1-david.marchand@redhat.com>
11/05/2026 18:11, David Marchand:
> GitHub started deprecating GHA actions based on Node 20.
> For now, only warnings are raised, but we can switch to more recent
> versions of the common actions, now:
> - cache v5
> - checkout v6
> - upload-artifact v7
> - setup-python v6
>
> Link: https://github.blog/changelog/2025-09-19-deprecation-of-node-20-on-github-actions-runners/
>
> Cc: stable@dpdk.org
>
> Signed-off-by: David Marchand <david.marchand@redhat.com>
Applied, thanks.
^ permalink raw reply
* Re: [PATCH v11] eal/x86: optimize memcpy of small sizes
From: Thomas Monjalon @ 2026-06-01 19:48 UTC (permalink / raw)
To: Morten Brørup
Cc: dev, Bruce Richardson, Konstantin Ananyev, Vipin Varghese,
Stephen Hemminger, Liangxing Wang, Thiyagarajan P,
Bala Murali Krishna, Anatoly Burakov, Vladimir Medvedkin,
Morten Brørup, Konstantin Ananyev
In-Reply-To: <20260521185631.116046-1-mb@smartsharesystems.com>
21/05/2026 20:56, Morten Brørup:
> The implementation for copying up to 64 bytes does not depend on address
> alignment with the size of the CPU's vector registers. Nonetheless, the
> exact same code for copying up to 64 bytes was present in both the aligned
> copy function and all the CPU vector register size specific variants of
> the unaligned copy functions.
> With this patch, the implementation for copying up to 64 bytes was
> consolidated into one instance, located in the common copy function,
> before checking alignment requirements.
> This provides three benefits:
> 1. No copy-paste in the source code.
> 2. A performance gain for copying up to 64 bytes, because the
> address alignment check is avoided in this case.
> 3. Reduced instruction memory footprint, because the compiler only
> generates one instance of the function for copying up to 64 bytes, instead
> of two instances (one in the unaligned copy function, and one in the
> aligned copy function).
>
> Furthermore, __rte_restrict was added to source and destination addresses.
>
> Also, the missing implementation of rte_mov48() was added.
>
> Until recently, some drivers required disabling stringop-overflow warnings
> when using rte_memcpy().
> For some strange reason, these warnings were disabled in the rte_memcpy
> header file, instead of in the problematic drivers.
> With series-38174 ("remove use of rte_memcpy from net/intel"), the
> problematic drivers were updated to use memcpy() instead of rte_memcpy(),
> so disabling these warnings is no longer required, and was removed.
>
> Regarding performance...
> The memcpy performance test (cache-to-cache copy) shows:
> Copying up to 15 bytes takes ca. 4.5 cycles, versus ca. 6.5 cycles before.
> Copying 8 bytes takes 4 cycles, versus 7 cycles before.
> Copying 16 bytes takes 2 cycles, versus 4 cycles before.
> Copying 64 bytes takes 4 cycles, versus 7 cycles before.
>
> Depends-on: series-38174 ("remove use of rte_memcpy from net/intel")
>
> Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
> Acked-by: Bruce Richardson <bruce.richardson@intel.com>
> Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
Applied, thanks.
^ permalink raw reply
* RE: [PATCH v2] doc, eal, devtools: discourage new __rte_always_inline
From: Morten Brørup @ 2026-06-01 18:50 UTC (permalink / raw)
To: Stephen Hemminger, dev; +Cc: Thomas Monjalon
In-Reply-To: <20260601172104.311909-1-stephen@networkplumber.org>
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Monday, 1 June 2026 19.21
>
> Modern compilers at -O2 make good inlining decisions for small
> static inline functions; forced inlining via __rte_always_inline
> should be reserved for cases where it is required for correctness
> or for documented measured performance reasons.
>
> Document the policy in the coding style guide and add a
> checkpatches.sh entry that flags when new uses of the attribute
> are introduced. Checkpatches is not an absolute blocker to
> acceptance, only an indicator that more review is needed.
>
> Add additional comments about use of __rte_always_inline,
> __rte_noinline, __rte_hot, and __rte_cold to the rte_common.h
> to aid developers.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
Reviewed-by: Morten Brørup <mb@smartsharesystems.com>
^ permalink raw reply
* [PATCH v7] net/idpf: update for new mempool cache algorithm
From: Morten Brørup @ 2026-06-01 18:36 UTC (permalink / raw)
To: dev, Andrew Rybchenko, Bruce Richardson, Jingjing Wu,
Praveen Shetty, Hemant Agrawal, Sachin Saxena
Cc: Morten Brørup
In-Reply-To: <20260408141315.904381-1-mb@smartsharesystems.com>
As a consequence of the improved mempool cache algorithm, the PMD was
updated regarding how much to backfill the mempool cache in the AVX512
code path.
Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
---
v7:
* Rebased.
v6:
* Moved driver changes out as separate patches, for easier review. (Bruce)
---
Depends-on: patch-164745 ("mempool: improve cache behaviour and performance")
---
.../net/intel/idpf/idpf_common_rxtx_avx512.c | 52 +++++++++++++++----
1 file changed, 42 insertions(+), 10 deletions(-)
diff --git a/drivers/net/intel/idpf/idpf_common_rxtx_avx512.c b/drivers/net/intel/idpf/idpf_common_rxtx_avx512.c
index 8db4c64106..5788a009ab 100644
--- a/drivers/net/intel/idpf/idpf_common_rxtx_avx512.c
+++ b/drivers/net/intel/idpf/idpf_common_rxtx_avx512.c
@@ -148,15 +148,31 @@ idpf_singleq_rearm(struct idpf_rx_queue *rxq)
/* Can this be satisfied from the cache? */
if (cache->len < IDPF_RXQ_REARM_THRESH) {
/* No. Backfill the cache first, and then fill from it */
- uint32_t req = IDPF_RXQ_REARM_THRESH + (cache->size -
- cache->len);
- /* How many do we require i.e. number to fill the cache + the request */
+ /* Backfill would exceed the cache bounce buffer limit? */
+ __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
+ if (unlikely(cache->size / 2 < IDPF_RXQ_REARM_THRESH)) {
+ idpf_singleq_rearm_common(rxq);
+ return;
+ }
+
+ /*
+ * Backfill the cache from the backend;
+ * move up the hot objects in the cache to the top half of the cache,
+ * and fetch (size / 2) objects to the bottom of the cache.
+ */
+ __rte_assume(cache->len < cache->size / 2);
+ rte_memcpy(&cache->objs[cache->size / 2], &cache->objs[0],
+ sizeof(void *) * cache->len);
int ret = rte_mempool_ops_dequeue_bulk
- (rxq->mp, &cache->objs[cache->len], req);
+ (rxq->mp, &cache->objs[0], cache->size / 2);
if (ret == 0) {
- cache->len += req;
+ cache->len += cache->size / 2;
} else {
+ /*
+ * No further action is required for roll back, as the objects moved
+ * in the cache were actually copied, and the cache remains intact.
+ */
if (rxq->rxrearm_nb + IDPF_RXQ_REARM_THRESH >=
rxq->nb_rx_desc) {
__m128i dma_addr0;
@@ -565,15 +581,31 @@ idpf_splitq_rearm(struct idpf_rx_queue *rx_bufq)
/* Can this be satisfied from the cache? */
if (cache->len < IDPF_RXQ_REARM_THRESH) {
/* No. Backfill the cache first, and then fill from it */
- uint32_t req = IDPF_RXQ_REARM_THRESH + (cache->size -
- cache->len);
- /* How many do we require i.e. number to fill the cache + the request */
+ /* Backfill would exceed the cache bounce buffer limit? */
+ __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
+ if (unlikely(cache->size / 2 < IDPF_RXQ_REARM_THRESH)) {
+ idpf_splitq_rearm_common(rx_bufq);
+ return;
+ }
+
+ /*
+ * Backfill the cache from the backend;
+ * move up the hot objects in the cache to the top half of the cache,
+ * and fetch (size / 2) objects to the bottom of the cache.
+ */
+ __rte_assume(cache->len < cache->size / 2);
+ rte_memcpy(&cache->objs[cache->size / 2], &cache->objs[0],
+ sizeof(void *) * cache->len);
int ret = rte_mempool_ops_dequeue_bulk
- (rx_bufq->mp, &cache->objs[cache->len], req);
+ (rx_bufq->mp, &cache->objs[0], cache->size / 2);
if (ret == 0) {
- cache->len += req;
+ cache->len += cache->size / 2;
} else {
+ /*
+ * No further action is required for roll back, as the objects moved
+ * in the cache were actually copied, and the cache remains intact.
+ */
if (rx_bufq->rxrearm_nb + IDPF_RXQ_REARM_THRESH >=
rx_bufq->nb_rx_desc) {
__m128i dma_addr0;
--
2.43.0
^ permalink raw reply related
* [PATCH v7] mempool/dpaa: update for new mempool cache algorithm
From: Morten Brørup @ 2026-06-01 18:36 UTC (permalink / raw)
To: dev, Andrew Rybchenko, Bruce Richardson, Jingjing Wu,
Praveen Shetty, Hemant Agrawal, Sachin Saxena
Cc: Morten Brørup
In-Reply-To: <20260601183621.252920-1-mb@smartsharesystems.com>
As a consequence of the improved mempool cache algorithm, the mempool
driver was updated to not modify the mempool cache's flushthresh field,
which is now obsolete, and modifying it has no effect.
Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
---
v7:
* Rebased.
v6:
* Moved driver changes out as separate patches, for easier review. (Bruce)
---
Depends-on: patch-164745 ("mempool: improve cache behaviour and performance")
---
drivers/mempool/dpaa/dpaa_mempool.c | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/drivers/mempool/dpaa/dpaa_mempool.c b/drivers/mempool/dpaa/dpaa_mempool.c
index 2f9395b3f4..2f8555a026 100644
--- a/drivers/mempool/dpaa/dpaa_mempool.c
+++ b/drivers/mempool/dpaa/dpaa_mempool.c
@@ -58,8 +58,6 @@ dpaa_mbuf_create_pool(struct rte_mempool *mp)
struct bman_pool_params params = {
.flags = BMAN_POOL_FLAG_DYNAMIC_BPID
};
- unsigned int lcore_id;
- struct rte_mempool_cache *cache;
MEMPOOL_INIT_FUNC_TRACE();
@@ -129,18 +127,6 @@ dpaa_mbuf_create_pool(struct rte_mempool *mp)
rte_memcpy(bp_info, (void *)&rte_dpaa_bpid_info[bpid],
sizeof(struct dpaa_bp_info));
mp->pool_data = (void *)bp_info;
- /* Update per core mempool cache threshold to optimal value which is
- * number of buffers that can be released to HW buffer pool in
- * a single API call.
- */
- for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
- cache = &mp->local_cache[lcore_id];
- DPAA_MEMPOOL_DEBUG("lCore %d: cache->flushthresh %d -> %d",
- lcore_id, cache->flushthresh,
- (uint32_t)(cache->size + DPAA_MBUF_MAX_ACQ_REL));
- if (cache->flushthresh)
- cache->flushthresh = cache->size + DPAA_MBUF_MAX_ACQ_REL;
- }
DPAA_MEMPOOL_INFO("BMAN pool created for bpid =%d", bpid);
return 0;
--
2.43.0
^ permalink raw reply related
* [PATCH v7] mempool/dpaa2: update for new mempool cache algorithm
From: Morten Brørup @ 2026-06-01 18:36 UTC (permalink / raw)
To: dev, Andrew Rybchenko, Bruce Richardson, Jingjing Wu,
Praveen Shetty, Hemant Agrawal, Sachin Saxena
Cc: Morten Brørup
In-Reply-To: <20260601183621.252920-1-mb@smartsharesystems.com>
As a consequence of the improved mempool cache algorithm, the mempool
driver was updated to not modify the mempool cache's flushthresh field,
which is now obsolete, and modifying it has no effect.
Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
---
v7:
* Rebased.
v6:
* Moved driver changes out as separate patches, for easier review. (Bruce)
---
Depends-on: patch-164745 ("mempool: improve cache behaviour and performance")
---
drivers/mempool/dpaa2/dpaa2_hw_mempool.c | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
index 02b6741853..ee001d8ce0 100644
--- a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
+++ b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
@@ -54,8 +54,6 @@ rte_hw_mbuf_create_pool(struct rte_mempool *mp)
struct dpaa2_bp_info *bp_info;
struct dpbp_attr dpbp_attr;
uint32_t bpid;
- unsigned int lcore_id;
- struct rte_mempool_cache *cache;
int ret;
avail_dpbp = dpaa2_alloc_dpbp_dev();
@@ -152,18 +150,6 @@ rte_hw_mbuf_create_pool(struct rte_mempool *mp)
DPAA2_MEMPOOL_DEBUG("BP List created for bpid =%d", dpbp_attr.bpid);
h_bp_list = bp_list;
- /* Update per core mempool cache threshold to optimal value which is
- * number of buffers that can be released to HW buffer pool in
- * a single API call.
- */
- for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
- cache = &mp->local_cache[lcore_id];
- DPAA2_MEMPOOL_DEBUG("lCore %d: cache->flushthresh %d -> %d",
- lcore_id, cache->flushthresh,
- (uint32_t)(cache->size + DPAA2_MBUF_MAX_ACQ_REL));
- if (cache->flushthresh)
- cache->flushthresh = cache->size + DPAA2_MBUF_MAX_ACQ_REL;
- }
return 0;
err4:
--
2.43.0
^ permalink raw reply related
* RE: [PATCH v4 02/27] eal: reimplement rte_smp_*mb with rte_atomic_thread_fence
From: Konstantin Ananyev @ 2026-06-01 18:24 UTC (permalink / raw)
To: Stephen Hemminger, dev@dpdk.org
Cc: Thomas Monjalon, Wathsala Vithanage, Bibo Mao, David Christensen,
Sun Yuechi, Bruce Richardson
In-Reply-To: <20260526232542.620966-3-stephen@networkplumber.org>
> The rte_smp_mb(), rte_smp_wmb() and rte_smp_rmb() functions were
> flagged as deprecated by commit 3ec965b6de12 ("doc: update atomic
> operation deprecation") in 2021 but nothing came of it.
>
> Reimplement them as inline wrappers over rte_atomic_thread_fence()
> and drop the deprecation notice.
> The API is preserved; only the implementation changes.
>
> The wrapper provides stronger guarantees than previous code
> because there is no C11 equivalent to old rte_smp_qmb().
> Generated code is unchanged on x86; on arm64,
> release/acquire emit dmb ish instead of dmb ishst/ishld;
> the difference is below measurement noise.
>
> Drop restrictions on rte_smp_XX in checkpatch since they are
> no longer on deprecation cycle.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
> devtools/checkpatches.sh | 8 --
> doc/guides/rel_notes/deprecation.rst | 8 --
> lib/eal/arm/include/rte_atomic_32.h | 6 --
> lib/eal/arm/include/rte_atomic_64.h | 6 --
> lib/eal/include/generic/rte_atomic.h | 130 +++++--------------------
> lib/eal/loongarch/include/rte_atomic.h | 6 --
> lib/eal/ppc/include/rte_atomic.h | 6 --
> lib/eal/riscv/include/rte_atomic.h | 6 --
> lib/eal/x86/include/rte_atomic.h | 33 +++----
> 9 files changed, 37 insertions(+), 172 deletions(-)
>
> diff --git a/devtools/checkpatches.sh b/devtools/checkpatches.sh
> index f5dd77443f..81bb0fe4e8 100755
> --- a/devtools/checkpatches.sh
> +++ b/devtools/checkpatches.sh
> @@ -121,14 +121,6 @@ check_forbidden_additions() { # <patch>
> -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
> "$1" || res=1
>
> - # refrain from new additions of rte_smp_[r/w]mb()
> - awk -v FOLDERS="lib drivers app examples" \
> - -v EXPRESSIONS="rte_smp_(r|w)?mb\\\(" \
> - -v RET_ON_FAIL=1 \
> - -v MESSAGE='Using rte_smp_[r/w]mb' \
> - -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
> - "$1" || res=1
> -
> # refrain from using compiler __sync_xxx builtins
> awk -v FOLDERS="lib drivers app examples" \
> -v EXPRESSIONS="__sync_.*\\\(" \
> diff --git a/doc/guides/rel_notes/deprecation.rst
> b/doc/guides/rel_notes/deprecation.rst
> index 35c9b4e06c..2190419f79 100644
> --- a/doc/guides/rel_notes/deprecation.rst
> +++ b/doc/guides/rel_notes/deprecation.rst
> @@ -47,14 +47,6 @@ Deprecation Notices
> operations must be used for patches that need to be merged in 20.08 onwards.
> This change will not introduce any performance degradation.
>
> -* rte_smp_*mb: These APIs provide full barrier functionality. However, many
> - use cases do not require full barriers. To support such use cases, DPDK has
> - adopted atomic operations from
> - https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html. These
> - operations and a new wrapper ``rte_atomic_thread_fence`` instead of
> - ``__atomic_thread_fence`` must be used for patches that need to be merged in
> - 20.08 onwards. This change will not introduce any performance degradation.
> -
> * lib: will fix extending some enum/define breaking the ABI. There are multiple
> samples in DPDK that enum/define terminated with a ``.*MAX.*`` value which is
> used by iterators, and arrays holding these values are sized with this
> diff --git a/lib/eal/arm/include/rte_atomic_32.h
> b/lib/eal/arm/include/rte_atomic_32.h
> index 696a539fef..4115271091 100644
> --- a/lib/eal/arm/include/rte_atomic_32.h
> +++ b/lib/eal/arm/include/rte_atomic_32.h
> @@ -17,12 +17,6 @@ extern "C" {
>
> #define rte_rmb() __sync_synchronize()
>
> -#define rte_smp_mb() rte_mb()
> -
> -#define rte_smp_wmb() rte_wmb()
> -
> -#define rte_smp_rmb() rte_rmb()
> -
> #define rte_io_mb() rte_mb()
>
> #define rte_io_wmb() rte_wmb()
> diff --git a/lib/eal/arm/include/rte_atomic_64.h
> b/lib/eal/arm/include/rte_atomic_64.h
> index 9f790238df..604e777bcd 100644
> --- a/lib/eal/arm/include/rte_atomic_64.h
> +++ b/lib/eal/arm/include/rte_atomic_64.h
> @@ -20,12 +20,6 @@ extern "C" {
>
> #define rte_rmb() asm volatile("dmb oshld" : : : "memory")
>
> -#define rte_smp_mb() asm volatile("dmb ish" : : : "memory")
> -
> -#define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory")
> -
> -#define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory")
> -
> #define rte_io_mb() rte_mb()
>
> #define rte_io_wmb() rte_wmb()
> diff --git a/lib/eal/include/generic/rte_atomic.h
> b/lib/eal/include/generic/rte_atomic.h
> index 292e52fade..1b04b43cbb 100644
> --- a/lib/eal/include/generic/rte_atomic.h
> +++ b/lib/eal/include/generic/rte_atomic.h
> @@ -59,55 +59,25 @@ static inline void rte_rmb(void);
> *
> * Guarantees that the LOAD and STORE operations that precede the
> * rte_smp_mb() call are globally visible across the lcores
> - * before the LOAD and STORE operations that follows it.
> - *
> - * @note
> - * This function is deprecated.
> - * It provides similar synchronization primitive as atomic fence,
> - * but has different syntax and memory ordering semantic. Hence
> - * deprecated for the simplicity of memory ordering semantics in use.
> - *
> - * rte_atomic_thread_fence(rte_memory_order_acq_rel) should be used
> instead.
> + * before the LOAD and STORE operations that follow it.
> */
> static inline void rte_smp_mb(void);
>
> /**
> * Write memory barrier between lcores
> *
> - * Guarantees that the STORE operations that precede the
> - * rte_smp_wmb() call are globally visible across the lcores
> - * before the STORE operations that follows it.
> - *
> - * @note
> - * This function is deprecated.
> - * It provides similar synchronization primitive as atomic fence,
> - * but has different syntax and memory ordering semantic. Hence
> - * deprecated for the simplicity of memory ordering semantics in use.
> - *
> - * rte_atomic_thread_fence(rte_memory_order_release) should be used
> instead.
> - * The fence also guarantees LOAD operations that precede the call
> - * are globally visible across the lcores before the STORE operations
> - * that follows it.
> + * Guarantees that the LOAD and STORE operations that precede the
> + * rte_smp_wmb() call are globally visible across the lcores before
> + * any STORE operations that follow it.
> */
> static inline void rte_smp_wmb(void);
>
> /**
> * Read memory barrier between lcores
> *
> - * Guarantees that the LOAD operations that precede the
> - * rte_smp_rmb() call are globally visible across the lcores
> - * before the LOAD operations that follows it.
> - *
> - * @note
> - * This function is deprecated.
> - * It provides similar synchronization primitive as atomic fence,
> - * but has different syntax and memory ordering semantic. Hence
> - * deprecated for the simplicity of memory ordering semantics in use.
> - *
> - * rte_atomic_thread_fence(rte_memory_order_acquire) should be used
> instead.
> - * The fence also guarantees LOAD operations that precede the call
> - * are globally visible across the lcores before the STORE operations
> - * that follows it.
> + * Guarantees that any LOAD operations that precede the rte_smp_rmb()
> + * call complete before LOAD and STORE operations that follow it
> + * become globally visible.
> */
> static inline void rte_smp_rmb(void);
> ///@}
> @@ -164,6 +134,24 @@ static inline void rte_io_rmb(void);
> */
> static inline void rte_atomic_thread_fence(rte_memory_order memorder);
>
> +static __rte_always_inline void
> +rte_smp_mb(void)
> +{
> + rte_atomic_thread_fence(rte_memory_order_seq_cst);
> +}
> +
> +static __rte_always_inline void
> +rte_smp_wmb(void)
> +{
> + rte_atomic_thread_fence(rte_memory_order_release);
> +}
> +
> +static __rte_always_inline void
> +rte_smp_rmb(void)
> +{
> + rte_atomic_thread_fence(rte_memory_order_acquire);
> +}
> +
> /*------------------------- 16 bit atomic operations -------------------------*/
>
> #ifndef RTE_TOOLCHAIN_MSVC
> @@ -184,9 +172,6 @@ static inline void
> rte_atomic_thread_fence(rte_memory_order memorder);
> * @return
> * Non-zero on success; 0 on failure.
> */
> -static inline int
> -rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
> -
> static inline int
> rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
> {
> @@ -303,9 +288,6 @@ rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic16_inc(rte_atomic16_t *v);
> -
> static inline void
> rte_atomic16_inc(rte_atomic16_t *v)
> {
> @@ -318,9 +300,6 @@ rte_atomic16_inc(rte_atomic16_t *v)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic16_dec(rte_atomic16_t *v);
> -
> static inline void
> rte_atomic16_dec(rte_atomic16_t *v)
> {
> @@ -379,8 +358,6 @@ rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
> * @return
> * True if the result after the increment operation is 0; false otherwise.
> */
> -static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
> -
> static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
> {
> return rte_atomic_fetch_add_explicit((volatile __rte_atomic int16_t *)&v-
> >cnt, 1,
> @@ -398,8 +375,6 @@ static inline int
> rte_atomic16_inc_and_test(rte_atomic16_t *v)
> * @return
> * True if the result after the decrement operation is 0; false otherwise.
> */
> -static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
> -
> static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
> {
> return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int16_t *)&v-
> >cnt, 1,
> @@ -417,8 +392,6 @@ static inline int
> rte_atomic16_dec_and_test(rte_atomic16_t *v)
> * @return
> * 0 if failed; else 1, success.
> */
> -static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
> -
> static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
> {
> return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
> @@ -453,9 +426,6 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
> * @return
> * Non-zero on success; 0 on failure.
> */
> -static inline int
> -rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
> -
> static inline int
> rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
> {
> @@ -572,9 +542,6 @@ rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic32_inc(rte_atomic32_t *v);
> -
> static inline void
> rte_atomic32_inc(rte_atomic32_t *v)
> {
> @@ -587,9 +554,6 @@ rte_atomic32_inc(rte_atomic32_t *v)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic32_dec(rte_atomic32_t *v);
> -
> static inline void
> rte_atomic32_dec(rte_atomic32_t *v)
> {
> @@ -648,8 +612,6 @@ rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
> * @return
> * True if the result after the increment operation is 0; false otherwise.
> */
> -static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
> -
> static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
> {
> return rte_atomic_fetch_add_explicit((volatile __rte_atomic int32_t *)&v-
> >cnt, 1,
> @@ -667,8 +629,6 @@ static inline int
> rte_atomic32_inc_and_test(rte_atomic32_t *v)
> * @return
> * True if the result after the decrement operation is 0; false otherwise.
> */
> -static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
> -
> static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
> {
> return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int32_t *)&v-
> >cnt, 1,
> @@ -686,8 +646,6 @@ static inline int
> rte_atomic32_dec_and_test(rte_atomic32_t *v)
> * @return
> * 0 if failed; else 1, success.
> */
> -static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
> -
> static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
> {
> return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
> @@ -721,9 +679,6 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
> * @return
> * Non-zero on success; 0 on failure.
> */
> -static inline int
> -rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
> -
> static inline int
> rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
> {
> @@ -770,9 +725,6 @@ typedef struct {
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic64_init(rte_atomic64_t *v);
> -
> static inline void
> rte_atomic64_init(rte_atomic64_t *v)
> {
> @@ -798,9 +750,6 @@ rte_atomic64_init(rte_atomic64_t *v)
> * @return
> * The value of the counter.
> */
> -static inline int64_t
> -rte_atomic64_read(rte_atomic64_t *v);
> -
> static inline int64_t
> rte_atomic64_read(rte_atomic64_t *v)
> {
> @@ -828,9 +777,6 @@ rte_atomic64_read(rte_atomic64_t *v)
> * @param new_value
> * The new value of the counter.
> */
> -static inline void
> -rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
> -
> static inline void
> rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> {
> @@ -856,9 +802,6 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> * @param inc
> * The value to be added to the counter.
> */
> -static inline void
> -rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
> -
> static inline void
> rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> {
> @@ -874,9 +817,6 @@ rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> * @param dec
> * The value to be subtracted from the counter.
> */
> -static inline void
> -rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
> -
> static inline void
> rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> {
> @@ -890,9 +830,6 @@ rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic64_inc(rte_atomic64_t *v);
> -
> static inline void
> rte_atomic64_inc(rte_atomic64_t *v)
> {
> @@ -905,9 +842,6 @@ rte_atomic64_inc(rte_atomic64_t *v)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void
> -rte_atomic64_dec(rte_atomic64_t *v);
> -
> static inline void
> rte_atomic64_dec(rte_atomic64_t *v)
> {
> @@ -927,9 +861,6 @@ rte_atomic64_dec(rte_atomic64_t *v)
> * @return
> * The value of v after the addition.
> */
> -static inline int64_t
> -rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
> -
> static inline int64_t
> rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
> {
> @@ -950,9 +881,6 @@ rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
> * @return
> * The value of v after the subtraction.
> */
> -static inline int64_t
> -rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
> -
> static inline int64_t
> rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
> {
> @@ -971,8 +899,6 @@ rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
> * @return
> * True if the result after the addition is 0; false otherwise.
> */
> -static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
> -
> static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
> {
> return rte_atomic64_add_return(v, 1) == 0;
> @@ -989,8 +915,6 @@ static inline int
> rte_atomic64_inc_and_test(rte_atomic64_t *v)
> * @return
> * True if the result after subtraction is 0; false otherwise.
> */
> -static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
> -
> static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
> {
> return rte_atomic64_sub_return(v, 1) == 0;
> @@ -1007,8 +931,6 @@ static inline int
> rte_atomic64_dec_and_test(rte_atomic64_t *v)
> * @return
> * 0 if failed; else 1, success.
> */
> -static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
> -
> static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
> {
> return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
> @@ -1020,8 +942,6 @@ static inline int
> rte_atomic64_test_and_set(rte_atomic64_t *v)
> * @param v
> * A pointer to the atomic counter.
> */
> -static inline void rte_atomic64_clear(rte_atomic64_t *v);
> -
> static inline void rte_atomic64_clear(rte_atomic64_t *v)
> {
> rte_atomic64_set(v, 0);
> diff --git a/lib/eal/loongarch/include/rte_atomic.h
> b/lib/eal/loongarch/include/rte_atomic.h
> index 785a452c9e..a789e3ab4d 100644
> --- a/lib/eal/loongarch/include/rte_atomic.h
> +++ b/lib/eal/loongarch/include/rte_atomic.h
> @@ -18,12 +18,6 @@ extern "C" {
>
> #define rte_rmb() rte_mb()
>
> -#define rte_smp_mb() rte_mb()
> -
> -#define rte_smp_wmb() rte_mb()
> -
> -#define rte_smp_rmb() rte_mb()
> -
> #define rte_io_mb() rte_mb()
>
> #define rte_io_wmb() rte_mb()
> diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
> index 64f4c3d670..0e64db2a35 100644
> --- a/lib/eal/ppc/include/rte_atomic.h
> +++ b/lib/eal/ppc/include/rte_atomic.h
> @@ -24,12 +24,6 @@ extern "C" {
>
> #define rte_rmb() asm volatile("sync" : : : "memory")
>
> -#define rte_smp_mb() rte_mb()
> -
> -#define rte_smp_wmb() rte_wmb()
> -
> -#define rte_smp_rmb() rte_rmb()
> -
> #define rte_io_mb() rte_mb()
>
> #define rte_io_wmb() rte_wmb()
> diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
> index 061b175f33..04c40e4e9b 100644
> --- a/lib/eal/riscv/include/rte_atomic.h
> +++ b/lib/eal/riscv/include/rte_atomic.h
> @@ -23,12 +23,6 @@ extern "C" {
>
> #define rte_rmb() asm volatile("fence r, r" : : : "memory")
>
> -#define rte_smp_mb() rte_mb()
> -
> -#define rte_smp_wmb() rte_wmb()
> -
> -#define rte_smp_rmb() rte_rmb()
> -
> #define rte_io_mb() asm volatile("fence iorw, iorw" : : : "memory")
>
> #define rte_io_wmb() asm volatile("fence orw, ow" : : : "memory")
> diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
> index 4f05302c9f..f4d39ce4fe 100644
> --- a/lib/eal/x86/include/rte_atomic.h
> +++ b/lib/eal/x86/include/rte_atomic.h
> @@ -23,10 +23,6 @@
>
> #define rte_rmb() _mm_lfence()
>
> -#define rte_smp_wmb() rte_compiler_barrier()
> -
> -#define rte_smp_rmb() rte_compiler_barrier()
> -
> #ifdef __cplusplus
> extern "C" {
> #endif
> @@ -63,20 +59,6 @@ extern "C" {
> * So below we use that technique for rte_smp_mb() implementation.
> */
>
> -static __rte_always_inline void
> -rte_smp_mb(void)
> -{
> -#ifdef RTE_TOOLCHAIN_MSVC
> - _mm_mfence();
> -#else
> -#ifdef RTE_ARCH_I686
> - asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
> -#else
> - asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
> -#endif
> -#endif
> -}
> -
> #define rte_io_mb() rte_mb()
>
> #define rte_io_wmb() rte_compiler_barrier()
> @@ -93,10 +75,19 @@ rte_smp_mb(void)
> static __rte_always_inline void
> rte_atomic_thread_fence(rte_memory_order memorder)
> {
> - if (memorder == rte_memory_order_seq_cst)
> - rte_smp_mb();
> - else
> + if (memorder == rte_memory_order_seq_cst) {
> +#ifdef RTE_TOOLCHAIN_MSVC
> + _mm_mfence();
> +#else
> +#ifdef RTE_ARCH_I686
> + asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
> +#else
> + asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
> +#endif
> +#endif
> + } else {
> __rte_atomic_thread_fence(memorder);
> + }
> }
>
> #ifdef __cplusplus
> --
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
> 2.53.0
^ permalink raw reply
* RE: [PATCH v4 01/27] eal: use intrinsics for rte_atomic on all platforms
From: Konstantin Ananyev @ 2026-06-01 18:23 UTC (permalink / raw)
To: Stephen Hemminger, dev@dpdk.org
Cc: Wathsala Vithanage, Bibo Mao, David Christensen, Sun Yuechi,
Bruce Richardson
In-Reply-To: <20260526232542.620966-2-stephen@networkplumber.org>
> Next step is to deprecate the rte_atomicNN_*() family. Rather than
> maintaining both the inline asm and intrinsic fallbacks, drop the
> asm paths and use intrinsics everywhere.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
> lib/eal/arm/include/rte_atomic_32.h | 4 -
> lib/eal/arm/include/rte_atomic_64.h | 4 -
> lib/eal/include/generic/rte_atomic.h | 76 +---------
> lib/eal/loongarch/include/rte_atomic.h | 4 -
> lib/eal/ppc/include/rte_atomic.h | 173 -----------------------
> lib/eal/riscv/include/rte_atomic.h | 4 -
> lib/eal/x86/include/rte_atomic.h | 172 ----------------------
> lib/eal/x86/include/rte_atomic_32.h | 188 -------------------------
> lib/eal/x86/include/rte_atomic_64.h | 157 ---------------------
> 9 files changed, 6 insertions(+), 776 deletions(-)
>
> diff --git a/lib/eal/arm/include/rte_atomic_32.h
> b/lib/eal/arm/include/rte_atomic_32.h
> index 0b9a0dfa30..696a539fef 100644
> --- a/lib/eal/arm/include/rte_atomic_32.h
> +++ b/lib/eal/arm/include/rte_atomic_32.h
> @@ -5,10 +5,6 @@
> #ifndef _RTE_ATOMIC_ARM32_H_
> #define _RTE_ATOMIC_ARM32_H_
>
> -#ifndef RTE_FORCE_INTRINSICS
> -# error Platform must be built with RTE_FORCE_INTRINSICS
> -#endif
> -
> #include "generic/rte_atomic.h"
>
> #ifdef __cplusplus
> diff --git a/lib/eal/arm/include/rte_atomic_64.h
> b/lib/eal/arm/include/rte_atomic_64.h
> index 181bb60929..9f790238df 100644
> --- a/lib/eal/arm/include/rte_atomic_64.h
> +++ b/lib/eal/arm/include/rte_atomic_64.h
> @@ -6,10 +6,6 @@
> #ifndef _RTE_ATOMIC_ARM64_H_
> #define _RTE_ATOMIC_ARM64_H_
>
> -#ifndef RTE_FORCE_INTRINSICS
> -# error Platform must be built with RTE_FORCE_INTRINSICS
> -#endif
> -
> #include "generic/rte_atomic.h"
> #include <rte_branch_prediction.h>
> #include <rte_debug.h>
> diff --git a/lib/eal/include/generic/rte_atomic.h
> b/lib/eal/include/generic/rte_atomic.h
> index 0a4f3f8528..292e52fade 100644
> --- a/lib/eal/include/generic/rte_atomic.h
> +++ b/lib/eal/include/generic/rte_atomic.h
> @@ -187,13 +187,11 @@ static inline void
> rte_atomic_thread_fence(rte_memory_order memorder);
> static inline int
> rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int
> rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
> {
> return __sync_bool_compare_and_swap(dst, exp, src);
> }
> -#endif
>
> /**
> * Atomic exchange.
> @@ -211,15 +209,11 @@ rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t
> exp, uint16_t src)
> * The original value at that location
> */
> static inline uint16_t
> -rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val);
> -
> -#ifdef RTE_FORCE_INTRINSICS
> -static inline uint16_t
> rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
> {
> - return rte_atomic_exchange_explicit(dst, val,
> rte_memory_order_seq_cst);
> + return rte_atomic_exchange_explicit((volatile __rte_atomic uint16_t
> *)dst,
> + val, rte_memory_order_seq_cst);
> }
> -#endif
>
> /**
> * The atomic counter structure.
> @@ -312,13 +306,11 @@ rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
> static inline void
> rte_atomic16_inc(rte_atomic16_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic16_inc(rte_atomic16_t *v)
> {
> rte_atomic16_add(v, 1);
> }
> -#endif
>
> /**
> * Atomically decrement a counter by one.
> @@ -329,13 +321,11 @@ rte_atomic16_inc(rte_atomic16_t *v)
> static inline void
> rte_atomic16_dec(rte_atomic16_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic16_dec(rte_atomic16_t *v)
> {
> rte_atomic16_sub(v, 1);
> }
> -#endif
>
> /**
> * Atomically add a 16-bit value to a counter and return the result.
> @@ -391,13 +381,11 @@ rte_atomic16_sub_return(rte_atomic16_t *v, int16_t
> dec)
> */
> static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
> {
> return rte_atomic_fetch_add_explicit((volatile __rte_atomic int16_t *)&v-
> >cnt, 1,
> rte_memory_order_seq_cst) + 1 == 0;
> }
> -#endif
>
> /**
> * Atomically decrement a 16-bit counter by one and test.
> @@ -412,13 +400,11 @@ static inline int
> rte_atomic16_inc_and_test(rte_atomic16_t *v)
> */
> static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
> {
> return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int16_t *)&v-
> >cnt, 1,
> rte_memory_order_seq_cst) - 1 == 0;
> }
> -#endif
>
> /**
> * Atomically test and set a 16-bit atomic counter.
> @@ -433,12 +419,10 @@ static inline int
> rte_atomic16_dec_and_test(rte_atomic16_t *v)
> */
> static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
> {
> return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
> }
> -#endif
>
> /**
> * Atomically set a 16-bit counter to 0.
> @@ -472,13 +456,11 @@ static inline void rte_atomic16_clear(rte_atomic16_t
> *v)
> static inline int
> rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int
> rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
> {
> return __sync_bool_compare_and_swap(dst, exp, src);
> }
> -#endif
>
> /**
> * Atomic exchange.
> @@ -496,15 +478,11 @@ rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t
> exp, uint32_t src)
> * The original value at that location
> */
> static inline uint32_t
> -rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val);
> -
> -#ifdef RTE_FORCE_INTRINSICS
> -static inline uint32_t
> rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
> {
> - return rte_atomic_exchange_explicit(dst, val,
> rte_memory_order_seq_cst);
> + return rte_atomic_exchange_explicit((volatile __rte_atomic uint32_t
> *)dst,
> + val, rte_memory_order_seq_cst);
> }
> -#endif
>
> /**
> * The atomic counter structure.
> @@ -597,13 +575,11 @@ rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
> static inline void
> rte_atomic32_inc(rte_atomic32_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic32_inc(rte_atomic32_t *v)
> {
> rte_atomic32_add(v, 1);
> }
> -#endif
>
> /**
> * Atomically decrement a counter by one.
> @@ -614,13 +590,11 @@ rte_atomic32_inc(rte_atomic32_t *v)
> static inline void
> rte_atomic32_dec(rte_atomic32_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic32_dec(rte_atomic32_t *v)
> {
> rte_atomic32_sub(v,1);
> }
> -#endif
>
> /**
> * Atomically add a 32-bit value to a counter and return the result.
> @@ -676,13 +650,11 @@ rte_atomic32_sub_return(rte_atomic32_t *v, int32_t
> dec)
> */
> static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
> {
> return rte_atomic_fetch_add_explicit((volatile __rte_atomic int32_t *)&v-
> >cnt, 1,
> rte_memory_order_seq_cst) + 1 == 0;
> }
> -#endif
>
> /**
> * Atomically decrement a 32-bit counter by one and test.
> @@ -697,13 +669,11 @@ static inline int
> rte_atomic32_inc_and_test(rte_atomic32_t *v)
> */
> static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
> {
> return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int32_t *)&v-
> >cnt, 1,
> rte_memory_order_seq_cst) - 1 == 0;
> }
> -#endif
>
> /**
> * Atomically test and set a 32-bit atomic counter.
> @@ -718,12 +688,10 @@ static inline int
> rte_atomic32_dec_and_test(rte_atomic32_t *v)
> */
> static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
> {
> return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
> }
> -#endif
>
> /**
> * Atomically set a 32-bit counter to 0.
> @@ -756,13 +724,11 @@ static inline void rte_atomic32_clear(rte_atomic32_t
> *v)
> static inline int
> rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int
> rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
> {
> return __sync_bool_compare_and_swap(dst, exp, src);
> }
> -#endif
>
> /**
> * Atomic exchange.
> @@ -780,15 +746,11 @@ rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t
> exp, uint64_t src)
> * The original value at that location
> */
> static inline uint64_t
> -rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val);
> -
> -#ifdef RTE_FORCE_INTRINSICS
> -static inline uint64_t
> rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
> {
> - return rte_atomic_exchange_explicit(dst, val,
> rte_memory_order_seq_cst);
> + return rte_atomic_exchange_explicit((volatile __rte_atomic uint64_t
> *)dst,
> + val, rte_memory_order_seq_cst);
> }
> -#endif
>
> /**
> * The atomic counter structure.
> @@ -811,7 +773,6 @@ typedef struct {
> static inline void
> rte_atomic64_init(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic64_init(rte_atomic64_t *v)
> {
> @@ -828,7 +789,6 @@ rte_atomic64_init(rte_atomic64_t *v)
> }
> #endif
> }
> -#endif
>
> /**
> * Atomically read a 64-bit counter.
> @@ -841,7 +801,6 @@ rte_atomic64_init(rte_atomic64_t *v)
> static inline int64_t
> rte_atomic64_read(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int64_t
> rte_atomic64_read(rte_atomic64_t *v)
> {
> @@ -860,7 +819,6 @@ rte_atomic64_read(rte_atomic64_t *v)
> return tmp;
> #endif
> }
> -#endif
>
> /**
> * Atomically set a 64-bit counter.
> @@ -873,7 +831,6 @@ rte_atomic64_read(rte_atomic64_t *v)
> static inline void
> rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> {
> @@ -890,7 +847,6 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> }
> #endif
> }
> -#endif
>
> /**
> * Atomically add a 64-bit value to a counter.
> @@ -903,14 +859,12 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t
> new_value)
> static inline void
> rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> {
> rte_atomic_fetch_add_explicit((volatile __rte_atomic int64_t *)&v->cnt,
> inc,
> rte_memory_order_seq_cst);
> }
> -#endif
>
> /**
> * Atomically subtract a 64-bit value from a counter.
> @@ -923,14 +877,12 @@ rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> static inline void
> rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> {
> rte_atomic_fetch_sub_explicit((volatile __rte_atomic int64_t *)&v->cnt,
> dec,
> rte_memory_order_seq_cst);
> }
> -#endif
>
> /**
> * Atomically increment a 64-bit counter by one and test.
> @@ -941,13 +893,11 @@ rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> static inline void
> rte_atomic64_inc(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic64_inc(rte_atomic64_t *v)
> {
> rte_atomic64_add(v, 1);
> }
> -#endif
>
> /**
> * Atomically decrement a 64-bit counter by one and test.
> @@ -958,13 +908,11 @@ rte_atomic64_inc(rte_atomic64_t *v)
> static inline void
> rte_atomic64_dec(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void
> rte_atomic64_dec(rte_atomic64_t *v)
> {
> rte_atomic64_sub(v, 1);
> }
> -#endif
>
> /**
> * Add a 64-bit value to an atomic counter and return the result.
> @@ -982,14 +930,12 @@ rte_atomic64_dec(rte_atomic64_t *v)
> static inline int64_t
> rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int64_t
> rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
> {
> return rte_atomic_fetch_add_explicit((volatile __rte_atomic int64_t *)&v-
> >cnt, inc,
> rte_memory_order_seq_cst) + inc;
> }
> -#endif
>
> /**
> * Subtract a 64-bit value from an atomic counter and return the result.
> @@ -1007,14 +953,12 @@ rte_atomic64_add_return(rte_atomic64_t *v, int64_t
> inc)
> static inline int64_t
> rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int64_t
> rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
> {
> return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int64_t *)&v-
> >cnt, dec,
> rte_memory_order_seq_cst) - dec;
> }
> -#endif
>
> /**
> * Atomically increment a 64-bit counter by one and test.
> @@ -1029,12 +973,10 @@ rte_atomic64_sub_return(rte_atomic64_t *v, int64_t
> dec)
> */
> static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
> {
> return rte_atomic64_add_return(v, 1) == 0;
> }
> -#endif
>
> /**
> * Atomically decrement a 64-bit counter by one and test.
> @@ -1049,12 +991,10 @@ static inline int
> rte_atomic64_inc_and_test(rte_atomic64_t *v)
> */
> static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
> {
> return rte_atomic64_sub_return(v, 1) == 0;
> }
> -#endif
>
> /**
> * Atomically test and set a 64-bit atomic counter.
> @@ -1069,12 +1009,10 @@ static inline int
> rte_atomic64_dec_and_test(rte_atomic64_t *v)
> */
> static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
> {
> return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
> }
> -#endif
>
> /**
> * Atomically set a 64-bit counter to 0.
> @@ -1084,12 +1022,10 @@ static inline int
> rte_atomic64_test_and_set(rte_atomic64_t *v)
> */
> static inline void rte_atomic64_clear(rte_atomic64_t *v);
>
> -#ifdef RTE_FORCE_INTRINSICS
> static inline void rte_atomic64_clear(rte_atomic64_t *v)
> {
> rte_atomic64_set(v, 0);
> }
> -#endif
>
> #endif
>
> diff --git a/lib/eal/loongarch/include/rte_atomic.h
> b/lib/eal/loongarch/include/rte_atomic.h
> index c8066a4612..785a452c9e 100644
> --- a/lib/eal/loongarch/include/rte_atomic.h
> +++ b/lib/eal/loongarch/include/rte_atomic.h
> @@ -5,10 +5,6 @@
> #ifndef RTE_ATOMIC_LOONGARCH_H
> #define RTE_ATOMIC_LOONGARCH_H
>
> -#ifndef RTE_FORCE_INTRINSICS
> -# error Platform must be built with RTE_FORCE_INTRINSICS
> -#endif
> -
> #include <rte_common.h>
> #include "generic/rte_atomic.h"
>
> diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
> index 10acc238f9..64f4c3d670 100644
> --- a/lib/eal/ppc/include/rte_atomic.h
> +++ b/lib/eal/ppc/include/rte_atomic.h
> @@ -43,179 +43,6 @@ rte_atomic_thread_fence(rte_memory_order memorder)
> }
>
> /*------------------------- 16 bit atomic operations -------------------------*/
> -#ifndef RTE_FORCE_INTRINSICS
> -static inline int
> -rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
> -{
> - return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src,
> rte_memory_order_acquire,
> - rte_memory_order_acquire) ? 1 : 0;
> -}
> -
> -static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
> -{
> - return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void
> -rte_atomic16_inc(rte_atomic16_t *v)
> -{
> - rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
> -}
> -
> -static inline void
> -rte_atomic16_dec(rte_atomic16_t *v)
> -{
> - rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
> -}
> -
> -static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
> -{
> - return rte_atomic_fetch_add_explicit(&v->cnt, 1,
> rte_memory_order_acquire) + 1 == 0;
> -}
> -
> -static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
> -{
> - return rte_atomic_fetch_sub_explicit(&v->cnt, 1,
> rte_memory_order_acquire) - 1 == 0;
> -}
> -
> -static inline uint16_t
> -rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
> -{
> - return __atomic_exchange_2(dst, val, rte_memory_order_seq_cst);
> -}
> -
> -/*------------------------- 32 bit atomic operations -------------------------*/
> -
> -static inline int
> -rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
> -{
> - return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src,
> rte_memory_order_acquire,
> - rte_memory_order_acquire) ? 1 : 0;
> -}
> -
> -static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
> -{
> - return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void
> -rte_atomic32_inc(rte_atomic32_t *v)
> -{
> - rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
> -}
> -
> -static inline void
> -rte_atomic32_dec(rte_atomic32_t *v)
> -{
> - rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
> -}
> -
> -static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
> -{
> - return rte_atomic_fetch_add_explicit(&v->cnt, 1,
> rte_memory_order_acquire) + 1 == 0;
> -}
> -
> -static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
> -{
> - return rte_atomic_fetch_sub_explicit(&v->cnt, 1,
> rte_memory_order_acquire) - 1 == 0;
> -}
> -
> -static inline uint32_t
> -rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
> -{
> - return __atomic_exchange_4(dst, val, rte_memory_order_seq_cst);
> -}
> -
> -/*------------------------- 64 bit atomic operations -------------------------*/
> -
> -static inline int
> -rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
> -{
> - return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src,
> rte_memory_order_acquire,
> - rte_memory_order_acquire) ? 1 : 0;
> -}
> -
> -static inline void
> -rte_atomic64_init(rte_atomic64_t *v)
> -{
> - v->cnt = 0;
> -}
> -
> -static inline int64_t
> -rte_atomic64_read(rte_atomic64_t *v)
> -{
> - return v->cnt;
> -}
> -
> -static inline void
> -rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> -{
> - v->cnt = new_value;
> -}
> -
> -static inline void
> -rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> -{
> - rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire);
> -}
> -
> -static inline void
> -rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> -{
> - rte_atomic_fetch_sub_explicit(&v->cnt, dec,
> rte_memory_order_acquire);
> -}
> -
> -static inline void
> -rte_atomic64_inc(rte_atomic64_t *v)
> -{
> - rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
> -}
> -
> -static inline void
> -rte_atomic64_dec(rte_atomic64_t *v)
> -{
> - rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
> -}
> -
> -static inline int64_t
> -rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
> -{
> - return rte_atomic_fetch_add_explicit(&v->cnt, inc,
> rte_memory_order_acquire) + inc;
> -}
> -
> -static inline int64_t
> -rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
> -{
> - return rte_atomic_fetch_sub_explicit(&v->cnt, dec,
> rte_memory_order_acquire) - dec;
> -}
> -
> -static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
> -{
> - return rte_atomic_fetch_add_explicit(&v->cnt, 1,
> rte_memory_order_acquire) + 1 == 0;
> -}
> -
> -static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
> -{
> - return rte_atomic_fetch_sub_explicit(&v->cnt, 1,
> rte_memory_order_acquire) - 1 == 0;
> -}
> -
> -static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
> -{
> - return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void rte_atomic64_clear(rte_atomic64_t *v)
> -{
> - v->cnt = 0;
> -}
> -
> -static inline uint64_t
> -rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
> -{
> - return __atomic_exchange_8(dst, val, rte_memory_order_seq_cst);
> -}
> -
> -#endif
>
> #ifdef __cplusplus
> }
> diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
> index 66346ad474..061b175f33 100644
> --- a/lib/eal/riscv/include/rte_atomic.h
> +++ b/lib/eal/riscv/include/rte_atomic.h
> @@ -8,10 +8,6 @@
> #ifndef RTE_ATOMIC_RISCV_H
> #define RTE_ATOMIC_RISCV_H
>
> -#ifndef RTE_FORCE_INTRINSICS
> -# error Platform must be built with RTE_FORCE_INTRINSICS
> -#endif
> -
> #include <stdint.h>
> #include <rte_common.h>
> #include <rte_config.h>
> diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
> index e071e4234e..4f05302c9f 100644
> --- a/lib/eal/x86/include/rte_atomic.h
> +++ b/lib/eal/x86/include/rte_atomic.h
> @@ -111,178 +111,6 @@ rte_atomic_thread_fence(rte_memory_order
> memorder)
> extern "C" {
> #endif
>
> -#ifndef RTE_FORCE_INTRINSICS
> -static inline int
> -rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
> -{
> - uint8_t res;
> -
> - asm volatile(
> - MPLOCKED
> - "cmpxchgw %[src], %[dst];"
> - "sete %[res];"
> - : [res] "=a" (res), /* output */
> - [dst] "=m" (*dst)
> - : [src] "r" (src), /* input */
> - "a" (exp),
> - "m" (*dst)
> - : "memory"); /* no-clobber list */
> - return res;
> -}
> -
> -static inline uint16_t
> -rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
> -{
> - asm volatile(
> - MPLOCKED
> - "xchgw %0, %1;"
> - : "=r" (val), "=m" (*dst)
> - : "0" (val), "m" (*dst)
> - : "memory"); /* no-clobber list */
> - return val;
> -}
> -
> -static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
> -{
> - return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void
> -rte_atomic16_inc(rte_atomic16_t *v)
> -{
> - asm volatile(
> - MPLOCKED
> - "incw %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : "m" (v->cnt) /* input */
> - );
> -}
> -
> -static inline void
> -rte_atomic16_dec(rte_atomic16_t *v)
> -{
> - asm volatile(
> - MPLOCKED
> - "decw %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : "m" (v->cnt) /* input */
> - );
> -}
> -
> -static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
> -{
> - uint8_t ret;
> -
> - asm volatile(
> - MPLOCKED
> - "incw %[cnt] ; "
> - "sete %[ret]"
> - : [cnt] "+m" (v->cnt), /* output */
> - [ret] "=qm" (ret)
> - );
> - return ret != 0;
> -}
> -
> -static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
> -{
> - uint8_t ret;
> -
> - asm volatile(MPLOCKED
> - "decw %[cnt] ; "
> - "sete %[ret]"
> - : [cnt] "+m" (v->cnt), /* output */
> - [ret] "=qm" (ret)
> - );
> - return ret != 0;
> -}
> -
> -/*------------------------- 32 bit atomic operations -------------------------*/
> -
> -static inline int
> -rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
> -{
> - uint8_t res;
> -
> - asm volatile(
> - MPLOCKED
> - "cmpxchgl %[src], %[dst];"
> - "sete %[res];"
> - : [res] "=a" (res), /* output */
> - [dst] "=m" (*dst)
> - : [src] "r" (src), /* input */
> - "a" (exp),
> - "m" (*dst)
> - : "memory"); /* no-clobber list */
> - return res;
> -}
> -
> -static inline uint32_t
> -rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
> -{
> - asm volatile(
> - MPLOCKED
> - "xchgl %0, %1;"
> - : "=r" (val), "=m" (*dst)
> - : "0" (val), "m" (*dst)
> - : "memory"); /* no-clobber list */
> - return val;
> -}
> -
> -static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
> -{
> - return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void
> -rte_atomic32_inc(rte_atomic32_t *v)
> -{
> - asm volatile(
> - MPLOCKED
> - "incl %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : "m" (v->cnt) /* input */
> - );
> -}
> -
> -static inline void
> -rte_atomic32_dec(rte_atomic32_t *v)
> -{
> - asm volatile(
> - MPLOCKED
> - "decl %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : "m" (v->cnt) /* input */
> - );
> -}
> -
> -static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
> -{
> - uint8_t ret;
> -
> - asm volatile(
> - MPLOCKED
> - "incl %[cnt] ; "
> - "sete %[ret]"
> - : [cnt] "+m" (v->cnt), /* output */
> - [ret] "=qm" (ret)
> - );
> - return ret != 0;
> -}
> -
> -static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
> -{
> - uint8_t ret;
> -
> - asm volatile(MPLOCKED
> - "decl %[cnt] ; "
> - "sete %[ret]"
> - : [cnt] "+m" (v->cnt), /* output */
> - [ret] "=qm" (ret)
> - );
> - return ret != 0;
> -}
> -
> -#endif /* !RTE_FORCE_INTRINSICS */
>
> #ifdef __cplusplus
> }
> diff --git a/lib/eal/x86/include/rte_atomic_32.h
> b/lib/eal/x86/include/rte_atomic_32.h
> index 0f25863aa5..37d139f30d 100644
> --- a/lib/eal/x86/include/rte_atomic_32.h
> +++ b/lib/eal/x86/include/rte_atomic_32.h
> @@ -20,193 +20,5 @@
>
> /*------------------------- 64 bit atomic operations -------------------------*/
>
> -#ifndef RTE_FORCE_INTRINSICS
> -static inline int
> -rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
> -{
> - uint8_t res;
> - union {
> - struct {
> - uint32_t l32;
> - uint32_t h32;
> - };
> - uint64_t u64;
> - } _exp, _src;
> -
> - _exp.u64 = exp;
> - _src.u64 = src;
> -
> -#ifndef __PIC__
> - asm volatile (
> - MPLOCKED
> - "cmpxchg8b (%[dst]);"
> - "setz %[res];"
> - : [res] "=a" (res) /* result in eax */
> - : [dst] "S" (dst), /* esi */
> - "b" (_src.l32), /* ebx */
> - "c" (_src.h32), /* ecx */
> - "a" (_exp.l32), /* eax */
> - "d" (_exp.h32) /* edx */
> - : "memory" ); /* no-clobber list */
> -#else
> - asm volatile (
> - "xchgl %%ebx, %%edi;\n"
> - MPLOCKED
> - "cmpxchg8b (%[dst]);"
> - "setz %[res];"
> - "xchgl %%ebx, %%edi;\n"
> - : [res] "=a" (res) /* result in eax */
> - : [dst] "S" (dst), /* esi */
> - "D" (_src.l32), /* ebx */
> - "c" (_src.h32), /* ecx */
> - "a" (_exp.l32), /* eax */
> - "d" (_exp.h32) /* edx */
> - : "memory" ); /* no-clobber list */
> -#endif
> -
> - return res;
> -}
> -
> -static inline uint64_t
> -rte_atomic64_exchange(volatile uint64_t *dest, uint64_t val)
> -{
> - uint64_t old;
> -
> - do {
> - old = *dest;
> - } while (rte_atomic64_cmpset(dest, old, val) == 0);
> -
> - return old;
> -}
> -
> -static inline void
> -rte_atomic64_init(rte_atomic64_t *v)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, 0);
> - }
> -}
> -
> -static inline int64_t
> -rte_atomic64_read(rte_atomic64_t *v)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - /* replace the value by itself */
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, tmp);
> - }
> - return tmp;
> -}
> -
> -static inline void
> -rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, new_value);
> - }
> -}
> -
> -static inline void
> -rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, tmp + inc);
> - }
> -}
> -
> -static inline void
> -rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, tmp - dec);
> - }
> -}
> -
> -static inline void
> -rte_atomic64_inc(rte_atomic64_t *v)
> -{
> - rte_atomic64_add(v, 1);
> -}
> -
> -static inline void
> -rte_atomic64_dec(rte_atomic64_t *v)
> -{
> - rte_atomic64_sub(v, 1);
> -}
> -
> -static inline int64_t
> -rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, tmp + inc);
> - }
> -
> - return tmp + inc;
> -}
> -
> -static inline int64_t
> -rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
> -{
> - int success = 0;
> - uint64_t tmp;
> -
> - while (success == 0) {
> - tmp = v->cnt;
> - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
> - tmp, tmp - dec);
> - }
> -
> - return tmp - dec;
> -}
> -
> -static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
> -{
> - return rte_atomic64_add_return(v, 1) == 0;
> -}
> -
> -static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
> -{
> - return rte_atomic64_sub_return(v, 1) == 0;
> -}
> -
> -static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
> -{
> - return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void rte_atomic64_clear(rte_atomic64_t *v)
> -{
> - rte_atomic64_set(v, 0);
> -}
> -#endif
>
> #endif /* _RTE_ATOMIC_I686_H_ */
> diff --git a/lib/eal/x86/include/rte_atomic_64.h
> b/lib/eal/x86/include/rte_atomic_64.h
> index 0a7a2131e0..1cd12695a2 100644
> --- a/lib/eal/x86/include/rte_atomic_64.h
> +++ b/lib/eal/x86/include/rte_atomic_64.h
> @@ -22,163 +22,6 @@
>
> /*------------------------- 64 bit atomic operations -------------------------*/
>
> -#ifndef RTE_FORCE_INTRINSICS
> -static inline int
> -rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
> -{
> - uint8_t res;
> -
> -
> - asm volatile(
> - MPLOCKED
> - "cmpxchgq %[src], %[dst];"
> - "sete %[res];"
> - : [res] "=a" (res), /* output */
> - [dst] "=m" (*dst)
> - : [src] "r" (src), /* input */
> - "a" (exp),
> - "m" (*dst)
> - : "memory"); /* no-clobber list */
> -
> - return res;
> -}
> -
> -static inline uint64_t
> -rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
> -{
> - asm volatile(
> - MPLOCKED
> - "xchgq %0, %1;"
> - : "=r" (val), "=m" (*dst)
> - : "0" (val), "m" (*dst)
> - : "memory"); /* no-clobber list */
> - return val;
> -}
> -
> -static inline void
> -rte_atomic64_init(rte_atomic64_t *v)
> -{
> - v->cnt = 0;
> -}
> -
> -static inline int64_t
> -rte_atomic64_read(rte_atomic64_t *v)
> -{
> - return v->cnt;
> -}
> -
> -static inline void
> -rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
> -{
> - v->cnt = new_value;
> -}
> -
> -static inline void
> -rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
> -{
> - asm volatile(
> - MPLOCKED
> - "addq %[inc], %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : [inc] "ir" (inc), /* input */
> - "m" (v->cnt)
> - );
> -}
> -
> -static inline void
> -rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
> -{
> - asm volatile(
> - MPLOCKED
> - "subq %[dec], %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : [dec] "ir" (dec), /* input */
> - "m" (v->cnt)
> - );
> -}
> -
> -static inline void
> -rte_atomic64_inc(rte_atomic64_t *v)
> -{
> - asm volatile(
> - MPLOCKED
> - "incq %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : "m" (v->cnt) /* input */
> - );
> -}
> -
> -static inline void
> -rte_atomic64_dec(rte_atomic64_t *v)
> -{
> - asm volatile(
> - MPLOCKED
> - "decq %[cnt]"
> - : [cnt] "=m" (v->cnt) /* output */
> - : "m" (v->cnt) /* input */
> - );
> -}
> -
> -static inline int64_t
> -rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
> -{
> - int64_t prev = inc;
> -
> - asm volatile(
> - MPLOCKED
> - "xaddq %[prev], %[cnt]"
> - : [prev] "+r" (prev), /* output */
> - [cnt] "=m" (v->cnt)
> - : "m" (v->cnt) /* input */
> - );
> - return prev + inc;
> -}
> -
> -static inline int64_t
> -rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
> -{
> - return rte_atomic64_add_return(v, -dec);
> -}
> -
> -static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
> -{
> - uint8_t ret;
> -
> - asm volatile(
> - MPLOCKED
> - "incq %[cnt] ; "
> - "sete %[ret]"
> - : [cnt] "+m" (v->cnt), /* output */
> - [ret] "=qm" (ret)
> - );
> -
> - return ret != 0;
> -}
> -
> -static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
> -{
> - uint8_t ret;
> -
> - asm volatile(
> - MPLOCKED
> - "decq %[cnt] ; "
> - "sete %[ret]"
> - : [cnt] "+m" (v->cnt), /* output */
> - [ret] "=qm" (ret)
> - );
> - return ret != 0;
> -}
> -
> -static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
> -{
> - return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
> -}
> -
> -static inline void rte_atomic64_clear(rte_atomic64_t *v)
> -{
> - v->cnt = 0;
> -}
> -#endif
>
> /*------------------------ 128 bit atomic operations -------------------------*/
>
> --
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
> 2.53.0
^ permalink raw reply
* RE: [PATCH v4 03/27] ring: unify memory model on C11, remove atomic32
From: Konstantin Ananyev @ 2026-06-01 18:18 UTC (permalink / raw)
To: Stephen Hemminger, dev@dpdk.org; +Cc: Wathsala Vithanage
In-Reply-To: <20260526232542.620966-4-stephen@networkplumber.org>
> Remove the RTE_USE_C11_MEM_MODEL build switch; C11 atomics are now
> the default for all platforms. Unifies __rte_ring_update_tail into
> the C11 form (atomic_store_release replaces the older rte_smp_wmb +
> plain store on the generic path) and renames rte_ring_generic_pvt.h
> to rte_ring_x86_pvt.h to reflect its new scope.
>
> Also splits the head-move helper into separate ST and MT variants,
> removing the runtime is_st branch from the MT retry loop.
> This gets small boost and scopes the following exception
> more tightly.
>
> Exception: on x86 with GCC, atomic_compare_exchange on the head CAS
> regresses MP/MC contended throughput by ~20% existing hand-written
> cmpxchg. As a workaround, GCC-on-x86 builds use the older
> __sync_bool_compare_and_swap builtin, which generates equivalent
> code to the original asm. Can be reverted if/when GCC gets
> fixed; similar issue was observed in Linux kernel.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
> lib/ring/meson.build | 2 +-
> lib/ring/rte_ring_c11_pvt.h | 75 +++--------
> lib/ring/rte_ring_elem_pvt.h | 125 ++++++++++++++++--
> ..._ring_generic_pvt.h => rte_ring_x86_pvt.h} | 61 ++-------
> lib/ring/soring.c | 15 ++-
> 5 files changed, 158 insertions(+), 120 deletions(-)
> rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_x86_pvt.h} (60%)
>
> diff --git a/lib/ring/meson.build b/lib/ring/meson.build
> index 21f2c12989..b178c963b8 100644
> --- a/lib/ring/meson.build
> +++ b/lib/ring/meson.build
> @@ -9,7 +9,7 @@ indirect_headers += files (
> 'rte_ring_elem.h',
> 'rte_ring_elem_pvt.h',
> 'rte_ring_c11_pvt.h',
> - 'rte_ring_generic_pvt.h',
> + 'rte_ring_x86_pvt.h',
> 'rte_ring_hts.h',
> 'rte_ring_hts_elem_pvt.h',
> 'rte_ring_peek.h',
> diff --git a/lib/ring/rte_ring_c11_pvt.h b/lib/ring/rte_ring_c11_pvt.h
> index 07b6efc416..3efe011f08 100644
> --- a/lib/ring/rte_ring_c11_pvt.h
> +++ b/lib/ring/rte_ring_c11_pvt.h
> @@ -15,35 +15,10 @@
> * @file rte_ring_c11_pvt.h
> * It is not recommended to include this file directly,
> * include <rte_ring.h> instead.
> - * Contains internal helper functions for MP/SP and MC/SC ring modes.
> + * Contains internal helper functions for MP and MC ring modes.
> * For more information please refer to <rte_ring.h>.
> */
>
> -/**
> - * @internal This function updates tail values.
> - */
> -static __rte_always_inline void
> -__rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,
> - uint32_t new_val, uint32_t single, uint32_t enqueue)
> -{
> - RTE_SET_USED(enqueue);
> -
> - /*
> - * If there are other enqueues/dequeues in progress that preceded us,
> - * we need to wait for them to complete
> - */
> - if (!single)
> - rte_wait_until_equal_32((uint32_t *)(uintptr_t)&ht->tail, old_val,
> - rte_memory_order_relaxed);
> -
> - /*
> - * R0: Establishes a synchronizing edge with load-acquire of tail at A1.
> - * Ensures that memory effects by this thread on ring elements array
> - * is observed by a different thread of the other type.
> - */
> - rte_atomic_store_explicit(&ht->tail, new_val,
> rte_memory_order_release);
> -}
> -
> /**
> * @internal This is a helper function that moves the producer/consumer head
> *
> @@ -72,14 +47,11 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,
> uint32_t old_val,
> * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only
> */
> static __rte_always_inline unsigned int
> -__rte_ring_headtail_move_head(struct rte_ring_headtail *d,
> +__rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
> const struct rte_ring_headtail *s, uint32_t capacity,
> - unsigned int is_st, unsigned int n,
> - enum rte_ring_queue_behavior behavior,
> + unsigned int n, enum rte_ring_queue_behavior behavior,
> uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
> {
> - uint32_t stail;
> - int success;
> unsigned int max = n;
>
> /*
> @@ -89,8 +61,7 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail
> *d,
> * d->head.
> * If not, an unsafe partial order may ensue.
> */
> - *old_head = rte_atomic_load_explicit(&d->head,
> - rte_memory_order_acquire);
> + *old_head = rte_atomic_load_explicit(&d->head, rte_memory_order_acquire);
> do {
> /* Reset n to the initial burst count */
> n = max;
> @@ -101,15 +72,14 @@ __rte_ring_headtail_move_head(struct
> rte_ring_headtail *d,
> * ring elements array is observed by the time
> * this thread observes its tail update.
> */
> - stail = rte_atomic_load_explicit(&s->tail,
> - rte_memory_order_acquire);
> + uint32_t stail = rte_atomic_load_explicit(&s->tail,
> rte_memory_order_acquire);
>
> /* The subtraction is done between two unsigned 32bits value
> * (the result is always modulo 32 bits even if we have
> * *old_head > s->tail). So 'entries' is always between 0
> * and capacity (which is < size).
> */
> - *entries = (capacity + stail - *old_head);
> + *entries = capacity + stail - *old_head;
>
> /* check that we have enough room in ring */
> if (unlikely(n > *entries))
> @@ -120,25 +90,20 @@ __rte_ring_headtail_move_head(struct
> rte_ring_headtail *d,
> return 0;
>
> *new_head = *old_head + n;
> - if (is_st) {
> - d->head = *new_head;
> - success = 1;
> - } else
> - /* on failure, *old_head is updated */
> - /*
> - * R1/A2.
> - * R1: Establishes a synchronizing edge with A0 of a
> - * different thread.
> - * A2: Establishes a synchronizing edge with R1 of a
> - * different thread to observe same value for stail
> - * observed by that thread on CAS failure (to retry
> - * with an updated *old_head).
> - */
> - success =
> rte_atomic_compare_exchange_strong_explicit(
> - &d->head, old_head, *new_head,
> - rte_memory_order_release,
> - rte_memory_order_acquire);
> - } while (unlikely(success == 0));
> +
> + /* on failure, *old_head is updated */
> + /*
> + * R1/A2.
> + * R1: Establishes a synchronizing edge with A0 of a
> + * different thread.
> + * A2: Establishes a synchronizing edge with R1 of a
> + * different thread to observe same value for stail
> + * observed by that thread on CAS failure (to retry
> + * with an updated *old_head).
> + */
> + } while (unlikely(!rte_atomic_compare_exchange_strong_explicit(
> + &d->head, old_head, *new_head,
> + rte_memory_order_release,
> rte_memory_order_acquire)));
> return n;
> }
>
> diff --git a/lib/ring/rte_ring_elem_pvt.h b/lib/ring/rte_ring_elem_pvt.h
> index 6eafae121f..9d1da12a92 100644
> --- a/lib/ring/rte_ring_elem_pvt.h
> +++ b/lib/ring/rte_ring_elem_pvt.h
> @@ -299,17 +299,108 @@ __rte_ring_dequeue_elems(struct rte_ring *r,
> uint32_t cons_head,
> cons_head & r->mask, esize, num);
> }
>
> -/* Between load and load. there might be cpu reorder in weak model
> - * (powerpc/arm).
> - * There are 2 choices for the users
> - * 1.use rmb() memory barrier
> - * 2.use one-direction load_acquire/store_release barrier
> - * It depends on performance test results.
> +/**
> + * @internal This function updates tail values.
> */
> -#ifdef RTE_USE_C11_MEM_MODEL
> -#include "rte_ring_c11_pvt.h"
> +static __rte_always_inline void
> +__rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,
> + uint32_t new_val, uint32_t single, uint32_t enqueue)
> +{
> + RTE_SET_USED(enqueue);
> +
> + /*
> + * If there are other enqueues/dequeues in progress that preceded us,
> + * we need to wait for them to complete
> + */
> + if (!single)
> + rte_wait_until_equal_32((uint32_t *)(uintptr_t)&ht->tail, old_val,
> + rte_memory_order_relaxed);
> +
> + /*
> + * R0: Establishes a synchronizing edge with load-acquire of tail at A1.
> + * Ensures that memory effects by this thread on ring elements array
> + * is observed by a different thread of the other type.
> + */
> + rte_atomic_store_explicit(&ht->tail, new_val,
> rte_memory_order_release);
> +}
> +
> +/**
> + * @internal This is a helper function that moves the producer/consumer head
> + *
> + *
> + * This optimized version for single threaded case.
> + *
> + * @param d
> + * A pointer to the headtail structure with head value to be moved
> + * @param s
> + * A pointer to the counter-part headtail structure. Note that this
> + * function only reads tail value from it
> + * @param capacity
> + * Either ring capacity value (for producer), or zero (for consumer)
> + * @param n
> + * The number of elements we want to move head value on
> + * @param behavior
> + * RTE_RING_QUEUE_FIXED: Move on a fixed number of items
> + * RTE_RING_QUEUE_VARIABLE: Move on as many items as possible
> + * @param old_head
> + * Returns head value as it was before the move
> + * @param new_head
> + * Returns the new head value
> + * @param entries
> + * Returns the number of ring entries available BEFORE head was moved
> + * @return
> + * Actual number of objects the head was moved on
> + * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only
> + */
> +static __rte_always_inline unsigned int
> +__rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,
> + const struct rte_ring_headtail *s, uint32_t capacity,
> + unsigned int n, enum rte_ring_queue_behavior behavior,
> + uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
> +{
> + uint32_t stail;
> +
I really like the idea to split _st and _mt move_head into separate functions.
That makes code much cleaner an easier to understand and maintain.
Few comments on actual '_st' implementation below:
> + /*
> + * A0: Establishes a synchronizing edge with R1.
> + * Ensure that this thread observes same values
> + * to stail observed by the thread that updated
> + * d->head.
> + * If not, an unsafe partial order may ensue.
> + */
I believe that comment is not relevant for '_st',
there is no R1 anymore for '_st' - see below,
and no other thread except that one can move the head.
So, there is probably no point to use '_acquire' order here.
> + *old_head = rte_atomic_load_explicit(&d->head,
> rte_memory_order_acquire);
> +
> + /*
> + * A1: Establishes a synchronizing edge with R0.
> + * Ensures that other thread's memory effects on
> + * ring elements array is observed by the time
> + * this thread observes its tail update.
> + */
> + stail = rte_atomic_load_explicit(&s->tail, rte_memory_order_acquire);
> +
> + /* The subtraction is done between two unsigned 32bits value
> + * (the result is always modulo 32 bits even if we have
> + * *old_head > s->tail). So 'entries' is always between 0
> + * and capacity (which is < size).
> + */
> + *entries = capacity + stail - *old_head;
> +
> + /* check that we have enough room in ring */
> + if (unlikely(n > *entries))
> + n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
> +
> + if (n > 0) {
> + *new_head = *old_head + n;
> + d->head = *new_head;
There is a bit of inconsistency with the 'load' operation above:
If we use atomic_load(&d->head. ...) then it would be better to use
atomic_store(&d->head,..., order_relaxed) here.
> + }
> +
> + return n;
> +}
> +
> +/* There are two choices because GCC optimizer does poorly on
> atomic_compare_exchange */
> +#if defined(RTE_TOOLCHAIN_GCC) && defined(RTE_ARCH_X86)
If we still need to use legacy code for x86, I think we need an explcit macro
to enable C11 for x86 (RTE_RING_FORCE_C11 or so):
to make sure that C11 version will still get tested and measured on x86.
> +#include "rte_ring_x86_pvt.h"
> #else
> -#include "rte_ring_generic_pvt.h"
> +#include "rte_ring_c11_pvt.h"
> #endif
I tried to look at compiler output for both cases, most of the code
looks nearly identical, one thing that I noticed:
C11 __rte_ring_headtail_move_head_mt() uses output
parameter: 'uint32_t *old_head' directly within CAS operation.
In x86_64 that cause gcc to generate extra instructions to
store return value of CAS (eax) within 'old_head' memory location,
even when CAS was not successfull and another attempt should be
performed. In some cases, even extra branch can be observed:
https://godbolt.org/z/4dTrqMjYe
In constrast, x86 specific version that uses
__sync_bool_compare_and_swap() doesn't exibit such problem,
as __sync_bool_compare_and_swap() doesn't update the 'old_head'
with new value, and we have to re-read it explicitly on each iteration.
I tried to overcome that problem by using local variable 'head' inside the loop,
and updaing '*old_head' value only at exit.
With such change gcc manages to avoid extra store(/branch),
see __rte_ring_headtail_move_head_mt_c11_v2() in the link above.
Can I ask you to re-run your perf test with the patch:
https://patchwork.dpdk.org/project/dpdk/patch/20260601181509.71007-1-konstantin.ananyev@huawei.com/
applied on top of your changes and see would it help in terms of performance?
From other side - if you'll point me to the exact tests you are running,
I am happy to repeat them on my box.
My preference would be to avoid arch/compiler specific versions, if possible.
> /**
> @@ -341,8 +432,12 @@ __rte_ring_move_prod_head(struct rte_ring *r,
> unsigned int is_sp,
> uint32_t *old_head, uint32_t *new_head,
> uint32_t *free_entries)
> {
> - return __rte_ring_headtail_move_head(&r->prod, &r->cons, r->capacity,
> - is_sp, n, behavior, old_head, new_head, free_entries);
> + if (is_sp)
> + return __rte_ring_headtail_move_head_st(&r->prod, &r->cons,
> r->capacity,
> + n, behavior, old_head, new_head, free_entries);
> + else
> + return __rte_ring_headtail_move_head_mt(&r->prod, &r->cons,
> r->capacity,
> + n, behavior, old_head, new_head, free_entries);
> }
>
> /**
> @@ -374,8 +469,12 @@ __rte_ring_move_cons_head(struct rte_ring *r,
> unsigned int is_sc,
> uint32_t *old_head, uint32_t *new_head,
> uint32_t *entries)
> {
> - return __rte_ring_headtail_move_head(&r->cons, &r->prod, 0,
> - is_sc, n, behavior, old_head, new_head, entries);
> + if (is_sc)
> + return __rte_ring_headtail_move_head_st(&r->cons, &r->prod,
> 0,
> + n, behavior, old_head, new_head, entries);
> + else
> + return __rte_ring_headtail_move_head_mt(&r->cons, &r->prod,
> 0,
> + n, behavior, old_head, new_head, entries);
> }
>
> /**
> diff --git a/lib/ring/rte_ring_generic_pvt.h b/lib/ring/rte_ring_x86_pvt.h
> similarity index 60%
> rename from lib/ring/rte_ring_generic_pvt.h
> rename to lib/ring/rte_ring_x86_pvt.h
> index affd2d5ba7..c8de108bbd 100644
> --- a/lib/ring/rte_ring_generic_pvt.h
> +++ b/lib/ring/rte_ring_x86_pvt.h
> @@ -7,39 +7,19 @@
> * Used as BSD-3 Licensed with permission from Kip Macy.
> */
>
> -#ifndef _RTE_RING_GENERIC_PVT_H_
> -#define _RTE_RING_GENERIC_PVT_H_
> +#ifndef _RTE_RING_X86_PVT_H_
> +#define _RTE_RING_X86_PVT_H_
>
> /**
> - * @file rte_ring_generic_pvt.h
> + * @file rte_ring_x86_pvt.h
> * It is not recommended to include this file directly,
> * include <rte_ring.h> instead.
> - * Contains internal helper functions for MP/SP and MC/SC ring modes.
> - * For more information please refer to <rte_ring.h>.
> + *
> + * Contains internal helper functions for MP and MC ring modes.
> + * It is GCC specific to workaround poor optimizer handling of C11 atomic
> + * compare_exchange.
> */
>
> -/**
> - * @internal This function updates tail values.
> - */
> -static __rte_always_inline void
> -__rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,
> - uint32_t new_val, uint32_t single, uint32_t enqueue)
> -{
> - if (enqueue)
> - rte_smp_wmb();
> - else
> - rte_smp_rmb();
> - /*
> - * If there are other enqueues/dequeues in progress that preceded us,
> - * we need to wait for them to complete
> - */
> - if (!single)
> - rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail,
> old_val,
> - rte_memory_order_relaxed);
> -
> - ht->tail = new_val;
> -}
> -
> /**
> * @internal This is a helper function that moves the producer/consumer head
> *
> @@ -50,8 +30,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,
> uint32_t old_val,
> * function only reads tail value from it
> * @param capacity
> * Either ring capacity value (for producer), or zero (for consumer)
> - * @param is_st
> - * Indicates whether multi-thread safe path is needed or not
> * @param n
> * The number of elements we want to move head value on
> * @param behavior
> @@ -68,14 +46,13 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,
> uint32_t old_val,
> * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only
> */
> static __rte_always_inline unsigned int
> -__rte_ring_headtail_move_head(struct rte_ring_headtail *d,
> +__rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
> const struct rte_ring_headtail *s, uint32_t capacity,
> - unsigned int is_st, unsigned int n,
> + unsigned int n,
> enum rte_ring_queue_behavior behavior,
> uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
> {
> unsigned int max = n;
> - int success;
>
> do {
> /* Reset n to the initial burst count */
> @@ -83,18 +60,13 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail
> *d,
>
> *old_head = d->head;
>
> - /* add rmb barrier to avoid load/load reorder in weak
> - * memory model. It is noop on x86
> - */
> - rte_smp_rmb();
> -
> /*
> * The subtraction is done between two unsigned 32bits value
> * (the result is always modulo 32 bits even if we have
> * *old_head > s->tail). So 'entries' is always between 0
> * and capacity (which is < size).
> */
> - *entries = (capacity + s->tail - *old_head);
> + *entries = capacity + s->tail - *old_head;
>
> /* check that we have enough room in ring */
> if (unlikely(n > *entries))
> @@ -105,15 +77,10 @@ __rte_ring_headtail_move_head(struct
> rte_ring_headtail *d,
> return 0;
>
> *new_head = *old_head + n;
> - if (is_st) {
> - d->head = *new_head;
> - success = 1;
> - } else
> - success = rte_atomic32_cmpset(
> - (uint32_t *)(uintptr_t)&d->head,
> - *old_head, *new_head);
> - } while (unlikely(success == 0));
> + } while (unlikely(!__sync_bool_compare_and_swap(
> + (uint32_t *)(uintptr_t)&d->head,
> + *old_head, *new_head)));
> return n;
> }
>
> -#endif /* _RTE_RING_GENERIC_PVT_H_ */
> +#endif /* _RTE_RING_X86_PVT_H_ */
> diff --git a/lib/ring/soring.c b/lib/ring/soring.c
> index 3b90521bdb..0e8bbc03c1 100644
> --- a/lib/ring/soring.c
> +++ b/lib/ring/soring.c
> @@ -135,9 +135,12 @@ __rte_soring_move_prod_head(struct rte_soring *r,
> uint32_t num,
>
> switch (st) {
> case RTE_RING_SYNC_ST:
> + n = __rte_ring_headtail_move_head_st(&r->prod.ht, &r-
> >cons.ht,
> + r->capacity, num, behavior, head, next, free);
> + break;
> case RTE_RING_SYNC_MT:
> - n = __rte_ring_headtail_move_head(&r->prod.ht, &r->cons.ht,
> - r->capacity, st, num, behavior, head, next, free);
> + n = __rte_ring_headtail_move_head_mt(&r->prod.ht, &r-
> >cons.ht,
> + r->capacity, num, behavior, head, next, free);
> break;
> case RTE_RING_SYNC_MT_RTS:
> n = __rte_ring_rts_move_head(&r->prod.rts, &r->cons.ht,
> @@ -168,9 +171,13 @@ __rte_soring_move_cons_head(struct rte_soring *r,
> uint32_t stage, uint32_t num,
>
> switch (st) {
> case RTE_RING_SYNC_ST:
> + n = __rte_ring_headtail_move_head_st(&r->cons.ht,
> + &r->stage[stage].ht, 0, num, behavior,
> + head, next, avail);
> + break;
> case RTE_RING_SYNC_MT:
> - n = __rte_ring_headtail_move_head(&r->cons.ht,
> - &r->stage[stage].ht, 0, st, num, behavior,
> + n = __rte_ring_headtail_move_head_mt(&r->cons.ht,
> + &r->stage[stage].ht, 0, num, behavior,
> head, next, avail);
> break;
> case RTE_RING_SYNC_MT_RTS:
> --
> 2.53.0
^ permalink raw reply
* [PATCH] ring: avoid extra store at move head
From: Konstantin Ananyev @ 2026-06-01 18:15 UTC (permalink / raw)
To: dev; +Cc: stephen
C11 __rte_ring_headtail_move_head_mt() uses output
parameter: 'uint32_t *old_head' directly within CAS operation.
In x86_64 that cause gcc to generate extra instructions to
store return value of CAS (eax) within 'old_head' memory location,
even when CAS was not successful and another attempt should be
performed. In some cases, even extra branch can be observed.
To be more specific the code like that is generated:
// start of 'do { } while();' loop
.L2
...
lock cmpxchgl %r8d, (%rdi)
jne .L17 //
.L1: // <---- successful completion of CAS, finish
movl %edx, %eax
ret
.L17: // <---- unsuccessful completion of CAS, repeat
movl %eax, (%r9)
jmp .L2
In constrast, x86 specific version that uses
__sync_bool_compare_and_swap() doesn't exibit such problem,
as __sync_bool_compare_and_swap() doesn't update the 'old_head'
with new value, and we have to re-read it explicitly on each iteration.
Overcome that problem by using local variable 'head' inside the loop,
and updaing '*old_head' value only at exit.
With such change gcc manages to avoid extra store(/branch).
Depends-on: series-38225 ("deprecate rte_atomicNN family")
Signed-off-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
lib/ring/rte_ring_c11_pvt.h | 19 +++++++++++--------
lib/ring/rte_ring_elem_pvt.h | 5 -----
2 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/lib/ring/rte_ring_c11_pvt.h b/lib/ring/rte_ring_c11_pvt.h
index 3efe011f08..ee98155bea 100644
--- a/lib/ring/rte_ring_c11_pvt.h
+++ b/lib/ring/rte_ring_c11_pvt.h
@@ -52,6 +52,7 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
unsigned int n, enum rte_ring_queue_behavior behavior,
uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
{
+ uint32_t head;
unsigned int max = n;
/*
@@ -61,7 +62,7 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
* d->head.
* If not, an unsafe partial order may ensue.
*/
- *old_head = rte_atomic_load_explicit(&d->head, rte_memory_order_acquire);
+ head = rte_atomic_load_explicit(&d->head, rte_memory_order_acquire);
do {
/* Reset n to the initial burst count */
n = max;
@@ -76,10 +77,10 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
/* The subtraction is done between two unsigned 32bits value
* (the result is always modulo 32 bits even if we have
- * *old_head > s->tail). So 'entries' is always between 0
+ * head > s->tail). So 'entries' is always between 0
* and capacity (which is < size).
*/
- *entries = capacity + stail - *old_head;
+ *entries = capacity + stail - head;
/* check that we have enough room in ring */
if (unlikely(n > *entries))
@@ -87,11 +88,11 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
0 : *entries;
if (n == 0)
- return 0;
+ break;
- *new_head = *old_head + n;
+ *new_head = head + n;
- /* on failure, *old_head is updated */
+ /* on failure, head is updated */
/*
* R1/A2.
* R1: Establishes a synchronizing edge with A0 of a
@@ -99,11 +100,13 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
* A2: Establishes a synchronizing edge with R1 of a
* different thread to observe same value for stail
* observed by that thread on CAS failure (to retry
- * with an updated *old_head).
+ * with an updated head).
*/
} while (unlikely(!rte_atomic_compare_exchange_strong_explicit(
- &d->head, old_head, *new_head,
+ &d->head, &head, *new_head,
rte_memory_order_release, rte_memory_order_acquire)));
+
+ *old_head = head;
return n;
}
diff --git a/lib/ring/rte_ring_elem_pvt.h b/lib/ring/rte_ring_elem_pvt.h
index 9d1da12a92..51176b0405 100644
--- a/lib/ring/rte_ring_elem_pvt.h
+++ b/lib/ring/rte_ring_elem_pvt.h
@@ -396,12 +396,7 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,
return n;
}
-/* There are two choices because GCC optimizer does poorly on atomic_compare_exchange */
-#if defined(RTE_TOOLCHAIN_GCC) && defined(RTE_ARCH_X86)
-#include "rte_ring_x86_pvt.h"
-#else
#include "rte_ring_c11_pvt.h"
-#endif
/**
* @internal This function updates the producer head for enqueue
--
2.51.0
^ permalink raw reply related
* [PATCH v2] doc, eal, devtools: discourage new __rte_always_inline
From: Stephen Hemminger @ 2026-06-01 17:21 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Thomas Monjalon
In-Reply-To: <20260527163734.599602-1-stephen@networkplumber.org>
Modern compilers at -O2 make good inlining decisions for small
static inline functions; forced inlining via __rte_always_inline
should be reserved for cases where it is required for correctness
or for documented measured performance reasons.
Document the policy in the coding style guide and add a
checkpatches.sh entry that flags when new uses of the attribute
are introduced. Checkpatches is not an absolute blocker to
acceptance, only an indicator that more review is needed.
Add additional comments about use of __rte_always_inline,
__rte_noinline, __rte_hot, and __rte_cold to the rte_common.h
to aid developers.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
devtools/checkpatches.sh | 8 +++++
doc/guides/contributing/coding_style.rst | 26 ++++++++++++++++-
lib/eal/include/rte_common.h | 37 ++++++++++++++++++++++--
3 files changed, 68 insertions(+), 3 deletions(-)
diff --git a/devtools/checkpatches.sh b/devtools/checkpatches.sh
index f5dd77443f..2a3d364178 100755
--- a/devtools/checkpatches.sh
+++ b/devtools/checkpatches.sh
@@ -137,6 +137,14 @@ check_forbidden_additions() { # <patch>
-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
"$1" || res=1
+ # forbid new use of __rte_always_inline
+ awk -v FOLDERS="lib drivers app examples" \
+ -v EXPRESSIONS='\\<__rte_always_inline\\>' \
+ -v RET_ON_FAIL=1 \
+ -v MESSAGE='Adding __rte_always_inline; prefer plain inline' \
+ -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
+ "$1" || res=1
+
# refrain from using compiler __rte_atomic_thread_fence()
# It should be avoided on x86 for SMP case.
awk -v FOLDERS="lib drivers app examples" \
diff --git a/doc/guides/contributing/coding_style.rst b/doc/guides/contributing/coding_style.rst
index 243a3c2959..8f9e77d1c1 100644
--- a/doc/guides/contributing/coding_style.rst
+++ b/doc/guides/contributing/coding_style.rst
@@ -747,11 +747,35 @@ Static Variables and Functions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* All functions and variables that are local to a file must be declared as ``static`` because it can often help the compiler to do some optimizations (such as, inlining the code).
-* Functions that should be inlined should to be declared as ``static inline`` and can be defined in a .c or a .h file.
+* Functions that should be inlined should be declared as ``static inline`` and can be defined in a .c or a .h file.
.. note::
Static functions defined in a header file must be declared as ``static inline`` in order to prevent compiler warnings about the function being unused.
+Use of ``__rte_always_inline``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``__rte_always_inline`` attribute forces the compiler to inline a function regardless of its size or call-graph heuristics.
+Prefer plain ``inline`` (or no annotation at all for static functions) and let the compiler decide.
+Modern compilers at ``-O2`` make good inlining decisions for small ``static inline`` functions in headers,
+and forced inlining can hurt performance by inflating function bodies, increasing register pressure, and overriding profile-guided optimization.
+
+``__rte_always_inline`` should only be used when one of the following applies:
+
+* The function contains ``__rte_constant()`` checks that gate a constant-folded fast path,
+ and the optimization is lost if the function is not inlined into the caller.
+ Examples include byte-order helpers and length-dispatched copy/compare routines.
+
+* The function wraps inline assembly or a compiler intrinsic whose correctness depends on being inlined into the caller's register context (for example, intrinsics requiring a compile-time constant argument).
+
+* Measurement on a representative workload shows that the annotation is required to retain performance, and the reason is documented in the commit message that introduces it.
+
+Each use must be justified at the point it is introduced. Adding ``__rte_always_inline`` because nearby code uses it is not a justification;
+if the constant or intrinsic that requires inlining is several call levels up the call chain,
+restructure the code rather than annotating the entire chain.
+
+The complementary attribute ``__rte_noinline`` is useful for explicitly marking cold paths (error handling, initialization, slow-path fallbacks) where outlining the function can reduce instruction-cache pressure on the hot path.
+
Const Attribute
~~~~~~~~~~~~~~~
diff --git a/lib/eal/include/rte_common.h b/lib/eal/include/rte_common.h
index 71415346cf..e358be7fcf 100644
--- a/lib/eal/include/rte_common.h
+++ b/lib/eal/include/rte_common.h
@@ -482,7 +482,22 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
#endif
/**
- * Force a function to be inlined
+ * Force a function to be inlined, regardless of the compiler's size and
+ * call-graph heuristics.
+ *
+ * Prefer plain @c inline (or no annotation on a static function) and let the compiler decide.
+ * Modern compilers at -O2 inline small static functions well,
+ * and forcing it can hurt by inflating call sites, raising register pressure,
+ * and overriding profile-guided optimization.
+ *
+ * Reserve this attribute for cases where inlining is required for
+ * correctness, or for a documented and measured performance reason, e.g.
+ * - a constant-folded fast path gated by @c __rte_constant() that is lost
+ * unless the function is inlined into the caller;
+ * - a wrapper around inline asm or an intrinsic that needs a
+ * compile-time-constant argument from the caller's context.
+ *
+ * See the DPDK coding style guide for the full policy.
*/
#ifdef RTE_TOOLCHAIN_MSVC
#define __rte_always_inline __forceinline
@@ -491,7 +506,11 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
#endif
/**
- * Force a function to be noinlined
+ * Force a function not to be inlined.
+ *
+ * Useful for explicitly outlining cold paths such as error handling,
+ * initialization, slow-path fallbacks, so they do not bloat the hot path
+ * or add to its instruction-cache footprint.
*/
#ifdef RTE_TOOLCHAIN_MSVC
#define __rte_noinline __declspec(noinline)
@@ -501,6 +520,12 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
/**
* Hint function in the hot path
+ *
+ * The compiler may optimize the function more aggressively, treat calls
+ * to it as likely for branch prediction, and group it with other hot
+ * functions to improve instruction-cache locality. This affects code
+ * placement and prediction, not inlining; combine with an inline
+ * annotation if both are wanted.
*/
#ifdef RTE_TOOLCHAIN_MSVC
#define __rte_hot
@@ -510,6 +535,14 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
/**
* Hint function in the cold path
+ *
+ * The compiler optimizes the function for size rather than speed,
+ * marks branches that reach it as unlikely, and may move it to a separate
+ * section to keep it off the hot path and reduce instruction-cache
+ * pressure there.
+ *
+ * Suitable for error handling, logging, and setup/teardown code.
+ * Functions marked @c __rte_noreturn are already treated as cold.
*/
#ifdef RTE_TOOLCHAIN_MSVC
#define __rte_cold
--
2.53.0
^ permalink raw reply related
* Re: [v2] dts: add support for no link topology
From: Andrew Bailey @ 2026-06-01 17:16 UTC (permalink / raw)
To: Patrick Robb; +Cc: dev, dmarx, luca.vizzarro, probb
In-Reply-To: <20260530203757.22139-1-patrickrobb1997@gmail.com>
[-- Attachment #1: Type: text/plain, Size: 822 bytes --]
Hi Patrick, Thanks for the review
> I don't understand the change above. Surely we do want to set the custom
> meson args for ICE DUTs when there is a sut_port_ingress, not when there
> isn't a sut_port_ingress, right?
>
> Yes, this was something I overlooked and will be updated in the next patch.
> Also, does the topology type affect whether we want to use these custom
> args, if the SUT driver is ice? I guess in a weird case we could have an
> ethdev test using an ice DUT which does not need a TG?
>
> My understanding is that if the link topology is no link, there will be no
ingress and the ports on the NIC would not be used for traffic. I do not
believe this argument will still be required if no traffic is being
handled. Please let me know if you disagree with this approach or if I am
missing something.
[-- Attachment #2: Type: text/html, Size: 1219 bytes --]
^ permalink raw reply
* Re: [PATCH v5 1/1] net/mana: add device reset support
From: Stephen Hemminger @ 2026-06-01 16:58 UTC (permalink / raw)
To: Wei Hu; +Cc: dev, longli, weh
In-Reply-To: <20260529142648.148407-2-weh@linux.microsoft.com>
On Fri, 29 May 2026 07:26:48 -0700
Wei Hu <weh@linux.microsoft.com> wrote:
> From: Wei Hu <weh@microsoft.com>
>
> Add support for handling hardware reset events in the MANA driver.
> When the MANA kernel driver receives a hardware service event, it
> initiates a device reset and notifies userspace via
> IBV_EVENT_DEVICE_FATAL. The DPDK driver handles this by performing
> an automatic teardown and recovery sequence.
>
> The reset flow has two phases. In the enter phase, running on the
> EAL interrupt thread, the driver transitions the device state,
> waits for data path threads to reach a quiescent state using RCU,
> stops queues, tears down IB resources, and frees per-queue MR
> caches. A control thread is then spawned to handle the exit phase:
> it waits for the hardware to recover, unregisters the interrupt
> handler, re-probes the PCI device, reinitializes MR caches, and
> restarts queues.
>
> A per-device mutex serializes the reset path with ethdev
> operations. The mutex uses PTHREAD_PROCESS_SHARED for multi-process
> support and is held across blocking IB verbs calls. Operations that
> cannot wait (configure, queue setup) return -EBUSY during reset,
> while dev_stop and dev_close join the reset thread before acquiring
> the lock to ensure proper sequencing. A CAS-based helper prevents
> double-join of the reset thread.
>
> Multi-process support is included: secondary processes unmap and
> remap doorbell pages via IPC during the reset enter and exit
> phases. Data path functions in both primary and secondary
> processes check the device state atomically and return early when
> the device is not active. RCU quiescent state tracking uses
> per-queue thread IDs in shared hugepage memory, covering both
> primary and secondary process data path threads.
>
> The driver uses ethdev recovery events to notify upper layers
> (e.g. netvsc) of the reset lifecycle: RTE_ETH_EVENT_ERR_RECOVERING
> on entry, RTE_ETH_EVENT_RECOVERY_SUCCESS or
> RTE_ETH_EVENT_RECOVERY_FAILED on completion. A PCI device removal
> event callback distinguishes hot-remove from service reset.
>
> Documentation for the device reset feature is added in the MANA
> NIC guide and the 26.07 release notes.
>
> Signed-off-by: Wei Hu <weh@microsoft.com>
> ---
I went a deeper with AI and tried to figure out a good way
to do what the driver is trying to do without reinventing so much.
This reset logic is considerably more complex than other DPDK drivers,
and most of the complexity looks self-inflicted. A few specific things.
The RCU use is not really RCU. thread_online/offline are called on every
rx/tx burst, and the "thread" token is the queue index, not a thread. So
it is a per-queue in-use flag paid for on the hottest path, plus a new
library dependency, to express "wait until no queue is mid-burst" -- which
the driver already half does by swapping the burst function to
mana_*_burst_removed and checking dev_state. Please drop the rcu use. If
you must drain readers, a per-queue atomic flag is lighter and local; the
fast path already has the dev_state acquire-load it needs.
The data path only needs the atomic, and that part is fine. The lock is
legitimate for serializing the teardown/rebuild against control ops, but
the way it is used is the problem: it is acquired in mana_intr_handler,
released in mana_reset_enter, re-acquired in mana_reset_thread, and
released in mana_reset_exit_delay. That cross-function, cross-thread
handoff is exactly why every function needs __rte_no_thread_safety_analysis.
Acquire and release the lock in the same function and the annotations all
go away. Turning off thread-safety analysis needs strong justification and
this does not have it.
Wrapping the ops in MANA_OPS_*_LOCK macros hides the lock/state protocol.
A single explicit helper at the top of each op is just as terse and stays
analyzable.
Two real bugs:
- Recovery and INTR_RMV events are sent via rte_eth_dev_callback_process
while reset_ops_lock is held. An app handling INTR_RMV or RECOVERY_FAILED
by calling dev_stop/dev_close will re-enter the lock (non-recursive) and
deadlock; on the recovery path the callback runs on the reset thread so
it also tries to join itself. Emit these events after dropping the lock,
as ERR_RECOVERING already does.
- thread_online is taken at the top of the burst functions but
thread_offline is only on some return paths. Any early return that
misses it leaves a token non-quiescent and rte_rcu_qsbr_check() in
mana_reset_enter spins forever. This is the kind of breakage the per-burst
bracketing invites -- another reason to drop it.
Also: reset_ops_lock is held across ibv_close_device and the PCI re-probe
(blocking under a sleeping mutex), and rte_alarm.h is included but no
longer used.
Please look at how hns3 or mlx5 structure reset/recovery. Matching the
common pattern means fixes can be made across drivers at once.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox