* [PATCH v6 08/11] test/bpf: test loading ELF file from memory
From: Marat Khalili @ 2026-06-17 19:44 UTC (permalink / raw)
To: Konstantin Ananyev; +Cc: dev
In-Reply-To: <20260617194425.12690-1-marat.khalili@huawei.com>
Run each subtest in test_bpf_elf twice: the old way loading ELF images
via temporary file, and using the new rte_bpf_load_ex API to load them
directly from memory.
In tests loading port/queue filters use new rte_bpf_eth_(rx|tx)_install
API to install an already loaded (via one of the ways) BPF program.
Signed-off-by: Marat Khalili <marat.khalili@huawei.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
app/test/test_bpf.c | 194 ++++++++++++++++++++++++++------------------
1 file changed, 114 insertions(+), 80 deletions(-)
diff --git a/app/test/test_bpf.c b/app/test/test_bpf.c
index 8c98bf111e5e..5cf40d334e7c 100644
--- a/app/test/test_bpf.c
+++ b/app/test/test_bpf.c
@@ -3977,12 +3977,61 @@ create_temp_bpf_file(const uint8_t *data, size_t size, const char *name)
#include "test_bpf_load.h"
+/* Function loading BPF program from ELF image in memory. */
+typedef struct rte_bpf *
+(*load_elf_image_t)(const void *data, size_t size, const char *section,
+ const struct rte_bpf_xsym *xsym, uint32_t nb_xsym, const struct rte_bpf_arg *prog_arg);
+
+/* Load BPF program by writing ELF image to temporary file and opening this file. */
+static struct rte_bpf *
+load_elf_image_temp_file(const void *data, size_t size, const char *section,
+ const struct rte_bpf_xsym *xsym, uint32_t nb_xsym, const struct rte_bpf_arg *prog_arg)
+{
+ /* Create temp file from embedded BPF object */
+ char *tmpfile = create_temp_bpf_file(data, size, "test");
+ if (tmpfile == NULL) {
+ rte_errno = EIO;
+ return NULL;
+ }
+
+ /* Try to load BPF program from temp file */
+ const struct rte_bpf_prm prm = {
+ .xsym = xsym,
+ .nb_xsym = nb_xsym,
+ .prog_arg = *prog_arg,
+ };
+
+ struct rte_bpf *bpf = rte_bpf_elf_load(&prm, tmpfile, section);
+ unlink(tmpfile);
+ free(tmpfile);
+
+ return bpf;
+}
+
+/* Load BPF program by calling rte_bpf_load_ex and specifying image as the origin. */
+static struct rte_bpf *
+load_elf_image_direct(const void *data, size_t size, const char *section,
+ const struct rte_bpf_xsym *xsym, uint32_t nb_xsym, const struct rte_bpf_arg *prog_arg)
+{
+ return rte_bpf_load_ex(&(struct rte_bpf_prm_ex){
+ .sz = sizeof(struct rte_bpf_prm_ex),
+ .origin = RTE_BPF_ORIGIN_ELF_MEMORY,
+ .elf_memory.data = data,
+ .elf_memory.size = size,
+ .elf_memory.section = section,
+ .xsym = xsym,
+ .nb_xsym = nb_xsym,
+ .prog_arg[0] = *prog_arg,
+ .nb_prog_arg = 1,
+ });
+}
+
/*
* Test loading BPF program from an object file.
* This test uses same arguments as previous test_call1 example.
*/
static int
-test_bpf_elf_load(void)
+test_bpf_elf_load(load_elf_image_t load_elf_image)
{
static const char test_section[] = "call1";
uint8_t tbuf[sizeof(struct dummy_vect8)];
@@ -4010,28 +4059,15 @@ test_bpf_elf_load(void)
},
},
};
- int ret;
-
- /* Create temp file from embedded BPF object */
- char *tmpfile = create_temp_bpf_file(app_test_bpf_load_o,
- app_test_bpf_load_o_len,
- "load");
- if (tmpfile == NULL)
- return -1;
-
- /* Try to load BPF program from temp file */
- const struct rte_bpf_prm prm = {
- .xsym = xsym,
- .nb_xsym = RTE_DIM(xsym),
- .prog_arg = {
- .type = RTE_BPF_ARG_PTR,
- .size = sizeof(tbuf),
- },
+ static const struct rte_bpf_arg prog_arg = {
+ .type = RTE_BPF_ARG_PTR,
+ .size = sizeof(tbuf),
};
+ struct rte_bpf *bpf;
+ int ret;
- struct rte_bpf *bpf = rte_bpf_elf_load(&prm, tmpfile, test_section);
- unlink(tmpfile);
- free(tmpfile);
+ bpf = load_elf_image(app_test_bpf_load_o, app_test_bpf_load_o_len, test_section,
+ xsym, RTE_DIM(xsym), &prog_arg);
/* If libelf support is not available */
if (bpf == NULL && rte_errno == ENOTSUP)
@@ -4174,22 +4210,28 @@ setup_mbufs(struct rte_mbuf *burst[], unsigned int n)
return tcp_count;
}
-static int bpf_tx_test(uint16_t port, const char *tmpfile, struct rte_mempool *pool,
- const char *section, uint32_t flags)
+static int bpf_tx_test(uint16_t port, struct rte_mempool *pool, load_elf_image_t load_elf_image,
+ const char *section, uint32_t flags)
{
- const struct rte_bpf_prm prm = {
- .prog_arg = {
- .type = RTE_BPF_ARG_PTR,
- .size = sizeof(struct dummy_net),
- },
+ static const struct rte_bpf_arg prog_arg = {
+ .type = RTE_BPF_ARG_PTR,
+ .size = sizeof(struct dummy_net),
};
+ struct rte_bpf *bpf;
int ret;
- /* Try to load BPF TX program from temp file */
- ret = rte_bpf_eth_tx_elf_load(port, 0, &prm, tmpfile, section, flags);
+ /* Try to load BPF program from image */
+ bpf = load_elf_image(app_test_bpf_filter_o, app_test_bpf_filter_o_len, section,
+ NULL, 0, &prog_arg);
+ TEST_ASSERT_NOT_NULL(bpf, "failed to load BPF filter from image, error=%d:(%s)\n",
+ rte_errno, rte_strerror(rte_errno));
+
+ /* Try to install loaded BPF program */
+ ret = rte_bpf_eth_tx_install(port, 0, bpf, flags);
if (ret != 0) {
- printf("%s@%d: failed to load BPF filter from file=%s error=%d:(%s)\n",
- __func__, __LINE__, tmpfile, rte_errno, rte_strerror(rte_errno));
+ printf("%s@%d: failed to install BPF filter, error=%d:(%s)\n",
+ __func__, __LINE__, rte_errno, rte_strerror(rte_errno));
+ rte_bpf_destroy(bpf);
return ret;
}
@@ -4217,10 +4259,9 @@ static int bpf_tx_test(uint16_t port, const char *tmpfile, struct rte_mempool *p
/* Test loading a transmit filter which only allows IPv4 packets */
static int
-test_bpf_elf_tx_load(void)
+test_bpf_elf_tx_load(load_elf_image_t load_elf_image)
{
static const char null_dev[] = "net_null_bpf0";
- char *tmpfile = NULL;
struct rte_mempool *mb_pool = NULL;
uint16_t port = UINT16_MAX;
int ret;
@@ -4237,27 +4278,17 @@ test_bpf_elf_tx_load(void)
if (ret != 0)
goto fail;
- /* Create temp file from embedded BPF object */
- tmpfile = create_temp_bpf_file(app_test_bpf_filter_o, app_test_bpf_filter_o_len, "tx");
- if (tmpfile == NULL)
- goto fail;
-
/* Do test with VM */
- ret = bpf_tx_test(port, tmpfile, mb_pool, "filter", 0);
+ ret = bpf_tx_test(port, mb_pool, load_elf_image, "filter", 0);
if (ret != 0)
goto fail;
/* Repeat with JIT */
- ret = bpf_tx_test(port, tmpfile, mb_pool, "filter", RTE_BPF_ETH_F_JIT);
+ ret = bpf_tx_test(port, mb_pool, load_elf_image, "filter", RTE_BPF_ETH_F_JIT);
if (ret == 0)
printf("%s: TX ELF load test passed\n", __func__);
fail:
- if (tmpfile) {
- unlink(tmpfile);
- free(tmpfile);
- }
-
if (port != UINT16_MAX)
rte_vdev_uninit(null_dev);
@@ -4272,23 +4303,29 @@ test_bpf_elf_tx_load(void)
}
/* Test loading a receive filter */
-static int bpf_rx_test(uint16_t port, const char *tmpfile, struct rte_mempool *pool,
- const char *section, uint32_t flags, uint16_t expected)
+static int bpf_rx_test(uint16_t port, struct rte_mempool *pool, load_elf_image_t load_elf_image,
+ const char *section, uint32_t flags, uint16_t expected)
{
- struct rte_mbuf *pkts[BPF_TEST_BURST];
- const struct rte_bpf_prm prm = {
- .prog_arg = {
- .type = RTE_BPF_ARG_PTR,
- .size = sizeof(struct dummy_net),
- },
+ static const struct rte_bpf_arg prog_arg = {
+ .type = RTE_BPF_ARG_PTR,
+ .size = sizeof(struct dummy_net),
};
+ struct rte_mbuf *pkts[BPF_TEST_BURST];
+ struct rte_bpf *bpf;
int ret;
- /* Load BPF program to drop all packets */
- ret = rte_bpf_eth_rx_elf_load(port, 0, &prm, tmpfile, section, flags);
+ /* Try to load BPF program from image */
+ bpf = load_elf_image(app_test_bpf_filter_o, app_test_bpf_filter_o_len, section,
+ NULL, 0, &prog_arg);
+ TEST_ASSERT_NOT_NULL(bpf, "failed to load BPF filter from image, error=%d:(%s)\n",
+ rte_errno, rte_strerror(rte_errno));
+
+ /* Try to install loaded BPF program */
+ ret = rte_bpf_eth_rx_install(port, 0, bpf, flags);
if (ret != 0) {
- printf("%s@%d: failed to load BPF filter from file=%s error=%d:(%s)\n",
- __func__, __LINE__, tmpfile, rte_errno, rte_strerror(rte_errno));
+ printf("%s@%d: failed to install BPF filter, error=%d:(%s)\n",
+ __func__, __LINE__, rte_errno, rte_strerror(rte_errno));
+ rte_bpf_destroy(bpf);
return ret;
}
@@ -4311,11 +4348,10 @@ static int bpf_rx_test(uint16_t port, const char *tmpfile, struct rte_mempool *p
/* Test loading a receive filters, first with drop all and then with allow all packets */
static int
-test_bpf_elf_rx_load(void)
+test_bpf_elf_rx_load(load_elf_image_t load_elf_image)
{
static const char null_dev[] = "net_null_bpf0";
struct rte_mempool *pool = NULL;
- char *tmpfile = NULL;
uint16_t port = UINT16_MAX;
int ret;
@@ -4331,28 +4367,23 @@ test_bpf_elf_rx_load(void)
if (ret != 0)
goto fail;
- /* Create temp file from embedded BPF object */
- tmpfile = create_temp_bpf_file(app_test_bpf_filter_o, app_test_bpf_filter_o_len, "rx");
- if (tmpfile == NULL)
- goto fail;
-
/* Do test with VM */
- ret = bpf_rx_test(port, tmpfile, pool, "drop", 0, 0);
+ ret = bpf_rx_test(port, pool, load_elf_image, "drop", 0, 0);
if (ret != 0)
goto fail;
/* Repeat with JIT */
- ret = bpf_rx_test(port, tmpfile, pool, "drop", RTE_BPF_ETH_F_JIT, 0);
+ ret = bpf_rx_test(port, pool, load_elf_image, "drop", RTE_BPF_ETH_F_JIT, 0);
if (ret != 0)
goto fail;
/* Repeat with allow all */
- ret = bpf_rx_test(port, tmpfile, pool, "allow", 0, BPF_TEST_BURST);
+ ret = bpf_rx_test(port, pool, load_elf_image, "allow", 0, BPF_TEST_BURST);
if (ret != 0)
goto fail;
/* Repeat with JIT */
- ret = bpf_rx_test(port, tmpfile, pool, "allow", RTE_BPF_ETH_F_JIT, BPF_TEST_BURST);
+ ret = bpf_rx_test(port, pool, load_elf_image, "allow", RTE_BPF_ETH_F_JIT, BPF_TEST_BURST);
if (ret != 0)
goto fail;
@@ -4364,11 +4395,6 @@ test_bpf_elf_rx_load(void)
"Mempool available %u != %u leaks?", avail, BPF_TEST_POOLSIZE);
fail:
- if (tmpfile) {
- unlink(tmpfile);
- free(tmpfile);
- }
-
if (port != UINT16_MAX)
rte_vdev_uninit(null_dev);
@@ -4381,13 +4407,21 @@ test_bpf_elf_rx_load(void)
static int
test_bpf_elf(void)
{
- int ret;
+ static const load_elf_image_t elf_image_loaders[] = {
+ load_elf_image_temp_file,
+ load_elf_image_direct,
+ };
- ret = test_bpf_elf_load();
- if (ret == TEST_SUCCESS)
- ret = test_bpf_elf_tx_load();
- if (ret == TEST_SUCCESS)
- ret = test_bpf_elf_rx_load();
+ int ret = TEST_SUCCESS;
+
+ for (int li = 0; li != RTE_DIM(elf_image_loaders); ++li) {
+ if (ret == TEST_SUCCESS)
+ ret = test_bpf_elf_load(elf_image_loaders[li]);
+ if (ret == TEST_SUCCESS)
+ ret = test_bpf_elf_tx_load(elf_image_loaders[li]);
+ if (ret == TEST_SUCCESS)
+ ret = test_bpf_elf_rx_load(elf_image_loaders[li]);
+ }
return ret;
}
--
2.43.0
^ permalink raw reply related
* [PATCH v6 10/11] doc: add load API to BPF programmer's guide
From: Marat Khalili @ 2026-06-17 19:44 UTC (permalink / raw)
To: Konstantin Ananyev; +Cc: dev
In-Reply-To: <20260617194425.12690-1-marat.khalili@huawei.com>
Rewrite the basic operations list to focus on a typical use. Provide an
end-to-end example demonstrating loading from an ELF file, executing via
JIT or the interpreter, and properly handling multiple custom arguments
using rte_bpf_prog_ctx.
Signed-off-by: Marat Khalili <marat.khalili@huawei.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
doc/guides/prog_guide/bpf_lib.rst | 75 ++++++++++++++++++++++++++++---
1 file changed, 68 insertions(+), 7 deletions(-)
diff --git a/doc/guides/prog_guide/bpf_lib.rst b/doc/guides/prog_guide/bpf_lib.rst
index 8c820328b984..df3782508829 100644
--- a/doc/guides/prog_guide/bpf_lib.rst
+++ b/doc/guides/prog_guide/bpf_lib.rst
@@ -15,17 +15,79 @@ for more information.
Also it introduces basic framework to load/unload BPF-based filters
on eth devices (right now only via SW RX/TX callbacks).
-The library API provides the following basic operations:
+The library API provides the following basic operations for working with BPF
+programs:
-* Create a new BPF execution context and load user provided eBPF code into it.
+* **Loading:** The extensible API (``rte_bpf_load_ex``) is the recommended
+ way to load a BPF program. By utilizing ``struct rte_bpf_prm_ex``, you can
+ load an eBPF program from an ELF file on disk, or load eBPF/cBPF bytecode
+ directly from memory buffers.
-* Destroy an BPF execution context and its runtime structures and free the associated memory.
+* **Execution via Callbacks:** Once loaded, a BPF program can be attached to
+ a specific ethernet device port and queue to automatically process incoming
+ or outgoing packets using ``rte_bpf_eth_rx_install`` or
+ ``rte_bpf_eth_tx_install``.
-* Execute eBPF bytecode associated with provided input parameter.
+* **Direct Execution:** You can execute a BPF program directly from your
+ application code using ``rte_bpf_exec_ex`` (or the burst variant
+ ``rte_bpf_exec_burst_ex``). This API allows passing an execution context
+ (``struct rte_bpf_prog_ctx``) containing up to 5 custom arguments.
-* Provide information about natively compiled code for given BPF context.
+* **JIT Execution:** For maximum performance, you can retrieve the natively
+ compiled (JIT) function pointer for a loaded program using
+ ``rte_bpf_get_jit_ex`` and call it directly from your code with the same
+ arguments.
-* Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue.
+* **Cleanup:** Destroy a BPF execution context and free the associated memory
+ using ``rte_bpf_destroy``.
+
+The following is a concise example of loading an eBPF program from an ELF file,
+and executing it directly, utilizing the JIT-compiled version if available:
+
+.. code-block:: c
+
+ struct rte_bpf_prm_ex prm = {
+ .sz = sizeof(struct rte_bpf_prm_ex),
+ .origin = RTE_BPF_ORIGIN_ELF_FILE,
+ .elf_file = {
+ .path = "ptype.o",
+ .section = ".text",
+ },
+ .nb_prog_arg = 2,
+ .prog_arg = {
+ [0] = {
+ .type = RTE_BPF_ARG_PTR_MBUF,
+ .size = sizeof(struct rte_mbuf),
+ .buf_size = RTE_MBUF_DEFAULT_BUF_SIZE,
+ },
+ [1] = {
+ .type = RTE_BPF_ARG_RAW,
+ .size = sizeof(uint64_t),
+ },
+ },
+ };
+ struct rte_bpf *bpf = rte_bpf_load_ex(&prm);
+ if (bpf == NULL) {
+ /* Handle load failure */
+ }
+
+ struct rte_bpf_prog_ctx ctx = {
+ .arg[0] = { .ptr = mbuf },
+ .arg[1] = { .u64 = RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK },
+ };
+
+ struct rte_bpf_jit_ex jit;
+ uint64_t ret;
+ if (rte_bpf_get_jit_ex(bpf, &jit) == 0 && jit.func2 != NULL) {
+ /* Call the JIT-compiled function directly for best performance */
+ ret = jit.func2(ctx.arg[0], ctx.arg[1]);
+ } else {
+ /* Fallback to interpreter */
+ uint64_t flags = 0;
+ ret = rte_bpf_exec_ex(bpf, &ctx, flags);
+ }
+
+ rte_bpf_destroy(bpf);
Packet data load instructions
-----------------------------
@@ -60,7 +122,6 @@ Not currently supported eBPF features
-------------------------------------
- JIT support only available for X86_64 and arm64 platforms
- - cBPF
- tail-pointer call
- eBPF MAP
- external function calls for 32-bit platforms
--
2.43.0
^ permalink raw reply related
* [PATCH v6 09/11] doc: add release notes for new extensible BPF API
From: Marat Khalili @ 2026-06-17 19:44 UTC (permalink / raw)
Cc: dev, Konstantin Ananyev
In-Reply-To: <20260617194425.12690-1-marat.khalili@huawei.com>
Document the following new eBPF features introduced in this release:
* Extensible BPF loading API (rte_bpf_load_ex, rte_bpf_prm_ex).
* Loading and executing eBPF programs with up to 5 arguments.
* Installing already loaded eBPF programs as port callbacks.
Signed-off-by: Marat Khalili <marat.khalili@huawei.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
doc/guides/rel_notes/release_26_07.rst | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/doc/guides/rel_notes/release_26_07.rst b/doc/guides/rel_notes/release_26_07.rst
index 5d7aa8d1bff5..3a151c8e83bb 100644
--- a/doc/guides/rel_notes/release_26_07.rst
+++ b/doc/guides/rel_notes/release_26_07.rst
@@ -155,6 +155,26 @@ New Features
Added AGENTS.md file for AI review
and supporting scripts to review patches and documentation.
+* **Added extensible BPF loading API.**
+
+ Added an extensible BPF loading API comprising the function
+ ``rte_bpf_load_ex`` and struct ``rte_bpf_prm_ex``. This enables new features
+ such as loading classic BPF (cBPF), loading ELF images directly from memory
+ buffers, and executing multi-argument programs, while avoiding future ABI
+ breakages.
+
+* **Added support for executing BPF programs with multiple arguments.**
+
+ Added support for loading and executing BPF programs with up to 5 arguments.
+ This introduces new API functions ``rte_bpf_exec_ex``,
+ ``rte_bpf_exec_burst_ex``, and ``rte_bpf_get_jit_ex``.
+
+* **Added BPF port callback installation API.**
+
+ Added new API functions ``rte_bpf_eth_rx_install`` and
+ ``rte_bpf_eth_tx_install`` for installing already loaded BPF programs as
+ port callbacks (as opposed to loading them directly from ELF files).
+
Removed Items
-------------
--
2.43.0
^ permalink raw reply related
* [PATCH v6 11/11] test/bpf: add tests for error handling contracts
From: Marat Khalili @ 2026-06-17 19:44 UTC (permalink / raw)
To: Konstantin Ananyev; +Cc: dev
In-Reply-To: <20260617194425.12690-1-marat.khalili@huawei.com>
Verify NULL parameter rejection in load APIs, graceful failure on
argument/flag mismatch in burst execution APIs, and safe return of the
libpcap stub.
Signed-off-by: Marat Khalili <marat.khalili@huawei.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
app/test/test_bpf.c | 128 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 127 insertions(+), 1 deletion(-)
diff --git a/app/test/test_bpf.c b/app/test/test_bpf.c
index 5cf40d334e7c..6b07e72295db 100644
--- a/app/test/test_bpf.c
+++ b/app/test/test_bpf.c
@@ -3514,6 +3514,121 @@ run_test(const struct bpf_test *tst)
}
+/* Test all eBPF load APIs with prm set to NULL. */
+static int
+test_bpf_load_null(void)
+{
+ struct rte_bpf *bpf;
+ int saved_errno;
+
+ rte_errno = 0;
+ bpf = rte_bpf_load(NULL);
+ saved_errno = rte_errno;
+ rte_bpf_destroy(bpf);
+ RTE_TEST_ASSERT_NULL(bpf, "rte_bpf_load(NULL) did not return NULL\n");
+ RTE_TEST_ASSERT_EQUAL(saved_errno, EINVAL,
+ "rte_bpf_load(NULL) did not set rte_errno to EINVAL\n");
+
+ rte_errno = 0;
+ bpf = rte_bpf_elf_load(NULL, "a", "b");
+ saved_errno = rte_errno;
+ rte_bpf_destroy(bpf);
+ RTE_TEST_ASSERT_NULL(bpf, "rte_bpf_elf_load(NULL, \"a\", \"b\") did not return NULL\n");
+ RTE_TEST_ASSERT_EQUAL(saved_errno, EINVAL,
+ "rte_bpf_elf_load(NULL, \"a\", \"b\") did not set rte_errno to EINVAL\n");
+
+ rte_errno = 0;
+ bpf = rte_bpf_load_ex(NULL);
+ saved_errno = rte_errno;
+ rte_bpf_destroy(bpf);
+ RTE_TEST_ASSERT_NULL(bpf, "rte_bpf_load_ex(NULL) did not return NULL\n");
+ RTE_TEST_ASSERT_EQUAL(saved_errno, EINVAL,
+ "rte_bpf_load_ex(NULL) did not set rte_errno to EINVAL\n");
+
+ return 0;
+}
+REGISTER_FAST_TEST(bpf_load_null_autotest, NOHUGE_OK, ASAN_OK, test_bpf_load_null);
+
+/* Test calling wrong API for execution of a multi-argument eBPF program. */
+static int
+test_bpf_exec_wrong_nb_prog_arg(void)
+{
+ static const struct ebpf_insn ins[] = {
+ { .code = (EBPF_ALU64 | EBPF_MOV | BPF_K), .dst_reg = EBPF_REG_0, .imm = 0 },
+ { .code = (BPF_JMP | EBPF_EXIT), }
+ };
+ static const struct rte_bpf_prm_ex prm = {
+ .sz = sizeof(struct rte_bpf_prm_ex),
+ .origin = RTE_BPF_ORIGIN_RAW,
+ .raw.ins = ins,
+ .raw.nb_ins = RTE_DIM(ins),
+ .prog_arg = {
+ { .type = RTE_BPF_ARG_RAW, .size = sizeof(uint64_t) },
+ { .type = RTE_BPF_ARG_RAW, .size = sizeof(uint64_t) },
+ },
+ .nb_prog_arg = 2, /* Intentionally mismatched: expects 2, burst gives 1 */
+ };
+
+ struct rte_bpf *bpf;
+ uint64_t rc[1];
+ void *ctx[1] = {NULL};
+ uint32_t result;
+ int saved_errno;
+
+ bpf = rte_bpf_load_ex(&prm);
+ RTE_TEST_ASSERT_NOT_NULL(bpf, "rte_bpf_load_ex failed\n");
+
+ rte_errno = 0;
+ result = rte_bpf_exec_burst(bpf, ctx, rc, 1);
+ saved_errno = rte_errno;
+ rte_bpf_destroy(bpf);
+ RTE_TEST_ASSERT_EQUAL(result, 0, "rte_bpf_exec_burst did not return 0\n");
+ RTE_TEST_ASSERT_EQUAL(saved_errno, EINVAL,
+ "rte_bpf_exec_burst did not set rte_errno to EINVAL\n");
+
+ return 0;
+}
+REGISTER_FAST_TEST(bpf_exec_wrong_nb_prog_arg_autotest, NOHUGE_OK, ASAN_OK,
+ test_bpf_exec_wrong_nb_prog_arg);
+
+/* Test passing unsupported flags when executing an eBPF program. */
+static int
+test_bpf_exec_wrong_flags(void)
+{
+ static const struct ebpf_insn ins[] = {
+ { .code = (EBPF_ALU64 | EBPF_MOV | BPF_K), .dst_reg = EBPF_REG_0, .imm = 0 },
+ { .code = (BPF_JMP | EBPF_EXIT), }
+ };
+ static const struct rte_bpf_prm_ex prm = {
+ .sz = sizeof(struct rte_bpf_prm_ex),
+ .origin = RTE_BPF_ORIGIN_RAW,
+ .raw.ins = ins,
+ .raw.nb_ins = RTE_DIM(ins),
+ .prog_arg = { { .type = RTE_BPF_ARG_RAW, .size = sizeof(uint64_t) } },
+ .nb_prog_arg = 1,
+ };
+
+ struct rte_bpf *bpf;
+ uint64_t rc[1];
+ struct rte_bpf_prog_ctx ctx_ex[1] = {};
+ uint32_t result;
+ int saved_errno;
+
+ bpf = rte_bpf_load_ex(&prm);
+ RTE_TEST_ASSERT_NOT_NULL(bpf, "rte_bpf_load_ex failed\n");
+
+ rte_errno = 0;
+ result = rte_bpf_exec_burst_ex(bpf, ctx_ex, rc, 1, UINT64_MAX);
+ saved_errno = rte_errno;
+ rte_bpf_destroy(bpf);
+ RTE_TEST_ASSERT_EQUAL(result, 0, "rte_bpf_exec_burst_ex did not return 0\n");
+ RTE_TEST_ASSERT_EQUAL(saved_errno, EINVAL,
+ "rte_bpf_exec_burst_ex did not set rte_errno to EINVAL\n");
+
+ return 0;
+}
+REGISTER_FAST_TEST(bpf_exec_wrong_flags_autotest, NOHUGE_OK, ASAN_OK, test_bpf_exec_wrong_flags);
+
static int
test_bpf(void)
{
@@ -4444,7 +4559,18 @@ REGISTER_FAST_TEST(bpf_elf_autotest, NOHUGE_OK, ASAN_OK, test_bpf_elf);
static int
test_bpf_convert(void)
{
- printf("BPF convert RTE_HAS_LIBPCAP is undefined, skipping test\n");
+ int dummy = 0;
+ struct rte_bpf_prm *prm;
+
+ prm = rte_bpf_convert(NULL);
+ rte_free(prm);
+ RTE_TEST_ASSERT_NULL(prm, "rte_bpf_convert(NULL) without libpcap did not return NULL\n");
+
+ prm = rte_bpf_convert((const struct bpf_program *)&dummy);
+ rte_free(prm);
+ RTE_TEST_ASSERT_NULL(prm, "rte_bpf_convert(&dummy) without libpcap did not return NULL\n");
+
+ printf("BPF convert RTE_HAS_LIBPCAP is undefined, skipping full test\n");
return TEST_SKIPPED;
}
--
2.43.0
^ permalink raw reply related
* [PATCH v6 03/11] bpf: support up to 5 arguments
From: Marat Khalili @ 2026-06-17 19:44 UTC (permalink / raw)
To: Konstantin Ananyev, Wathsala Vithanage; +Cc: dev
In-Reply-To: <20260617194425.12690-1-marat.khalili@huawei.com>
When using rte_bpf_load_ex allow up to 5 arguments for a BPF program.
Particularly useful for call-backs and other internal functions.
Signed-off-by: Marat Khalili <marat.khalili@huawei.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
lib/bpf/bpf.c | 32 ++++++++++-
lib/bpf/bpf_exec.c | 124 +++++++++++++++++++++++++++++++++++++++-
lib/bpf/bpf_impl.h | 2 +-
lib/bpf/bpf_jit_arm64.c | 2 +-
lib/bpf/bpf_jit_x86.c | 2 +-
lib/bpf/bpf_load.c | 8 ++-
lib/bpf/bpf_validate.c | 45 +++++++++++----
lib/bpf/rte_bpf.h | 121 +++++++++++++++++++++++++++++++++++++--
8 files changed, 311 insertions(+), 25 deletions(-)
diff --git a/lib/bpf/bpf.c b/lib/bpf/bpf.c
index 5239b3e11e0e..67dededd9ae8 100644
--- a/lib/bpf/bpf.c
+++ b/lib/bpf/bpf.c
@@ -16,8 +16,8 @@ void
rte_bpf_destroy(struct rte_bpf *bpf)
{
if (bpf != NULL) {
- if (bpf->jit.func != NULL)
- munmap(bpf->jit.func, bpf->jit.sz);
+ if (bpf->jit.raw != NULL)
+ munmap(bpf->jit.raw, bpf->jit.sz);
munmap(bpf, bpf->sz);
}
}
@@ -29,7 +29,33 @@ rte_bpf_get_jit(const struct rte_bpf *bpf, struct rte_bpf_jit *jit)
if (bpf == NULL || jit == NULL)
return -EINVAL;
- jit[0] = bpf->jit;
+ if (bpf->prm.nb_prog_arg != 1) {
+ RTE_BPF_LOG_LINE(ERR,
+ "this program takes %d arguments, use rte_bpf_get_jit_ex",
+ bpf->prm.nb_prog_arg);
+ return -EINVAL;
+ }
+
+ *jit = (struct rte_bpf_jit) {
+ .func = bpf->jit.raw,
+ .sz = bpf->jit.sz,
+ };
+ return 0;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_bpf_get_jit_ex, 26.11)
+int
+rte_bpf_get_jit_ex(const struct rte_bpf *bpf, struct rte_bpf_jit_ex *jit)
+{
+ if (bpf == NULL || jit == NULL)
+ return -EINVAL;
+
+ if (bpf->jit.raw == NULL) {
+ RTE_BPF_LOG_LINE(ERR, "no JIT-compiled version");
+ return -ENOENT;
+ }
+
+ *jit = bpf->jit;
return 0;
}
diff --git a/lib/bpf/bpf_exec.c b/lib/bpf/bpf_exec.c
index e4668ba10b64..20f8d0fa29de 100644
--- a/lib/bpf/bpf_exec.c
+++ b/lib/bpf/bpf_exec.c
@@ -10,6 +10,7 @@
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_byteorder.h>
+#include <rte_errno.h>
#include "bpf_impl.h"
@@ -502,6 +503,12 @@ rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
uint64_t reg[EBPF_REG_NUM];
uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
+ if (bpf->prm.nb_prog_arg != 1) {
+ /* Use rte_bpf_exec_burst_ex with this program. */
+ rte_errno = EINVAL;
+ return 0;
+ }
+
for (i = 0; i != num; i++) {
reg[EBPF_REG_1] = (uintptr_t)ctx[i];
@@ -513,12 +520,127 @@ rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
return i;
}
+static uint32_t
+exec_vm_burst_ex(const struct rte_bpf *bpf, const struct rte_bpf_prog_ctx *ctx,
+ uint64_t rc[], uint32_t num)
+{
+ uint32_t i;
+ uint64_t reg[EBPF_REG_NUM];
+ uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
+
+ for (i = 0; i != num; i++) {
+
+ switch (bpf->prm.nb_prog_arg) {
+ case 5:
+ reg[EBPF_REG_5] = ctx[i].arg[4].u64;
+ /* FALLTHROUGH */
+ case 4:
+ reg[EBPF_REG_4] = ctx[i].arg[3].u64;
+ /* FALLTHROUGH */
+ case 3:
+ reg[EBPF_REG_3] = ctx[i].arg[2].u64;
+ /* FALLTHROUGH */
+ case 2:
+ reg[EBPF_REG_2] = ctx[i].arg[1].u64;
+ /* FALLTHROUGH */
+ case 1:
+ reg[EBPF_REG_1] = ctx[i].arg[0].u64;
+ /* FALLTHROUGH */
+ case 0:
+ break;
+ }
+
+ reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack));
+
+ rc[i] = bpf_exec(bpf, reg);
+ }
+
+ return i;
+}
+
+static uint32_t
+exec_jit_burst_ex(const struct rte_bpf *bpf, const struct rte_bpf_prog_ctx *ctx,
+ uint64_t rc[], uint32_t num)
+{
+ uint32_t i = 0;
+ const struct rte_bpf_jit_ex jit = bpf->jit;
+
+ /*
+ * Fast path: assumes application pre-validated RTE_BPF_EXEC_FLAG_JIT
+ * and successful JIT generation. No explicit NULL checks here.
+ */
+ switch (bpf->prm.nb_prog_arg) {
+ case 0:
+ for (i = 0; i != num; i++)
+ rc[i] = jit.func0();
+ break;
+ case 1:
+ for (i = 0; i != num; i++) {
+ const union rte_bpf_func_arg *const arg = ctx[i].arg;
+ rc[i] = jit.func1(arg[0]);
+ }
+ break;
+ case 2:
+ for (i = 0; i != num; i++) {
+ const union rte_bpf_func_arg *const arg = ctx[i].arg;
+ rc[i] = jit.func2(arg[0], arg[1]);
+ }
+ break;
+ case 3:
+ for (i = 0; i != num; i++) {
+ const union rte_bpf_func_arg *const arg = ctx[i].arg;
+ rc[i] = jit.func3(arg[0], arg[1], arg[2]);
+ }
+ break;
+ case 4:
+ for (i = 0; i != num; i++) {
+ const union rte_bpf_func_arg *const arg = ctx[i].arg;
+ rc[i] = jit.func4(arg[0], arg[1], arg[2], arg[3]);
+ }
+ break;
+ case 5:
+ for (i = 0; i != num; i++) {
+ const union rte_bpf_func_arg *const arg = ctx[i].arg;
+ rc[i] = jit.func5(arg[0], arg[1], arg[2], arg[3], arg[4]);
+ }
+ break;
+ }
+
+ return i;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_bpf_exec_burst_ex, 26.11)
+uint32_t
+rte_bpf_exec_burst_ex(const struct rte_bpf *bpf, const struct rte_bpf_prog_ctx *ctx,
+ uint64_t rc[], uint32_t num, uint64_t flags)
+{
+ if ((flags & ~RTE_BPF_EXEC_FLAG_MASK) != 0) {
+ rte_errno = EINVAL;
+ return 0;
+ }
+
+ return (flags & RTE_BPF_EXEC_FLAG_JIT) != 0 ?
+ exec_jit_burst_ex(bpf, ctx, rc, num) :
+ exec_vm_burst_ex(bpf, ctx, rc, num);
+}
+
RTE_EXPORT_SYMBOL(rte_bpf_exec)
uint64_t
rte_bpf_exec(const struct rte_bpf *bpf, void *ctx)
{
- uint64_t rc;
+ uint64_t rc = UINT64_MAX;
rte_bpf_exec_burst(bpf, &ctx, &rc, 1);
return rc;
}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_bpf_exec_ex, 26.11)
+uint64_t
+rte_bpf_exec_ex(const struct rte_bpf *bpf, const struct rte_bpf_prog_ctx *ctx,
+ uint64_t flags)
+{
+ uint64_t rc = UINT64_MAX;
+
+ rte_bpf_exec_burst_ex(bpf, ctx, &rc, 1, flags);
+ return rc;
+}
diff --git a/lib/bpf/bpf_impl.h b/lib/bpf/bpf_impl.h
index 1cee109bc98a..4a98b3373067 100644
--- a/lib/bpf/bpf_impl.h
+++ b/lib/bpf/bpf_impl.h
@@ -12,7 +12,7 @@
struct rte_bpf {
struct rte_bpf_prm_ex prm;
- struct rte_bpf_jit jit;
+ struct rte_bpf_jit_ex jit;
size_t sz;
uint32_t stack_sz;
};
diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c
index 9e5e142c13ba..ba7ae4d680c5 100644
--- a/lib/bpf/bpf_jit_arm64.c
+++ b/lib/bpf/bpf_jit_arm64.c
@@ -1471,7 +1471,7 @@ __rte_bpf_jit_arm64(struct rte_bpf *bpf)
/* Flush the icache */
__builtin___clear_cache((char *)ctx.ins, (char *)(ctx.ins + ctx.idx));
- bpf->jit.func = (void *)ctx.ins;
+ bpf->jit.raw = ctx.ins;
bpf->jit.sz = size;
goto finish;
diff --git a/lib/bpf/bpf_jit_x86.c b/lib/bpf/bpf_jit_x86.c
index 6f4235d43499..54eb279643b9 100644
--- a/lib/bpf/bpf_jit_x86.c
+++ b/lib/bpf/bpf_jit_x86.c
@@ -1568,7 +1568,7 @@ __rte_bpf_jit_x86(struct rte_bpf *bpf)
if (rc != 0)
munmap(st.ins, st.sz);
else {
- bpf->jit.func = (void *)st.ins;
+ bpf->jit.raw = st.ins;
bpf->jit.sz = st.sz;
}
diff --git a/lib/bpf/bpf_load.c b/lib/bpf/bpf_load.c
index 57544fb814c6..41b13e28ba23 100644
--- a/lib/bpf/bpf_load.c
+++ b/lib/bpf/bpf_load.c
@@ -149,7 +149,8 @@ rte_bpf_load(const struct rte_bpf_prm *prm)
.raw.nb_ins = prm->nb_ins,
.xsym = prm->xsym,
.nb_xsym = prm->nb_xsym,
- .prog_arg = prm->prog_arg,
+ .prog_arg[0] = prm->prog_arg,
+ .nb_prog_arg = 1,
});
}
@@ -170,7 +171,8 @@ rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
.elf_file.section = sname,
.xsym = prm->xsym,
.nb_xsym = prm->nb_xsym,
- .prog_arg = prm->prog_arg,
+ .prog_arg[0] = prm->prog_arg,
+ .nb_prog_arg = 1,
});
}
@@ -271,6 +273,6 @@ rte_bpf_load_ex(const struct rte_bpf_prm_ex *prm)
}
RTE_BPF_LOG_FUNC_LINE(INFO, "successfully creates %p(jit={.func=%p,.sz=%zu});",
- load.bpf, load.bpf->jit.func, load.bpf->jit.sz);
+ load.bpf, load.bpf->jit.raw, load.bpf->jit.sz);
return load.bpf;
}
diff --git a/lib/bpf/bpf_validate.c b/lib/bpf/bpf_validate.c
index 5bfc59296d05..bf8a4abb5a5a 100644
--- a/lib/bpf/bpf_validate.c
+++ b/lib/bpf/bpf_validate.c
@@ -2425,10 +2425,14 @@ evaluate(struct bpf_verifier *bvf)
.s = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
};
- bvf->evst->rv[EBPF_REG_1].v = bvf->prm->prog_arg;
- bvf->evst->rv[EBPF_REG_1].mask = UINT64_MAX;
- if (bvf->prm->prog_arg.type == RTE_BPF_ARG_RAW)
- eval_max_bound(bvf->evst->rv + EBPF_REG_1, UINT64_MAX);
+ for (uint32_t pai = 0; pai != bvf->prm->nb_prog_arg; ++pai) {
+ struct bpf_reg_val *reg = &bvf->evst->rv[EBPF_REG_1 + pai];
+
+ reg->v = bvf->prm->prog_arg[pai];
+ reg->mask = UINT64_MAX;
+ if (reg->v.type == RTE_BPF_ARG_RAW)
+ eval_max_bound(reg, UINT64_MAX);
+ }
bvf->evst->rv[EBPF_REG_10] = rvfp;
@@ -2521,21 +2525,42 @@ evaluate(struct bpf_verifier *bvf)
return rc;
}
+static bool
+prog_arg_is_valid(const struct rte_bpf_arg *prog_arg)
+{
+ /* check input argument type, don't allow mbuf ptr on 32-bit */
+ if (prog_arg->type != RTE_BPF_ARG_RAW &&
+ prog_arg->type != RTE_BPF_ARG_PTR &&
+ (sizeof(uint64_t) != sizeof(uintptr_t) ||
+ prog_arg->type != RTE_BPF_ARG_PTR_MBUF)) {
+ RTE_BPF_LOG_FUNC_LINE(ERR, "unsupported argument type");
+ return false;
+ }
+
+ return true;
+}
+
int
__rte_bpf_validate(const struct rte_bpf_prm_ex *prm, uint32_t *stack_sz)
{
int32_t rc;
struct bpf_verifier bvf;
- /* check input argument type, don't allow mbuf ptr on 32-bit */
- if (prm->prog_arg.type != RTE_BPF_ARG_RAW &&
- prm->prog_arg.type != RTE_BPF_ARG_PTR &&
- (sizeof(uint64_t) != sizeof(uintptr_t) ||
- prm->prog_arg.type != RTE_BPF_ARG_PTR_MBUF)) {
- RTE_BPF_LOG_FUNC_LINE(ERR, "unsupported argument type");
+ if (prm->nb_prog_arg > EBPF_FUNC_MAX_ARGS) {
+ RTE_BPF_LOG_FUNC_LINE(ERR,
+ "support up to %u arguments, found %u",
+ EBPF_FUNC_MAX_ARGS, prm->nb_prog_arg);
return -ENOTSUP;
}
+ for (uint32_t pai = 0; pai != prm->nb_prog_arg; ++pai)
+ if (!prog_arg_is_valid(&prm->prog_arg[pai])) {
+ RTE_BPF_LOG_FUNC_LINE(ERR,
+ "unsupported argument %d (r%d) type",
+ pai, EBPF_REG_1 + pai);
+ return -ENOTSUP;
+ }
+
memset(&bvf, 0, sizeof(bvf));
bvf.prm = prm;
bvf.in = calloc(prm->raw.nb_ins, sizeof(bvf.in[0]));
diff --git a/lib/bpf/rte_bpf.h b/lib/bpf/rte_bpf.h
index bf58a418191e..0e7eaa3c18eb 100644
--- a/lib/bpf/rte_bpf.h
+++ b/lib/bpf/rte_bpf.h
@@ -25,6 +25,11 @@
extern "C" {
#endif
+#define RTE_BPF_EXEC_FLAG_JIT RTE_BIT64(0) /**< use JIT-compiled version */
+
+/** Mask with all supported `RTE_BPF_EXEC_FLAG_*` flags set. */
+#define RTE_BPF_EXEC_FLAG_MASK RTE_BPF_EXEC_FLAG_JIT
+
/**
* Possible types for function/BPF program arguments.
*/
@@ -122,7 +127,8 @@ struct rte_bpf_prm_ex {
/**< array of external symbols that eBPF code is allowed to reference */
uint32_t nb_xsym; /**< number of elements in xsym */
- struct rte_bpf_arg prog_arg; /**< input arg description */
+ struct rte_bpf_arg prog_arg[EBPF_FUNC_MAX_ARGS]; /**< program arguments */
+ uint32_t nb_prog_arg; /**< program argument count */
};
/**
@@ -138,13 +144,49 @@ struct rte_bpf_prm {
};
/**
- * Information about compiled into native ISA eBPF code.
+ * Information about compiled into native ISA eBPF code accepting 1 argument.
*/
struct rte_bpf_jit {
uint64_t (*func)(void *); /**< JIT-ed native code */
size_t sz; /**< size of JIT-ed code */
};
+union rte_bpf_func_arg {
+ uint64_t u64;
+ void *ptr;
+};
+
+typedef uint64_t (*rte_bpf_jit_func0_t)(void);
+typedef uint64_t (*rte_bpf_jit_func1_t)(union rte_bpf_func_arg);
+typedef uint64_t (*rte_bpf_jit_func2_t)(union rte_bpf_func_arg, union rte_bpf_func_arg);
+typedef uint64_t (*rte_bpf_jit_func3_t)(union rte_bpf_func_arg, union rte_bpf_func_arg,
+ union rte_bpf_func_arg);
+typedef uint64_t (*rte_bpf_jit_func4_t)(union rte_bpf_func_arg, union rte_bpf_func_arg,
+ union rte_bpf_func_arg, union rte_bpf_func_arg);
+typedef uint64_t (*rte_bpf_jit_func5_t)(union rte_bpf_func_arg, union rte_bpf_func_arg,
+ union rte_bpf_func_arg, union rte_bpf_func_arg, union rte_bpf_func_arg);
+
+/**
+ * JIT-ed native code, member depends on number of program arguments.
+ */
+struct rte_bpf_jit_ex {
+ union {
+ void *raw;
+ rte_bpf_jit_func0_t func0; /* nullary function */
+ rte_bpf_jit_func1_t func1; /* unary function */
+ rte_bpf_jit_func2_t func2; /* binary function */
+ rte_bpf_jit_func3_t func3; /* ternary function */
+ rte_bpf_jit_func4_t func4; /* quaternary function */
+ rte_bpf_jit_func5_t func5; /* quinary function */
+ };
+ size_t sz;
+};
+
+/* Tuple of eBPF program arguments. */
+struct rte_bpf_prog_ctx {
+ union rte_bpf_func_arg arg[EBPF_FUNC_MAX_ARGS];
+};
+
struct rte_bpf;
/**
@@ -224,7 +266,7 @@ rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
__rte_malloc __rte_dealloc(rte_bpf_destroy, 1);
/**
- * Execute given BPF bytecode.
+ * Execute given BPF bytecode accepting 1 argument.
*
* @param bpf
* handle for the BPF code to execute.
@@ -237,7 +279,29 @@ uint64_t
rte_bpf_exec(const struct rte_bpf *bpf, void *ctx);
/**
- * Execute given BPF bytecode over a set of input contexts.
+ * @warning
+ * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
+ *
+ * Execute given BPF bytecode accepting any number of arguments.
+ *
+ * @param bpf
+ * handle for the BPF code to execute.
+ * @param ctx
+ * program arguments tuple.
+ * @param flags
+ * bitwise OR of `RTE_BPF_EXEC_FLAG_*` values controlling execution.
+ * Flag RTE_BPF_EXEC_FLAG_JIT requires presence of JIT version (can be checked
+ * with rte_bpf_get_jit_ex).
+ * @return
+ * BPF execution return value.
+ */
+__rte_experimental
+uint64_t
+rte_bpf_exec_ex(const struct rte_bpf *bpf, const struct rte_bpf_prog_ctx *ctx,
+ uint64_t flags);
+
+/**
+ * Execute given BPF bytecode accepting 1 argument over a set of input contexts.
*
* @param bpf
* handle for the BPF code to execute.
@@ -255,7 +319,35 @@ rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
uint32_t num);
/**
- * Provide information about natively compiled code for given BPF handle.
+ * @warning
+ * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
+ *
+ * Execute given BPF program accepting any number of arguments over a set of
+ * input contexts.
+ *
+ * @param bpf
+ * handle for the BPF code to execute.
+ * @param ctx
+ * pointer to array of program argument tuples, can be NULL for nullary programs.
+ * @param rc
+ * array of return values (one per input).
+ * @param num
+ * number executions, number of elements in arrays ctx and rc[].
+ * @param flags
+ * bitwise OR of `RTE_BPF_EXEC_FLAG_*` values controlling execution.
+ * Flag RTE_BPF_EXEC_FLAG_JIT requires presence of JIT version (can be checked
+ * with rte_bpf_get_jit_ex).
+ * @return
+ * number of successfully processed inputs.
+ */
+__rte_experimental
+uint32_t
+rte_bpf_exec_burst_ex(const struct rte_bpf *bpf, const struct rte_bpf_prog_ctx *ctx,
+ uint64_t rc[], uint32_t num, uint64_t flags);
+
+/**
+ * Provide information about natively compiled code for given BPF program
+ * accepting 1 argument.
*
* @param bpf
* handle for the BPF code.
@@ -268,6 +360,25 @@ rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
int
rte_bpf_get_jit(const struct rte_bpf *bpf, struct rte_bpf_jit *jit);
+/**
+ * @warning
+ * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
+ *
+ * Get function JIT-compiled from the BPF program.
+ *
+ * @param bpf
+ * handle for the BPF code.
+ * @param jit
+ * pointer to the struct rte_bpf_jit_ex.
+ * @return
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if there is no JIT-compiled version.
+ * - Zero if operation completed successfully.
+ */
+__rte_experimental
+int
+rte_bpf_get_jit_ex(const struct rte_bpf *bpf, struct rte_bpf_jit_ex *jit);
+
/**
* Dump epf instructions to a file.
*
--
2.43.0
^ permalink raw reply related
* [PATCH v6 02/11] bpf: introduce extensible load API
From: Marat Khalili @ 2026-06-17 19:44 UTC (permalink / raw)
To: Konstantin Ananyev, Wathsala Vithanage; +Cc: dev
In-Reply-To: <20260617194425.12690-1-marat.khalili@huawei.com>
Introduce new BPF load parameters struct rte_bpf_prm_ex that can be
extended without breaking backward or forward compatibility. Introduce
new function rte_bpf_load_ex consolidating in one code path loading from
both ELF file and raw memory image, with possibility to add more options
in the future.
Some changes in code layout and sequence:
* Both old APIs now only forwarding calls to a new single entry point.
* There is now a centralized cleanup point for all temporary resources
created during the load process.
* External symbols (xsyms) are now checked for validity just after the
load started, not after they were already used for relocation.
* File bpf_load_elf.c now only handles opening ELF file and providing
patched instruction array to the load process. These are left as two
separate functions to support other ELF sources like memory image in
the future.
* Function stubs for the case libelf is not available are moved to
bpf_load_elf.c to make keeping track of them easier (forgetting to
update stubs is a common problem).
Signed-off-by: Marat Khalili <marat.khalili@huawei.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>
---
lib/bpf/bpf_exec.c | 10 +--
lib/bpf/bpf_impl.h | 32 ++++++-
lib/bpf/bpf_jit_arm64.c | 12 +--
lib/bpf/bpf_jit_x86.c | 8 +-
lib/bpf/bpf_load.c | 195 +++++++++++++++++++++++++++++++++++-----
lib/bpf/bpf_load_elf.c | 151 ++++++++++++++++++-------------
lib/bpf/bpf_stub.c | 17 ----
lib/bpf/bpf_validate.c | 32 +++----
lib/bpf/meson.build | 4 +-
lib/bpf/rte_bpf.h | 68 +++++++++++++-
10 files changed, 392 insertions(+), 137 deletions(-)
diff --git a/lib/bpf/bpf_exec.c b/lib/bpf/bpf_exec.c
index 18013753b147..e4668ba10b64 100644
--- a/lib/bpf/bpf_exec.c
+++ b/lib/bpf/bpf_exec.c
@@ -47,7 +47,7 @@
RTE_BPF_LOG_LINE(ERR, \
"%s(%p): division by 0 at pc: %#zx;", \
__func__, bpf, \
- (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
+ (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.raw.ins); \
return 0; \
} \
} while (0)
@@ -81,7 +81,7 @@
RTE_BPF_LOG_LINE(ERR, \
"%s(%p): unsupported atomic operation at pc: %#zx;", \
__func__, bpf, \
- (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
+ (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.raw.ins); \
return 0; \
} \
} while (0)
@@ -157,7 +157,7 @@ bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
RTE_BPF_LOG_LINE(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): "
"load beyond packet boundary at pc: %#zx;",
__func__, bpf, mb, off, len,
- (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);
+ (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.raw.ins);
return p;
}
@@ -166,7 +166,7 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
{
const struct ebpf_insn *ins;
- for (ins = bpf->prm.ins; ; ins++) {
+ for (ins = bpf->prm.raw.ins; ; ins++) {
switch (ins->code) {
/* 32 bit ALU IMM operations */
case (BPF_ALU | BPF_ADD | BPF_K):
@@ -483,7 +483,7 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
RTE_BPF_LOG_LINE(ERR,
"%s(%p): invalid opcode %#x at pc: %#zx;",
__func__, bpf, ins->code,
- (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
+ (uintptr_t)ins - (uintptr_t)bpf->prm.raw.ins);
return 0;
}
}
diff --git a/lib/bpf/bpf_impl.h b/lib/bpf/bpf_impl.h
index fb5ec3c4d65f..1cee109bc98a 100644
--- a/lib/bpf/bpf_impl.h
+++ b/lib/bpf/bpf_impl.h
@@ -11,17 +11,45 @@
#define MAX_BPF_STACK_SIZE 0x200
struct rte_bpf {
- struct rte_bpf_prm prm;
+ struct rte_bpf_prm_ex prm;
struct rte_bpf_jit jit;
size_t sz;
uint32_t stack_sz;
};
+/* Temporary copies etc. used by the load process. */
+struct __rte_bpf_load {
+ struct rte_bpf_prm_ex prm;
+
+ /* Loading ELF and applying relocations. */
+ int elf_fd; /* ELF fd, must be negative (not zero) by default. */
+ void *elf; /* Using void to avoid dependency on libelf. */
+
+ /* Value we are going to return, if any. */
+ struct rte_bpf *bpf;
+};
+
/*
* Use '__rte' prefix for non-static internal functions
* to avoid potential name conflict with other libraries.
*/
-int __rte_bpf_validate(struct rte_bpf *bpf);
+
+/* Free temporary resources created by opening ELF. */
+void
+__rte_bpf_load_elf_cleanup(struct __rte_bpf_load *load);
+
+/* Open the ELF file. */
+int
+__rte_bpf_load_elf_file(struct __rte_bpf_load *load);
+
+/* Get code from ELF and apply relocations to it. */
+int
+__rte_bpf_load_elf_code(struct __rte_bpf_load *load);
+
+/* Validate final BPF code and calculate stack size. */
+int
+__rte_bpf_validate(const struct rte_bpf_prm_ex *prm, uint32_t *stack_sz);
+
int __rte_bpf_jit(struct rte_bpf *bpf);
int __rte_bpf_jit_x86(struct rte_bpf *bpf);
int __rte_bpf_jit_arm64(struct rte_bpf *bpf);
diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c
index 4bbb97da1b89..9e5e142c13ba 100644
--- a/lib/bpf/bpf_jit_arm64.c
+++ b/lib/bpf/bpf_jit_arm64.c
@@ -111,12 +111,12 @@ jump_offset_init(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
uint32_t i;
- ctx->map = malloc(bpf->prm.nb_ins * sizeof(ctx->map[0]));
+ ctx->map = malloc(bpf->prm.raw.nb_ins * sizeof(ctx->map[0]));
if (ctx->map == NULL)
return -ENOMEM;
/* Fill with fake offsets */
- for (i = 0; i != bpf->prm.nb_ins; i++) {
+ for (i = 0; i != bpf->prm.raw.nb_ins; i++) {
ctx->map[i].off = INT32_MAX;
ctx->map[i].off_to_b = 0;
}
@@ -1130,8 +1130,8 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
uint8_t op;
uint32_t i;
- for (i = 0; i != bpf->prm.nb_ins; i++) {
- ins = bpf->prm.ins + i;
+ for (i = 0; i != bpf->prm.raw.nb_ins; i++) {
+ ins = bpf->prm.raw.ins + i;
op = ins->code;
switch (op) {
@@ -1168,10 +1168,10 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
emit_prologue(ctx);
- for (i = 0; i != bpf->prm.nb_ins; i++) {
+ for (i = 0; i != bpf->prm.raw.nb_ins; i++) {
jump_offset_update(ctx, i);
- ins = bpf->prm.ins + i;
+ ins = bpf->prm.raw.ins + i;
op = ins->code;
off = ins->off;
imm = ins->imm;
diff --git a/lib/bpf/bpf_jit_x86.c b/lib/bpf/bpf_jit_x86.c
index 88b1b5aeab1a..6f4235d43499 100644
--- a/lib/bpf/bpf_jit_x86.c
+++ b/lib/bpf/bpf_jit_x86.c
@@ -1324,12 +1324,12 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
emit_prolog(st, bpf->stack_sz);
- for (i = 0; i != bpf->prm.nb_ins; i++) {
+ for (i = 0; i != bpf->prm.raw.nb_ins; i++) {
st->idx = i;
st->off[i] = st->sz;
- ins = bpf->prm.ins + i;
+ ins = bpf->prm.raw.ins + i;
dr = ebpf2x86[ins->dst_reg];
sr = ebpf2x86[ins->src_reg];
@@ -1532,13 +1532,13 @@ __rte_bpf_jit_x86(struct rte_bpf *bpf)
/* init state */
memset(&st, 0, sizeof(st));
- st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
+ st.off = malloc(bpf->prm.raw.nb_ins * sizeof(st.off[0]));
if (st.off == NULL)
return -ENOMEM;
/* fill with fake offsets */
st.exit.off = INT32_MAX;
- for (i = 0; i != bpf->prm.nb_ins; i++)
+ for (i = 0; i != bpf->prm.raw.nb_ins; i++)
st.off[i] = INT32_MAX;
/*
diff --git a/lib/bpf/bpf_load.c b/lib/bpf/bpf_load.c
index b8a0426fe2ed..57544fb814c6 100644
--- a/lib/bpf/bpf_load.c
+++ b/lib/bpf/bpf_load.c
@@ -14,14 +14,14 @@
#include "bpf_impl.h"
static struct rte_bpf *
-bpf_load(const struct rte_bpf_prm *prm)
+bpf_load(const struct rte_bpf_prm_ex *prm)
{
uint8_t *buf;
struct rte_bpf *bpf;
size_t sz, bsz, insz, xsz;
xsz = prm->nb_xsym * sizeof(prm->xsym[0]);
- insz = prm->nb_ins * sizeof(prm->ins[0]);
+ insz = prm->raw.nb_ins * sizeof(prm->raw.ins[0]);
bsz = sizeof(bpf[0]);
sz = insz + xsz + bsz;
@@ -37,10 +37,10 @@ bpf_load(const struct rte_bpf_prm *prm)
if (xsz > 0)
memcpy(buf + bsz, prm->xsym, xsz);
- memcpy(buf + bsz + xsz, prm->ins, insz);
+ memcpy(buf + bsz + xsz, prm->raw.ins, insz);
bpf->prm.xsym = (void *)(buf + bsz);
- bpf->prm.ins = (void *)(buf + bsz + xsz);
+ bpf->prm.raw.ins = (void *)(buf + bsz + xsz);
return bpf;
}
@@ -80,37 +80,44 @@ bpf_check_xsym(const struct rte_bpf_xsym *xsym)
return 0;
}
-RTE_EXPORT_SYMBOL(rte_bpf_load)
-struct rte_bpf *
-rte_bpf_load(const struct rte_bpf_prm *prm)
+static int
+bpf_check_xsyms(const struct rte_bpf_xsym *xsym, uint32_t nb_xsym)
{
- struct rte_bpf *bpf;
int32_t rc;
uint32_t i;
- if (prm == NULL || prm->ins == NULL || prm->nb_ins == 0 ||
- (prm->nb_xsym != 0 && prm->xsym == NULL)) {
- rte_errno = EINVAL;
- return NULL;
- }
+ if (nb_xsym != 0 && xsym == NULL)
+ return -EINVAL;
rc = 0;
- for (i = 0; i != prm->nb_xsym && rc == 0; i++)
- rc = bpf_check_xsym(prm->xsym + i);
+ for (i = 0; i != nb_xsym && rc == 0; i++)
+ rc = bpf_check_xsym(xsym + i);
if (rc != 0) {
- rte_errno = -rc;
RTE_BPF_LOG_FUNC_LINE(ERR, "%d-th xsym is invalid", i);
- return NULL;
+ return rc;
}
+ return 0;
+}
+
+static int
+bpf_load_raw(struct __rte_bpf_load *load)
+{
+ const struct rte_bpf_prm_ex *const prm = &load->prm;
+ struct rte_bpf *bpf;
+ int32_t rc;
+
+ RTE_ASSERT(prm->origin == RTE_BPF_ORIGIN_RAW);
+
+ if (prm->raw.ins == NULL || prm->raw.nb_ins == 0)
+ return -EINVAL;
+
bpf = bpf_load(prm);
- if (bpf == NULL) {
- rte_errno = ENOMEM;
- return NULL;
- }
+ if (bpf == NULL)
+ return -ENOMEM;
- rc = __rte_bpf_validate(bpf);
+ rc = __rte_bpf_validate(&load->prm, &bpf->stack_sz);
if (rc == 0) {
__rte_bpf_jit(bpf);
if (mprotect(bpf, bpf->sz, PROT_READ) != 0)
@@ -119,9 +126,151 @@ rte_bpf_load(const struct rte_bpf_prm *prm)
if (rc != 0) {
rte_bpf_destroy(bpf);
+ return rc;
+ }
+
+ load->bpf = bpf;
+ return 0;
+}
+
+RTE_EXPORT_SYMBOL(rte_bpf_load)
+struct rte_bpf *
+rte_bpf_load(const struct rte_bpf_prm *prm)
+{
+ if (prm == NULL) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ return rte_bpf_load_ex(&(struct rte_bpf_prm_ex){
+ .sz = sizeof(struct rte_bpf_prm_ex),
+ .origin = RTE_BPF_ORIGIN_RAW,
+ .raw.ins = prm->ins,
+ .raw.nb_ins = prm->nb_ins,
+ .xsym = prm->xsym,
+ .nb_xsym = prm->nb_xsym,
+ .prog_arg = prm->prog_arg,
+ });
+}
+
+RTE_EXPORT_SYMBOL(rte_bpf_elf_load)
+struct rte_bpf *
+rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
+ const char *sname)
+{
+ if (prm == NULL) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ return rte_bpf_load_ex(&(struct rte_bpf_prm_ex){
+ .sz = sizeof(struct rte_bpf_prm_ex),
+ .origin = RTE_BPF_ORIGIN_ELF_FILE,
+ .elf_file.path = fname,
+ .elf_file.section = sname,
+ .xsym = prm->xsym,
+ .nb_xsym = prm->nb_xsym,
+ .prog_arg = prm->prog_arg,
+ });
+}
+
+/*
+ * Check extensible opts for invalid size or non-zero unsupported members.
+ *
+ * This code provides forward compatibility with applications compiled against
+ * newer version of this library. `opts_sz` is the size of struct `opts` in the
+ * version used for compiling the application, read from the member `sz`;
+ * `type_sz` is the size of the same struct in the version used for compiling
+ * the library.
+ *
+ * If new fields were added to the struct in the application version, `opts_sz`
+ * will be greater than `type_sz`. In this case we are making sure all bytes we
+ * don't know how to interpret are zeroes, that is any new features that are
+ * there are not being used.
+ *
+ * This function can be used to check any struct following this convention.
+ */
+static bool
+opts_valid(const void *opts, size_t opts_sz, size_t type_sz)
+{
+ if (opts == NULL)
+ return true;
+
+ if (opts_sz < sizeof(opts_sz))
+ /* Size of the struct is too small even for sz member. */
+ return false;
+
+ /* Verify that all extra bytes are zeroed. */
+ for (size_t offset = type_sz; offset < opts_sz; ++offset)
+ if (((const char *)opts)[offset] != 0)
+ return false;
+
+ return true;
+}
+
+static int
+load_try(struct __rte_bpf_load *load, const struct rte_bpf_prm_ex *app_prm)
+{
+ int rc;
+
+ if (app_prm == NULL || !opts_valid(app_prm, app_prm->sz, sizeof(load->prm)))
+ return -EINVAL;
+
+ /*
+ * Convert extensible prm of application size to the size known to us.
+ *
+ * This code provides compatibility with applications compiled against
+ * different version of this library. `app_prm->sz` is the size of
+ * struct `rte_bpf_prm_ex` in the version used for compiling the
+ * application; `sizeof(load->prm)` is the size of the same struct in
+ * the version used for compiling the library.
+ *
+ * We are copying only the fields known to the application and leave
+ * the rest filled with zeroes. Any features not known to the
+ * application will have backward-compatible default behaviour.
+ */
+ memcpy(&load->prm, app_prm, RTE_MIN(app_prm->sz, sizeof(load->prm)));
+ load->prm.sz = sizeof(load->prm);
+
+ rc = bpf_check_xsyms(load->prm.xsym, load->prm.nb_xsym);
+
+ /* Convert prm origin to raw unless it already is. */
+ switch (load->prm.origin) {
+ case RTE_BPF_ORIGIN_RAW:
+ break;
+ case RTE_BPF_ORIGIN_ELF_FILE:
+ rc = rc < 0 ? rc : __rte_bpf_load_elf_file(load);
+ rc = rc < 0 ? rc : __rte_bpf_load_elf_code(load);
+ break;
+ default:
+ rc = rc < 0 ? rc : -EINVAL;
+ }
+
+ /* Now that it is raw load it as such. */
+ rc = rc < 0 ? rc : bpf_load_raw(load);
+
+ return rc;
+}
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_bpf_load_ex, 26.11)
+struct rte_bpf *
+rte_bpf_load_ex(const struct rte_bpf_prm_ex *prm)
+{
+ struct __rte_bpf_load load = { .elf_fd = -1 };
+
+ const int rc = load_try(&load, prm);
+
+ __rte_bpf_load_elf_cleanup(&load);
+
+ RTE_ASSERT((rc < 0) == (load.bpf == NULL));
+
+ if (rc < 0) {
+ RTE_BPF_LOG_FUNC_LINE(ERR, "failed, error code: %d", -rc);
rte_errno = -rc;
return NULL;
}
- return bpf;
+ RTE_BPF_LOG_FUNC_LINE(INFO, "successfully creates %p(jit={.func=%p,.sz=%zu});",
+ load.bpf, load.bpf->jit.func, load.bpf->jit.sz);
+ return load.bpf;
}
diff --git a/lib/bpf/bpf_load_elf.c b/lib/bpf/bpf_load_elf.c
index 2390823cbf30..4ae7492351ae 100644
--- a/lib/bpf/bpf_load_elf.c
+++ b/lib/bpf/bpf_load_elf.c
@@ -2,6 +2,13 @@
* Copyright(c) 2018 Intel Corporation
*/
+#include "bpf_impl.h"
+
+#include <errno.h>
+
+#ifdef RTE_LIBRTE_BPF_ELF
+
+#include <inttypes.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
@@ -26,8 +33,6 @@
#include <rte_byteorder.h>
#include <rte_errno.h>
-#include "bpf_impl.h"
-
/* To overcome compatibility issue */
#ifndef EM_BPF
#define EM_BPF 247
@@ -56,7 +61,7 @@ bpf_find_xsym(const char *sn, enum rte_bpf_xtype type,
*/
static int
resolve_xsym(const char *sn, size_t ofs, struct ebpf_insn *ins, size_t ins_sz,
- const struct rte_bpf_prm *prm)
+ const struct rte_bpf_prm_ex *prm)
{
uint32_t idx, fidx;
enum rte_bpf_xtype type;
@@ -183,7 +188,7 @@ find_elf_code(Elf *elf, const char *section, Elf_Data **psd, size_t *pidx)
*/
static int
process_reloc(Elf *elf, size_t sym_idx, Elf64_Rel *re, size_t re_sz,
- struct ebpf_insn *ins, size_t ins_sz, const struct rte_bpf_prm *prm)
+ struct ebpf_insn *ins, size_t ins_sz, const struct rte_bpf_prm_ex *prm)
{
int32_t rc;
uint32_t i, n;
@@ -232,8 +237,8 @@ process_reloc(Elf *elf, size_t sym_idx, Elf64_Rel *re, size_t re_sz,
* and update bpf code.
*/
static int
-elf_reloc_code(Elf *elf, Elf_Data *ed, size_t sidx,
- const struct rte_bpf_prm *prm)
+elf_reloc_code(Elf *elf, struct ebpf_insn *ins, size_t ins_sz, size_t sidx,
+ const struct rte_bpf_prm_ex *prm)
{
Elf64_Rel *re;
Elf_Scn *sc;
@@ -256,7 +261,7 @@ elf_reloc_code(Elf *elf, Elf_Data *ed, size_t sidx,
sd->d_size % sizeof(re[0]) != 0)
return -EINVAL;
rc = process_reloc(elf, sh->sh_link,
- sd->d_buf, sd->d_size, ed->d_buf, ed->d_size,
+ sd->d_buf, sd->d_size, ins, ins_sz,
prm);
}
}
@@ -264,72 +269,96 @@ elf_reloc_code(Elf *elf, Elf_Data *ed, size_t sidx,
return rc;
}
-static struct rte_bpf *
-bpf_load_elf(const struct rte_bpf_prm *prm, int32_t fd, const char *section)
+void
+__rte_bpf_load_elf_cleanup(struct __rte_bpf_load *load)
{
- Elf *elf;
- Elf_Data *sd;
- size_t sidx;
- int32_t rc;
- struct rte_bpf *bpf;
- struct rte_bpf_prm np;
+ elf_end(load->elf);
- elf_version(EV_CURRENT);
- elf = elf_begin(fd, ELF_C_READ, NULL);
+ if (load->elf_fd >= 0 && close(load->elf_fd) < 0) {
+ const int close_errno = errno;
+ RTE_BPF_LOG_FUNC_LINE(ERR, "error %d closing: %s",
+ close_errno, strerror(close_errno));
+ }
+}
- rc = find_elf_code(elf, section, &sd, &sidx);
- if (rc == 0)
- rc = elf_reloc_code(elf, sd, sidx, prm);
+int
+__rte_bpf_load_elf_file(struct __rte_bpf_load *load)
+{
+ const struct rte_bpf_prm_ex *const prm = &load->prm;
- if (rc == 0) {
- np = prm[0];
- np.ins = sd->d_buf;
- np.nb_ins = sd->d_size / sizeof(struct ebpf_insn);
- bpf = rte_bpf_load(&np);
- } else {
- bpf = NULL;
- rte_errno = -rc;
+ RTE_ASSERT(prm->origin == RTE_BPF_ORIGIN_ELF_FILE);
+
+ if (prm->elf_file.path == NULL || prm->elf_file.section == NULL)
+ return -EINVAL;
+
+ if (elf_version(EV_CURRENT) == EV_NONE)
+ return -ENOTSUP;
+
+ load->elf_fd = open(prm->elf_file.path, O_RDONLY);
+ if (load->elf_fd < 0) {
+ const int open_errno = errno;
+ RTE_BPF_LOG_FUNC_LINE(ERR, "error %d opening \"%s\": %s",
+ open_errno, prm->elf_file.path, strerror(open_errno));
+ return -open_errno;
+ }
+
+ load->elf = elf_begin(load->elf_fd, ELF_C_READ, NULL);
+ if (load->elf == NULL) {
+ const int rc = elf_errno();
+ RTE_BPF_LOG_FUNC_LINE(ERR, "error %d opening ELF \"%s\": %s",
+ rc, prm->elf_file.path, elf_errmsg(rc));
+ return -EINVAL;
}
- elf_end(elf);
- return bpf;
+ return 0;
}
-RTE_EXPORT_SYMBOL(rte_bpf_elf_load)
-struct rte_bpf *
-rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
- const char *sname)
+int
+__rte_bpf_load_elf_code(struct __rte_bpf_load *load)
{
- int32_t fd, rc;
- struct rte_bpf *bpf;
+ struct rte_bpf_prm_ex *const prm = &load->prm;
+ Elf_Data *sd;
+ size_t sidx;
+ int rc;
- if (prm == NULL || fname == NULL || sname == NULL) {
- rte_errno = EINVAL;
- return NULL;
- }
+ rc = find_elf_code(load->elf, prm->elf_file.section, &sd, &sidx);
+ if (rc < 0)
+ return rc;
- fd = open(fname, O_RDONLY);
- if (fd < 0) {
- rc = errno;
- RTE_BPF_LOG_LINE(ERR, "%s(%s) error code: %d(%s)",
- __func__, fname, rc, strerror(rc));
- rte_errno = EINVAL;
- return NULL;
- }
+ prm->origin = RTE_BPF_ORIGIN_RAW;
+ prm->raw.ins = sd->d_buf;
+ prm->raw.nb_ins = sd->d_size / sizeof(struct ebpf_insn);
- bpf = bpf_load_elf(prm, fd, sname);
- close(fd);
+ rc = elf_reloc_code(load->elf, sd->d_buf, sd->d_size, sidx, prm);
+ if (rc < 0)
+ return -EINVAL;
- if (bpf == NULL) {
- RTE_BPF_LOG_LINE(ERR,
- "%s(fname=\"%s\", sname=\"%s\") failed, "
- "error code: %d",
- __func__, fname, sname, rte_errno);
- return NULL;
- }
+ return 0;
+}
+
+#else /* RTE_LIBRTE_BPF_ELF */
+
+void
+__rte_bpf_load_elf_cleanup(struct __rte_bpf_load *load)
+{
+ RTE_ASSERT(load->elf == NULL);
+ RTE_ASSERT(load->elf_fd < 0);
+}
- RTE_BPF_LOG_LINE(INFO, "%s(fname=\"%s\", sname=\"%s\") "
- "successfully creates %p(jit={.func=%p,.sz=%zu});",
- __func__, fname, sname, bpf, bpf->jit.func, bpf->jit.sz);
- return bpf;
+int
+__rte_bpf_load_elf_file(struct __rte_bpf_load *load)
+{
+ RTE_SET_USED(load);
+ RTE_BPF_LOG_FUNC_LINE(ERR, "not supported, rebuild with libelf installed");
+ return -ENOTSUP;
}
+
+int
+__rte_bpf_load_elf_code(struct __rte_bpf_load *load)
+{
+ RTE_SET_USED(load);
+ RTE_BPF_LOG_FUNC_LINE(ERR, "not supported, rebuild with libelf installed");
+ return -ENOTSUP;
+}
+
+#endif /* RTE_LIBRTE_BPF_ELF */
diff --git a/lib/bpf/bpf_stub.c b/lib/bpf/bpf_stub.c
index e06e820d8327..4c329832c264 100644
--- a/lib/bpf/bpf_stub.c
+++ b/lib/bpf/bpf_stub.c
@@ -10,23 +10,6 @@
* Contains stubs for unimplemented public API functions
*/
-#ifndef RTE_LIBRTE_BPF_ELF
-RTE_EXPORT_SYMBOL(rte_bpf_elf_load)
-struct rte_bpf *
-rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
- const char *sname)
-{
- if (prm == NULL || fname == NULL || sname == NULL) {
- rte_errno = EINVAL;
- return NULL;
- }
-
- RTE_BPF_LOG_FUNC_LINE(ERR, "not supported, rebuild with libelf installed");
- rte_errno = ENOTSUP;
- return NULL;
-}
-#endif
-
#ifndef RTE_HAS_LIBPCAP
RTE_EXPORT_SYMBOL(rte_bpf_convert)
struct rte_bpf_prm *
diff --git a/lib/bpf/bpf_validate.c b/lib/bpf/bpf_validate.c
index a7f4f576c9d6..5bfc59296d05 100644
--- a/lib/bpf/bpf_validate.c
+++ b/lib/bpf/bpf_validate.c
@@ -80,7 +80,7 @@ struct evst_pool {
};
struct bpf_verifier {
- const struct rte_bpf_prm *prm;
+ const struct rte_bpf_prm_ex *prm;
struct inst_node *in;
uint64_t stack_sz;
uint32_t nb_nodes;
@@ -1837,7 +1837,7 @@ add_edge(struct bpf_verifier *bvf, struct inst_node *node, uint32_t nidx)
{
uint32_t ne;
- if (nidx >= bvf->prm->nb_ins) {
+ if (nidx >= bvf->prm->raw.nb_ins) {
RTE_BPF_LOG_FUNC_LINE(ERR,
"program boundary violation at pc: %u, next pc: %u",
get_node_idx(bvf, node), nidx);
@@ -1946,10 +1946,10 @@ log_unreachable(const struct bpf_verifier *bvf)
struct inst_node *node;
const struct ebpf_insn *ins;
- for (i = 0; i != bvf->prm->nb_ins; i++) {
+ for (i = 0; i != bvf->prm->raw.nb_ins; i++) {
node = bvf->in + i;
- ins = bvf->prm->ins + i;
+ ins = bvf->prm->raw.ins + i;
if (node->colour == WHITE &&
ins->code != (BPF_LD | BPF_IMM | EBPF_DW))
@@ -1966,7 +1966,7 @@ log_loop(const struct bpf_verifier *bvf)
uint32_t i, j;
struct inst_node *node;
- for (i = 0; i != bvf->prm->nb_ins; i++) {
+ for (i = 0; i != bvf->prm->raw.nb_ins; i++) {
node = bvf->in + i;
if (node->colour != BLACK)
@@ -1998,9 +1998,9 @@ validate(struct bpf_verifier *bvf)
const char *err;
rc = 0;
- for (i = 0; i < bvf->prm->nb_ins; i++) {
+ for (i = 0; i < bvf->prm->raw.nb_ins; i++) {
- ins = bvf->prm->ins + i;
+ ins = bvf->prm->raw.ins + i;
node = bvf->in + i;
err = check_syntax(ins);
@@ -2432,7 +2432,7 @@ evaluate(struct bpf_verifier *bvf)
bvf->evst->rv[EBPF_REG_10] = rvfp;
- ins = bvf->prm->ins;
+ ins = bvf->prm->raw.ins;
node = bvf->in;
next = node;
rc = 0;
@@ -2522,23 +2522,23 @@ evaluate(struct bpf_verifier *bvf)
}
int
-__rte_bpf_validate(struct rte_bpf *bpf)
+__rte_bpf_validate(const struct rte_bpf_prm_ex *prm, uint32_t *stack_sz)
{
int32_t rc;
struct bpf_verifier bvf;
/* check input argument type, don't allow mbuf ptr on 32-bit */
- if (bpf->prm.prog_arg.type != RTE_BPF_ARG_RAW &&
- bpf->prm.prog_arg.type != RTE_BPF_ARG_PTR &&
+ if (prm->prog_arg.type != RTE_BPF_ARG_RAW &&
+ prm->prog_arg.type != RTE_BPF_ARG_PTR &&
(sizeof(uint64_t) != sizeof(uintptr_t) ||
- bpf->prm.prog_arg.type != RTE_BPF_ARG_PTR_MBUF)) {
+ prm->prog_arg.type != RTE_BPF_ARG_PTR_MBUF)) {
RTE_BPF_LOG_FUNC_LINE(ERR, "unsupported argument type");
return -ENOTSUP;
}
memset(&bvf, 0, sizeof(bvf));
- bvf.prm = &bpf->prm;
- bvf.in = calloc(bpf->prm.nb_ins, sizeof(bvf.in[0]));
+ bvf.prm = prm;
+ bvf.in = calloc(prm->raw.nb_ins, sizeof(bvf.in[0]));
if (bvf.in == NULL)
return -ENOMEM;
@@ -2555,11 +2555,11 @@ __rte_bpf_validate(struct rte_bpf *bpf)
/* copy collected info */
if (rc == 0) {
- bpf->stack_sz = bvf.stack_sz;
+ *stack_sz = bvf.stack_sz;
/* for LD_ABS/LD_IND, we'll need extra space on the stack */
if (bvf.nb_ldmb_nodes != 0)
- bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +
+ *stack_sz = RTE_ALIGN_CEIL(*stack_sz +
sizeof(uint64_t), sizeof(uint64_t));
}
diff --git a/lib/bpf/meson.build b/lib/bpf/meson.build
index 28df7f469a4c..4901b6ee1463 100644
--- a/lib/bpf/meson.build
+++ b/lib/bpf/meson.build
@@ -19,6 +19,7 @@ sources = files('bpf.c',
'bpf_dump.c',
'bpf_exec.c',
'bpf_load.c',
+ 'bpf_load_elf.c',
'bpf_pkt.c',
'bpf_stub.c',
'bpf_validate.c')
@@ -38,10 +39,9 @@ deps += ['mbuf', 'net', 'ethdev']
dep = dependency('libelf', required: false, method: 'pkg-config')
if dep.found()
dpdk_conf.set('RTE_LIBRTE_BPF_ELF', 1)
- sources += files('bpf_load_elf.c')
ext_deps += dep
else
- warning('libelf is missing, rte_bpf_elf_load API will be disabled')
+ warning('libelf is missing, ELF API will be disabled')
endif
if dpdk_conf.has('RTE_HAS_LIBPCAP')
diff --git a/lib/bpf/rte_bpf.h b/lib/bpf/rte_bpf.h
index 309d84bc516a..bf58a418191e 100644
--- a/lib/bpf/rte_bpf.h
+++ b/lib/bpf/rte_bpf.h
@@ -86,7 +86,47 @@ struct rte_bpf_xsym {
};
/**
- * Input parameters for loading eBPF code.
+ * Possible origins of eBPF program code.
+ */
+enum rte_bpf_origin {
+ RTE_BPF_ORIGIN_RAW, /**< code loaded from raw array */
+ RTE_BPF_ORIGIN_RESERVED, /**< reserved for cBPF */
+ RTE_BPF_ORIGIN_ELF_FILE, /**< code loaded from elf_file */
+};
+
+/**
+ * Input parameters for loading eBPF code, extensible version.
+ *
+ * Follows libbpf conventions for extensible structs.
+ */
+struct rte_bpf_prm_ex {
+ size_t sz; /**< size of this struct for backward compatibility */
+
+ uint32_t flags; /**< flags controlling eBPF load and other options */
+
+ enum rte_bpf_origin origin; /**< origin of eBPF program code */
+
+ /** program origin parameters, member in use depends on origin */
+ union {
+ struct {
+ const struct ebpf_insn *ins; /**< eBPF instructions */
+ uint32_t nb_ins; /**< number of instructions in ins */
+ } raw;
+ struct {
+ const char *path; /**< path to the ELF file */
+ const char *section; /**< ELF section with the code */
+ } elf_file;
+ };
+
+ const struct rte_bpf_xsym *xsym;
+ /**< array of external symbols that eBPF code is allowed to reference */
+ uint32_t nb_xsym; /**< number of elements in xsym */
+
+ struct rte_bpf_arg prog_arg; /**< input arg description */
+};
+
+/**
+ * Input parameters for loading eBPF code, legacy version.
*/
struct rte_bpf_prm {
const struct ebpf_insn *ins; /**< array of eBPF instructions */
@@ -116,6 +156,32 @@ struct rte_bpf;
void
rte_bpf_destroy(struct rte_bpf *bpf);
+/**
+ * @warning
+ * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
+ *
+ * Create a new eBPF execution context, load code from specified origin into it.
+ *
+ * @param prm
+ * Parameters used to create and initialise the BPF execution context.
+ *
+ * Member sz must be set to the struct size as known to the application.
+ * If it exceeds the size known to the library, and the extra part has
+ * non-zero bytes, parameter is rejected. If it's smaller than the size known
+ * to the library, defaults are used for the members that are not present.
+ * @return
+ * BPF handle that is used in future BPF operations,
+ * or NULL on error, with error code set in rte_errno.
+ * Possible rte_errno errors include:
+ * - EINVAL - invalid parameter passed to function
+ * - ENOMEM - can't reserve enough memory
+ * - ENOTSUP - requested feature is not supported (e.g. no libelf to load ELF)
+ */
+__rte_experimental
+struct rte_bpf *
+rte_bpf_load_ex(const struct rte_bpf_prm_ex *prm)
+ __rte_malloc __rte_dealloc(rte_bpf_destroy, 1);
+
/**
* Create a new eBPF execution context and load given BPF code into it.
*
--
2.43.0
^ permalink raw reply related
* RE: [PATCH v5 00/11] bpf: introduce extensible load API
From: Marat Khalili @ 2026-06-17 19:48 UTC (permalink / raw)
To: thomas@monjalon.net; +Cc: dev@dpdk.org, Stephen Hemminger
In-Reply-To: <20260612090636.10d8bf26@phoenix.local>
> Detailed AI review found some minor things:
>
> 02/11: comment grammar nits — "Any features that not known to
> the application" -> "that are not known" (twice); "the size of
> same struct" -> "the size of the same struct".
>
> 05/11: Doxygen for rte_bpf_eth_rx_install and
> rte_bpf_eth_tx_install references rte_bpf_eth_unload, which does
> not exist. Use rte_bpf_eth_rx_unload and rte_bpf_eth_tx_unload
> respectively.
>
> 07/11: stray space before comma in the test_bpf_filter log
> string ("for \"%s\" ,").
Sent v6 with typos corrected.
Will not re-send dependent patchset with updated Depends-on
For now since it doesn't do much anyway.
^ permalink raw reply
* RE: [v4] net/cksum: compute raw cksum for several segments
From: Marat Khalili @ 2026-06-17 20:02 UTC (permalink / raw)
To: Su Sai; +Cc: dev@dpdk.org, stephen@networkplumber.org
In-Reply-To: <20260616123812.21048-1-spiderdetective.ss@gmail.com>
> -----Original Message-----
> From: Su Sai <spiderdetective.ss@gmail.com>
> Sent: Tuesday 16 June 2026 13:38
> To: stephen@networkplumber.org
> Cc: dev@dpdk.org; spiderdetective.ss@gmail.com
> Subject: [v4] net/cksum: compute raw cksum for several segments
>
> The rte_raw_cksum_mbuf function is used to compute
> the raw checksum of a packet.
> If the packet payload stored in multi mbuf, the function
> will goto the hard case. In hard case,
> the variable 'tmp' is a type of uint32_t,
> so rte_bswap16 will drop high 16 bit.
> Meanwhile, the variable 'sum' is a type of uint32_t,
> so 'sum += tmp' will drop the carry when overflow.
> Both drop will make cksum incorrect.
> This commit fixes the above bug.
>
> Signed-off-by: Su Sai <spiderdetective.ss@gmail.com>
You can add:
Acked-by: Marat Khalili <marat.khalili@huawei.com>
> ---
> .mailmap | 1 +
> app/test/test_cksum.c | 102 ++++++++++++++++++++++++++++++++++++++++++
> lib/net/rte_cksum.h | 27 +++++++++--
> 3 files changed, 126 insertions(+), 4 deletions(-)
>
> diff --git a/.mailmap b/.mailmap
> index 4001e5fb0e..bcf73cb902 100644
> --- a/.mailmap
> +++ b/.mailmap
> @@ -1630,6 +1630,7 @@ Sylvia Grundw眉rmer <sylvia.grundwuermer@b-plus.com>
> Sylwester Dziedziuch <sylwesterx.dziedziuch@intel.com>
> Sylwia Wnuczko <sylwia.wnuczko@intel.com>
> Szymon Sliwa <szs@semihalf.com>
> +Su Sai <spiderdetective.ss@gmail.com> <susai.ss@bytedance.com>
> Szymon T Cudzilo <szymon.t.cudzilo@intel.com>
> Tadhg Kearney <tadhg.kearney@intel.com>
> Taekyung Kim <kim.tae.kyung@navercorp.com>
> diff --git a/app/test/test_cksum.c b/app/test/test_cksum.c
> index ea443382a1..5bd9723fbd 100644
> --- a/app/test/test_cksum.c
> +++ b/app/test/test_cksum.c
> @@ -85,6 +85,42 @@ static const char test_cksum_ipv4_opts_udp[] = {
> 0x00, 0x35, 0x00, 0x09, 0x89, 0x6f, 0x78,
> };
>
> +/*
> + * generated in scapy with
> + * Ether()/IP()/TCP(options=[NOP,NOP,Timestamps])/os.urandom(113))
> + */
> +static const char test_cksum_ipv4_tcp_multi_segs[] = {
> + 0x00, 0x16, 0x3e, 0x0b, 0x6b, 0xd2, 0xee, 0xff,
> + 0xff, 0xff, 0xff, 0xff, 0x08, 0x00, 0x45, 0x00,
> + 0x00, 0xa5, 0x46, 0x10, 0x40, 0x00, 0x40, 0x06,
> + 0x80, 0xb5, 0xc0, 0xa8, 0xf9, 0x1d, 0xc0, 0xa8,
> + 0xf9, 0x1e, 0xdc, 0xa2, 0x14, 0x51, 0xbb, 0x8f,
> + 0xa0, 0x00, 0xe4, 0x7c, 0xe4, 0xb8, 0x80, 0x10,
> + 0x02, 0x00, 0x4b, 0xc1, 0x00, 0x00, 0x01, 0x01,
> + 0x08, 0x0a, 0x90, 0x60, 0xf4, 0xff, 0x03, 0xc5,
> + 0xb4, 0x19, 0x77, 0x34, 0xd4, 0xdc, 0x84, 0x86,
> + 0xff, 0x44, 0x09, 0x63, 0x36, 0x2e, 0x26, 0x9b,
> + 0x90, 0x70, 0xf2, 0xed, 0xc8, 0x5b, 0x87, 0xaa,
> + 0xb4, 0x67, 0x6b, 0x32, 0x3d, 0xc4, 0xbf, 0x15,
> + 0xa9, 0x16, 0x6c, 0x2a, 0x9d, 0xb2, 0xb7, 0x6b,
> + 0x58, 0x44, 0x58, 0x12, 0x4b, 0x8f, 0xe5, 0x12,
> + 0x11, 0x90, 0x94, 0x68, 0x37, 0xad, 0x0a, 0x9b,
> + 0xd6, 0x79, 0xf2, 0xb7, 0x31, 0xcf, 0x44, 0x22,
> + 0xc8, 0x99, 0x3f, 0xe5, 0xe7, 0xac, 0xc7, 0x0b,
> + 0x86, 0xdf, 0xda, 0xed, 0x0a, 0x0f, 0x86, 0xd7,
> + 0x48, 0xe2, 0xf1, 0xc2, 0x43, 0xed, 0x47, 0x3a,
> + 0xea, 0x25, 0x2d, 0xd6, 0x60, 0x38, 0x30, 0x07,
> + 0x28, 0xdd, 0x1f, 0x0c, 0xdd, 0x7b, 0x7c, 0xd9,
> + 0x35, 0x9d, 0x14, 0xaa, 0xc6, 0x35, 0xd1, 0x03,
> + 0x38, 0xb1, 0xf5,
> +};
> +
> +static const uint8_t test_cksum_ipv4_tcp_multi_segs_len[] = {
> + 66, /* the first seg contains all headers, including L2 to L4 */
> + 61, /* the second seg length is odd, test byte order independent */
> + 52, /* three segs are sufficient to test the most complex scenarios */
> +};
> +
> /* test l3/l4 checksum api */
> static int
> test_l4_cksum(struct rte_mempool *pktmbuf_pool, const char *pktdata, size_t len)
> @@ -223,6 +259,66 @@ test_l4_cksum(struct rte_mempool *pktmbuf_pool, const char *pktdata, size_t len)
> return -1;
> }
>
> +/* test l4 checksum api for a packet with multiple mbufs */
> +static int
> +test_l4_cksum_multi_mbufs(struct rte_mempool *pktmbuf_pool, const char *pktdata, size_t len,
> + const uint8_t *segs, size_t segs_len)
> +{
> + struct rte_mbuf *m[NB_MBUF] = {0};
> + struct rte_mbuf *m_hdr = NULL;
> + struct rte_net_hdr_lens hdr_lens;
> + size_t i, off = 0;
> + uint32_t packet_type, l3;
> + void *l3_hdr;
> + char *data;
> +
> + for (i = 0; i < segs_len; i++) {
> + m[i] = rte_pktmbuf_alloc(pktmbuf_pool);
> + if (m[i] == NULL)
> + GOTO_FAIL("Cannot allocate mbuf");
> +
> + data = rte_pktmbuf_append(m[i], segs[i]);
> + if (data == NULL)
> + GOTO_FAIL("Cannot append data");
> +
> + memcpy(data, pktdata + off, segs[i]);
> + off += segs[i];
> +
> + if (m_hdr) {
> + if (rte_pktmbuf_chain(m_hdr, m[i]))
> + GOTO_FAIL("Cannot chain mbuf");
> + } else {
> + m_hdr = m[i];
> + }
> + }
> +
> + if (off != len)
> + GOTO_FAIL("Invalid segs");
> +
> + packet_type = rte_net_get_ptype(m_hdr, &hdr_lens, RTE_PTYPE_ALL_MASK);
> + l3 = packet_type & RTE_PTYPE_L3_MASK;
> +
> + l3_hdr = rte_pktmbuf_mtod_offset(m_hdr, void *, hdr_lens.l2_len);
> + off = hdr_lens.l2_len + hdr_lens.l3_len;
> +
> + if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) {
> + if (rte_ipv4_udptcp_cksum_mbuf_verify(m_hdr, l3_hdr, off) != 0)
> + GOTO_FAIL("Invalid L4 checksum verification for multiple mbufs");
> + } else if (l3 == RTE_PTYPE_L3_IPV6 || l3 == RTE_PTYPE_L3_IPV6_EXT) {
> + if (rte_ipv6_udptcp_cksum_mbuf_verify(m_hdr, l3_hdr, off) != 0)
> + GOTO_FAIL("Invalid L4 checksum verification for multiple mbufs");
> + }
> +
> + rte_pktmbuf_free_bulk(m, segs_len);
> +
> + return 0;
> +
> +fail:
> + rte_pktmbuf_free_bulk(m, segs_len);
> +
> + return -1;
> +}
> +
> static int
> test_cksum(void)
> {
> @@ -256,6 +352,12 @@ test_cksum(void)
> sizeof(test_cksum_ipv4_opts_udp)) < 0)
> GOTO_FAIL("checksum error on ipv4_opts_udp");
>
> + if (test_l4_cksum_multi_mbufs(pktmbuf_pool, test_cksum_ipv4_tcp_multi_segs,
> + sizeof(test_cksum_ipv4_tcp_multi_segs),
> + test_cksum_ipv4_tcp_multi_segs_len,
> + sizeof(test_cksum_ipv4_tcp_multi_segs_len)) < 0)
> + GOTO_FAIL("checksum error on multi mbufs check");
> +
> rte_mempool_free(pktmbuf_pool);
>
> return 0;
> diff --git a/lib/net/rte_cksum.h b/lib/net/rte_cksum.h
> index a8e8927952..679ba82eb6 100644
> --- a/lib/net/rte_cksum.h
> +++ b/lib/net/rte_cksum.h
> @@ -80,6 +80,25 @@ __rte_raw_cksum_reduce(uint32_t sum)
> return (uint16_t)sum;
> }
>
> +/**
> + * @internal Reduce a sum to the non-complemented checksum.
> + * Helper routine for the rte_raw_cksum_mbuf().
> + *
> + * @param sum
> + * Value of the sum.
> + * @return
> + * The non-complemented checksum.
> + */
> +static inline uint16_t
> +__rte_raw_cksum_reduce_u64(uint64_t sum)
> +{
> + uint32_t tmp;
> +
> + tmp = __rte_raw_cksum_reduce((uint32_t)sum);
> + tmp += __rte_raw_cksum_reduce((uint32_t)(sum >> 32));
> + return __rte_raw_cksum_reduce(tmp);
> +}
> +
> /**
> * Process the non-complemented checksum of a buffer.
> *
> @@ -119,8 +138,8 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len,
> {
> const struct rte_mbuf *seg;
> const char *buf;
> - uint32_t sum, tmp;
> - uint32_t seglen, done;
> + uint32_t seglen, done, tmp;
> + uint64_t sum;
>
> /* easy case: all data in the first segment */
> if (off + len <= rte_pktmbuf_data_len(m)) {
> @@ -157,7 +176,7 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len,
> for (;;) {
> tmp = __rte_raw_cksum(buf, seglen, 0);
> if (done & 1)
> - tmp = rte_bswap16((uint16_t)tmp);
> + tmp = rte_bswap32(tmp);
> sum += tmp;
> done += seglen;
> if (done == len)
> @@ -169,7 +188,7 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len,
> seglen = len - done;
> }
>
> - *cksum = __rte_raw_cksum_reduce(sum);
> + *cksum = __rte_raw_cksum_reduce_u64(sum);
> return 0;
> }
>
> --
> 2.20.1
^ permalink raw reply
* Re: [PATCH v5] dts: add retry loop to trex traffic generation
From: Patrick Robb @ 2026-06-17 20:55 UTC (permalink / raw)
To: Andrew Bailey; +Cc: luca.vizzarro, lylavoie, knimoji, dev
In-Reply-To: <20260617155748.59724-1-abailey@iol.unh.edu>
[-- Attachment #1: Type: text/plain, Size: 235 bytes --]
Looks good, I will merge to next-dts this evening. Remind me, it was Intel
NICs that were having link issues with TREX before you added this retry
loop, right? Thanks for the fix.
Reviewed-by: Patrick Robb <patrickrobb1997@gmail.com>
[-- Attachment #2: Type: text/html, Size: 334 bytes --]
^ permalink raw reply
* Re: [PATCH v5] dts: add retry loop to trex traffic generation
From: Andrew Bailey @ 2026-06-17 20:58 UTC (permalink / raw)
To: Patrick Robb; +Cc: luca.vizzarro, lylavoie, knimoji, dev
In-Reply-To: <CAK6Duxt3ZSCrHQyFp-n1OrEiWH7-4YNPzC8LP_U1HxvbouSwvg@mail.gmail.com>
[-- Attachment #1: Type: text/plain, Size: 65 bytes --]
Thanks Patrick,
yes it was intel NICs having these link issues.
[-- Attachment #2: Type: text/html, Size: 114 bytes --]
^ permalink raw reply
* Re: [PATCH v5 0/6] net/gve: add hardware timestamping support
From: Stephen Hemminger @ 2026-06-17 21:11 UTC (permalink / raw)
To: Mark Blasko; +Cc: dev, joshwash, jtranoleary
In-Reply-To: <20260617000712.2195506-1-blasko@google.com>
On Wed, 17 Jun 2026 00:07:04 +0000
Mark Blasko <blasko@google.com> wrote:
> This patch series introduces support for GVE hardware timestamping
> on DQO queues. To support concurrent access, a mutex lock is introduced
> to protect admin queue operations. A mechanism is then added to
> periodically synchronize the NIC clock via a dedicated control thread,
> and support is introduced for the read_clock ethdev operation.
> Finally, the RX datapath is updated to reconstruct full 64-bit
> timestamps from the 32-bit values in DQO descriptors.
Applied to next-net
^ permalink raw reply
* Re: [PATCH] app/testpmd: include IP fields in UDP RSS option
From: Stephen Hemminger @ 2026-06-17 21:12 UTC (permalink / raw)
To: Maxime Leroy
Cc: dev, stable, Aman Singh, Heqing Zhu, Jijiang Liu, Helin Zhang,
Cunming Liang, Jing Chen
In-Reply-To: <20260616093912.685791-1-maxime@leroys.fr>
On Tue, 16 Jun 2026 11:39:12 +0200
Maxime Leroy <maxime@leroys.fr> wrote:
> The --rss-udp option is documented as enabling IPv4/IPv6 + UDP RSS, but it
> currently sets the RSS hash functions to RTE_ETH_RSS_UDP only.
>
> On PMDs that translate this directly to L4 port extracts, this can build a
> L4-only RSS key. Add RTE_ETH_RSS_IP when --rss-udp is selected so the
> configured key matches the documented IPv4/IPv6 + UDP behavior.
>
> Make --rss-ip additive as well, so combining --rss-ip and --rss-udp is
> order-independent.
>
> Fixes: 8a387fa85f02 ("ethdev: more RSS flags")
> Cc: stable@dpdk.org
> Signed-off-by: Maxime Leroy <maxime@leroys.fr>
> ---
Applied to next-net
^ permalink raw reply
* Re: [PATCH 0/2] ethdev: fix fast-path ops on a stopped port
From: Stephen Hemminger @ 2026-06-17 21:13 UTC (permalink / raw)
To: Maxime Leroy; +Cc: dev
In-Reply-To: <20260616094259.686555-1-maxime@leroys.fr>
On Tue, 16 Jun 2026 11:42:56 +0200
Maxime Leroy <maxime@leroys.fr> wrote:
> Two small fixes for fast-path ops on a stopped port:
> patch 1 stops rte_eth_rx_queue_count() from dereferencing NULL after a port
> stop, patch 2 makes the dummy queue count return 0 (empty queue) instead of
> -ENOTSUP.
>
> Maxime Leroy (2):
> ethdev: keep fast-path ops valid after port stop
> ethdev: return 0 from dummy queue count
>
> lib/ethdev/ethdev_driver.c | 2 +-
> lib/ethdev/ethdev_private.c | 7 +++++++
> 2 files changed, 8 insertions(+), 1 deletion(-)
>
Applied to next-net
^ permalink raw reply
* Re: [PATCH v3] dts: clean cryptodev environment after a test run
From: Patrick Robb @ 2026-06-17 21:13 UTC (permalink / raw)
To: Andrew Bailey; +Cc: luca.vizzarro, lylavoie, ahassick, knimoji, dev
In-Reply-To: <20260513154055.134729-1-abailey@iol.unh.edu>
[-- Attachment #1: Type: text/plain, Size: 196 bytes --]
So, some cryptodevs must be unbound from their DPDK driver before they can
be destroyed by linux? Out of curiosity, which devices are these?
Reviewed-by: Patrick Robb <patrickrobb1997@gmail.com>
[-- Attachment #2: Type: text/html, Size: 296 bytes --]
^ permalink raw reply
* Re: [PATCH 0/4] bpf/arm64: add BPF_ABS/BPF_IND packet load support
From: Stephen Hemminger @ 2026-06-17 21:17 UTC (permalink / raw)
To: Marat Khalili; +Cc: dev@dpdk.org
In-Reply-To: <e8f68b2da12a4bb58ccb580c49557510@huawei.com>
On Wed, 17 Jun 2026 17:37:39 +0000
Marat Khalili <marat.khalili@huawei.com> wrote:
> I think this should CC participants of the previous discussion:
> https://inbox.dpdk.org/dev/20260319114500.9757-1-cfontain@redhat.com/
>
> (Humans may think they submit patches independently, but weights of their LLMs
> were already contaminated with the other effort. Hello brave new world.)
Didn't see previous thread. This effort was more targeted at
why can't capture which uses pcap_compile -> bpf_convert flow be JIT'd?
Had not looked back at other overlaps.
^ permalink raw reply
* Re: [PATCH v5] dts: add retry loop to trex traffic generation
From: Patrick Robb @ 2026-06-18 1:46 UTC (permalink / raw)
To: Andrew Bailey; +Cc: luca.vizzarro, lylavoie, knimoji, dev
In-Reply-To: <CABJ3N2Wxr0wAmuF7FGThGgfssacuHQhgU7vs15rHh5dCedLdvQ@mail.gmail.com>
[-- Attachment #1: Type: text/plain, Size: 195 bytes --]
Applied to next-dts, thanks Andrew.
On Wed, Jun 17, 2026 at 4:59 PM Andrew Bailey <abailey@iol.unh.edu> wrote:
> Thanks Patrick,
>
> yes it was intel NICs having these link issues.
>
[-- Attachment #2: Type: text/html, Size: 530 bytes --]
^ permalink raw reply
* 回复：回复：[PATCH] gpu/metax: add new driver for Metax GPU
From: 许玲燕 @ 2026-06-18 2:00 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev, eagostini
In-Reply-To: <VJ0LOsEwRLSFadcujOiwrA@monjalon.net>
[-- Attachment #1: Type: text/plain, Size: 3718 bytes --]
Hi,
Thank you for your reply. I would like to clarify the current setup for accessing the download portal:
Website Language: At present, the Metax Software Download Center is only available in Chinese, and we do not have an English version at the moment. For non-Chinese speakers, we highly recommend using your browser's built-in translation feature, which works perfectly for navigating the site and downloading the files.
Download Access: Currently, there are no direct, open download links. Registration and login are required to access and download the library files. This is our standard process for distributing the SDK and kernel modules.
I understand this might be an extra step, but please rest assured that the registration process is straightforward. If you encounter any issues during the registration or translation process, please let me know and I will be happy to assist you.
Best regards,
lingyxu
------------------------------------------------------------------
发件人：Thomas Monjalon <thomas@monjalon.net>
发送时间：2026年6月12日(周五) 22:49
收件人："许玲燕"<lingyan.xu@metax-tech.com>
抄 送：dev<dev@dpdk.org>; eagostini<eagostini@nvidia.com>; "王冬冬"<dongdong.wang@metax-tech.com>
主 题：Re: 回复：[PATCH] gpu/metax: add new driver for Metax GPU
Would it be possible to have a direct download link
not requiring any registration? At least for the library?
Do you have an english version of the website?
12/06/2026 09:19, 许玲燕:
> Hi,
> Thank you for the follow-up. Please find the clarifications below regarding the kernel dependency and availability:
> 1. Kernel Dependency
> While it is not upstreamed into the mainline Linux kernel, it is actively maintained by Metax to interface with our hardware.
> 2. Availability and Download Link
> Yes, both the proprietary user-space libraries and the corresponding kernel modules are freely available for download. You can access them via the Metax Software Download Center:
> Download Link: https://sw-download.metax-tech.com/ <https://sw-download.metax-tech.com/ > <https://sw-download.metax-tech.com/ <https://sw-download.metax-tech.com/ > >
> How to obtain the files:
>
> *
> Register and log in to the portal.
>
> *
> Navigate to "SDK Development Kit".
>
> *
> Select your specific GPU Type.
>
> *
> Choose your target OS. We currently support Linux (aarch64 & x86_64) across mainstream distributions, including but not limited to:
> *
> Ubuntu (18.04 - 24.04)
>
> *
> RHEL / CentOS / RockyOS (8.x / 9.x)
>
> *
> Domestic distros: KylinV10/V11, OpenCloudOS, TencentOS, etc.
> Best regards,
> ------------------------------------------------------------------
> 发件人：Thomas Monjalon <thomas@monjalon.net>
> 发送时间：2026年6月11日(周四) 17:17
> 收件人："许玲燕"<lingyan.xu@metax-tech.com>
> 抄 送：dev<dev@dpdk.org>; eagostini<eagostini@nvidia.com>; "王冬冬"<dongdong.wang@metax-tech.com>
> 主 题：Re: [PATCH] gpu/metax: add new driver for Metax GPU
> 11/06/2026 09:10:
> > Both libmcruntime.so and the corresponding gdrapi libraries
> > are proprietary user-space libraries provided by Metax.
> > They are not upstreamed to the DPDK mainline repository.
> > However, please rest assured that the current patch interacts
> > with them via standard dlopen (dynamic loading) at runtime.
> > We do not link directly against their source code
> > or require them as hard build-time dependencies.
> > Therefore, this approach will not introduce any additional
> > compilation dependencies or licensing issues to the DPDK main tree.
> What about the kernel dependency?
> Are libraries and kernel module freely available for download?
> Can you provide a link?
>
[-- Attachment #2: Type: text/html, Size: 7082 bytes --]
^ permalink raw reply
* RE: ARM v8 rte_power_pause
From: Hemant Agrawal @ 2026-06-18 6:17 UTC (permalink / raw)
To: Wathsala Vithanage, Morten Brørup
Cc: dev@dpdk.org, Maxime Leroy, Gagandeep Singh
In-Reply-To: <09dbd9b1-f4a6-48e2-9424-546eafdfa01b@arm.com>
Hi Watshala,
I think WFET is not available on A72 core.
Can you update your answer w.r.t Cortex-A72/Arm v8.0 architecture?
Regards
Hemant
> -----Original Message-----
> From: Wathsala Vithanage <wathsala.vithanage@arm.com>
> Sent: 17 June 2026 17:27
> To: Hemant Agrawal <hemant.agrawal@nxp.com>; Morten Brørup
> <mb@smartsharesystems.com>
> Cc: dev@dpdk.org; Maxime Leroy <maxime@leroys.fr>; Gagandeep Singh
> <G.Singh@nxp.com>
> Subject: Re: ARM v8 rte_power_pause
> Importance: High
>
> Hi Morten and Hemant,
>
> YIELD is a NOP on non-SMT CPUs, such as Neoverse.
>
> WFE is universally available on AArch64, but it comes with a caveat: the CPU
> can remain in a low-power state indefinitely unless an event is triggered. That
> event can be generated explicitly via SEV/SEVL by a different CPU, or implicitly
> through address monitoring (LDAXR).
>
> WFET is the safer variant because it includes a timeout, so explicit or implicit
> event-register manipulation is not required.
>
> --wathsala
>
> On 6/12/26 01:11, Hemant Agrawal wrote:
> > Hi Morten,
> > On Cortex‑A72 (ARMv8), the only architectural primitives available are
> YIELD, WFE, and WFI:
> >
> > YIELD is the only deterministic, low-overhead option (pure CPU relax,
> no entry into low-power state)
> > WFE can be used as a low-power idle hint, but it is event-driven and
> not time-based (it may return immediately)
> > WFI depends on interrupt wakeup and is therefore not suitable for
> > tight latency loops
> >
> > For ~1 µs latency targets, the practical approach is a hybrid strategy:
> >
> > Short waits → spin using YIELD
> > Slightly longer waits → opportunistically use WFE for power reduction
> >
> > A simple implementation could look like (not tested):
> >
> > static inline void rte_armv8_pause(unsigned int iters) {
> > if (iters < 64) {
> > for (unsigned int i = 0; i < iters; i++)
> > asm volatile("yield");
> > } else {
> > asm volatile("sevl");
> > asm volatile("wfe");
> > }
> > }
> >
> > @Wathsala Vithanage — would appreciate your thoughts, especially if there
> are any micro-architectural nuances we should consider.
> >
> > Regards,
> > Hemant
> >
> >> -----Original Message-----
> >> From: Morten Brørup <mb@smartsharesystems.com>
> >> Sent: 03 June 2026 17:26
> >> To: Wathsala Vithanage <wathsala.vithanage@arm.com>; Hemant Agrawal
> >> <hemant.agrawal@nxp.com>; Sachin Saxena (OSS)
> >> <sachin.saxena@oss.nxp.com>
> >> Cc: dev@dpdk.org; Maxime Leroy <maxime@leroys.fr>
> >> Subject: ARM v8 rte_power_pause
> >> Importance: High
> >>
> >> Hi Wathsala, Hemant and Sachin,
> >>
> >> Over at the Grout project, we are discussing power management in the
> >> context of 100 Gbit/s latency deadlines [1].
> >>
> >> rte_power_pause() is not implemented for ARM v8 / Cortex-A72.
> >> Syscalls such as nanosleep() have too much overhead, and cannot be used.
> >>
> >> Any suggestions for a power-reducing method to make a CPU core "sleep"
> (i.e.
> >> do nothing) for durations in the order of 1 microsecond?
> >>
> >> [1]:
> >> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit
> >>
> hu%2F&data=05%7C02%7Chemant.agrawal%40nxp.com%7C06a651571db
> 545d47d7a0
> >>
> 8decc67908e%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C639
> 172942353
> >>
> 967617%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYi
> OiIwLjAuMD
> >>
> AwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7
> C%7C&s
> >>
> data=7NCh3%2BS3TAu1sRLYgqGNAaTwqdgwjqhAs2awPixIeEM%3D&reserve
> d=0
> >> b.com%2FDPDK%2Fgrout%2Fpull%2F624%23issuecomment-
> >>
> 4602036364&data=05%7C02%7Chemant.agrawal%40nxp.com%7Cdbff5f2e
> >>
> 8db1406f0c4008dec1671791%7C686ea1d3bc2b4c6fa92cd99c5c301635%7
> >>
> C0%7C0%7C639160845728472826%7CUnknown%7CTWFpbGZsb3d8eyJFb
> >>
> XB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTW
> >>
> FpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=DRpJWjm2yaF3Cnhk0b
> >> bFFhmGbKRweOOiWdsWco2NbX0%3D&reserved=0
> >>
> >> -Morten
^ permalink raw reply
* [v4] net/cksum: compute raw cksum for several segments
From: Su Sai @ 2026-06-18 6:32 UTC (permalink / raw)
To: marat.khalili; +Cc: stephen, dev, spiderdetective.ss
In-Reply-To: <1f2b21bd87a24610bc09b3b8da8100b5@huawei.com>
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=y, Size: 7063 bytes --]
The rte_raw_cksum_mbuf function is used to compute
the raw checksum of a packet.
If the packet payload stored in multi mbuf, the function
will goto the hard case. In hard case,
the variable 'tmp' is a type of uint32_t,
so rte_bswap16 will drop high 16 bit.
Meanwhile, the variable 'sum' is a type of uint32_t,
so 'sum += tmp' will drop the carry when overflow.
Both drop will make cksum incorrect.
This commit fixes the above bug.
Signed-off-by: Su Sai <spiderdetective.ss@gmail.com>
Acked-by: Marat Khalili <marat.khalili@huawei.com>
---
.mailmap | 1 +
app/test/test_cksum.c | 102 ++++++++++++++++++++++++++++++++++++++++++
lib/net/rte_cksum.h | 27 +++++++++--
3 files changed, 126 insertions(+), 4 deletions(-)
diff --git a/.mailmap b/.mailmap
index 4001e5fb0e..bcf73cb902 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1630,6 +1630,7 @@ Sylvia Grundwürmer <sylvia.grundwuermer@b-plus.com>
Sylwester Dziedziuch <sylwesterx.dziedziuch@intel.com>
Sylwia Wnuczko <sylwia.wnuczko@intel.com>
Szymon Sliwa <szs@semihalf.com>
+Su Sai <spiderdetective.ss@gmail.com> <susai.ss@bytedance.com>
Szymon T Cudzilo <szymon.t.cudzilo@intel.com>
Tadhg Kearney <tadhg.kearney@intel.com>
Taekyung Kim <kim.tae.kyung@navercorp.com>
diff --git a/app/test/test_cksum.c b/app/test/test_cksum.c
index ea443382a1..5bd9723fbd 100644
--- a/app/test/test_cksum.c
+++ b/app/test/test_cksum.c
@@ -85,6 +85,42 @@ static const char test_cksum_ipv4_opts_udp[] = {
0x00, 0x35, 0x00, 0x09, 0x89, 0x6f, 0x78,
};
+/*
+ * generated in scapy with
+ * Ether()/IP()/TCP(options=[NOP,NOP,Timestamps])/os.urandom(113))
+ */
+static const char test_cksum_ipv4_tcp_multi_segs[] = {
+ 0x00, 0x16, 0x3e, 0x0b, 0x6b, 0xd2, 0xee, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0x08, 0x00, 0x45, 0x00,
+ 0x00, 0xa5, 0x46, 0x10, 0x40, 0x00, 0x40, 0x06,
+ 0x80, 0xb5, 0xc0, 0xa8, 0xf9, 0x1d, 0xc0, 0xa8,
+ 0xf9, 0x1e, 0xdc, 0xa2, 0x14, 0x51, 0xbb, 0x8f,
+ 0xa0, 0x00, 0xe4, 0x7c, 0xe4, 0xb8, 0x80, 0x10,
+ 0x02, 0x00, 0x4b, 0xc1, 0x00, 0x00, 0x01, 0x01,
+ 0x08, 0x0a, 0x90, 0x60, 0xf4, 0xff, 0x03, 0xc5,
+ 0xb4, 0x19, 0x77, 0x34, 0xd4, 0xdc, 0x84, 0x86,
+ 0xff, 0x44, 0x09, 0x63, 0x36, 0x2e, 0x26, 0x9b,
+ 0x90, 0x70, 0xf2, 0xed, 0xc8, 0x5b, 0x87, 0xaa,
+ 0xb4, 0x67, 0x6b, 0x32, 0x3d, 0xc4, 0xbf, 0x15,
+ 0xa9, 0x16, 0x6c, 0x2a, 0x9d, 0xb2, 0xb7, 0x6b,
+ 0x58, 0x44, 0x58, 0x12, 0x4b, 0x8f, 0xe5, 0x12,
+ 0x11, 0x90, 0x94, 0x68, 0x37, 0xad, 0x0a, 0x9b,
+ 0xd6, 0x79, 0xf2, 0xb7, 0x31, 0xcf, 0x44, 0x22,
+ 0xc8, 0x99, 0x3f, 0xe5, 0xe7, 0xac, 0xc7, 0x0b,
+ 0x86, 0xdf, 0xda, 0xed, 0x0a, 0x0f, 0x86, 0xd7,
+ 0x48, 0xe2, 0xf1, 0xc2, 0x43, 0xed, 0x47, 0x3a,
+ 0xea, 0x25, 0x2d, 0xd6, 0x60, 0x38, 0x30, 0x07,
+ 0x28, 0xdd, 0x1f, 0x0c, 0xdd, 0x7b, 0x7c, 0xd9,
+ 0x35, 0x9d, 0x14, 0xaa, 0xc6, 0x35, 0xd1, 0x03,
+ 0x38, 0xb1, 0xf5,
+};
+
+static const uint8_t test_cksum_ipv4_tcp_multi_segs_len[] = {
+ 66, /* the first seg contains all headers, including L2 to L4 */
+ 61, /* the second seg length is odd, test byte order independent */
+ 52, /* three segs are sufficient to test the most complex scenarios */
+};
+
/* test l3/l4 checksum api */
static int
test_l4_cksum(struct rte_mempool *pktmbuf_pool, const char *pktdata, size_t len)
@@ -223,6 +259,66 @@ test_l4_cksum(struct rte_mempool *pktmbuf_pool, const char *pktdata, size_t len)
return -1;
}
+/* test l4 checksum api for a packet with multiple mbufs */
+static int
+test_l4_cksum_multi_mbufs(struct rte_mempool *pktmbuf_pool, const char *pktdata, size_t len,
+ const uint8_t *segs, size_t segs_len)
+{
+ struct rte_mbuf *m[NB_MBUF] = {0};
+ struct rte_mbuf *m_hdr = NULL;
+ struct rte_net_hdr_lens hdr_lens;
+ size_t i, off = 0;
+ uint32_t packet_type, l3;
+ void *l3_hdr;
+ char *data;
+
+ for (i = 0; i < segs_len; i++) {
+ m[i] = rte_pktmbuf_alloc(pktmbuf_pool);
+ if (m[i] == NULL)
+ GOTO_FAIL("Cannot allocate mbuf");
+
+ data = rte_pktmbuf_append(m[i], segs[i]);
+ if (data == NULL)
+ GOTO_FAIL("Cannot append data");
+
+ memcpy(data, pktdata + off, segs[i]);
+ off += segs[i];
+
+ if (m_hdr) {
+ if (rte_pktmbuf_chain(m_hdr, m[i]))
+ GOTO_FAIL("Cannot chain mbuf");
+ } else {
+ m_hdr = m[i];
+ }
+ }
+
+ if (off != len)
+ GOTO_FAIL("Invalid segs");
+
+ packet_type = rte_net_get_ptype(m_hdr, &hdr_lens, RTE_PTYPE_ALL_MASK);
+ l3 = packet_type & RTE_PTYPE_L3_MASK;
+
+ l3_hdr = rte_pktmbuf_mtod_offset(m_hdr, void *, hdr_lens.l2_len);
+ off = hdr_lens.l2_len + hdr_lens.l3_len;
+
+ if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) {
+ if (rte_ipv4_udptcp_cksum_mbuf_verify(m_hdr, l3_hdr, off) != 0)
+ GOTO_FAIL("Invalid L4 checksum verification for multiple mbufs");
+ } else if (l3 == RTE_PTYPE_L3_IPV6 || l3 == RTE_PTYPE_L3_IPV6_EXT) {
+ if (rte_ipv6_udptcp_cksum_mbuf_verify(m_hdr, l3_hdr, off) != 0)
+ GOTO_FAIL("Invalid L4 checksum verification for multiple mbufs");
+ }
+
+ rte_pktmbuf_free_bulk(m, segs_len);
+
+ return 0;
+
+fail:
+ rte_pktmbuf_free_bulk(m, segs_len);
+
+ return -1;
+}
+
static int
test_cksum(void)
{
@@ -256,6 +352,12 @@ test_cksum(void)
sizeof(test_cksum_ipv4_opts_udp)) < 0)
GOTO_FAIL("checksum error on ipv4_opts_udp");
+ if (test_l4_cksum_multi_mbufs(pktmbuf_pool, test_cksum_ipv4_tcp_multi_segs,
+ sizeof(test_cksum_ipv4_tcp_multi_segs),
+ test_cksum_ipv4_tcp_multi_segs_len,
+ sizeof(test_cksum_ipv4_tcp_multi_segs_len)) < 0)
+ GOTO_FAIL("checksum error on multi mbufs check");
+
rte_mempool_free(pktmbuf_pool);
return 0;
diff --git a/lib/net/rte_cksum.h b/lib/net/rte_cksum.h
index a8e8927952..679ba82eb6 100644
--- a/lib/net/rte_cksum.h
+++ b/lib/net/rte_cksum.h
@@ -80,6 +80,25 @@ __rte_raw_cksum_reduce(uint32_t sum)
return (uint16_t)sum;
}
+/**
+ * @internal Reduce a sum to the non-complemented checksum.
+ * Helper routine for the rte_raw_cksum_mbuf().
+ *
+ * @param sum
+ * Value of the sum.
+ * @return
+ * The non-complemented checksum.
+ */
+static inline uint16_t
+__rte_raw_cksum_reduce_u64(uint64_t sum)
+{
+ uint32_t tmp;
+
+ tmp = __rte_raw_cksum_reduce((uint32_t)sum);
+ tmp += __rte_raw_cksum_reduce((uint32_t)(sum >> 32));
+ return __rte_raw_cksum_reduce(tmp);
+}
+
/**
* Process the non-complemented checksum of a buffer.
*
@@ -119,8 +138,8 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len,
{
const struct rte_mbuf *seg;
const char *buf;
- uint32_t sum, tmp;
- uint32_t seglen, done;
+ uint32_t seglen, done, tmp;
+ uint64_t sum;
/* easy case: all data in the first segment */
if (off + len <= rte_pktmbuf_data_len(m)) {
@@ -157,7 +176,7 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len,
for (;;) {
tmp = __rte_raw_cksum(buf, seglen, 0);
if (done & 1)
- tmp = rte_bswap16((uint16_t)tmp);
+ tmp = rte_bswap32(tmp);
sum += tmp;
done += seglen;
if (done == len)
@@ -169,7 +188,7 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len,
seglen = len - done;
}
- *cksum = __rte_raw_cksum_reduce(sum);
+ *cksum = __rte_raw_cksum_reduce_u64(sum);
return 0;
}
--
2.20.1
^ permalink raw reply related
* Re: [v4] net/cksum: compute raw cksum for several segments
From: su sai @ 2026-06-18 6:37 UTC (permalink / raw)
To: Marat Khalili; +Cc: dev@dpdk.org, stephen@networkplumber.org
In-Reply-To: <1f2b21bd87a24610bc09b3b8da8100b5@huawei.com>
[-- Attachment #1: Type: text/plain, Size: 9441 bytes --]
Appreciate your review. I've resent the updated patch with your Acked-by
tag included.
On Thu, Jun 18, 2026 at 4:02 AM Marat Khalili <marat.khalili@huawei.com>
wrote:
> > -----Original Message-----
> > From: Su Sai <spiderdetective.ss@gmail.com>
> > Sent: Tuesday 16 June 2026 13:38
> > To: stephen@networkplumber.org
> > Cc: dev@dpdk.org; spiderdetective.ss@gmail.com
> > Subject: [v4] net/cksum: compute raw cksum for several segments
> >
> > The rte_raw_cksum_mbuf function is used to compute
> > the raw checksum of a packet.
> > If the packet payload stored in multi mbuf, the function
> > will goto the hard case. In hard case,
> > the variable 'tmp' is a type of uint32_t,
> > so rte_bswap16 will drop high 16 bit.
> > Meanwhile, the variable 'sum' is a type of uint32_t,
> > so 'sum += tmp' will drop the carry when overflow.
> > Both drop will make cksum incorrect.
> > This commit fixes the above bug.
> >
> > Signed-off-by: Su Sai <spiderdetective.ss@gmail.com>
>
> You can add:
>
> Acked-by: Marat Khalili <marat.khalili@huawei.com>
>
> > ---
> > .mailmap | 1 +
> > app/test/test_cksum.c | 102 ++++++++++++++++++++++++++++++++++++++++++
> > lib/net/rte_cksum.h | 27 +++++++++--
> > 3 files changed, 126 insertions(+), 4 deletions(-)
> >
> > diff --git a/.mailmap b/.mailmap
> > index 4001e5fb0e..bcf73cb902 100644
> > --- a/.mailmap
> > +++ b/.mailmap
> > @@ -1630,6 +1630,7 @@ Sylvia Grundw眉rmer <sylvia.grundwuermer@b-plus.com
> >
> > Sylwester Dziedziuch <sylwesterx.dziedziuch@intel.com>
> > Sylwia Wnuczko <sylwia.wnuczko@intel.com>
> > Szymon Sliwa <szs@semihalf.com>
> > +Su Sai <spiderdetective.ss@gmail.com> <susai.ss@bytedance.com>
> > Szymon T Cudzilo <szymon.t.cudzilo@intel.com>
> > Tadhg Kearney <tadhg.kearney@intel.com>
> > Taekyung Kim <kim.tae.kyung@navercorp.com>
> > diff --git a/app/test/test_cksum.c b/app/test/test_cksum.c
> > index ea443382a1..5bd9723fbd 100644
> > --- a/app/test/test_cksum.c
> > +++ b/app/test/test_cksum.c
> > @@ -85,6 +85,42 @@ static const char test_cksum_ipv4_opts_udp[] = {
> > 0x00, 0x35, 0x00, 0x09, 0x89, 0x6f, 0x78,
> > };
> >
> > +/*
> > + * generated in scapy with
> > + * Ether()/IP()/TCP(options=[NOP,NOP,Timestamps])/os.urandom(113))
> > + */
> > +static const char test_cksum_ipv4_tcp_multi_segs[] = {
> > + 0x00, 0x16, 0x3e, 0x0b, 0x6b, 0xd2, 0xee, 0xff,
> > + 0xff, 0xff, 0xff, 0xff, 0x08, 0x00, 0x45, 0x00,
> > + 0x00, 0xa5, 0x46, 0x10, 0x40, 0x00, 0x40, 0x06,
> > + 0x80, 0xb5, 0xc0, 0xa8, 0xf9, 0x1d, 0xc0, 0xa8,
> > + 0xf9, 0x1e, 0xdc, 0xa2, 0x14, 0x51, 0xbb, 0x8f,
> > + 0xa0, 0x00, 0xe4, 0x7c, 0xe4, 0xb8, 0x80, 0x10,
> > + 0x02, 0x00, 0x4b, 0xc1, 0x00, 0x00, 0x01, 0x01,
> > + 0x08, 0x0a, 0x90, 0x60, 0xf4, 0xff, 0x03, 0xc5,
> > + 0xb4, 0x19, 0x77, 0x34, 0xd4, 0xdc, 0x84, 0x86,
> > + 0xff, 0x44, 0x09, 0x63, 0x36, 0x2e, 0x26, 0x9b,
> > + 0x90, 0x70, 0xf2, 0xed, 0xc8, 0x5b, 0x87, 0xaa,
> > + 0xb4, 0x67, 0x6b, 0x32, 0x3d, 0xc4, 0xbf, 0x15,
> > + 0xa9, 0x16, 0x6c, 0x2a, 0x9d, 0xb2, 0xb7, 0x6b,
> > + 0x58, 0x44, 0x58, 0x12, 0x4b, 0x8f, 0xe5, 0x12,
> > + 0x11, 0x90, 0x94, 0x68, 0x37, 0xad, 0x0a, 0x9b,
> > + 0xd6, 0x79, 0xf2, 0xb7, 0x31, 0xcf, 0x44, 0x22,
> > + 0xc8, 0x99, 0x3f, 0xe5, 0xe7, 0xac, 0xc7, 0x0b,
> > + 0x86, 0xdf, 0xda, 0xed, 0x0a, 0x0f, 0x86, 0xd7,
> > + 0x48, 0xe2, 0xf1, 0xc2, 0x43, 0xed, 0x47, 0x3a,
> > + 0xea, 0x25, 0x2d, 0xd6, 0x60, 0x38, 0x30, 0x07,
> > + 0x28, 0xdd, 0x1f, 0x0c, 0xdd, 0x7b, 0x7c, 0xd9,
> > + 0x35, 0x9d, 0x14, 0xaa, 0xc6, 0x35, 0xd1, 0x03,
> > + 0x38, 0xb1, 0xf5,
> > +};
> > +
> > +static const uint8_t test_cksum_ipv4_tcp_multi_segs_len[] = {
> > + 66, /* the first seg contains all headers, including L2 to L4 */
> > + 61, /* the second seg length is odd, test byte order independent
> */
> > + 52, /* three segs are sufficient to test the most complex
> scenarios */
> > +};
> > +
> > /* test l3/l4 checksum api */
> > static int
> > test_l4_cksum(struct rte_mempool *pktmbuf_pool, const char *pktdata,
> size_t len)
> > @@ -223,6 +259,66 @@ test_l4_cksum(struct rte_mempool *pktmbuf_pool,
> const char *pktdata, size_t len)
> > return -1;
> > }
> >
> > +/* test l4 checksum api for a packet with multiple mbufs */
> > +static int
> > +test_l4_cksum_multi_mbufs(struct rte_mempool *pktmbuf_pool, const char
> *pktdata, size_t len,
> > + const uint8_t *segs, size_t segs_len)
> > +{
> > + struct rte_mbuf *m[NB_MBUF] = {0};
> > + struct rte_mbuf *m_hdr = NULL;
> > + struct rte_net_hdr_lens hdr_lens;
> > + size_t i, off = 0;
> > + uint32_t packet_type, l3;
> > + void *l3_hdr;
> > + char *data;
> > +
> > + for (i = 0; i < segs_len; i++) {
> > + m[i] = rte_pktmbuf_alloc(pktmbuf_pool);
> > + if (m[i] == NULL)
> > + GOTO_FAIL("Cannot allocate mbuf");
> > +
> > + data = rte_pktmbuf_append(m[i], segs[i]);
> > + if (data == NULL)
> > + GOTO_FAIL("Cannot append data");
> > +
> > + memcpy(data, pktdata + off, segs[i]);
> > + off += segs[i];
> > +
> > + if (m_hdr) {
> > + if (rte_pktmbuf_chain(m_hdr, m[i]))
> > + GOTO_FAIL("Cannot chain mbuf");
> > + } else {
> > + m_hdr = m[i];
> > + }
> > + }
> > +
> > + if (off != len)
> > + GOTO_FAIL("Invalid segs");
> > +
> > + packet_type = rte_net_get_ptype(m_hdr, &hdr_lens,
> RTE_PTYPE_ALL_MASK);
> > + l3 = packet_type & RTE_PTYPE_L3_MASK;
> > +
> > + l3_hdr = rte_pktmbuf_mtod_offset(m_hdr, void *, hdr_lens.l2_len);
> > + off = hdr_lens.l2_len + hdr_lens.l3_len;
> > +
> > + if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) {
> > + if (rte_ipv4_udptcp_cksum_mbuf_verify(m_hdr, l3_hdr, off)
> != 0)
> > + GOTO_FAIL("Invalid L4 checksum verification for
> multiple mbufs");
> > + } else if (l3 == RTE_PTYPE_L3_IPV6 || l3 == RTE_PTYPE_L3_IPV6_EXT)
> {
> > + if (rte_ipv6_udptcp_cksum_mbuf_verify(m_hdr, l3_hdr, off)
> != 0)
> > + GOTO_FAIL("Invalid L4 checksum verification for
> multiple mbufs");
> > + }
> > +
> > + rte_pktmbuf_free_bulk(m, segs_len);
> > +
> > + return 0;
> > +
> > +fail:
> > + rte_pktmbuf_free_bulk(m, segs_len);
> > +
> > + return -1;
> > +}
> > +
> > static int
> > test_cksum(void)
> > {
> > @@ -256,6 +352,12 @@ test_cksum(void)
> > sizeof(test_cksum_ipv4_opts_udp)) < 0)
> > GOTO_FAIL("checksum error on ipv4_opts_udp");
> >
> > + if (test_l4_cksum_multi_mbufs(pktmbuf_pool,
> test_cksum_ipv4_tcp_multi_segs,
> > + sizeof(test_cksum_ipv4_tcp_multi_segs),
> > + test_cksum_ipv4_tcp_multi_segs_len,
> > + sizeof(test_cksum_ipv4_tcp_multi_segs_len)) < 0)
> > + GOTO_FAIL("checksum error on multi mbufs check");
> > +
> > rte_mempool_free(pktmbuf_pool);
> >
> > return 0;
> > diff --git a/lib/net/rte_cksum.h b/lib/net/rte_cksum.h
> > index a8e8927952..679ba82eb6 100644
> > --- a/lib/net/rte_cksum.h
> > +++ b/lib/net/rte_cksum.h
> > @@ -80,6 +80,25 @@ __rte_raw_cksum_reduce(uint32_t sum)
> > return (uint16_t)sum;
> > }
> >
> > +/**
> > + * @internal Reduce a sum to the non-complemented checksum.
> > + * Helper routine for the rte_raw_cksum_mbuf().
> > + *
> > + * @param sum
> > + * Value of the sum.
> > + * @return
> > + * The non-complemented checksum.
> > + */
> > +static inline uint16_t
> > +__rte_raw_cksum_reduce_u64(uint64_t sum)
> > +{
> > + uint32_t tmp;
> > +
> > + tmp = __rte_raw_cksum_reduce((uint32_t)sum);
> > + tmp += __rte_raw_cksum_reduce((uint32_t)(sum >> 32));
> > + return __rte_raw_cksum_reduce(tmp);
> > +}
> > +
> > /**
> > * Process the non-complemented checksum of a buffer.
> > *
> > @@ -119,8 +138,8 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m,
> uint32_t off, uint32_t len,
> > {
> > const struct rte_mbuf *seg;
> > const char *buf;
> > - uint32_t sum, tmp;
> > - uint32_t seglen, done;
> > + uint32_t seglen, done, tmp;
> > + uint64_t sum;
> >
> > /* easy case: all data in the first segment */
> > if (off + len <= rte_pktmbuf_data_len(m)) {
> > @@ -157,7 +176,7 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m,
> uint32_t off, uint32_t len,
> > for (;;) {
> > tmp = __rte_raw_cksum(buf, seglen, 0);
> > if (done & 1)
> > - tmp = rte_bswap16((uint16_t)tmp);
> > + tmp = rte_bswap32(tmp);
> > sum += tmp;
> > done += seglen;
> > if (done == len)
> > @@ -169,7 +188,7 @@ rte_raw_cksum_mbuf(const struct rte_mbuf *m,
> uint32_t off, uint32_t len,
> > seglen = len - done;
> > }
> >
> > - *cksum = __rte_raw_cksum_reduce(sum);
> > + *cksum = __rte_raw_cksum_reduce_u64(sum);
> > return 0;
> > }
> >
> > --
> > 2.20.1
>
>
[-- Attachment #2: Type: text/html, Size: 14120 bytes --]
^ permalink raw reply
* [PATCH v3 00/20] net/sxe2: added Linkdata sxe2 ethernet driver
From: liujie5 @ 2026-06-18 8:27 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260614092328.201826-21-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
V3:
Remove the `drv-sw-stats` devarg
Jie Liu (20):
net/sxe2: support AVX512 vectorized path for Rx and Tx
net/sxe2: add AVX2 vector data path for Rx and Tx
drivers: add supported packet types and link state
net/sxe2: support L2 filtering and MAC config
drivers: support RSS feature
net/sxe2: support TM hierarchy and shaping
net/sxe2: support IPsec inline protocol offload
net/sxe2: support statistics and multi-process
drivers: interrupt handling
net/sxe2: add NEON vec Rx/Tx burst functions
drivers: add support for VF representors
net/sxe2: add support for custom UDP tunnel ports
net/sxe2: support firmware version reading
net/sxe2: implement get monitor address
common/sxe2: add shared SFP module definitions
net/sxe2: support SFP module info and EEPROM access
net/sxe2: implement private dump info
net/sxe2: add mbuf validation in Tx debug mode
drivers: add parameters parsed using rte_kvargs
net/sxe2: update sxe2 feature matrix docs
doc/guides/nics/features/sxe2.ini | 56 +
doc/guides/nics/sxe2.rst | 132 ++
drivers/common/sxe2/sxe2_common.c | 156 ++
drivers/common/sxe2/sxe2_common.h | 4 +
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++
drivers/common/sxe2/sxe2_ioctl_chnl.c | 178 +-
drivers/common/sxe2/sxe2_ioctl_chnl_func.h | 18 +
drivers/common/sxe2/sxe2_msg.h | 118 ++
drivers/net/sxe2/meson.build | 54 +-
drivers/net/sxe2/sxe2_cmd_chnl.c | 1587 +++++++++++++++-
drivers/net/sxe2/sxe2_cmd_chnl.h | 139 ++
drivers/net/sxe2/sxe2_drv_cmd.h | 523 +++++-
drivers/net/sxe2/sxe2_dump.c | 302 +++
drivers/net/sxe2/sxe2_dump.h | 12 +
drivers/net/sxe2/sxe2_ethdev.c | 1515 ++++++++++++++-
drivers/net/sxe2/sxe2_ethdev.h | 110 +-
drivers/net/sxe2/sxe2_ethdev_repr.c | 609 ++++++
drivers/net/sxe2/sxe2_ethdev_repr.h | 32 +
drivers/net/sxe2/sxe2_filter.c | 895 +++++++++
drivers/net/sxe2/sxe2_filter.h | 100 +
drivers/net/sxe2/sxe2_flow.c | 1394 ++++++++++++++
drivers/net/sxe2/sxe2_flow.h | 30 +
drivers/net/sxe2/sxe2_flow_define.h | 144 ++
drivers/net/sxe2/sxe2_flow_parse_action.c | 1182 ++++++++++++
drivers/net/sxe2/sxe2_flow_parse_action.h | 23 +
drivers/net/sxe2/sxe2_flow_parse_engine.c | 106 ++
drivers/net/sxe2/sxe2_flow_parse_engine.h | 13 +
drivers/net/sxe2/sxe2_flow_parse_pattern.c | 1935 +++++++++++++++++++
drivers/net/sxe2/sxe2_flow_parse_pattern.h | 46 +
drivers/net/sxe2/sxe2_ipsec.c | 1565 ++++++++++++++++
drivers/net/sxe2/sxe2_ipsec.h | 254 +++
drivers/net/sxe2/sxe2_irq.c | 1026 ++++++++++
drivers/net/sxe2/sxe2_irq.h | 25 +
drivers/net/sxe2/sxe2_mac.c | 530 ++++++
drivers/net/sxe2/sxe2_mac.h | 84 +
drivers/net/sxe2/sxe2_mp.c | 414 ++++
drivers/net/sxe2/sxe2_mp.h | 67 +
drivers/net/sxe2/sxe2_queue.c | 17 +-
drivers/net/sxe2/sxe2_queue.h | 15 +-
drivers/net/sxe2/sxe2_rss.c | 584 ++++++
drivers/net/sxe2/sxe2_rss.h | 81 +
drivers/net/sxe2/sxe2_rx.c | 93 +-
drivers/net/sxe2/sxe2_rx.h | 2 +
drivers/net/sxe2/sxe2_security.c | 335 ++++
drivers/net/sxe2/sxe2_security.h | 77 +
drivers/net/sxe2/sxe2_stats.c | 586 ++++++
drivers/net/sxe2/sxe2_stats.h | 39 +
drivers/net/sxe2/sxe2_switchdev.c | 332 ++++
drivers/net/sxe2/sxe2_switchdev.h | 33 +
drivers/net/sxe2/sxe2_tm.c | 1169 ++++++++++++
drivers/net/sxe2/sxe2_tm.h | 78 +
drivers/net/sxe2/sxe2_tx.c | 7 +
drivers/net/sxe2/sxe2_txrx.c | 1969 +++++++++++++++++++-
drivers/net/sxe2/sxe2_txrx.h | 8 +
drivers/net/sxe2/sxe2_txrx_check_mbuf.c | 595 ++++++
drivers/net/sxe2/sxe2_txrx_check_mbuf.h | 38 +
drivers/net/sxe2/sxe2_txrx_poll.c | 281 ++-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 38 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 748 ++++++++
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 868 +++++++++
drivers/net/sxe2/sxe2_txrx_vec_common.h | 53 +-
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 691 +++++++
drivers/net/sxe2/sxe2_txrx_vec_sse.c | 29 +-
drivers/net/sxe2/sxe2_vsi.c | 146 ++
drivers/net/sxe2/sxe2_vsi.h | 12 +-
drivers/net/sxe2/sxe2vf_regs.h | 85 +
67 files changed, 24800 insertions(+), 266 deletions(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/common/sxe2/sxe2_msg.h
create mode 100644 drivers/net/sxe2/sxe2_dump.c
create mode 100644 drivers/net/sxe2/sxe2_dump.h
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.c
create mode 100644 drivers/net/sxe2/sxe2_ethdev_repr.h
create mode 100644 drivers/net/sxe2/sxe2_filter.c
create mode 100644 drivers/net/sxe2/sxe2_filter.h
create mode 100644 drivers/net/sxe2/sxe2_flow.c
create mode 100644 drivers/net/sxe2/sxe2_flow.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_action.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_engine.h
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.c
create mode 100644 drivers/net/sxe2/sxe2_flow_parse_pattern.h
create mode 100644 drivers/net/sxe2/sxe2_ipsec.c
create mode 100644 drivers/net/sxe2/sxe2_ipsec.h
create mode 100644 drivers/net/sxe2/sxe2_irq.c
create mode 100644 drivers/net/sxe2/sxe2_mac.c
create mode 100644 drivers/net/sxe2/sxe2_mac.h
create mode 100644 drivers/net/sxe2/sxe2_mp.c
create mode 100644 drivers/net/sxe2/sxe2_mp.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
create mode 100644 drivers/net/sxe2/sxe2_security.c
create mode 100644 drivers/net/sxe2/sxe2_security.h
create mode 100644 drivers/net/sxe2/sxe2_stats.c
create mode 100644 drivers/net/sxe2/sxe2_stats.h
create mode 100644 drivers/net/sxe2/sxe2_switchdev.c
create mode 100644 drivers/net/sxe2/sxe2_switchdev.h
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_check_mbuf.h
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
create mode 100644 drivers/net/sxe2/sxe2vf_regs.h
--
2.31.1
^ permalink raw reply
* [PATCH v3 02/20] net/sxe2: add AVX2 vector data path for Rx and Tx
From: liujie5 @ 2026-06-18 8:27 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260618082723.571054-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Added AVX256 vectorized versions of Rx and Tx data path functions to
improve packet processing performance.
The vector path uses AVX2 SIMD instructions to process multiple
descriptors per loop, significantly reducing the per-packet overhead.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
net/sxe2: support AVX512 vectorized path for Rx and Tx
---
drivers/net/sxe2/meson.build | 9 +
drivers/net/sxe2/sxe2_txrx.c | 38 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 12 +-
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 748 ++++++++++++++++++++++++++
4 files changed, 802 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx2.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 7bd0d8120c..c225dd7cd8 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -39,6 +39,15 @@ if arch_subdir == 'x86'
c_args: avx512_args)
objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
endif
+ sxe2_avx2_lib = static_library('sxe2_avx2_lib',
+ 'sxe2_txrx_vec_avx2.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev,
+ static_rte_bus_pci],
+ include_directories: includes,
+ c_args: [cflags, '-mavx2'])
+ objs += sxe2_avx2_lib.extract_objects('sxe2_txrx_vec_avx2.c')
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index aa1c474088..eaf95259a5 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -167,8 +167,14 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
#endif
}
- if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
- tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+ if (((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX2;
+
+ if ((0 == (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK)))
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
@@ -197,6 +203,13 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
}
#endif
+ } else if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX2) {
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx2_simple;
+ }
} else {
if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
@@ -231,6 +244,10 @@ static const struct {
{ sxe2_tx_pkts_vec_avx512_simple,
"Vector AVX512 Simple" },
#endif
+ { sxe2_tx_pkts_vec_avx2,
+ "Vector AVX2" },
+ { sxe2_tx_pkts_vec_avx2_simple,
+ "Vector AVX2 Simple" },
{ sxe2_tx_pkts_vec_sse,
"Vector SSE" },
{ sxe2_tx_pkts_vec_sse_simple,
@@ -330,7 +347,13 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
#endif
}
- if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
+ ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) ||
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)) &&
+ (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256))
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX2;
+
+ if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
#endif
@@ -354,6 +377,11 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
else
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
#endif
+ } else if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX2) {
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx2;
} else {
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
}
@@ -381,6 +409,10 @@ static const struct {
{ sxe2_rx_pkts_scattered_vec_avx512_offload,
"Offload Vector AVX512 Scattered" },
#endif
+ { sxe2_rx_pkts_scattered_vec_avx2,
+ "Vector AVX2 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx2_offload,
+ "Offload Vector AVX2 Scattered" },
{ sxe2_rx_pkts_scattered_vec_sse_offload,
"Vector SSE Scattered" },
#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index af7c8d12b2..369777606f 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,19 +11,21 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
- SXE2_RX_MODE_VEC_AVX512)
+ SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
- SXE2_TX_MODE_VEC_AVX512)
+ SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -68,6 +70,12 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
new file mode 100644
index 0000000000..3d83551bb5
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
@@ -0,0 +1,748 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static inline void
+sxe2_tx_desc_fill_one_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m256i desc_group0;
+ __m256i desc_group1;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ if (((uint64_t)desc & 0x1F) != 0 && pkts_num != 0) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+
+ while (pkts_num > 3) {
+ desc3_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len)
+ << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group1 = _mm256_set_epi64x(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]));
+
+ desc_group0 = _mm256_set_epi64x(desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc + 2), desc_group1);
+ _mm256_store_si256(RTE_CAST_PTR(__m256i *, desc), desc_group0);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx2 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx2(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+ }
+
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx2_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num = sxe2_tx_pkts_vec_avx2_batch(txq,
+ tx_pkts + tx_done_num, tx_need_num, with_offloads);
+
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx2(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read), dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ return;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ?
+ (rxq->ring_depth - 1) : (rxq->realloc_start - 1);
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx2(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m256i mbufs6_7, mbufs4_5, mbufs2_3, mbufs0_1;
+ uint32_t bit_num;
+ uint16_t done_num;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+ done_num = 0;
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx2(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m256i crc_adjust =
+ _mm256_set_epi16(0, 0, 0, -rxq->crc_len,
+ 0, -rxq->crc_len, 0,
+ 0, 0, 0, 0,
+ -rxq->crc_len, 0, -rxq->crc_len, 0, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+ const __m256i rvp_shuf_mask =
+ _mm256_set_epi8(7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 7, 6, 5, 4,
+ 3, 2, 13, 12,
+ 0xFF, 0xFF, 13, 12,
+ 0xFF, 0xFF, 0xFF, 0xFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 8, 0, 10, 2,
+ 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ if (split_rxe_flags) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs6_7 = _mm256_shuffle_epi8(descs6_7, rvp_shuf_mask);
+ mbufs4_5 = _mm256_shuffle_epi8(descs4_5, rvp_shuf_mask);
+
+ mbufs6_7 = _mm256_add_epi16(mbufs6_7, crc_adjust);
+ mbufs4_5 = _mm256_add_epi16(mbufs4_5, crc_adjust);
+
+ const __m256i ptype_mask = _mm256_set1_epi32(SXE2_RX_DESC_PTYPE_MASK);
+
+ const __m256i staterrs4_7 = _mm256_unpackhi_epi32(descs6_7, descs4_5);
+
+ __m256i ptypes4_7 = _mm256_and_si256(staterrs4_7, ptype_mask);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes4_7, 9);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes4_7, 1);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_7, 11);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_7, 3);
+
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype7], 4);
+ mbufs6_7 = _mm256_insert_epi32(mbufs6_7, ptype_tbl[ptype6], 0);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype5], 4);
+ mbufs4_5 = _mm256_insert_epi32(mbufs4_5, ptype_tbl[ptype4], 0);
+
+ mbufs2_3 = _mm256_shuffle_epi8(descs2_3, rvp_shuf_mask);
+ mbufs0_1 = _mm256_shuffle_epi8(descs0_1, rvp_shuf_mask);
+
+ mbufs2_3 = _mm256_add_epi16(mbufs2_3, crc_adjust);
+ mbufs0_1 = _mm256_add_epi16(mbufs0_1, crc_adjust);
+
+ const __m256i staterrs0_3 = _mm256_unpackhi_epi32(descs2_3, descs0_1);
+
+ __m256i ptypes0_3 = _mm256_and_si256(staterrs0_3, ptype_mask);
+
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes0_3, 9);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes0_3, 1);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_3, 11);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_3, 3);
+
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype3], 4);
+ mbufs2_3 = _mm256_insert_epi32(mbufs2_3, ptype_tbl[ptype2], 0);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype1], 4);
+ mbufs0_1 = _mm256_insert_epi32(mbufs0_1, ptype_tbl[ptype0], 0);
+
+ __m256i staterrs0_7 = _mm256_unpacklo_epi64(staterrs4_7, staterrs0_3);
+
+ __m256i stu_len0_7 = _mm256_unpackhi_epi64(staterrs4_7, staterrs0_3);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0x00001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8
+ (0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+ const __m256i vlan_mask =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32
+ (RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 8), 4);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 4), 4);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 4);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(mbuf_flags, 4), 4);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] =
+ _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] =
+ _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] =
+ _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags != NULL) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 = _mm256_and_si256(staterrs0_7,
+ umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+
+ __m128i umbcast_bits_128 = _mm_packs_epi32
+ (_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256
+ (umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags != NULL) {
+ const __m256i eop_rxe_mask = _mm256_set1_epi32
+ (SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m128i eop_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits = _mm_or_si128
+ (not_eop_bits,
+ _mm_srli_epi16
+ (_mm_and_si128(eop_rxe_bits, rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((1 == (rxq->processing_idx & 1)) && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
+ split_rxe_flags, umbcast_flags, do_offload);
+ if (rx_done_num == 0)
+ goto l_end;
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num =
+ sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM,
+ do_offload);
+ done_num += once_num;
+ nb_pkts -= once_num;
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
+ rx_pkts + done_num, nb_pkts, do_offload);
+l_end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
--
2.31.1
^ permalink raw reply related
* [PATCH v3 01/20] net/sxe2: support AVX512 vectorized path for Rx and Tx
From: liujie5 @ 2026-06-18 8:27 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260618082723.571054-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add AVX512 vector data path for Rx and Tx burst functions.
The decision to use AVX512 is based on:
1. CPU hardware flags (AVX512F, AVX512BW).
2. Compiler support (CC_AVX512_SUPPORT).
3. Max SIMD bitwidth configuration.
Performance shows approximately X% improvement in small packet
forwarding scenarios.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 24 +
drivers/net/sxe2/sxe2_drv_cmd.h | 80 +--
drivers/net/sxe2/sxe2_ethdev.c | 2 +-
drivers/net/sxe2/sxe2_ethdev.h | 3 +-
drivers/net/sxe2/sxe2_queue.h | 15 +-
drivers/net/sxe2/sxe2_rx.c | 55 +-
drivers/net/sxe2/sxe2_txrx.c | 92 ++-
drivers/net/sxe2/sxe2_txrx_poll.c | 38 +-
drivers/net/sxe2/sxe2_txrx_vec.c | 46 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 18 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 868 ++++++++++++++++++++++++
drivers/net/sxe2/sxe2_txrx_vec_common.h | 52 +-
drivers/net/sxe2/sxe2_txrx_vec_sse.c | 29 +-
13 files changed, 1130 insertions(+), 192 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_avx512.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index 6b2eb75b0e..7bd0d8120c 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -15,6 +15,30 @@ includes += include_directories('../../common/sxe2')
if arch_subdir == 'x86'
sources += files('sxe2_txrx_vec_sse.c')
+
+ sxe2_avx512_cpu_support =(
+ cc.get_define('__AVX512F__', args: machine_args) != '' and
+ cc.get_define('__AVX512BW__', args: machine_args) != '')
+
+ sxe2_avx512_cc_support = (
+ not machine_args.contains('-mno-avx512f') and
+ cc.has_argument('-mavx512f') and
+ cc.has_argument('-mavx512bw'))
+
+ if sxe2_avx512_cpu_support == true or sxe2_avx512_cc_support == true
+ cflags += ['-DCC_AVX512_SUPPORT']
+ avx512_args = [cflags, '-mavx512f', '-mavx512bw']
+ if cc.has_argument('-march=skylake-avx512')
+ avx512_args += '-march=skylake-avx512'
+ endif
+ sxe2_avx512_lib = static_library('sxe2_avx512_lib', 'sxe2_txrx_vec_avx512.c',
+ dependencies: [static_rte_ethdev,
+ static_rte_kvargs, static_rte_hash,
+ static_rte_security, static_rte_cryptodev, static_rte_bus_pci],
+ include_directories: includes,
+ c_args: avx512_args)
+ objs += sxe2_avx512_lib.extract_objects('sxe2_txrx_vec_avx512.c')
+ endif
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index bba6476c2e..ccc9c20ef4 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -67,20 +67,20 @@ enum sxe2_dev_type {
SXE2_DEV_T_MAX,
};
-struct sxe2_drv_queue_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_queue_caps {
uint16_t queues_cnt;
uint16_t base_idx_in_pf;
-};
+} __rte_packed_end;
-struct sxe2_drv_msix_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_msix_caps {
uint16_t msix_vectors_cnt;
uint16_t base_idx_in_func;
-};
+} __rte_packed_end;
-struct sxe2_drv_rss_hash_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rss_hash_caps {
uint16_t hash_key_size;
uint16_t lut_key_size;
-};
+} __rte_packed_end;
enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_BOTH = 0,
@@ -89,18 +89,18 @@ enum sxe2_vf_vsi_valid {
SXE2_VF_VSI_MAX,
};
-struct sxe2_drv_vsi_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_caps {
uint16_t func_id;
uint16_t dpdk_vsi_id;
uint16_t kernel_vsi_id;
uint16_t vsi_type;
-};
+} __rte_packed_end;
-struct sxe2_drv_representor_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_representor_caps {
uint16_t cnt_repr_vf;
uint8_t rsv[2];
struct sxe2_drv_vsi_caps repr_vf_id[256];
-};
+} __rte_packed_end;
enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_NOTSET = 0,
@@ -111,25 +111,25 @@ enum sxe2_phys_port_name_type {
SXE2_PHYS_PORT_NAME_TYPE_UNKNOWN,
};
-struct sxe2_switchdev_mode_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_mode_info {
uint8_t pf_id;
uint8_t is_switchdev;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_switchdev_cpvsi_info {
+struct __rte_aligned(4) __rte_packed_begin sxe2_switchdev_cpvsi_info {
uint16_t cp_vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_txsch_caps {
+struct __rte_aligned(4) __rte_packed_begin sxe2_txsch_caps {
uint8_t layer_cap;
uint8_t tm_mid_node_num;
uint8_t prio_num;
uint8_t rev;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_caps_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_caps_resp {
struct sxe2_drv_queue_caps queue_caps;
struct sxe2_drv_msix_caps msix_caps;
struct sxe2_drv_rss_hash_caps rss_hash_caps;
@@ -141,24 +141,24 @@ struct sxe2_drv_dev_caps_resp {
uint8_t dev_type;
uint8_t rev;
uint32_t cap_flags;
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_info_resp {
uint64_t dsn;
uint16_t vsi_id;
uint8_t rsv[2];
uint8_t mac_addr[SXE2_ETH_ALEN];
uint8_t rsv2[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_dev_fw_info_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_dev_fw_info_resp {
uint8_t main_version_id;
uint8_t sub_version_id;
uint8_t fix_version_id;
uint8_t build_id;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_ctxt {
uint64_t dma_addr;
uint32_t max_lro_size;
uint32_t split_type_mask;
@@ -170,62 +170,62 @@ struct sxe2_drv_rxq_ctxt {
uint8_t keep_crc_en;
uint8_t split_en;
uint8_t desc_size;
-};
+} __rte_packed_end;
-struct sxe2_drv_rxq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_rxq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
uint16_t max_frame_size;
uint8_t rsv[2];
struct sxe2_drv_rxq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_ctxt {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_ctxt {
uint64_t dma_addr;
uint32_t sched_mode;
uint16_t queue_id;
uint16_t depth;
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_txq_cfg_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_txq_cfg_req {
uint16_t q_cnt;
uint16_t vsi_id;
struct sxe2_drv_txq_ctxt cfg[];
-};
+} __rte_packed_end;
-struct sxe2_drv_q_switch_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_q_switch_req {
uint16_t q_idx;
uint16_t vsi_id;
uint8_t is_enable;
uint8_t sched_mode;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_create_req_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_create_req_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_free_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_free_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_req {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_req {
uint16_t vsi_id;
uint8_t rsv[2];
-};
+} __rte_packed_end;
-struct sxe2_drv_vsi_info_get_resp {
+struct __rte_aligned(4) __rte_packed_begin sxe2_drv_vsi_info_get_resp {
uint16_t vsi_id;
uint16_t vsi_type;
struct sxe2_drv_queue_caps used_queues;
struct sxe2_drv_msix_caps used_msix;
-};
+} __rte_packed_end;
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index b6cc8703a7..066e1faf7e 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -891,7 +891,7 @@ static int32_t sxe2_eth_pmd_probe_pf(struct sxe2_common_device *cdev,
static int32_t sxe2_parse_eth_devargs(struct rte_device *dev,
struct rte_eth_devargs *eth_da)
{
- int ret = 0;
+ int32_t ret = 0;
if (dev->devargs == NULL)
return 0;
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index a3706945e8..8015d9a064 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -130,9 +130,8 @@ struct sxe2_devargs {
uint8_t flow_dup_pattern_mode;
uint8_t func_flow_direct_en;
uint8_t fnav_stat_type;
- uint8_t high_performance_mode;
+ uint8_t no_sched_mode;
uint8_t sched_layer_mode;
- uint8_t sw_stats_en;
uint8_t rx_low_latency;
};
diff --git a/drivers/net/sxe2/sxe2_queue.h b/drivers/net/sxe2/sxe2_queue.h
index adb4be1214..a300b66771 100644
--- a/drivers/net/sxe2/sxe2_queue.h
+++ b/drivers/net/sxe2/sxe2_queue.h
@@ -7,7 +7,6 @@
#include <rte_ethdev.h>
#include <rte_io.h>
-#include <rte_stdatomic.h>
#include <ethdev_driver.h>
#include "sxe2_drv_cmd.h"
@@ -123,13 +122,13 @@ struct sxe2_rxq_stats {
};
struct sxe2_rxq_sw_stats {
- RTE_ATOMIC(uint64_t)pkts;
- RTE_ATOMIC(uint64_t)bytes;
- RTE_ATOMIC(uint64_t)drop_pkts;
- RTE_ATOMIC(uint64_t)drop_bytes;
- RTE_ATOMIC(uint64_t)unicast_pkts;
- RTE_ATOMIC(uint64_t)multicast_pkts;
- RTE_ATOMIC(uint64_t)broadcast_pkts;
+ uint64_t pkts;
+ uint64_t bytes;
+ uint64_t drop_pkts;
+ uint64_t drop_bytes;
+ uint64_t unicast_pkts;
+ uint64_t multicast_pkts;
+ uint64_t broadcast_pkts;
};
struct sxe2_rx_queue {
diff --git a/drivers/net/sxe2/sxe2_rx.c b/drivers/net/sxe2/sxe2_rx.c
index 28832d5f71..543d825166 100644
--- a/drivers/net/sxe2/sxe2_rx.c
+++ b/drivers/net/sxe2/sxe2_rx.c
@@ -479,20 +479,13 @@ int32_t __rte_cold sxe2_rxqs_all_start(struct rte_eth_dev *dev)
goto l_free_started_queue;
}
- rte_atomic_store_explicit(&rxq->sw_stats.pkts, 0,
- rte_memory_order_relaxed);
- rte_atomic_store_explicit(&rxq->sw_stats.bytes, 0,
- rte_memory_order_relaxed);
- rte_atomic_store_explicit(&rxq->sw_stats.drop_pkts, 0,
- rte_memory_order_relaxed);
- rte_atomic_store_explicit(&rxq->sw_stats.drop_bytes, 0,
- rte_memory_order_relaxed);
- rte_atomic_store_explicit(&rxq->sw_stats.unicast_pkts, 0,
- rte_memory_order_relaxed);
- rte_atomic_store_explicit(&rxq->sw_stats.broadcast_pkts, 0,
- rte_memory_order_relaxed);
- rte_atomic_store_explicit(&rxq->sw_stats.multicast_pkts, 0,
- rte_memory_order_relaxed);
+ rxq->sw_stats.pkts = 0;
+ rxq->sw_stats.bytes = 0;
+ rxq->sw_stats.drop_pkts = 0;
+ rxq->sw_stats.drop_bytes = 0;
+ rxq->sw_stats.unicast_pkts = 0;
+ rxq->sw_stats.broadcast_pkts = 0;
+ rxq->sw_stats.multicast_pkts = 0;
}
ret = 0;
goto l_end;
@@ -524,31 +517,15 @@ void __rte_cold sxe2_rxqs_all_stop(struct rte_eth_dev *dev)
rxq = dev->data->rx_queues[nb_rxq];
if (rxq) {
- sw_stats_prev->ipackets +=
- rte_atomic_load_explicit(&rxq->sw_stats.pkts,
- rte_memory_order_relaxed);
- sw_stats_prev->ierrors +=
- rte_atomic_load_explicit(&rxq->sw_stats.drop_pkts,
- rte_memory_order_relaxed);
- sw_stats_prev->ibytes +=
- rte_atomic_load_explicit(&rxq->sw_stats.bytes,
- rte_memory_order_relaxed);
-
- sw_stats_prev->rx_sw_unicast_packets +=
- rte_atomic_load_explicit(&rxq->sw_stats.unicast_pkts,
- rte_memory_order_relaxed);
- sw_stats_prev->rx_sw_broadcast_packets +=
- rte_atomic_load_explicit(&rxq->sw_stats.broadcast_pkts,
- rte_memory_order_relaxed);
- sw_stats_prev->rx_sw_multicast_packets +=
- rte_atomic_load_explicit(&rxq->sw_stats.multicast_pkts,
- rte_memory_order_relaxed);
- sw_stats_prev->rx_sw_drop_packets +=
- rte_atomic_load_explicit(&rxq->sw_stats.drop_pkts,
- rte_memory_order_relaxed);
- sw_stats_prev->rx_sw_drop_bytes +=
- rte_atomic_load_explicit(&rxq->sw_stats.drop_bytes,
- rte_memory_order_relaxed);
+ sw_stats_prev->ipackets += rxq->sw_stats.pkts;
+ sw_stats_prev->ierrors += rxq->sw_stats.drop_pkts;
+ sw_stats_prev->ibytes += rxq->sw_stats.bytes;
+
+ sw_stats_prev->rx_sw_unicast_packets += rxq->sw_stats.unicast_pkts;
+ sw_stats_prev->rx_sw_broadcast_packets += rxq->sw_stats.broadcast_pkts;
+ sw_stats_prev->rx_sw_multicast_packets += rxq->sw_stats.multicast_pkts;
+ sw_stats_prev->rx_sw_drop_packets += rxq->sw_stats.drop_pkts;
+ sw_stats_prev->rx_sw_drop_bytes += rxq->sw_stats.drop_bytes;
}
}
}
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 8d17535301..aa1c474088 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -157,6 +157,19 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
tx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ tx_mode_flags |= SXE2_TX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(TX, "AVX512 is not supported in build env.");
+#endif
+ }
+ if ((tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) == 0)
+ tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
if (ret != 0)
@@ -172,14 +185,25 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
tx_mode_flags = adapter->q_ctxt.tx_mode_flags;
}
-#ifdef RTE_ARCH_X86
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
- if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
- dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ dev->tx_pkt_prepare = NULL;
+#ifdef RTE_ARCH_X86
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_avx512_simple;
+ }
+#endif
} else {
- dev->tx_pkt_prepare = NULL;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
+ }
}
} else {
#endif
@@ -201,8 +225,16 @@ static const struct {
} sxe2_tx_burst_infos[] = {
{ sxe2_tx_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
- { sxe2_tx_pkts_vec_sse, "Vector SSE" },
- { sxe2_tx_pkts_vec_sse_simple, "Vector SSE Simple" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_tx_pkts_vec_avx512,
+ "Vector AVX512" },
+ { sxe2_tx_pkts_vec_avx512_simple,
+ "Vector AVX512 Simple" },
+#endif
+ { sxe2_tx_pkts_vec_sse,
+ "Vector SSE" },
+ { sxe2_tx_pkts_vec_sse_simple,
+ "Vector SSE Simple" },
#endif
};
@@ -288,6 +320,20 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (ret == 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
rx_mode_flags = vec_flags;
+#ifdef RTE_ARCH_X86
+ if ((rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
+ (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)) {
+#ifdef CC_AVX512_SUPPORT
+ rx_mode_flags |= SXE2_RX_MODE_VEC_AVX512;
+#else
+ PMD_LOG_INFO(RX, "AVX512 support detected but not enabled");
+#endif
+ }
+ if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0 &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
+ rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
if (ret != 0)
@@ -301,7 +347,16 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
#ifdef RTE_ARCH_X86
if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
- dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_AVX512) {
+#ifdef CC_AVX512_SUPPORT
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_avx512;
+#endif
+ } else {
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_sse_offload;
+ }
return;
}
#endif
@@ -315,19 +370,30 @@ static const struct {
eth_rx_burst_t rx_burst;
const char *info;
} sxe2_rx_burst_infos[] = {
- { sxe2_rx_pkts_scattered, "Scalar Scattered" },
- { sxe2_rx_pkts_scattered_split, "Scalar Scattered split" },
+ { sxe2_rx_pkts_scattered,
+ "Scalar Scattered" },
+ { sxe2_rx_pkts_scattered_split,
+ "Scalar Scattered split" },
#ifdef RTE_ARCH_X86
- { sxe2_rx_pkts_scattered_vec_sse_offload, "Vector SSE Scattered" },
+#ifdef CC_AVX512_SUPPORT
+ { sxe2_rx_pkts_scattered_vec_avx512,
+ "Vector AVX512 Scattered" },
+ { sxe2_rx_pkts_scattered_vec_avx512_offload,
+ "Offload Vector AVX512 Scattered" },
+#endif
+ { sxe2_rx_pkts_scattered_vec_sse_offload,
+ "Vector SSE Scattered" },
#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
- __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode)
+ __rte_unused uint16_t queue_id,
+ struct rte_eth_burst_mode *mode)
{
eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
int32_t ret = -EINVAL;
uint32_t i, size;
+
size = RTE_DIM(sxe2_rx_burst_infos);
for (i = 0; i < size; ++i) {
if (pkt_burst == sxe2_rx_burst_infos[i].rx_burst) {
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index b9d34afb31..947a5247ed 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -682,23 +682,17 @@ sxe2_rx_sw_stats_update(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf,
union sxe2_rx_desc *rxd)
{
uint64_t qword1 = rte_le_to_cpu_64(rxd->wb.status_err_ptype_len);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.pkts, 1,
- rte_memory_order_relaxed);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.bytes,
- mbuf->pkt_len + RTE_ETHER_CRC_LEN,
- rte_memory_order_relaxed);
+ rxq->sw_stats.pkts += 1;
+ rxq->sw_stats.bytes += mbuf->pkt_len + RTE_ETHER_CRC_LEN;
switch (SXE2_RX_DESC_STATUS_UMBCAST_VAL_GET(qword1)) {
case SXE2_RX_DESC_STATUS_UNICAST:
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.unicast_pkts, 1,
- rte_memory_order_relaxed);
+ rxq->sw_stats.unicast_pkts += 1;
break;
case SXE2_RX_DESC_STATUS_MULTICAST:
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.multicast_pkts, 1,
- rte_memory_order_relaxed);
+ rxq->sw_stats.multicast_pkts += 1;
break;
case SXE2_RX_DESC_STATUS_BROADCAST:
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.broadcast_pkts, 1,
- rte_memory_order_relaxed);
+ rxq->sw_stats.broadcast_pkts += 1;
break;
default:
break;
@@ -787,11 +781,9 @@ uint16_t sxe2_rx_pkts_scattered(void *rx_queue, struct rte_mbuf **rx_pkts, uint1
if (unlikely(qword1 & SXE2_RX_DESC_ERROR_RXE_MASK) ||
unlikely(qword1 & SXE2_RX_DESC_ERROR_OVERSIZE_MASK)) {
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_pkts, 1,
- rte_memory_order_relaxed);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_bytes,
- first_seg->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN,
- rte_memory_order_relaxed);
+ rxq->sw_stats.drop_pkts += 1;
+ rxq->sw_stats.drop_bytes +=
+ first_seg->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN;
rte_pktmbuf_free(first_seg);
first_seg = NULL;
continue;
@@ -822,8 +814,7 @@ uint16_t sxe2_rx_pkts_scattered(void *rx_queue, struct rte_mbuf **rx_pkts, uint1
sxe2_rx_mbuf_common_fields_fill(rxq, first_seg, &desc_tmp);
- if (rxq->vsi->adapter->devargs.sw_stats_en)
- sxe2_rx_sw_stats_update(rxq, first_seg, &desc_tmp);
+ sxe2_rx_sw_stats_update(rxq, first_seg, &desc_tmp);
rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, first_seg->data_off));
@@ -990,11 +981,9 @@ uint16_t sxe2_rx_pkts_scattered_split(void *rx_queue, struct rte_mbuf **rx_pkts,
if (unlikely(qword1 & SXE2_RX_DESC_ERROR_RXE_MASK) ||
unlikely(qword1 & SXE2_RX_DESC_ERROR_OVERSIZE_MASK)) {
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_pkts, 1,
- rte_memory_order_relaxed);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_bytes,
- first_seg->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN,
- rte_memory_order_relaxed);
+ rxq->sw_stats.drop_pkts += 1;
+ rxq->sw_stats.drop_bytes +=
+ first_seg->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN;
rte_pktmbuf_free(first_seg);
first_seg = NULL;
continue;
@@ -1023,8 +1012,7 @@ uint16_t sxe2_rx_pkts_scattered_split(void *rx_queue, struct rte_mbuf **rx_pkts,
first_seg->port = rxq->port_id;
sxe2_rx_mbuf_common_fields_fill(rxq, first_seg, &desc_tmp);
- if (rxq->vsi->adapter->devargs.sw_stats_en)
- sxe2_rx_sw_stats_update(rxq, first_seg, &desc_tmp);
+ sxe2_rx_sw_stats_update(rxq, first_seg, &desc_tmp);
rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, first_seg->data_off));
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index 8df4954d86..cf004f5eb2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -165,16 +165,54 @@ static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
return;
}
i = txq->next_dd - (txq->rs_thresh - 1);
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
+#ifdef CC_AVX512_SUPPORT
+ struct rte_eth_dev *dev;
+ struct sxe2_tx_buffer_vec *buffer_vec;
+
+ dev = &rte_eth_devices[txq->port_id];
+
+ if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
+ dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
+ buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for ( ; i < txq->next_use; ++i) {
+ if (buffer_vec[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
+ }
+ }
+ } else {
+#endif
+ buffer = txq->buffer_ring;
+ buffer = txq->buffer_ring;
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ if (buffer[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(buffer[i].mbuf);
+ buffer[i].mbuf = NULL;
+ }
+ }
+ i = 0;
+ }
+ for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
buffer[i].mbuf = NULL;
}
}
- i = 0;
+#ifdef CC_AVX512_SUPPORT
}
+#endif
+
for (; i < txq->next_use; ++i) {
if (buffer[i].mbuf != NULL) {
rte_pktmbuf_free_seg(buffer[i].mbuf);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 04ff4d96a5..af7c8d12b2 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -11,15 +11,19 @@
#define SXE2_RX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_RX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
- SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE)
+ SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
+ SXE2_RX_MODE_VEC_AVX512)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
+#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
- SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE)
+ SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
+ SXE2_TX_MODE_VEC_AVX512)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -54,6 +58,16 @@ uint16_t sxe2_tx_pkts_vec_sse(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_
uint16_t sxe2_tx_pkts_vec_sse_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_sse_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_avx512_ctx_offload(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
new file mode 100644
index 0000000000..c67e8e5090
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -0,0 +1,868 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef SXE2_TEST
+#include <rte_vect.h>
+
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_queue.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_vsi.h"
+
+static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_queue *txq)
+{
+ struct sxe2_tx_buffer_vec *buffer;
+ struct rte_mbuf *mbuf;
+ struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
+ struct rte_mempool *mp;
+ struct rte_mempool_cache *cache;
+ void **cache_objs;
+ uint32_t copied;
+ uint32_t i;
+ int32_t ret;
+ uint16_t rs_thresh;
+ uint16_t free_num;
+
+ if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
+ (txq->desc_ring[txq->next_dd].wb.dd &
+ rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
+ ret = 0;
+ goto l_end;
+ }
+
+ rs_thresh = txq->rs_thresh;
+
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += txq->next_dd - (rs_thresh - 1);
+
+ if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
+ (rs_thresh & 31) == 0) {
+ mp = buffer[0].mbuf->pool;
+ cache = rte_mempool_default_cache(mp, rte_lcore_id());
+
+ if (cache == NULL || cache->len)
+ goto normal;
+
+ if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer, rs_thresh);
+ goto done;
+ }
+ cache_objs = &cache->objs[cache->len];
+
+ copied = 0;
+ while (copied < rs_thresh) {
+ const __m512i objs0 = _mm512_loadu_si512(&buffer[copied]);
+ const __m512i objs1 = _mm512_loadu_si512(&buffer[copied + 8]);
+ const __m512i objs2 = _mm512_loadu_si512(&buffer[copied + 16]);
+ const __m512i objs3 = _mm512_loadu_si512(&buffer[copied + 24]);
+
+ _mm512_storeu_si512(&cache_objs[copied], objs0);
+ _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
+ _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
+ _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
+ copied += 32;
+ }
+ cache->len += rs_thresh;
+
+ if (cache->len >= cache->flushthresh) {
+ (void)rte_mempool_ops_enqueue_bulk(mp,
+ &cache->objs[cache->size], cache->len - cache->size);
+ cache->len = cache->size;
+ }
+ goto done;
+ }
+
+normal:
+ mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
+
+ if (likely(mbuf)) {
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+
+ if (likely(mbuf)) {
+ if (likely(mbuf->pool == mbuf_free_arr[0]->pool)) {
+ mbuf_free_arr[free_num] = mbuf;
+ free_num++;
+ } else {
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+
+ mbuf_free_arr[0] = mbuf;
+ free_num = 1;
+ }
+ }
+ }
+
+ rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
+ (void *)mbuf_free_arr, free_num);
+ } else {
+ for (i = 1; i < rs_thresh; ++i) {
+ mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
+ if (mbuf != NULL)
+ rte_mempool_put(mbuf->pool, mbuf);
+ }
+ }
+
+done:
+ txq->desc_free_num += txq->rs_thresh;
+ txq->next_dd += txq->rs_thresh;
+ if (txq->next_dd >= txq->ring_depth)
+ txq->next_dd = txq->rs_thresh - 1;
+ ret = rs_thresh;
+
+l_end:
+ return ret;
+}
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
+ uint64_t desc_cmd, bool with_offloads)
+{
+ __m128i data_desc;
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
+}
+
+static __rte_always_inline
+void sxe2_tx_desc_fill_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
+ uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
+{
+ __m512i desc_group;
+ uint64_t desc0_qw1;
+ uint64_t desc1_qw1;
+ uint64_t desc2_qw1;
+ uint64_t desc3_qw1;
+
+ const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
+ uint32_t desc_offset[4] = {0};
+
+ while (pkts_num > 3) {
+ desc3_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+
+ desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
+ desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);
+
+ desc2_qw1 = desc_qw1_com |
+ ((uint64_t)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT;
+ desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
+ desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);
+
+ desc1_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
+ desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);
+
+ desc0_qw1 = (desc_qw1_com |
+ ((uint64_t)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+ desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
+ desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);
+
+ desc_group =
+ _mm512_set_epi64(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
+ desc2_qw1, rte_pktmbuf_iova(pkts[2]),
+ desc1_qw1, rte_pktmbuf_iova(pkts[1]),
+ desc0_qw1, rte_pktmbuf_iova(pkts[0]));
+
+ _mm512_storeu_si512(RTE_CAST_PTR(void *, desc), desc_group);
+
+ pkts_num -= 4;
+ desc += 4;
+ pkts += 4;
+ }
+
+ while (pkts_num) {
+ sxe2_tx_desc_fill_one_avx512(desc, *pkts, desc_cmd, with_offloads);
+
+ pkts_num--;
+ desc++;
+ pkts++;
+ }
+}
+
+static __rte_always_inline void
+sxe2_tx_pkts_mbuf_fill_avx512(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ uint16_t i;
+
+ for (i = 0; i < nb_pkts; ++i)
+ buffer[i].mbuf = tx_pkts[i];
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer_vec *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec_avx512(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts avx512 batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer += next_use;
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ tx_pkts += (res_num - 1);
+ desc += (res_num - 1);
+
+ sxe2_tx_desc_fill_one_avx512(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = txq->desc_ring;
+ buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ }
+
+ sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+
+ sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
+ PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
+ txq->port_id, txq->queue_id, next_use, nb_pkts);
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_avx512_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num =
+ sxe2_tx_pkts_vec_avx512_batch(txq, tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_avx512_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static inline void sxe2_rx_queue_rearm_avx512(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ __m128i dma_addr0, dma_addr1;
+ __m128i virt_addr0, virt_addr1;
+ __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ dma_addr0 = _mm_setzero_si128();
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read),
+ dma_addr0);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+
+#if RTE_IOVA_IN_MBUF
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
+ virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+
+ dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
+#else
+
+ dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
+ dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
+#endif
+
+ dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
+ dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
+
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
+ _mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ? (rxq->ring_depth - 1) :
+ (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags,
+ uint8_t *umbcast_flags, bool do_offload)
+{
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+ const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
+ struct rte_mbuf **buffer;
+ volatile union sxe2_rx_desc *desc;
+ __m512i mbufs4_7;
+ __m512i mbufs0_3;
+ __m256i mbufs6_7;
+ __m256i mbufs4_5;
+ __m256i mbufs2_3;
+ __m256i mbufs0_1;
+ uint32_t bit_num = 0;
+ uint16_t done_num = 0;
+ uint16_t i = 0;
+ uint16_t j = 0;
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ desc = &rxq->desc_ring[rxq->processing_idx];
+
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_avx512(rxq);
+
+ if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) & SXE2_RX_DESC_STATUS_DD_MASK))
+ goto l_end;
+
+ const __m512i crc_adjust =
+ _mm512_set4_epi32(0, -rxq->crc_len, -rxq->crc_len, 0);
+
+ const __m256i dd_mask = _mm256_set1_epi32(1);
+
+ const __m512i rvp_shuf_mask =
+ _mm512_set4_epi32((7 << 24) | (6 << 16) | (5 << 8) | 4,
+ (3 << 24) | (2 << 16) | (13 << 8) | 12,
+ (0xFFU << 24) | (0xFF << 16) | (13 << 8) | 12,
+ 0xFFFFFFFF);
+
+ const __m128i eop_shuf_mask =
+ _mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 8, 0, 10, 2, 12, 4, 14, 6);
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+ offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
+ desc += SXE2_RX_NUM_PER_LOOP_AVX) {
+ _mm256_storeu_si256((void *)&rx_pkts[i],
+ _mm256_loadu_si256((void *)&buffer[i]));
+#ifdef RTE_ARCH_X86_64
+ _mm256_storeu_si256((void *)&rx_pkts[i + 4],
+ _mm256_loadu_si256((void *)&buffer[i + 4]));
+#endif
+
+ const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
+ rte_compiler_barrier();
+ const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
+ rte_compiler_barrier();
+ const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
+ rte_compiler_barrier();
+ const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
+ rte_compiler_barrier();
+ const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
+ rte_compiler_barrier();
+ const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
+ rte_compiler_barrier();
+ const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
+ rte_compiler_barrier();
+ const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));
+
+ const __m256i descs6_7 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
+ const __m256i descs4_5 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
+ const __m256i descs2_3 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
+ const __m256i descs0_1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);
+
+ const __m512i descs4_7 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs4_5), descs6_7, 1);
+ const __m512i descs0_3 =
+ _mm512_inserti64x4(_mm512_castsi256_si512(descs0_1), descs2_3, 1);
+
+ if (split_rxe_flags != NULL) {
+ for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
+ rte_mbuf_prefetch_part2(rx_pkts[i + j]);
+ }
+
+ mbufs4_7 = _mm512_shuffle_epi8(descs4_7, rvp_shuf_mask);
+ mbufs0_3 = _mm512_shuffle_epi8(descs0_3, rvp_shuf_mask);
+
+ mbufs4_7 = _mm512_add_epi32(mbufs4_7, crc_adjust);
+ mbufs0_3 = _mm512_add_epi32(mbufs0_3, crc_adjust);
+
+ const __m512i ptype_mask = _mm512_set1_epi64(SXE2_RX_FLEX_DESC_PTYPE_M <<
+ SXE2_RX_FLEX_DESC_PTYPE_S);
+
+ __m512i ptypes4_7 = _mm512_and_si512(descs4_7, ptype_mask);
+ __m512i ptypes0_3 = _mm512_and_si512(descs0_3, ptype_mask);
+
+ const __m256i ptypes6_7 = _mm512_extracti64x4_epi64(ptypes4_7, 1);
+ const __m256i ptypes4_5 = _mm512_extracti64x4_epi64(ptypes4_7, 0);
+ const __m256i ptypes2_3 = _mm512_extracti64x4_epi64(ptypes0_3, 1);
+ const __m256i ptypes0_1 = _mm512_extracti64x4_epi64(ptypes0_3, 0);
+
+ const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 13);
+ const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 5);
+ const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 13);
+ const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 5);
+ const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 13);
+ const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 5);
+ const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 13);
+ const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 5);
+
+ const __m512i ptype_mask4_7 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype7],
+ 0, 0, 0, ptype_tbl[ptype6],
+ 0, 0, 0, ptype_tbl[ptype5],
+ 0, 0, 0, ptype_tbl[ptype4]);
+ const __m512i ptype_mask0_3 =
+ _mm512_set_epi32(0, 0, 0, ptype_tbl[ptype3],
+ 0, 0, 0, ptype_tbl[ptype2],
+ 0, 0, 0, ptype_tbl[ptype1],
+ 0, 0, 0, ptype_tbl[ptype0]);
+
+ mbufs4_7 = _mm512_or_si512(mbufs4_7, ptype_mask4_7);
+ mbufs0_3 = _mm512_or_si512(mbufs0_3, ptype_mask0_3);
+
+ mbufs6_7 = _mm512_extracti64x4_epi64(mbufs4_7, 1);
+ mbufs4_5 = _mm512_extracti64x4_epi64(mbufs4_7, 0);
+ mbufs2_3 = _mm512_extracti64x4_epi64(mbufs0_3, 1);
+ mbufs0_1 = _mm512_extracti64x4_epi64(mbufs0_3, 0);
+
+ const __m512i staterr_per_mask =
+ _mm512_set_epi32(0x17, 0x1F, 0x07, 0x0F,
+ 0x13, 0x1B, 0x03, 0x0B,
+ 0x16, 0x1E, 0x06, 0x0E,
+ 0x12, 0x1A, 0x02, 0x0A);
+ __m512i qw1_0_7 = _mm512_permutex2var_epi32(descs4_7,
+ staterr_per_mask,
+ descs0_3);
+
+ __m256i staterrs0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 0);
+
+ __m256i stu_len0_7 = _mm512_extracti64x4_epi64(qw1_0_7, 1);
+ __m256i mbuf_flags = _mm256_setzero_si256();
+
+ if (do_offload) {
+ const __m256i desc_flags_mask = _mm256_set1_epi32(0xC0001C04);
+ const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
+ const __m256i vlan_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0, 0, 0, 0);
+
+ const __m256i rss_flags =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH,
+ 0, 0, 0, 0);
+
+ const __m256i cksum_flags =
+ _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0,
+ 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));
+
+ const __m256i cksum_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_IP_CKSUM_MASK |
+ RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
+
+ const __m256i vlan_mask =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED);
+
+ __m256i tmp_flags;
+ __m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
+ stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);
+
+ tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
+ mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);
+
+ descs_flags = _mm256_srli_epi32(descs_flags, 10);
+ tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
+ tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
+ tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
+ tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ __m256i fnav_vld0_3, fnav_vld4_7;
+ __m256i fnav_vld0_7;
+ __m256i v_zeros, v_ffff, v_u32_one;
+ const __m256i fdir_flags =
+ _mm256_set1_epi32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+ fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
+ fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);
+
+ fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);
+
+ fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
+ fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);
+
+ v_zeros = _mm256_setzero_si256();
+ v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
+ v_u32_one = _mm256_srli_epi32(v_ffff, 31);
+
+ tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);
+
+ tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);
+
+ mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);
+
+ rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
+ rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
+ rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
+ rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
+ }
+#endif
+ }
+
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+ offsetof(struct rte_mbuf, rearm_data) + 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
+ offsetof(struct rte_mbuf, rearm_data) + 16);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+ RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+
+ __m256i rearm_arr[8];
+
+ rearm_arr[6] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 8), 0x04);
+ rearm_arr[4] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(mbuf_flags, 4), 0x04);
+ rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04);
+ rearm_arr[0] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(mbuf_flags, 4), 0x04);
+
+ rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
+ rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
+ rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
+ rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);
+
+ const __m256i tmp_mbuf_flags =
+ _mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));
+
+ rearm_arr[7] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
+ rearm_arr[5] = _mm256_blend_epi32(mbuf_init,
+ _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
+ rearm_arr[3] = _mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
+ rearm_arr[1] = _mm256_blend_epi32(mbuf_init,
+ _mm256_srli_si256(tmp_mbuf_flags, 4), 4);
+
+ rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
+ rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
+ rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
+ rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);
+
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
+ _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);
+
+ if (umbcast_flags) {
+ const __m256i umbcast_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
+ __m256i umbcast_bits_256 =
+ _mm256_and_si256(staterrs0_7, umbcast_mask);
+
+ umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);
+ __m128i umbcast_bits_128 =
+ _mm_packs_epi32(_mm256_castsi256_si128(umbcast_bits_256),
+ _mm256_extractf128_si256(umbcast_bits_256, 1));
+
+ umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);
+
+ *(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ if (split_rxe_flags) {
+ const __m256i eop_rxe_mask =
+ _mm256_set1_epi32(SXE2_RX_DESC_STATUS_EOP_MASK |
+ SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+ const __m128i eop_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
+ const __m128i rxe_mask_128 =
+ _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
+ SXE2_RX_DESC_ERROR_OVERSIZE_MASK);
+
+ const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);
+
+ const __m128i eop_rxe_bits = _mm_packs_epi32
+ (_mm256_castsi256_si128(tmp_stats),
+ _mm256_extractf128_si256(tmp_stats, 1));
+
+ __m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);
+
+ not_eop_bits =
+ _mm_or_si128(not_eop_bits,
+ _mm_srli_epi16(_mm_and_si128(eop_rxe_bits,
+ rxe_mask_128),
+ 7));
+
+ not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);
+
+ *(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
+ }
+
+ staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);
+
+ staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
+
+ bit_num = rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
+ bit_num += rte_popcount64
+ (_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));
+ done_num += bit_num;
+
+ if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ if ((rxq->processing_idx & 1) == 1 && done_num > 1) {
+ rxq->processing_idx--;
+ done_num--;
+ }
+ rxq->realloc_num += done_num;
+
+l_end:
+ PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
+ rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_avx512(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool do_offload)
+{
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ rx_done_num = sxe2_rx_pkts_common_vec_avx512(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags,
+ umbcast_flags, do_offload);
+ if (rx_done_num == 0)
+ goto l_end;
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+
+ return rx_pkt_done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_common_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool offload)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num = 0;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue, rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM, offload);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
+ goto end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_avx512(rx_queue,
+ rx_pkts + done_num, nb_pkts, offload);
+
+end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
+ rx_pkts, nb_pkts, true);
+}
+
+#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_common.h b/drivers/net/sxe2/sxe2_txrx_vec_common.h
index 6b1649c390..cc74f6e582 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_common.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec_common.h
@@ -130,27 +130,20 @@ sxe2_tx_desc_fill_offloads(struct rte_mbuf *mbuf, uint64_t *desc_qw1)
static inline void sxe2_vf_rx_vec_sw_stats_cnt(struct sxe2_rx_queue *rxq,
struct rte_mbuf *mbuf, uint8_t umbcast_flag)
{
- if (rxq->vsi->adapter->devargs.sw_stats_en) {
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.pkts, 1,
- rte_memory_order_relaxed);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.bytes,
- mbuf->pkt_len + RTE_ETHER_CRC_LEN, rte_memory_order_relaxed);
- switch (SXE2_RX_UMBCAST_FLAGS_VAL_GET(umbcast_flag)) {
- case SXE2_RX_DESC_STATUS_UNICAST:
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.unicast_pkts, 1,
- rte_memory_order_relaxed);
- break;
- case SXE2_RX_DESC_STATUS_MULTICAST:
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.multicast_pkts, 1,
- rte_memory_order_relaxed);
- break;
- case SXE2_RX_DESC_STATUS_BROADCAST:
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.broadcast_pkts, 1,
- rte_memory_order_relaxed);
- break;
- default:
- break;
- }
+ rxq->sw_stats.pkts += 1;
+ rxq->sw_stats.bytes += mbuf->pkt_len + RTE_ETHER_CRC_LEN;
+ switch (SXE2_RX_UMBCAST_FLAGS_VAL_GET(umbcast_flag)) {
+ case SXE2_RX_DESC_STATUS_UNICAST:
+ rxq->sw_stats.unicast_pkts += 1;
+ break;
+ case SXE2_RX_DESC_STATUS_MULTICAST:
+ rxq->sw_stats.multicast_pkts += 1;
+ break;
+ case SXE2_RX_DESC_STATUS_BROADCAST:
+ rxq->sw_stats.broadcast_pkts += 1;
+ break;
+ default:
+ break;
}
}
@@ -196,11 +189,9 @@ sxe2_rx_pkts_refactor(struct sxe2_rx_queue *rxq,
} else if (split_rxe_flags[buf_idx] & SXE2_RX_DESC_STATUS_EOP_MASK) {
continue;
} else {
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_pkts, 1,
- rte_memory_order_relaxed);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_bytes,
- first_seg->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN,
- rte_memory_order_relaxed);
+ rxq->sw_stats.drop_pkts += 1;
+ rxq->sw_stats.drop_bytes +=
+ first_seg->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN;
rte_pktmbuf_free(first_seg);
first_seg = NULL;
last_seg = NULL;
@@ -218,11 +209,10 @@ sxe2_rx_pkts_refactor(struct sxe2_rx_queue *rxq,
mbuf_bufs[buf_idx]->data_len += rxq->crc_len;
mbuf_bufs[buf_idx]->pkt_len += rxq->crc_len;
} else {
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_pkts, 1,
- rte_memory_order_relaxed);
- rte_atomic_fetch_add_explicit(&rxq->sw_stats.drop_bytes,
- mbuf_bufs[buf_idx]->pkt_len - rxq->crc_len + RTE_ETHER_CRC_LEN,
- rte_memory_order_relaxed);
+ rxq->sw_stats.drop_pkts += 1;
+ rxq->sw_stats.drop_bytes +=
+ mbuf_bufs[buf_idx]->pkt_len - rxq->crc_len +
+ RTE_ETHER_CRC_LEN;
rte_pktmbuf_free_seg(mbuf_bufs[buf_idx]);
continue;
}
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_sse.c b/drivers/net/sxe2/sxe2_txrx_vec_sse.c
index f6e3f45937..182a7dfc17 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_sse.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_sse.c
@@ -483,41 +483,16 @@ static __rte_always_inline uint16_t
sxe2_rx_pkts_scattered_batch_vec_sse(struct sxe2_rx_queue *rxq,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
- const uint64_t *split_rxe_flags64;
uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
uint16_t rx_done_num;
uint16_t rx_pkt_done_num;
rx_pkt_done_num = 0;
- if (rxq->vsi->adapter->devargs.sw_stats_en) {
- rx_done_num = sxe2_rx_pkts_common_vec_sse(rxq, rx_pkts,
- nb_pkts, split_rxe_flags, umbcast_flags);
- } else {
- rx_done_num = sxe2_rx_pkts_common_vec_sse(rxq, rx_pkts,
- nb_pkts, split_rxe_flags, NULL);
- }
+ rx_done_num = sxe2_rx_pkts_common_vec_sse(rxq, rx_pkts,
+ nb_pkts, split_rxe_flags, umbcast_flags);
if (rx_done_num == 0)
goto l_end;
- if (!rxq->vsi->adapter->devargs.sw_stats_en) {
- split_rxe_flags64 = (uint64_t *)split_rxe_flags;
- if (rxq->pkt_first_seg == NULL &&
- split_rxe_flags64[0] == 0 &&
- split_rxe_flags64[1] == 0 &&
- split_rxe_flags64[2] == 0 &&
- split_rxe_flags64[3] == 0) {
- rx_pkt_done_num = rx_done_num;
- goto l_end;
- }
- if (rxq->pkt_first_seg == NULL) {
- while (rx_pkt_done_num < rx_done_num &&
- split_rxe_flags[rx_pkt_done_num] == 0)
- rx_pkt_done_num++;
- if (rx_pkt_done_num == rx_done_num)
- goto l_end;
- rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
- }
- }
rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
&umbcast_flags[rx_pkt_done_num]);
--
2.31.1
^ permalink raw reply related
* [PATCH v3 05/20] drivers: support RSS feature
From: liujie5 @ 2026-06-18 8:27 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260618082723.571054-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
Add support for Receive Side Scaling (RSS) to distribute incoming
traffic across multiple receive queues.
- Implement rss_hash_update and rss_hash_conf_get.
- Implement reta_update and reta_query.
- Support RSS hash configuration for IPv4, IPv6, TCP and UDP.
- Default hash key is initialized during port start.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/common/sxe2/sxe2_flow_public.h | 633 +++++++++++++++++++++++++
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 173 +++++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 16 +
drivers/net/sxe2/sxe2_drv_cmd.h | 29 ++
drivers/net/sxe2/sxe2_ethdev.c | 37 ++
drivers/net/sxe2/sxe2_ethdev.h | 8 +
drivers/net/sxe2/sxe2_flow_define.h | 143 ++++++
drivers/net/sxe2/sxe2_queue.c | 11 +
| 584 +++++++++++++++++++++++
| 81 ++++
drivers/net/sxe2/sxe2_txrx.h | 3 +
drivers/net/sxe2/sxe2_txrx_poll.c | 85 +++-
13 files changed, 1803 insertions(+), 1 deletion(-)
create mode 100644 drivers/common/sxe2/sxe2_flow_public.h
create mode 100644 drivers/net/sxe2/sxe2_flow_define.h
create mode 100644 drivers/net/sxe2/sxe2_rss.c
create mode 100644 drivers/net/sxe2/sxe2_rss.h
diff --git a/drivers/common/sxe2/sxe2_flow_public.h b/drivers/common/sxe2/sxe2_flow_public.h
new file mode 100644
index 0000000000..32ab2a9713
--- /dev/null
+++ b/drivers/common/sxe2/sxe2_flow_public.h
@@ -0,0 +1,633 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FLOW_PUBLIC_H__
+#define __SXE2_FLOW_PUBLIC_H__
+#include "sxe2_osal.h"
+
+enum sxe2_flow_type {
+ SXE2_FLOW_TYPE_NONE = 0,
+ SXE2_FLOW_MAC_PAY = 1,
+ SXE2_FLOW_MAC_IPV4_FRAG_PAY = 22,
+ SXE2_FLOW_MAC_IPV4_PAY = 23,
+ SXE2_FLOW_MAC_IPV4_UDP_PAY = 24,
+ SXE2_FLOW_MAC_IPV4_TCP_PAY = 26,
+ SXE2_FLOW_MAC_IPV4_SCTP_PAY = 27,
+ SXE2_FLOW_MAC_IPV4_IPV4_FRAG_PAY = 29,
+ SXE2_FLOW_MAC_IPV4_IPV4_PAY = 30,
+ SXE2_FLOW_MAC_IPV4_IPV4_UDP_PAY = 31,
+ SXE2_FLOW_MAC_IPV4_IPV4_TCP_PAY = 33,
+ SXE2_FLOW_MAC_IPV4_IPV4_SCTP_PAY = 34,
+ SXE2_FLOW_MAC_IPV4_IPV6_FRAG_PAY = 36,
+ SXE2_FLOW_MAC_IPV4_IPV6_PAY = 37,
+ SXE2_FLOW_MAC_IPV4_IPV6_UDP_PAY = 38,
+ SXE2_FLOW_MAC_IPV4_IPV6_TCP_PAY = 40,
+ SXE2_FLOW_MAC_IPV4_IPV6_SCTP_PAY = 41,
+ SXE2_FLOW_MAC_IPV4_GRE_PAY = 43,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_FRAG_PAY = 44,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_PAY = 45,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_UDP_PAY = 46,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_TCP_PAY = 48,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV4_SCTP_PAY = 49,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_FRAG_PAY = 51,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_PAY = 52,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_UDP_PAY = 53,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_TCP_PAY = 55,
+ SXE2_FLOW_MAC_IPV4_GRE_IPV6_SCTP_PAY = 56,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_PAY = 58,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_FRAG_PAY = 59,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_PAY = 60,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_UDP_PAY = 61,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_TCP_PAY = 63,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV4_SCTP_PAY = 64,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_FRAG_PAY = 66,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_PAY = 67,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_UDP_PAY = 68,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_TCP_PAY = 70,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_IPV6_SCTP_PAY = 71,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_PAY = 73,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_FRAG_PAY = 74,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_PAY = 75,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_UDP_PAY = 76,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_TCP_PAY = 78,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV4_SCTP_PAY = 79,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_FRAG_PAY = 81,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_PAY = 82,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_UDP_PAY = 83,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_TCP_PAY = 85,
+ SXE2_FLOW_MAC_IPV4_GRE_MAC_VLAN_IPV6_SCTP_PAY = 86,
+ SXE2_FLOW_MAC_IPV6_FRAG_PAY = 88,
+ SXE2_FLOW_MAC_IPV6_PAY = 89,
+ SXE2_FLOW_MAC_IPV6_UDP_PAY = 90,
+ SXE2_FLOW_MAC_IPV6_TCP_PAY = 92,
+ SXE2_FLOW_MAC_IPV6_SCTP_PAY = 93,
+ SXE2_FLOW_MAC_IPV6_IPV4_FRAG_PAY = 95,
+ SXE2_FLOW_MAC_IPV6_IPV4_PAY = 96,
+ SXE2_FLOW_MAC_IPV6_IPV4_UDP_PAY = 97,
+ SXE2_FLOW_MAC_IPV6_IPV4_TCP_PAY = 99,
+ SXE2_FLOW_MAC_IPV6_IPV4_SCTP_PAY = 100,
+ SXE2_FLOW_MAC_IPV6_IPV6_FRAG_PAY = 102,
+ SXE2_FLOW_MAC_IPV6_IPV6_PAY = 103,
+ SXE2_FLOW_MAC_IPV6_IPV6_UDP_PAY = 104,
+ SXE2_FLOW_MAC_IPV6_IPV6_TCP_PAY = 106,
+ SXE2_FLOW_MAC_IPV6_IPV6_SCTP_PAY = 107,
+ SXE2_FLOW_MAC_IPV6_GRE_PAY = 109,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_FRAG_PAY = 110,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_PAY = 111,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_UDP_PAY = 112,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_TCP_PAY = 114,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV4_SCTP_PAY = 115,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_FRAG_PAY = 117,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_PAY = 118,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_UDP_PAY = 119,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_TCP_PAY = 121,
+ SXE2_FLOW_MAC_IPV6_GRE_IPV6_SCTP_PAY = 122,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_PAY = 124,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_FRAG_PAY = 125,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_PAY = 126,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_UDP_PAY = 127,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_TCP_PAY = 129,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV4_SCTP_PAY = 130,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_FRAG_PAY = 132,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_PAY = 133,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_UDP_PAY = 134,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_TCP_PAY = 136,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_IPV6_SCTP_PAY = 137,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_PAY = 139,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_FRAG_PAY = 140,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_PAY = 141,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_UDP_PAY = 142,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_TCP_PAY = 144,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV4_SCTP_PAY = 145,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_FRAG_PAY = 147,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_PAY = 148,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_UDP_PAY = 149,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_TCP_PAY = 151,
+ SXE2_FLOW_MAC_IPV6_GRE_MAC_VLAN_IPV6_SCTP_PAY = 152,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_PAY = 329,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_PAY = 330,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_FRAG_PAY = 331,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_PAY = 332,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_UDP_PAY = 333,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_TCP_PAY = 334,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV4_SCTP_PAY = 335,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_FRAG_PAY = 336,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_PAY = 337,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_UDP_PAY = 338,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_TCP_PAY = 339,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV4_SCTP_PAY = 340,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_FRAG_PAY = 341,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_PAY = 342,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_UDP_PAY = 343,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_TCP_PAY = 344,
+ SXE2_FLOW_MAC_IPV4_UDP_GTPU_IPV6_SCTP_PAY = 345,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_FRAG_PAY = 346,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_PAY = 347,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_UDP_PAY = 348,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_TCP_PAY = 349,
+ SXE2_FLOW_MAC_IPV6_UDP_GTPU_IPV6_SCTP_PAY = 350,
+ SXE2_FLOW_MAC_IPV6_MAC_PAY = 820,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_FRAG_PAY = 821,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_PAY = 822,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_UDP_PAY = 823,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_TCP_PAY = 824,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV4_SCTP_PAY = 825,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_FRAG_PAY = 827,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_PAY = 828,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_UDP_PAY = 829,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_TCP_PAY = 830,
+ SXE2_FLOW_MAC_IPV6_MAC_IPV6_SCTP_PAY = 831,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_PAY = 835,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_FRAG_PAY = 836,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_PAY = 837,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_UDP_PAY = 838,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_TCP_PAY = 839,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV4_SCTP_PAY = 840,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_FRAG_PAY = 842,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_PAY = 843,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_UDP_PAY = 844,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_TCP_PAY = 845,
+ SXE2_FLOW_MAC_IPV6_MAC_VLAN_IPV6_SCTP_PAY = 846,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_PAY = 878,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_FRAG_PAY = 877,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_PAY = 876,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_UDP_PAY = 879,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_TCP_PAY = 880,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV4_SCTP_PAY = 875,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_FRAG_PAY = 871,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_PAY = 870,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_UDP_PAY = 872,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_TCP_PAY = 873,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_IPV6_SCTP_PAY = 869,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_PAY = 891,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_FRAG_PAY = 890,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_PAY = 889,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_UDP_PAY = 892,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_TCP_PAY = 893,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV4_SCTP_PAY = 888,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_FRAG_PAY = 884,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_PAY = 883,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_UDP_PAY = 885,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_TCP_PAY = 886,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_IPV6_SCTP_PAY = 882,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_PAY = 904,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_FRAG_PAY = 903,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_PAY = 902,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_UDP_PAY = 905,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_TCP_PAY = 906,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV4_SCTP_PAY = 901,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_FRAG_PAY = 897,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_PAY = 896,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_UDP_PAY = 898,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_TCP_PAY = 899,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_IPV6_SCTP_PAY = 895,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_PAY = 917,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_FRAG_PAY = 916,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_PAY = 915,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_UDP_PAY = 918,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_TCP_PAY = 919,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV4_SCTP_PAY = 914,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_FRAG_PAY = 910,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_PAY = 909,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_UDP_PAY = 911,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_TCP_PAY = 912,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_IPV6_SCTP_PAY = 908,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_PAY = 930,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY = 929,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_PAY = 928,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY = 931,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY = 932,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY = 927,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY = 923,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_PAY = 922,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY = 924,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY = 925,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY = 921,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_PAY = 943,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_FRAG_PAY = 942,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_PAY = 941,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_UDP_PAY = 944,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_TCP_PAY = 945,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV4_SCTP_PAY = 940,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_FRAG_PAY = 936,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_PAY = 935,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_UDP_PAY = 937,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_TCP_PAY = 938,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_VLAN_IPV6_SCTP_PAY = 934,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_PAY = 956,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY = 955,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_PAY = 954,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY = 957,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY = 958,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY = 953,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY = 949,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_PAY = 948,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY = 950,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY = 951,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY = 947,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_PAY = 969,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_FRAG_PAY = 968,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_PAY = 967,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_UDP_PAY = 970,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_TCP_PAY = 971,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV4_SCTP_PAY = 966,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_FRAG_PAY = 962,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_PAY = 961,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_UDP_PAY = 963,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_TCP_PAY = 964,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_VLAN_IPV6_SCTP_PAY = 960,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_PAY = 982,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_FRAG_PAY = 981,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_PAY = 980,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_UDP_PAY = 983,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_TCP_PAY = 984,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV4_SCTP_PAY = 979,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_FRAG_PAY = 975,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_PAY = 974,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_UDP_PAY = 976,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_TCP_PAY = 977,
+ SXE2_FLOW_MAC_IPV6_UDP_VXGEN_MAC_IPV6_SCTP_PAY = 973,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_PAY = 995,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_FRAG_PAY = 994,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_PAY = 993,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_UDP_PAY = 996,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_TCP_PAY = 997,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV4_SCTP_PAY = 992,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_FRAG_PAY = 988,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_PAY = 987,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_UDP_PAY = 989,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_TCP_PAY = 990,
+ SXE2_FLOW_MAC_IPV4_UDP_VXGEN_MAC_IPV6_SCTP_PAY = 986,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_PAY = 1008,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_FRAG_PAY = 1007,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_PAY = 1006,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_UDP_PAY = 1009,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_TCP_PAY = 1010,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV4_SCTP_PAY = 1005,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_FRAG_PAY = 1001,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_PAY = 1000,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_UDP_PAY = 1002,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_TCP_PAY = 1003,
+ SXE2_FLOW_MAC_IPV6_UDP_GRE_MAC_IPV6_SCTP_PAY = 999,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_PAY = 1021,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_FRAG_PAY = 1020,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_PAY = 1019,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_UDP_PAY = 1022,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_TCP_PAY = 1023,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV4_SCTP_PAY = 1018,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_FRAG_PAY = 1014,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_PAY = 1013,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_UDP_PAY = 1015,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_TCP_PAY = 1016,
+ SXE2_FLOW_MAC_IPV4_UDP_GRE_MAC_IPV6_SCTP_PAY = 1012,
+ SXE2_FLOW_TYPE_MAX = 2048,
+};
+
+enum sxe2_rss_cfg_hdr_type {
+ SXE2_RSS_OUTER_HEADERS,
+ SXE2_RSS_INNER_HEADERS,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GRE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_VXLAN,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_VXLAN,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GENEVE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GENEVE,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV4_UDP_GTPU,
+
+ SXE2_RSS_INNER_HEADERS_WITH_OUTER_IPV6_UDP_GTPU,
+ SXE2_RSS_ANY_HEADERS
+};
+
+enum sxe2_flow_hdr {
+ SXE2_FLOW_HDR_ETH = 0,
+ SXE2_FLOW_HDR_VLAN,
+ SXE2_FLOW_HDR_QINQ,
+ SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_ICMP = 5,
+ SXE2_FLOW_HDR_TCP,
+ SXE2_FLOW_HDR_UDP,
+ SXE2_FLOW_HDR_SCTP,
+ SXE2_FLOW_HDR_GRE,
+ SXE2_FLOW_HDR_VXLAN = 10,
+ SXE2_FLOW_HDR_GENEVE,
+ SXE2_FLOW_HDR_GTPU,
+
+ SXE2_FLOW_HDR_IPV_FRAG,
+
+ SXE2_FLOW_HDR_IPV_OTHER,
+
+ SXE2_FLOW_HDR_ETH_NON_IP = 15,
+ SXE2_FLOW_HDR_MAX = 128,
+};
+
+enum sxe2_flow_fld_id {
+ SXE2_FLOW_FLD_ID_ETH_DA = 0,
+ SXE2_FLOW_FLD_ID_ETH_SA,
+ SXE2_FLOW_FLD_ID_S_TCI,
+ SXE2_FLOW_FLD_ID_C_TCI,
+ SXE2_FLOW_FLD_ID_S_TPID,
+ SXE2_FLOW_FLD_ID_C_TPID = 5,
+ SXE2_FLOW_FLD_ID_S_VID,
+ SXE2_FLOW_FLD_ID_C_VID,
+ SXE2_FLOW_FLD_ID_ETH_TYPE,
+
+ SXE2_FLOW_FLD_ID_IPV4_TOS,
+ SXE2_FLOW_FLD_ID_IPV6_DSCP = 10,
+ SXE2_FLOW_FLD_ID_IPV4_TTL,
+ SXE2_FLOW_FLD_ID_IPV4_PROT,
+ SXE2_FLOW_FLD_ID_IPV6_TTL,
+ SXE2_FLOW_FLD_ID_IPV6_PROT,
+ SXE2_FLOW_FLD_ID_IPV4_SA = 15,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_IPV4_CHKSUM,
+ SXE2_FLOW_FLD_ID_IPV4_ID = 20,
+ SXE2_FLOW_FLD_ID_IPV6_ID,
+ SXE2_FLOW_FLD_ID_IPV6_PRE32_SA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE32_DA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE48_SA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE48_DA = 25,
+ SXE2_FLOW_FLD_ID_IPV6_PRE64_SA,
+ SXE2_FLOW_FLD_ID_IPV6_PRE64_DA,
+
+ SXE2_FLOW_FLD_ID_TCP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_TCP_DST_PORT,
+ SXE2_FLOW_FLD_ID_UDP_SRC_PORT = 30,
+ SXE2_FLOW_FLD_ID_UDP_DST_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_DST_PORT,
+ SXE2_FLOW_FLD_ID_TCP_FLAGS,
+ SXE2_FLOW_FLD_ID_TCP_CHKSUM = 35,
+ SXE2_FLOW_FLD_ID_UDP_CHKSUM,
+ SXE2_FLOW_FLD_ID_SCTP_CHKSUM,
+
+ SXE2_FLOW_FLD_ID_VXLAN_VNI,
+
+ SXE2_FLOW_FLD_ID_GENEVE_VNI,
+
+ SXE2_FLOW_FLD_ID_GTPU_TEID = 40,
+
+ SXE2_FLOW_FLD_ID_NVGRE_TNI,
+
+ SXE2_FLOW_FLD_ID_MAX = 128,
+};
+
+struct sxe2_ether_hdr {
+ uint8_t dst_addr[SXE2_ETH_ALEN];
+ uint8_t src_addr[SXE2_ETH_ALEN];
+ uint16_t ether_type;
+};
+
+struct sxe2_vlan_hdr {
+ uint16_t type;
+ uint16_t vlan;
+};
+
+struct sxe2_ipv4_hdr {
+ uint8_t ver_ihl;
+ uint8_t tos;
+ uint16_t tot_len;
+ uint16_t id;
+ uint16_t frag_off;
+ uint8_t ttl;
+ uint8_t protocol;
+ uint16_t check;
+ uint32_t saddr;
+ uint32_t daddr;
+};
+#define SXE2_IPV6_ADDR_LENGTH (16)
+#define SXE2_IPV6_TC_SHIFT (20)
+#define SXE2_IPV6_TC_MASK (0xFF)
+
+struct sxe2_ipv6_hdr {
+ uint32_t pri_ver_flow;
+ uint16_t payload_len;
+ uint8_t nexthdr;
+ uint8_t hop_limit;
+ union {
+ uint8_t saddr[16];
+ uint16_t saddr16[8];
+ uint32_t saddr32[4];
+ };
+ union {
+ uint8_t daddr[16];
+ uint16_t daddr16[8];
+ uint32_t daddr32[4];
+ };
+};
+
+struct sxe2_tcp_hdr {
+ uint16_t source;
+ uint16_t dest;
+ uint32_t seq;
+ uint32_t ack_seq;
+ uint16_t flag;
+ uint16_t window;
+ uint16_t check;
+ uint16_t urg_ptr;
+};
+
+struct sxe2_udp_hdr {
+ uint16_t source;
+ uint16_t dest;
+ uint16_t len;
+ uint16_t check;
+};
+
+struct sxe2_sctp_hdr {
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+struct sxe2_nvgre_hdr {
+ uint16_t flags;
+ uint16_t protocol;
+ uint32_t tni;
+};
+struct sxe2_geneve_hdr {
+ uint16_t flags;
+ uint16_t protocol;
+ uint32_t vni;
+};
+struct sxe2_gtpu_hdr {
+ uint8_t flag;
+ uint8_t msg_type;
+ uint16_t msg_len;
+ uint32_t teid;
+};
+struct sxe2_vxlan_hdr {
+ uint8_t flag;
+ uint8_t resvd0;
+ uint8_t resvd1;
+ uint8_t protocol;
+ uint32_t vni;
+};
+
+enum sxe2_flow_act_type {
+ SXE2_FLOW_ACTION_DROP = 0,
+ SXE2_FLOW_ACTION_TC_REDIRECT,
+ SXE2_FLOW_ACTION_TO_VSI,
+ SXE2_FLOW_ACTION_TO_VSI_LIST,
+ SXE2_FLOW_ACTION_PASSTHRU,
+ SXE2_FLOW_ACTION_QUEUE,
+ SXE2_FLOW_ACTION_Q_REGION,
+ SXE2_FLOW_ACTION_MARK,
+ SXE2_FLOW_ACTION_COUNT,
+ SXE2_FLOW_ACTION_RSS,
+ SXE2_FLOW_ACTION_MAX = 32,
+};
+
+enum sxe2_rss_hash_key_func {
+ SXE2_RSS_HASH_FUNC_TOEPLITZ = 0,
+ SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ = 1,
+ SXE2_RSS_HASH_FUNC_XOR = 2,
+ SXE2_RSS_HASH_FUNC_JEKINS = 3
+};
+
+struct sxe2_flow_action_rss {
+ DECLARE_BITMAP(hdr_out, SXE2_FLOW_HDR_MAX);
+ DECLARE_BITMAP(hdr_in, SXE2_FLOW_HDR_MAX);
+ DECLARE_BITMAP(fld, SXE2_FLOW_FLD_ID_MAX);
+ uint8_t is_inner;
+ uint8_t func;
+ uint8_t hdr_type;
+};
+
+struct sxe2_flow_action_queue {
+ uint16_t vsi_index;
+ uint16_t q_index;
+};
+
+struct sxe2_flow_action_queue_region {
+ uint16_t vsi_index;
+ uint16_t q_index;
+ uint8_t region;
+};
+
+struct sxe2_flow_action_passthru {
+ uint16_t vsi_index;
+};
+
+struct sxe2_flow_action_mark {
+ uint32_t mark_id;
+};
+
+#define SXE2_VSI_MAX (2048)
+struct sxe2_flow_action_vsi {
+ uint16_t vsi_index;
+};
+
+struct sxe2_flow_action_vsi_list {
+ DECLARE_BITMAP(vsi_list_map, SXE2_VSI_MAX);
+ uint16_t vsi_cnt;
+};
+
+enum sxe2_fnav_stat_ctrl_type {
+ SXE2_FNAV_STAT_ENA_NONE = 0,
+ SXE2_FNAV_STAT_ENA_PKTS,
+ SXE2_FNAV_STAT_ENA_BYTES,
+ SXE2_FNAV_STAT_ENA_ALL,
+};
+
+struct sxe2_flow_action_count {
+ uint32_t user_id;
+ uint32_t driver_id;
+ uint32_t stat_index;
+ uint32_t stat_ctrl;
+};
+
+enum sxe2_flow_engine_type {
+ SXE2_FLOW_ENGINE_ACL,
+ SXE2_FLOW_ENGINE_SWITCH,
+ SXE2_FLOW_ENGINE_FNAV,
+ SXE2_FLOW_ENGINE_RSS,
+ SXE2_FLOW_ENGINE_MAX,
+};
+
+struct sxe2_flow_item {
+ struct sxe2_ether_hdr eth;
+ struct sxe2_vlan_hdr vlan;
+ struct sxe2_vlan_hdr qinq;
+ struct sxe2_ipv4_hdr ipv4;
+ struct sxe2_ipv6_hdr ipv6;
+ struct sxe2_udp_hdr udp;
+ struct sxe2_tcp_hdr tcp;
+ struct sxe2_sctp_hdr sctp;
+ struct sxe2_gtpu_hdr gtpu;
+ struct sxe2_vxlan_hdr vxlan;
+ struct sxe2_nvgre_hdr nvgre;
+ struct sxe2_geneve_hdr geneve;
+};
+
+enum sxe2_flow_sw_direct_type {
+ SXE2_FLOW_SW_DIRECT_TX,
+ SXE2_FLOW_SW_DIRECT_RX,
+ SXE2_FLOW_SW_DIRECT_MAX,
+};
+enum sxe2_flow_sw_pattern_type {
+ SXE2_FLOW_SW_PATTERN_ONLY,
+ SXE2_FLOW_SW_PATTERN_LAST,
+ SXE2_FLOW_SW_PATTERN_FIRST,
+ SXE2_FLOW_SW_PATTERN_MAX,
+};
+
+enum sxe2_flow_tunnel_type {
+ SXE2_FLOW_TUNNEL_TYPE_NONE,
+ SXE2_FLOW_TUNNEL_TYPE_PARENT,
+ SXE2_FLOW_TUNNEL_TYPE_VXLAN,
+ SXE2_FLOW_TUNNEL_TYPE_GTPU,
+ SXE2_FLOW_TUNNEL_TYPE_GENEVE,
+ SXE2_FLOW_TUNNEL_TYPE_GRE,
+ SXE2_FLOW_TUNNEL_TYPE_IPIP,
+};
+
+struct sxe2_flow_meta {
+ uint8_t switch_pattern_dup_allow;
+ uint8_t switch_src_direct;
+ uint16_t flow_src_vsi;
+ uint16_t flow_rule_vsi;
+ uint32_t flow_prio;
+ uint16_t flow_type;
+ uint8_t tunnel_type;
+ uint8_t rsv;
+};
+
+struct sxe2_flow_pattern {
+ DECLARE_BITMAP(hdrs, SXE2_FLOW_HDR_MAX);
+ DECLARE_BITMAP(map_spec, SXE2_FLOW_FLD_ID_MAX);
+ DECLARE_BITMAP(map_mask, SXE2_FLOW_FLD_ID_MAX);
+ struct sxe2_flow_item item_spec;
+ struct sxe2_flow_item item_mask;
+ uint64_t rss_type_allow;
+};
+
+struct sxe2_flow_action {
+ DECLARE_BITMAP(act_types, SXE2_FLOW_ACTION_MAX);
+ struct sxe2_flow_action_rss rss;
+ struct sxe2_flow_action_queue queue;
+ struct sxe2_flow_action_queue_region q_region;
+ struct sxe2_flow_action_passthru passthru;
+ struct sxe2_flow_action_vsi vsi;
+ struct sxe2_flow_action_vsi_list vsi_list;
+ struct sxe2_flow_action_mark mark;
+ struct sxe2_flow_action_count count;
+};
+#endif /* __SXE2_FLOW_PUBLIC_H__ */
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index b661e3cbf4..da7a690063 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -62,4 +62,5 @@ sources += files(
'sxe2_txrx_vec.c',
'sxe2_mac.c',
'sxe2_filter.c',
+ 'sxe2_rss.c',
)
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index 1fa9ad718e..b997e7b044 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -541,3 +541,176 @@ int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on)
return ret;
}
+
+int32_t sxe2_drv_rss_key_set(struct sxe2_adapter *adapter, uint8_t *key, uint16_t key_size)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_key_req *req = NULL;
+ int32_t ret = 0;
+ uint16_t buf_size = sizeof(*req) + key_size;
+
+ req = rte_zmalloc("drv_cmd_rss_key", buf_size, 0);
+ if (!req) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to alloc rss key");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ req->vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req->key_size = rte_cpu_to_le_16(key_size);
+ rte_memcpy(req->key, key, key_size);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_KEY_SET,
+ req, buf_size, NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd set rss key, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ if (req) {
+ rte_free(req);
+ req = NULL;
+ }
+ return ret;
+}
+
+int32_t sxe2_drv_rss_lut_set(struct sxe2_adapter *adapter, uint8_t *lut, uint16_t lut_size)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_lut_req *req = NULL;
+ int32_t ret = 0;
+ uint16_t buf_size = sizeof(struct sxe2_rss_lut_req) + lut_size;
+
+ req = rte_zmalloc("drv_cmd_rss_lut", buf_size, 0);
+ if (!req) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to alloc rss lut");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ req->vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req->lut_size = rte_cpu_to_le_16(lut_size);
+ rte_memcpy(req->lut, lut, lut_size);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_LUT_SET,
+ req, buf_size, NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd set rss lut, ret=%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ if (req) {
+ rte_free(req);
+ req = NULL;
+ }
+ return ret;
+}
+
+int32_t sxe2_drv_rss_hash_ctrl_func(struct sxe2_adapter *adapter, enum sxe2_rss_hash_key_func func)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_func_req req = {0};
+ int32_t ret = 0;
+
+ req.vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req.func = func;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_FUNC_SET,
+ &req, sizeof(req), NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd set rss func, ret=%d", ret);
+ return ret;
+}
+
+static void sxe2_drv_flow_bitmap_fill(uint32_t *bitmap, uint16_t *list)
+{
+ uint16_t index = 0;
+ uint16_t i = 0;
+ uint16_t map_size = sizeof(*bitmap) * SXE2_BITS_PER_BYTE;
+
+ while (list[i] != SXE2_FLOW_END) {
+ index = list[i] / map_size;
+ bitmap[index] |= (1UL << (list[i] % map_size));
+ i++;
+ }
+}
+
+int32_t sxe2_drv_rss_hf_add(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_hf_req req = {0};
+ int32_t ret = 0;
+
+ req.vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req.symm = rss_conf->symm;
+ req.hdr_type = rte_cpu_to_le_32(SXE2_RSS_OUTER_HEADERS);
+ sxe2_drv_flow_bitmap_fill(req.headers, rss_conf->hdrs);
+ sxe2_drv_flow_bitmap_fill(req.hash_flds, rss_conf->flds);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_ADD,
+ &req, sizeof(req), NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd add rss hf, ret=%d", ret);
+ return ret;
+}
+
+int32_t sxe2_drv_rss_hf_del(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_rss_hf_req req = {0};
+ int32_t ret = 0;
+
+ req.vsi_id = rte_cpu_to_le_16(adapter->vsi_ctxt.dpdk_vsi_id);
+ req.symm = rss_conf->symm;
+ req.hdr_type = rte_cpu_to_le_32(SXE2_RSS_OUTER_HEADERS);
+ sxe2_drv_flow_bitmap_fill(req.headers, rss_conf->hdrs);
+ sxe2_drv_flow_bitmap_fill(req.hash_flds, rss_conf->flds);
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_DEL,
+ &req, sizeof(req), NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd del rss hf, ret=%d", ret);
+ return ret;
+}
+
+int32_t sxe2_drv_rss_hf_clear(struct sxe2_adapter *adapter)
+{
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ int32_t ret = 0;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_RSS_HF_CLEAR,
+ NULL, 0, NULL, 0);
+
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret)
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to cmd clear rss hf, ret=%d", ret);
+
+ return ret;
+}
+
+int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue *rxq)
+{
+ (void)adapter;
+ (void)rxq;
+ return 0;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index c93bc2b0c9..2546c65a6c 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -53,4 +53,20 @@ int32_t sxe2_drv_vlan_insert_strip_cfg(struct sxe2_adapter *adapter);
int32_t sxe2_drv_vlan_filter_switch(struct sxe2_adapter *adapter, bool on);
+int32_t sxe2_drv_rss_key_set(struct sxe2_adapter *adapter, uint8_t *key, uint16_t key_size);
+
+int32_t sxe2_drv_rss_lut_set(struct sxe2_adapter *adapter, uint8_t *lut, uint16_t lut_size);
+
+int32_t sxe2_drv_rss_hash_ctrl_func(struct sxe2_adapter *adapter, enum sxe2_rss_hash_key_func func);
+
+int32_t sxe2_drv_rss_hf_add(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf);
+
+int32_t sxe2_drv_rss_hf_del(struct sxe2_adapter *adapter,
+ struct sxe2_rss_hf_config *rss_conf);
+
+int32_t sxe2_drv_rss_hf_clear(struct sxe2_adapter *adapter);
+
+int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue *rxq);
+
#endif /* SXE2_CMD_CHNL_H */
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index d69d650148..9998f241f0 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -6,6 +6,7 @@
#define SXE2_DRV_CMD_H
#include "sxe2_osal.h"
+#include "sxe2_flow_public.h"
#define SXE2_DRV_CMD_MODULE_S (16)
#define SXE2_MK_DRV_CMD(module, cmd) (((module) << SXE2_DRV_CMD_MODULE_S) | ((cmd) & 0xFFFF))
@@ -320,6 +321,34 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_vlan_filter_switch_req {
uint8_t rsv;
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_key_req {
+ uint16_t vsi_id;
+ uint16_t key_size;
+ uint8_t key[];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_lut_req {
+ uint16_t vsi_id;
+ uint16_t lut_size;
+ uint8_t lut[];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_func_req {
+ uint16_t vsi_id;
+ uint8_t func;
+ uint8_t rsv[1];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_rss_hf_req {
+ uint16_t vsi_id;
+ uint8_t rsv[2];
+ uint32_t headers[BITS_TO_U32(SXE2_FLOW_HDR_MAX)];
+ uint32_t hash_flds[BITS_TO_U32(SXE2_FLOW_FLD_ID_MAX)];
+ uint32_t hdr_type;
+ uint8_t symm;
+ uint8_t rsv1[3];
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index 23d30250d1..b24174d8f0 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -124,6 +124,11 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.vlan_filter_set = sxe2_dev_vlan_filter_set,
.vlan_offload_set = sxe2_dev_vlan_offload_set,
+
+ .reta_update = sxe2_dev_rss_reta_update,
+ .reta_query = sxe2_dev_rss_reta_query,
+ .rss_hash_update = sxe2_dev_rss_hash_update,
+ .rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -140,6 +145,12 @@ static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
goto end;
}
+ ret = sxe2_rss_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init rss, ret=%d", ret);
+ goto end;
+ }
+
end:
return ret;
}
@@ -280,6 +291,22 @@ static int32_t sxe2_dev_infos_get(struct rte_eth_dev *dev,
RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO |
RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO;
+
+ if (adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_PTP)
+ dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TIMESTAMP;
+
+ if (adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_RSS) {
+ dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
+ dev_info->flow_type_rss_offloads |= SXE2_RSS_HF_SUPPORT_ALL;
+ dev_info->reta_size = adapter->rss_ctxt.rss_lut_size;
+ dev_info->hash_key_size = adapter->rss_ctxt.rss_key_size;
+ dev_info->rss_algo_capa =
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_DEFAULT) |
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_TOEPLITZ) |
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) |
+ RTE_ETH_HASH_ALGO_TO_CAPA(RTE_ETH_HASH_FUNCTION_SIMPLE_XOR);
+ }
+
dev_info->default_rxconf = (struct rte_eth_rxconf) {
.rx_thresh = {
.pthresh = SXE2_DEFAULT_RX_PTHRESH,
@@ -552,6 +579,8 @@ static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
sxe2_sw_queue_ctx_hw_cap_set(adapter, &dev_caps.queue_caps);
+ sxe2_sw_rss_ctx_hw_cap_set(adapter, &dev_caps.rss_hash_caps);
+
sxe2_sw_vsi_ctx_hw_cap_set(adapter, &dev_caps.vsi_caps);
l_end:
@@ -939,8 +968,15 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_eth_err;
}
+ ret = sxe2_rss_disable(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to disable rss, ret=%d", ret);
+ goto init_rss_err;
+ }
+
goto l_end;
+init_rss_err:
init_eth_err:
init_dev_info_err:
sxe2_vsi_uninit(dev);
@@ -954,6 +990,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
{
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
+ (void)sxe2_rss_disable(dev);
sxe2_vsi_uninit(dev);
sxe2_dev_pci_map_uinit(dev);
sxe2_eth_uinit(dev);
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index bb015ea723..fd106eb867 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -15,6 +15,7 @@
#include "sxe2_common.h"
#include "sxe2_vsi.h"
+#include "sxe2_rss.h"
#include "sxe2_irq.h"
#include "sxe2_queue.h"
#include "sxe2_mac.h"
@@ -122,6 +123,11 @@ enum {
SXE2_FLAGS_NBITS
};
+struct sxe2_ptp_context {
+ uint64_t mbuf_rx_ts_flag;
+ int32_t mbuf_rx_ts_offset;
+};
+
struct sxe2_devargs {
uint8_t flow_dup_pattern_mode;
uint8_t func_flow_direct_en;
@@ -299,7 +305,9 @@ struct sxe2_adapter {
struct sxe2_queue_context q_ctxt;
struct sxe2_vsi_context vsi_ctxt;
struct sxe2_filter_context filter_ctxt;
+ struct sxe2_rss_context rss_ctxt;
struct sxe2_link_context link_ctxt;
+ struct sxe2_ptp_context ptp_ctxt;
struct sxe2_devargs devargs;
struct sxe2_switchdev_info switchdev_info;
bool rule_started;
diff --git a/drivers/net/sxe2/sxe2_flow_define.h b/drivers/net/sxe2/sxe2_flow_define.h
new file mode 100644
index 0000000000..d2f6000efa
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_flow_define.h
@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_FLOW_DEFINE_H__
+#define __SXE2_FLOW_DEFINE_H__
+#include <rte_tailq.h>
+#include <rte_eal.h>
+#include <rte_flow_driver.h>
+
+#include "sxe2_osal.h"
+#include "sxe2_flow_public.h"
+
+#define SXE2_FLOW_ETH_TYPE_MIN (1500)
+
+enum sxe2_expansion {
+ SXE2_EXPANSION_ERROR = 0,
+ SXE2_EXPANSION_OUTER_ETH,
+ SXE2_EXPANSION_OUTER_VLAN,
+ SXE2_EXPANSION_OUTER_QINQ,
+ SXE2_EXPANSION_OUTER_IPV4,
+ SXE2_EXPANSION_OUTER_IPV4_FRAG_EXT,
+ SXE2_EXPANSION_OUTER_IPV6,
+ SXE2_EXPANSION_OUTER_IPV6_FRAG_EXT,
+ SXE2_EXPANSION_OUTER_UDP,
+ SXE2_EXPANSION_OUTER_TCP,
+ SXE2_EXPANSION_OUTER_SCTP,
+
+ SXE2_EXPANSION_VXLAN,
+ SXE2_EXPANSION_VXLAN_GPE,
+ SXE2_EXPANSION_GRE,
+ SXE2_EXPANSION_NVGRE,
+ SXE2_EXPANSION_GENEVE,
+ SXE2_EXPANSION_GTPU,
+ SXE2_EXPANSION_IPIP,
+ SXE2_EXPANSION_OUTER_END,
+
+ SXE2_EXPANSION_ETH,
+ SXE2_EXPANSION_VLAN,
+ SXE2_EXPANSION_IPV4,
+ SXE2_EXPANSION_IPV4_FRAG_EXT,
+ SXE2_EXPANSION_IPV6,
+ SXE2_EXPANSION_IPV6_FRAG_EXT,
+ SXE2_EXPANSION_UDP,
+ SXE2_EXPANSION_TCP,
+ SXE2_EXPANSION_SCTP,
+
+ SXE2_EXPANSION_END,
+ SXE2_EXPANSION_MAX,
+};
+
+enum sxe2_flow_udp_tunnel_protocol {
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_VXLAN_GPE,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GENEVE,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_GTP_U,
+ SXE2_FLOW_UDP_TUNNEL_PROTOCOL_NVGRE,
+ SXE2_FLOW_UDP_TUNNEL_MAX,
+};
+
+enum {
+ SXE2_FLOW_ETH_TYPE_IPV4 = 0x0800,
+ SXE2_FLOW_ETH_TYPE_IPV6 = 0x86DD,
+ SXE2_FLOW_IP_PROTOCOL_GRE = 0x2F,
+ SXE2_FLOW_IP_PROTOCOL_IPV4 = 0x04,
+ SXE2_FLOW_IP_PROTOCOL_IPV6 = 0x29,
+ SXE2_FLOW_IP_PROTOCOL_ETH = 0x3B,
+ SXE2_FLOW_IP_PROTOCOL_UDP = 0x11,
+ SXE2_FLOW_IP_PROTOCOL_TCP = 0x06,
+ SXE2_FLOW_IP_PROTOCOL_SCTP = 0x84,
+};
+
+union sxe2_flow_item_raw {
+ struct sxe2_flow_item item;
+ uint8_t raw[sizeof(struct sxe2_flow_item)];
+};
+
+struct sxe2_flow {
+ TAILQ_ENTRY(sxe2_flow) next;
+ enum sxe2_flow_engine_type engine_type;
+ struct sxe2_flow_pattern pattern_outer;
+ struct sxe2_flow_pattern pattern_inner;
+ uint8_t has_mask;
+ uint8_t has_spec;
+ uint8_t has_hdr;
+ struct sxe2_flow_meta meta;
+ struct sxe2_flow_action action;
+ uint32_t flow_id;
+ int32_t create_err;
+ DECLARE_BITMAP(flow_type, SXE2_EXPANSION_MAX);
+};
+
+TAILQ_HEAD(sxe2_flow_list_t, sxe2_flow);
+
+struct rte_flow {
+ TAILQ_ENTRY(rte_flow) next;
+ struct sxe2_flow_list_t sxe2_flow_list;
+};
+TAILQ_HEAD(rte_flow_list_t, rte_flow);
+
+struct sxe2_fnav_cid_mgr {
+ TAILQ_ENTRY(sxe2_fnav_cid_mgr) next;
+ uint16_t stat_index;
+ uint32_t user_id;
+ uint32_t driver_id;
+ uint32_t count_type;
+ uint64_t hits;
+ uint64_t bytes;
+};
+TAILQ_HEAD(sxe2_fnav_cid_mgr_list_t, sxe2_fnav_cid_mgr);
+
+struct sxe2_fnav_count_resource {
+ uint32_t count_type;
+ uint32_t global_index;
+ struct sxe2_fnav_cid_mgr_list_t fnav_cid_mgr_list;
+};
+
+struct sxe2_flow_context {
+ struct rte_flow_list_t rte_flow_list;
+ rte_spinlock_t flow_list_lock;
+ struct sxe2_fnav_count_resource hw_res;
+ uint32_t fnav_inited;
+};
+#define SXE2_INVALID_RSS_ATTR \
+ (RTE_ETH_RSS_L3_PRE40 | RTE_ETH_RSS_L3_PRE56 | RTE_ETH_RSS_L3_PRE96)
+#define SXE2_VALID_RSS_IPV4_L4 \
+ (RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_NONFRAG_IPV4_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_SCTP)
+
+#define SXE2_VALID_RSS_IPV6_L4 \
+ (RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_NONFRAG_IPV6_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP)
+#define SXE2_VALID_RSS_IPV4 \
+ (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | \
+ RTE_ETH_RSS_NONFRAG_IPV4_OTHER | SXE2_VALID_RSS_IPV4_L4)
+#define SXE2_VALID_RSS_IPV6 \
+ (RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | \
+ RTE_ETH_RSS_NONFRAG_IPV6_OTHER | SXE2_VALID_RSS_IPV6_L4)
+
+#define SXE2_VALID_RSS_L3 (SXE2_VALID_RSS_IPV4 | SXE2_VALID_RSS_IPV6)
+#define SXE2_VALID_RSS_L4 (SXE2_VALID_RSS_IPV4_L4 | SXE2_VALID_RSS_IPV6_L4)
+
+#endif /* __SXE2_FLOW_DEFINE_H__ */
diff --git a/drivers/net/sxe2/sxe2_queue.c b/drivers/net/sxe2/sxe2_queue.c
index 1786d6ea4f..220cab6fce 100644
--- a/drivers/net/sxe2/sxe2_queue.c
+++ b/drivers/net/sxe2/sxe2_queue.c
@@ -17,6 +17,7 @@ void sxe2_sw_queue_ctx_hw_cap_set(struct sxe2_adapter *adapter,
int32_t sxe2_queues_init(struct rte_eth_dev *dev)
{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
int32_t ret = 0;
uint16_t buf_size;
uint16_t frame_size;
@@ -36,6 +37,16 @@ int32_t sxe2_queues_init(struct rte_eth_dev *dev)
dev->data->scattered_rx = 1;
}
+ adapter->ptp_ctxt.mbuf_rx_ts_offset = -1;
+ adapter->ptp_ctxt.mbuf_rx_ts_flag = 0;
+ if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
+ ret = rte_mbuf_dyn_rx_timestamp_register
+ (&adapter->ptp_ctxt.mbuf_rx_ts_offset,
+ (uint64_t *)&adapter->ptp_ctxt.mbuf_rx_ts_flag);
+ if (ret)
+ PMD_LOG_ERR(INIT, "Failed to enable timestamp offloads, ret=%d", ret);
+ }
+
return ret;
}
--git a/drivers/net/sxe2/sxe2_rss.c b/drivers/net/sxe2/sxe2_rss.c
new file mode 100644
index 0000000000..1d56613043
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_rss.c
@@ -0,0 +1,584 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include "sxe2_rss.h"
+#include "sxe2_common_log.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_cmd_chnl.h"
+
+void sxe2_sw_rss_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_rss_hash_caps *rss_caps)
+{
+ adapter->rss_ctxt.rss_key_size = rss_caps->hash_key_size;
+ adapter->rss_ctxt.rss_lut_size = rss_caps->lut_key_size;
+}
+
+int32_t sxe2_rss_hash_key_init(struct rte_eth_dev *dev)
+{
+ struct rte_eth_rss_conf *rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i = 0;
+
+ if (rss_ctxt->rss_key == NULL) {
+ rss_ctxt->rss_key = (uint8_t *)rte_zmalloc("rss_key", rss_ctxt->rss_key_size, 0);
+ if (rss_ctxt->rss_key == NULL) {
+ PMD_LOG_ERR(INIT, "Failed to allocate rss key");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ }
+
+ if (!rss_conf->rss_key) {
+ for (i = 0; i < rss_ctxt->rss_key_size; i++)
+ rss_ctxt->rss_key[i] = (uint8_t)rte_rand();
+ } else {
+ rte_memcpy(rss_ctxt->rss_key, rss_conf->rss_key,
+ RTE_MIN(rss_conf->rss_key_len, rss_ctxt->rss_key_size));
+ }
+
+ ret = sxe2_drv_rss_key_set(adapter, rss_ctxt->rss_key,
+ rss_ctxt->rss_key_size);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss key, ret:%d", ret);
+ rte_free(rss_ctxt->rss_key);
+ rss_ctxt->rss_key = NULL;
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+void sxe2_rss_hash_key_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+
+ if (rss_ctxt->rss_key) {
+ rte_free(rss_ctxt->rss_key);
+ rss_ctxt->rss_key = NULL;
+ }
+}
+
+int32_t sxe2_rss_lut_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i;
+
+ if (rss_ctxt->rss_lut == NULL) {
+ rss_ctxt->rss_lut = (uint8_t *)rte_zmalloc("rss_lut", rss_ctxt->rss_lut_size, 0);
+ if (rss_ctxt->rss_lut == NULL) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to allocate rss lut");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ }
+
+ for (i = 0; i < rss_ctxt->rss_lut_size; i++)
+ rss_ctxt->rss_lut[i] = (uint8_t)(i % dev->data->nb_rx_queues);
+
+ ret = sxe2_drv_rss_lut_set(adapter, rss_ctxt->rss_lut, rss_ctxt->rss_lut_size);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss lut, ret:%d", ret);
+ rte_free(rss_ctxt->rss_lut);
+ rss_ctxt->rss_lut = NULL;
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+void sxe2_rss_lut_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+
+ if (rss_ctxt->rss_lut) {
+ rte_free(rss_ctxt->rss_lut);
+ rss_ctxt->rss_lut = NULL;
+ }
+}
+
+static struct sxe2_rss_hf_config sxe2_rss_default_hf_config[] = {
+ {
+ .rss_hf = RTE_ETH_RSS_L2_PAYLOAD,
+ .hdrs = {SXE2_FLOW_HDR_ETH,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_ETH_TYPE,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_IPV4,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_IPV6,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_FRAG_IPV4,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_IPV_FRAG,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_FRAG_IPV6,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_IPV_FRAG,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_IPV_OTHER,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_IPV_OTHER,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_UDP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_UDP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_UDP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_UDP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_UDP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_UDP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_TCP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_TCP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_TCP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_TCP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_TCP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_TCP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV4_SCTP,
+ .hdrs = {SXE2_FLOW_HDR_IPV4,
+ SXE2_FLOW_HDR_SCTP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV4_SA,
+ SXE2_FLOW_FLD_ID_IPV4_DA,
+ SXE2_FLOW_FLD_ID_SCTP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+ {
+ .rss_hf = RTE_ETH_RSS_NONFRAG_IPV6_SCTP,
+ .hdrs = {SXE2_FLOW_HDR_IPV6,
+ SXE2_FLOW_HDR_SCTP,
+ SXE2_FLOW_END},
+ .flds = {SXE2_FLOW_FLD_ID_IPV6_SA,
+ SXE2_FLOW_FLD_ID_IPV6_DA,
+ SXE2_FLOW_FLD_ID_SCTP_SRC_PORT,
+ SXE2_FLOW_FLD_ID_SCTP_DST_PORT,
+ SXE2_FLOW_END},
+ },
+};
+
+int32_t sxe2_rss_hf_type_set(struct rte_eth_dev *dev, uint64_t rss_hf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint32_t i;
+ uint8_t symm = 0;
+
+ if (0 == (rss_hf & SXE2_RSS_HF_SUPPORT_ALL) && rss_hf != 0) {
+ PMD_DEV_LOG_ERR(adapter, DRV,
+ "Failed to set unsupported rss_hf:0x%016" PRIx64,
+ rss_hf);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < RTE_DIM(sxe2_rss_default_hf_config); i++) {
+ if (rss_ctxt->rss_hf & sxe2_rss_default_hf_config[i].rss_hf) {
+ sxe2_rss_default_hf_config[i].symm = rss_ctxt->symm;
+ ret = sxe2_drv_rss_hf_del(adapter, &sxe2_rss_default_hf_config[i]);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT,
+ "Failed to del rss hf cfg[%d], ret:%d", i, ret);
+ goto l_end;
+ }
+ }
+ }
+
+ if (rss_ctxt->hash_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
+ symm = 1;
+
+ for (i = 0; i < RTE_DIM(sxe2_rss_default_hf_config); i++) {
+ if (rss_hf & sxe2_rss_default_hf_config[i].rss_hf) {
+ sxe2_rss_default_hf_config[i].symm = symm;
+ ret = sxe2_drv_rss_hf_add(adapter, &sxe2_rss_default_hf_config[i]);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT,
+ "Failed to add rss hf cfg[%d], ret:%d", i, ret);
+ goto l_end;
+ }
+ }
+ }
+
+ rss_ctxt->rss_hf = rss_hf & SXE2_RSS_HF_SUPPORT_ALL;
+ rss_ctxt->symm = symm;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_rss_hash_function_set(struct rte_eth_dev *dev, enum rte_eth_hash_function func)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ enum sxe2_rss_hash_key_func hash_func = SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ;
+ int32_t ret = 0;
+
+ switch (func) {
+ case RTE_ETH_HASH_FUNCTION_DEFAULT:
+ case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
+ case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ:
+ hash_func = SXE2_RSS_HASH_FUNC_SYM_TOEPLITZ;
+ break;
+ case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR:
+ hash_func = SXE2_RSS_HASH_FUNC_XOR;
+ break;
+ default:
+ PMD_DEV_LOG_ERR(adapter, DRV, "RSS hash function[%d] not support.", func);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_drv_rss_hash_ctrl_func(adapter, hash_func);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss hash function, ret=[%d]", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_rss_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct rte_eth_rss_conf *rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
+ enum rte_eth_hash_function rss_func = RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
+ int32_t ret = 0;
+
+ adapter->rss_ctxt.inited = false;
+
+ if (dev->data->nb_rx_queues <= 1) {
+ PMD_DEV_LOG_DEBUG(adapter, INIT, "No need to init rss, rx queues %d.",
+ dev->data->nb_rx_queues);
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_RSS) == 0) {
+ PMD_DEV_LOG_WARN(adapter, INIT, "RSS not supported");
+ goto l_end;
+ }
+
+ ret = sxe2_rss_hash_key_init(dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init rss key");
+ goto l_end;
+ }
+
+ ret = sxe2_rss_lut_init(dev);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init rss lut");
+ goto l_err_key;
+ }
+
+ rss_func = rss_conf->algorithm;
+ ret = sxe2_rss_hash_function_set(dev, rss_func);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to init rss hash function");
+ goto l_err_lut;
+ }
+ ret = sxe2_rss_hf_type_set(dev, rss_conf->rss_hf);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, INIT, "Failed to set rss hf type");
+ goto l_err_lut;
+ }
+ adapter->rss_ctxt.inited = true;
+ goto l_end;
+
+l_err_lut:
+ sxe2_rss_lut_uninit(dev);
+l_err_key:
+ sxe2_rss_hash_key_uninit(dev);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_rss_disable(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_RSS) == 0)
+ goto l_end;
+
+ ret = sxe2_drv_rss_hf_clear(adapter);
+ if (ret)
+ PMD_LOG_ERR(INIT, "Failed to clear rss hf");
+
+ sxe2_rss_hash_key_uninit(dev);
+
+ sxe2_rss_lut_uninit(dev);
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_rss_reta_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ uint8_t *lut_tmp = NULL;
+ int32_t ret = 0;
+ uint16_t i;
+ uint16_t shift;
+ uint16_t idx;
+
+ if (!adapter->rss_ctxt.inited) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (reta_size != rss_ctxt->rss_lut_size) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "The size of hash lookup table configured "
+ "(%d) doesn't match the number of hardware can "
+ "support (%d)", reta_size, rss_ctxt->rss_lut_size);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ lut_tmp = rte_zmalloc("rss_lut_temp", reta_size, 0);
+ if (!lut_tmp) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "No memory can be allocated");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ rte_memcpy(lut_tmp, rss_ctxt->rss_lut, reta_size);
+
+ for (i = 0; i < reta_size; i++) {
+ idx = i / RTE_ETH_RETA_GROUP_SIZE;
+ shift = i % RTE_ETH_RETA_GROUP_SIZE;
+ if (reta_conf[idx].mask & (1ULL << shift))
+ lut_tmp[i] = reta_conf[idx].reta[shift];
+ }
+
+ ret = sxe2_drv_rss_lut_set(adapter, lut_tmp, reta_size);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss lut");
+ goto l_end;
+ }
+
+ rte_memcpy(rss_ctxt->rss_lut, lut_tmp, reta_size);
+
+l_end:
+ if (lut_tmp)
+ rte_free(lut_tmp);
+ return ret;
+}
+
+int32_t sxe2_dev_rss_reta_query(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i;
+ uint16_t shift;
+ uint16_t idx;
+
+ if (!adapter->rss_ctxt.inited) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (reta_size != rss_ctxt->rss_lut_size) {
+ PMD_LOG_ERR(INIT, "The size of hash lookup table configured "
+ "(%d) doesn't match the number of hardware can "
+ "support (%d)", reta_size, rss_ctxt->rss_lut_size);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < reta_size; i++) {
+ idx = i / RTE_ETH_RETA_GROUP_SIZE;
+ shift = i % RTE_ETH_RETA_GROUP_SIZE;
+ if (reta_conf[idx].mask & (1ULL << shift))
+ reta_conf[idx].reta[shift] = rss_ctxt->rss_lut[i];
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_rss_hash_key_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+ uint16_t i;
+
+ if (rss_conf->rss_key_len == 0 || rss_conf->rss_key == NULL)
+ goto l_end;
+
+ if (rss_conf->rss_key_len != rss_ctxt->rss_key_size) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "The size of hash key configured "
+ "(%d) doesn't match the size of hardware can "
+ "support (%d)", rss_conf->rss_key_len,
+ rss_ctxt->rss_key_size);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ for (i = 0; i < rss_conf->rss_key_len; i++) {
+ if (rss_conf->rss_key[i] != rss_ctxt->rss_key[i])
+ break;
+ }
+ if (i == rss_conf->rss_key_len)
+ goto l_end;
+
+ ret = sxe2_drv_rss_key_set(adapter, rss_conf->rss_key,
+ rss_conf->rss_key_len);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss key");
+ goto l_end;
+ }
+
+ rte_memcpy(rss_ctxt->rss_key, rss_conf->rss_key, rss_conf->rss_key_len);
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_rss_hash_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = -1;
+
+ if (!adapter->rss_ctxt.inited) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ ret = sxe2_rss_hash_key_update(dev, rss_conf);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss hash key");
+ goto l_end;
+ }
+
+ if (rss_conf->algorithm != rss_ctxt->hash_func) {
+ ret = sxe2_rss_hash_function_set(dev, rss_conf->algorithm);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss hash function");
+ goto l_end;
+ }
+ rss_ctxt->hash_func = rss_conf->algorithm;
+ }
+
+ if ((rss_conf->rss_hf & SXE2_RSS_HF_SUPPORT_ALL)
+ != rss_ctxt->rss_hf) {
+ ret = sxe2_rss_hf_type_set(dev, rss_conf->rss_hf);
+ if (ret) {
+ PMD_DEV_LOG_ERR(adapter, DRV, "Failed to set rss hf type");
+ goto l_end;
+ }
+ }
+ ret = 0;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_rss_context *rss_ctxt = &adapter->rss_ctxt;
+ int32_t ret = 0;
+
+ if (adapter->rss_ctxt.inited == 0) {
+ PMD_DEV_LOG_INFO(adapter, DRV, "RSS not inited.");
+ ret = -ENOTSUP;
+ goto l_end;
+ }
+
+ if (rss_conf->rss_key) {
+ rss_conf->rss_key_len = rss_ctxt->rss_key_size;
+ rte_memcpy(rss_conf->rss_key, rss_ctxt->rss_key, rss_ctxt->rss_key_size);
+ }
+ rss_conf->rss_hf = rss_ctxt->rss_hf;
+ rss_conf->algorithm = rss_ctxt->hash_func;
+l_end:
+ return ret;
+}
--git a/drivers/net/sxe2/sxe2_rss.h b/drivers/net/sxe2/sxe2_rss.h
new file mode 100644
index 0000000000..2a454ac1b3
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_rss.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_RSS_H__
+#define __SXE2_RSS_H__
+#include <ethdev_driver.h>
+#include "sxe2_osal.h"
+#include "sxe2_drv_cmd.h"
+#include "sxe2_flow_define.h"
+
+#define SXE2_FLOW_END (0xFFFF)
+
+struct sxe2_rss_context {
+ enum rte_eth_hash_function hash_func;
+ uint16_t rss_key_size;
+ uint16_t rss_lut_size;
+ uint8_t *rss_key;
+ uint8_t *rss_lut;
+ uint64_t rss_hf;
+ uint8_t symm;
+ bool inited;
+};
+
+struct sxe2_rss_hf_config {
+ uint64_t rss_hf;
+ uint16_t hdrs[SXE2_FLOW_HDR_MAX];
+ uint16_t flds[SXE2_FLOW_FLD_ID_MAX];
+ uint8_t symm;
+};
+
+#define SXE2_RSS_HF_SUPPORT_ALL ( \
+ RTE_ETH_RSS_IPV4 | \
+ RTE_ETH_RSS_FRAG_IPV4 | \
+ RTE_ETH_RSS_NONFRAG_IPV4_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_UDP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_SCTP | \
+ RTE_ETH_RSS_NONFRAG_IPV4_OTHER | \
+ RTE_ETH_RSS_IPV6 | \
+ RTE_ETH_RSS_FRAG_IPV6 | \
+ RTE_ETH_RSS_NONFRAG_IPV6_TCP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_UDP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_SCTP | \
+ RTE_ETH_RSS_NONFRAG_IPV6_OTHER | \
+ RTE_ETH_RSS_L2_PAYLOAD)
+
+struct sxe2_adapter;
+
+void sxe2_sw_rss_ctx_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_drv_rss_hash_caps *rss_caps);
+
+int32_t sxe2_rss_hash_key_init(struct rte_eth_dev *dev);
+
+void sxe2_rss_hash_key_uninit(struct rte_eth_dev *dev);
+
+int32_t sxe2_rss_lut_init(struct rte_eth_dev *dev);
+
+void sxe2_rss_lut_uninit(struct rte_eth_dev *dev);
+
+int32_t sxe2_rss_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_rss_hash_function_set(struct rte_eth_dev *dev, enum rte_eth_hash_function func);
+
+int32_t sxe2_rss_hf_type_set(struct rte_eth_dev *dev, uint64_t rss_hf);
+
+int32_t sxe2_rss_disable(struct rte_eth_dev *dev);
+
+int32_t sxe2_dev_rss_reta_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size);
+
+int32_t sxe2_dev_rss_reta_query(struct rte_eth_dev *dev,
+ struct rte_eth_rss_reta_entry64 *reta_conf,
+ uint16_t reta_size);
+
+int32_t sxe2_dev_rss_hash_update(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf);
+
+int32_t sxe2_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
+ struct rte_eth_rss_conf *rss_conf);
+#endif /* __SXE2_RSS_H__ */
diff --git a/drivers/net/sxe2/sxe2_txrx.h b/drivers/net/sxe2/sxe2_txrx.h
index 467bd5aec1..dc5297accd 100644
--- a/drivers/net/sxe2/sxe2_txrx.h
+++ b/drivers/net/sxe2/sxe2_txrx.h
@@ -27,5 +27,8 @@ void sxe2_init_ptype_tbl(struct rte_eth_dev *dev);
const uint32_t *sxe2_dev_supported_ptypes_get(struct rte_eth_dev *dev,
size_t *no_of_elements);
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+int32_t sxe2_rx_update_ptp_time(struct sxe2_rx_queue *rxq);
+#endif
#endif /* SXE2_TXRX_H */
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.c b/drivers/net/sxe2/sxe2_txrx_poll.c
index 8e55f10cfb..d5ca17d33f 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.c
+++ b/drivers/net/sxe2/sxe2_txrx_poll.c
@@ -17,6 +17,7 @@
#include "sxe2_queue.h"
#include "sxe2_ethdev.h"
#include "sxe2_common_log.h"
+#include "sxe2_cmd_chnl.h"
static __rte_always_inline int32_t
sxe2_tx_bufs_free(struct sxe2_tx_queue *txq)
@@ -282,6 +283,30 @@ sxe2_tx_desc_checksum_fill(uint64_t offloads, uint32_t *desc_cmd, uint32_t *desc
return;
}
+static __rte_always_inline void sxe2_desc_tso_fill(struct rte_mbuf *tx_pkt,
+ uint64_t *desc_type_cmd_tso_mss,
+ union sxe2_tx_offload_info ol_info)
+{
+ uint32_t hdr_len;
+ uint32_t tso_len;
+
+ if (!ol_info.l4_len) {
+ PMD_LOG_DEBUG(TX, "TSO ERROR: L4 length is 0");
+ goto l_end;
+ }
+ hdr_len = ol_info.l2_len + ol_info.l3_len + ol_info.l4_len;
+ if (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
+ hdr_len += ol_info.outer_l2_len + ol_info.outer_l3_len;
+
+ tso_len = tx_pkt->pkt_len - hdr_len;
+ *desc_type_cmd_tso_mss |=
+ ((uint64_t)SXE2_TX_CTXT_DESC_CMD_TSO << SXE2_TX_CTXT_DESC_CMD_SHIFT) |
+ ((uint64_t)tso_len << SXE2_TX_CTXT_DESC_TSO_LEN_SHIFT) |
+ ((uint64_t)tx_pkt->tso_segsz << SXE2_TX_CTXT_DESC_MSS_SHIFT);
+l_end:
+ return;
+}
+
static __rte_always_inline uint64_t
sxe2_tx_data_desc_build_cobt(uint32_t cmd, uint32_t offset, uint16_t buf_size, uint16_t l2tag)
{
@@ -395,6 +420,11 @@ uint16_t sxe2_tx_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkt
rte_pktmbuf_free_seg(buffer->mbuf);
buffer->mbuf = NULL;
}
+ if (offloads & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
+ sxe2_desc_tso_fill(tx_pkt,
+ &desc_type_cmd_tso_mss, ol_info);
+ else if (offloads & RTE_MBUF_F_TX_IEEE1588_TMST)
+ desc_type_cmd_tso_mss |= SXE2_TX_CTXT_DESC_CMD_TSYN_MASK;
if (offloads & RTE_MBUF_F_TX_QINQ) {
desc_l2tag2 = tx_pkt->vlan_tci_outer;
@@ -707,6 +737,57 @@ sxe2_rx_desc_filter_para_fill(struct sxe2_rx_queue *rxq __rte_unused,
#endif
}
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+int32_t sxe2_rx_update_ptp_time(struct sxe2_rx_queue *rxq)
+{
+ struct sxe2_adapter *adapter;
+ uint64_t cur_time_ms;
+ int32_t ret = 0;
+ cur_time_ms = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
+
+ if (likely((cur_time_ms - rxq->update_time) < SXE2_RX_PKTS_TS_TIMEOUT_VAL))
+ goto l_end;
+ rxq->update_time = cur_time_ms;
+ adapter = rxq->vsi->adapter;
+ rxq->ts_need_update = true;
+ ret = sxe2_drv_ptp_gettime(adapter, rxq);
+ if (rxq->desc_ts < rxq->ts_low)
+ rxq->ts_need_update = false;
+
+ PMD_LOG_INFO(RX, "rxq update time ret=%d, cur time=%" PRIu64 ", rxqh=%" PRIu64 ", rxql=%d",
+ ret, cur_time_ms, rxq->ts_high, rxq->ts_low);
+l_end:
+ return ret;
+}
+
+static inline void sxe2_rx_desc_ptp_para_fill(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf *mbuf,
+ union sxe2_rx_desc *desc)
+{
+ struct sxe2_adapter *adapter = rxq->vsi->adapter;
+ uint64_t ts_ns;
+
+ if (adapter->ptp_ctxt.mbuf_rx_ts_flag != 0 &&
+ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) &&
+ SXE2_RX_DESC_RXDID_VAL_GET(desc->wb.rxdid_src) == SXE2_RX_DESC_RXDID_1588) {
+ rxq->desc_ts = rte_le_to_cpu_32(desc->wb_ts.ts_h);
+ (void)sxe2_rx_update_ptp_time(rxq);
+ if (rxq->ts_need_update && rxq->desc_ts < rxq->ts_low)
+ rxq->ts_high += 1;
+
+ rxq->ts_need_update = true;
+ rxq->ts_low = rxq->desc_ts;
+ rxq->update_time = rte_get_timer_cycles() /
+ (rte_get_timer_hz() / 1000);
+ ts_ns = rxq->ts_high * NSEC_PER_SEC + rxq->ts_low;
+ *RTE_MBUF_DYNFIELD(mbuf, adapter->ptp_ctxt.mbuf_rx_ts_offset, uint64_t *) = ts_ns;
+ mbuf->ol_flags |= adapter->ptp_ctxt.mbuf_rx_ts_flag;
+ PMD_LOG_INFO(RX, "receive ptp pkt,ts_s=%" PRIu64 ", ts_ns=%d", rxq->ts_high,
+ rxq->ts_low);
+ }
+}
+#endif
+
static __rte_always_inline void
sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf,
union sxe2_rx_desc *rxd)
@@ -718,10 +799,12 @@ sxe2_rx_mbuf_common_fields_fill(struct sxe2_rx_queue *rxq, struct rte_mbuf *mbuf
mbuf->ol_flags = 0;
mbuf->packet_type = ptype_tbl[SXE2_RX_DESC_PTYPE_VAL_GET(qword1)];
-
pkt_flags = sxe2_rx_desc_error_para(rxq, rxd);
sxe2_rx_desc_vlan_para_fill(mbuf, rxd);
sxe2_rx_desc_filter_para_fill(rxq, mbuf, rxd);
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ sxe2_rx_desc_ptp_para_fill(rxq, mbuf, rxd);
+#endif
mbuf->ol_flags |= pkt_flags;
}
--
2.31.1
^ permalink raw reply related
* [PATCH v3 06/20] net/sxe2: support TM hierarchy and shaping
From: liujie5 @ 2026-06-18 8:27 UTC (permalink / raw)
To: stephen; +Cc: dev, Jie Liu
In-Reply-To: <20260618082723.571054-1-liujie5@linkdatatechnology.com>
From: Jie Liu <liujie5@linkdatatechnology.com>
This patch implements the traffic management ops for example PMD.
It supports a 4-level hierarchy: port, vsi, queue group and queue.
- Support node add/delete and hierarchy commit.
- Support private shaper and rate limiting on each node.
The hardware requires all nodes to be configured before the hierarchy
is committed to the global registers.
Signed-off-by: Jie Liu <liujie5@linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 1 +
drivers/net/sxe2/sxe2_cmd_chnl.c | 163 +++++
drivers/net/sxe2/sxe2_cmd_chnl.h | 6 +
drivers/net/sxe2/sxe2_drv_cmd.h | 28 +-
drivers/net/sxe2/sxe2_ethdev.c | 82 +++
drivers/net/sxe2/sxe2_ethdev.h | 5 +
drivers/net/sxe2/sxe2_tm.c | 1151 ++++++++++++++++++++++++++++++
drivers/net/sxe2/sxe2_tm.h | 76 ++
drivers/net/sxe2/sxe2_tx.c | 1 -
9 files changed, 1511 insertions(+), 2 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_tm.c
create mode 100644 drivers/net/sxe2/sxe2_tm.h
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index da7a690063..f03ea15356 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -63,4 +63,5 @@ sources += files(
'sxe2_mac.c',
'sxe2_filter.c',
'sxe2_rss.c',
+ 'sxe2_tm.c',
)
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.c b/drivers/net/sxe2/sxe2_cmd_chnl.c
index b997e7b044..19323ffcc4 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.c
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.c
@@ -230,6 +230,7 @@ static void sxe2_txq_ctxt_cfg_fill(struct sxe2_tx_queue *txq,
struct sxe2_drv_txq_cfg_req *req,
uint16_t txq_cnt)
{
+ struct sxe2_adapter *adapter = txq->vsi->adapter;
struct sxe2_drv_txq_ctxt *ctxt = req->cfg;
uint16_t q_idx = 0;
@@ -241,6 +242,8 @@ static void sxe2_txq_ctxt_cfg_fill(struct sxe2_tx_queue *txq,
ctxt->depth = txq[q_idx].ring_depth;
ctxt->dma_addr = txq[q_idx].base_addr;
ctxt->queue_id = txq[q_idx].queue_id;
+
+ ctxt->sched_mode = sxe2_sched_mode_get(adapter);
}
}
@@ -310,6 +313,7 @@ int32_t sxe2_drv_txq_switch(struct sxe2_adapter *adapter, struct sxe2_tx_queue *
req.q_idx = txq->queue_id;
req.is_enable = (uint8_t)enable;
+ req.sched_mode = sxe2_sched_mode_get(adapter);
sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_TXQ_DISABLE,
&req, sizeof(req), NULL, 0);
@@ -714,3 +718,162 @@ int32_t sxe2_drv_ptp_gettime(struct sxe2_adapter *adapter, struct sxe2_rx_queue
(void)rxq;
return 0;
}
+
+int32_t sxe2_drv_root_tree_alloc(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_tm_res tm_resp;
+ int32_t ret;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SCHED_ROOT_TREE_ALLOC,
+ NULL, 0,
+ &tm_resp, sizeof(tm_resp));
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "add sched root failed, ret:%d", ret);
+ goto l_end;
+ }
+
+ tm_ctxt->root_teid = tm_resp.teid;
+
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_root_tree_release(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_drv_cmd_params param = {0};
+ struct sxe2_tm_res tm_res = {0};
+ int32_t ret;
+
+ tm_res.teid = adapter->tm_ctxt.root_teid;
+
+ sxe2_drv_cmd_params_fill(adapter, ¶m, SXE2_DRV_CMD_SCHED_ROOT_TREE_RELEASE,
+ &tm_res, sizeof(tm_res),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, ¶m);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "release sched root failed, ret:%d", ret);
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static void sxe2_drv_tm_node_to_info(struct sxe2_adapter *adapter,
+ struct sxe2_tm_node *node, struct sxe2_tm_info *info)
+{
+ uint32_t rate = 0;
+
+ if (node->shaper_profile->profile.committed.rate == UINT64_MAX)
+ rate = UINT32_MAX;
+ else
+ rate = (uint32_t)(node->shaper_profile->profile.committed.rate * 8 / 1000);
+
+ info->committed = rte_cpu_to_le_32(rate);
+
+ if (node->shaper_profile->profile.peak.rate == UINT64_MAX)
+ rate = UINT32_MAX;
+ else
+ rate = (uint32_t)(node->shaper_profile->profile.peak.rate * 8 / 1000);
+
+ info->peak = rte_cpu_to_le_32(rate);
+
+ info->priority = (adapter->tm_ctxt.prio_max - 1 - node->priority);
+
+ info->weight = rte_cpu_to_le_16(node->hw_weight);
+}
+
+static int32_t sxe2_drv_tm_commit_node(struct sxe2_adapter *adapter,
+ struct sxe2_tm_node *tm_node)
+{
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_tm_add_mid_msg msg_mid = {0};
+ struct sxe2_tm_add_queue_msg msg_queue = {0};
+ struct sxe2_tm_res res = {0};
+ struct sxe2_common_device *cdev = adapter->cdev;
+ int32_t ret;
+ uint32_t i;
+
+ if (tm_node->type == SXE2_TM_NODE_TYPE_VSIG) {
+ goto l_add;
+ } else if (tm_node->type == SXE2_TM_NODE_TYPE_MID) {
+ sxe2_drv_tm_node_to_info(adapter, tm_node, &msg_mid.info);
+ msg_mid.parent_teid = rte_cpu_to_le_16(tm_node->parent->teid);
+ msg_mid.adj_lvl = adapter->sched_ctxt.adj_lvl;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd,
+ SXE2_DRV_CMD_SCHED_TM_ADD_MID_NODE,
+ &msg_mid, sizeof(msg_mid),
+ &res, sizeof(res));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "add tm mid node failed, ret:%d", ret);
+ goto l_end;
+ }
+ } else if (tm_node->type == SXE2_TM_NODE_TYPE_QUEUE) {
+ sxe2_drv_tm_node_to_info(adapter, tm_node, &msg_queue.info);
+ msg_queue.parent_teid = rte_cpu_to_le_16(tm_node->parent->teid);
+ msg_queue.queue_id = rte_cpu_to_le_16(tm_node->id);
+ msg_queue.adj_lvl = adapter->sched_ctxt.adj_lvl;
+
+ sxe2_drv_cmd_params_fill(adapter, &cmd,
+ SXE2_DRV_CMD_SCHED_TM_ADD_QUEUE_NODE,
+ &msg_queue, sizeof(msg_queue),
+ &res, sizeof(res));
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "add tm queue failed, ret:%d", ret);
+ goto l_end;
+ }
+ } else {
+ PMD_LOG_ERR(DRV, "commit tm node failed, type:%d", tm_node->type);
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ tm_node->teid = rte_le_to_cpu_16(res.teid);
+
+l_add:
+ for (i = 0; i < tm_node->child_cnt; i++) {
+ ret = sxe2_drv_tm_commit_node(adapter, tm_node->children[i]);
+ if (ret)
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_drv_tm_commit(struct sxe2_adapter *adapter)
+{
+ struct sxe2_drv_cmd_params cmd = { 0 };
+ struct sxe2_common_device *cdev = adapter->cdev;
+ struct sxe2_tm_res tm_res = {0};
+ int32_t ret;
+
+ tm_res.teid = adapter->tm_ctxt.root_teid;
+ sxe2_drv_cmd_params_fill(adapter, &cmd, SXE2_DRV_CMD_SCHED_ROOT_CHILDREN_DELETE,
+ &tm_res, sizeof(tm_res),
+ NULL, 0);
+ ret = sxe2_drv_cmd_exec(cdev, &cmd);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "del tm root failed, ret:%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_drv_tm_commit_node(adapter, adapter->tm_ctxt.root);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "commit tm node failed, ret:%d", ret);
+ goto l_end;
+ }
+
+ PMD_LOG_DEBUG(DRV, "commit tm success");
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_cmd_chnl.h b/drivers/net/sxe2/sxe2_cmd_chnl.h
index 2546c65a6c..77e689abcd 100644
--- a/drivers/net/sxe2/sxe2_cmd_chnl.h
+++ b/drivers/net/sxe2/sxe2_cmd_chnl.h
@@ -38,6 +38,12 @@ int32_t sxe2_drv_mac_link_status_get(struct sxe2_adapter *adapter);
int32_t sxe2_drv_promisc_config(struct sxe2_adapter *adapter, bool set);
+int32_t sxe2_drv_root_tree_release(struct rte_eth_dev *dev);
+
+int32_t sxe2_drv_root_tree_alloc(struct rte_eth_dev *dev);
+
+int32_t sxe2_drv_tm_commit(struct sxe2_adapter *adapter);
+
int32_t sxe2_drv_allmulti_config(struct sxe2_adapter *adapter, bool set);
int32_t sxe2_drv_uc_config(struct sxe2_adapter *adapter, struct rte_ether_addr *addr, bool add);
diff --git a/drivers/net/sxe2/sxe2_drv_cmd.h b/drivers/net/sxe2/sxe2_drv_cmd.h
index 9998f241f0..0f7839823e 100644
--- a/drivers/net/sxe2/sxe2_drv_cmd.h
+++ b/drivers/net/sxe2/sxe2_drv_cmd.h
@@ -29,7 +29,7 @@
#define SXE2_SCHED_MODE_DEFAULT 0
#define SXE2_SCHED_MODE_TM 1
-#define SXE2_SCHED_MODE_HIGH_PERFORMANCE 2
+#define SXE2_SCHED_MODE_NO_SCHED 2
#define SXE2_SCHED_MODE_INVALID 3
#define SXE2_SRCVSI_PRUNE_MAX_NUM 2
@@ -349,6 +349,32 @@ struct __rte_aligned(4) __rte_packed_begin sxe2_rss_hf_req {
uint8_t rsv1[3];
} __rte_packed_end;
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_res {
+ uint16_t teid;
+ uint8_t rsv[2];
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_info {
+ uint32_t committed;
+ uint32_t peak;
+ uint8_t priority;
+ uint8_t reserve;
+ uint16_t weight;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_add_mid_msg {
+ uint16_t parent_teid;
+ uint8_t adj_lvl;
+ struct sxe2_tm_info info;
+} __rte_packed_end;
+
+struct __rte_aligned(4) __rte_packed_begin sxe2_tm_add_queue_msg {
+ uint16_t parent_teid;
+ uint16_t queue_id;
+ uint8_t adj_lvl;
+ struct sxe2_tm_info info;
+} __rte_packed_end;
+
enum sxe2_drv_cmd_module {
SXE2_DRV_CMD_MODULE_HANDSHAKE = 0,
SXE2_DRV_CMD_MODULE_DEV = 1,
diff --git a/drivers/net/sxe2/sxe2_ethdev.c b/drivers/net/sxe2/sxe2_ethdev.c
index b24174d8f0..d66bbed83f 100644
--- a/drivers/net/sxe2/sxe2_ethdev.c
+++ b/drivers/net/sxe2/sxe2_ethdev.c
@@ -129,6 +129,8 @@ static const struct eth_dev_ops sxe2_eth_dev_ops = {
.reta_query = sxe2_dev_rss_reta_query,
.rss_hash_update = sxe2_dev_rss_hash_update,
.rss_hash_conf_get = sxe2_dev_rss_hash_conf_get,
+
+ .tm_ops_get = sxe2_tm_ops_get,
};
static int32_t sxe2_dev_configure(struct rte_eth_dev *dev)
@@ -564,6 +566,14 @@ static void sxe2_drv_dev_caps_set(struct sxe2_adapter *adapter,
adapter->cap_flags |= SXE2_DEV_CAPS_OFFLOAD_FC_STATE;
}
+static void sxe2_sw_sched_hw_cap_set(struct sxe2_adapter *adapter,
+ struct sxe2_txsch_caps *txsch_caps)
+{
+ adapter->sched_ctxt.tm_layers = txsch_caps->layer_cap;
+ adapter->sched_ctxt.root_max_children = txsch_caps->tm_mid_node_num;
+ adapter->sched_ctxt.prio_max = txsch_caps->prio_num;
+}
+
static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
{
int32_t ret = -1;
@@ -583,6 +593,8 @@ static int32_t sxe2_func_caps_get(struct sxe2_adapter *adapter)
sxe2_sw_vsi_ctx_hw_cap_set(adapter, &dev_caps.vsi_caps);
+ sxe2_sw_sched_hw_cap_set(adapter, &dev_caps.txsch_caps);
+
l_end:
return ret;
}
@@ -919,6 +931,68 @@ void sxe2_dev_pci_map_uinit(struct rte_eth_dev *dev)
adapter->dev_info.dev_data = NULL;
}
+uint32_t sxe2_sched_mode_get(struct sxe2_adapter *adapter)
+{
+ uint32_t ret_mode = SXE2_SCHED_MODE_INVALID;
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+
+ if (adapter->devargs.no_sched_mode)
+ ret_mode = SXE2_SCHED_MODE_NO_SCHED;
+ else if (tm_ctxt->committed)
+ ret_mode = SXE2_SCHED_MODE_TM;
+ else
+ ret_mode = SXE2_SCHED_MODE_DEFAULT;
+
+ return ret_mode;
+}
+
+static int32_t sxe2_sched_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ adapter->sched_ctxt.adj_lvl = adapter->devargs.sched_layer_mode;
+
+ if (adapter->devargs.no_sched_mode) {
+ PMD_LOG_DEBUG(DRV, "TM feature will be disabled in high-performance mode.");
+ adapter->cap_flags &= ~(SXE2_DEV_CAPS_OFFLOAD_TM);
+ } else {
+ ret = sxe2_tm_init(dev);
+ if (ret)
+ goto l_end;
+
+ ret = sxe2_drv_root_tree_alloc(dev);
+ if (ret)
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_sched_uinit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+
+ if (!adapter->devargs.no_sched_mode) {
+ ret = sxe2_tm_uninit(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to uninit tm, ret=%d", ret);
+ goto l_end;
+ }
+
+ ret = sxe2_drv_root_tree_release(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to release root tree, ret=%d", ret);
+ goto l_end;
+ }
+ }
+
+l_end:
+ return ret;
+}
+
static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
struct sxe2_dev_kvargs_info *kvargs __rte_unused)
{
@@ -974,8 +1048,15 @@ static int32_t sxe2_dev_init(struct rte_eth_dev *dev,
goto init_rss_err;
}
+ ret = sxe2_sched_init(dev);
+ if (ret) {
+ PMD_LOG_ERR(INIT, "Failed to init sched, ret=%d", ret);
+ goto init_sched_err;
+ }
+
goto l_end;
+init_sched_err:
init_rss_err:
init_eth_err:
init_dev_info_err:
@@ -991,6 +1072,7 @@ static int32_t sxe2_dev_close(struct rte_eth_dev *dev)
(void)sxe2_dev_stop(dev);
(void)sxe2_queues_release(dev);
(void)sxe2_rss_disable(dev);
+ (void)sxe2_sched_uinit(dev);
sxe2_vsi_uninit(dev);
sxe2_dev_pci_map_uinit(dev);
sxe2_eth_uinit(dev);
diff --git a/drivers/net/sxe2/sxe2_ethdev.h b/drivers/net/sxe2/sxe2_ethdev.h
index fd106eb867..0176ae538a 100644
--- a/drivers/net/sxe2/sxe2_ethdev.h
+++ b/drivers/net/sxe2/sxe2_ethdev.h
@@ -20,6 +20,7 @@
#include "sxe2_queue.h"
#include "sxe2_mac.h"
#include "sxe2_osal.h"
+#include "sxe2_tm.h"
#include "sxe2_filter.h"
struct sxe2_link_msg {
@@ -308,6 +309,8 @@ struct sxe2_adapter {
struct sxe2_rss_context rss_ctxt;
struct sxe2_link_context link_ctxt;
struct sxe2_ptp_context ptp_ctxt;
+ struct sxe2_sched_hw_cap sched_ctxt;
+ struct sxe2_tm_context tm_ctxt;
struct sxe2_devargs devargs;
struct sxe2_switchdev_info switchdev_info;
bool rule_started;
@@ -331,6 +334,8 @@ void *sxe2_pci_map_addr_get(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type,
uint16_t idx_in_func);
+uint32_t sxe2_sched_mode_get(struct sxe2_adapter *adapter);
+
struct sxe2_pci_map_bar_info *sxe2_dev_get_bar_info(struct sxe2_adapter *adapter,
enum sxe2_pci_map_resource res_type);
diff --git a/drivers/net/sxe2/sxe2_tm.c b/drivers/net/sxe2/sxe2_tm.c
new file mode 100644
index 0000000000..4c4f793cd5
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_tm.c
@@ -0,0 +1,1151 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#include <rte_ethdev.h>
+#include <rte_tm_driver.h>
+
+#include "sxe2_tm.h"
+#include "sxe2_ethdev.h"
+#include "sxe2_common_log.h"
+#include "sxe2_cmd_chnl.h"
+
+static uint16_t sxe2_tm_level_node_num_get(uint8_t level)
+{
+ uint16_t node_num = 0;
+
+ switch (level) {
+ case 0:
+ node_num = SXE2_TM_1L_NODE_NUM_MAX;
+ break;
+ case 1:
+ node_num = SXE2_TM_2L_NODE_NUM_MAX;
+ break;
+ case 2:
+ node_num = SXE2_TM_3L_NODE_NUM_MAX;
+ break;
+ case 3:
+ node_num = SXE2_TM_4L_NODE_NUM_MAX;
+ break;
+ }
+ return node_num;
+}
+
+static int32_t sxe2_tm_capabilities_get(struct rte_eth_dev *dev,
+ struct rte_tm_capabilities *cap,
+ struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = 0;
+ uint32_t i;
+
+ if (!cap || !error) {
+ PMD_LOG_ERR(DRV, "sxe2 get tm cap failed, cap or error is NULL.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ error->type = RTE_TM_ERROR_TYPE_NONE;
+ memset(cap, 0, sizeof(struct rte_tm_capabilities));
+
+ for (i = 0; i < adapter->tm_ctxt.tm_layers; i++)
+ cap->n_nodes_max += sxe2_tm_level_node_num_get(i);
+
+ cap->n_levels_max = adapter->tm_ctxt.tm_layers;
+
+ cap->non_leaf_nodes_identical = 1;
+
+ cap->leaf_nodes_identical = 1;
+
+ cap->shaper_n_max = cap->n_nodes_max;
+
+ cap->shaper_private_n_max = cap->n_nodes_max;
+
+ cap->shaper_private_dual_rate_n_max = cap->n_nodes_max;
+
+ cap->shaper_private_rate_min = 0;
+
+ cap->shaper_private_rate_max = 12500000000ull;
+
+ cap->shaper_private_packet_mode_supported = 0;
+ cap->shaper_private_byte_mode_supported = 1;
+
+ cap->shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD;
+
+ cap->shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
+
+ cap->shaper_shared_n_max = 0;
+ cap->shaper_shared_n_nodes_per_shaper_max = 0;
+ cap->shaper_shared_n_shapers_per_node_max = 0;
+ cap->shaper_shared_dual_rate_n_max = 0;
+ cap->shaper_shared_rate_min = 0;
+ cap->shaper_shared_rate_max = 0;
+ cap->shaper_shared_packet_mode_supported = 0;
+ cap->shaper_shared_byte_mode_supported = 0;
+
+ cap->sched_n_children_max = dev->data->nb_tx_queues;
+
+ cap->sched_sp_n_priorities_max = 7;
+
+ cap->sched_wfq_n_children_per_group_max = 1;
+ cap->sched_wfq_n_groups_max = 0;
+ cap->sched_wfq_weight_max = 0;
+ cap->sched_wfq_packet_mode_supported = 0;
+ cap->sched_wfq_byte_mode_supported = 0;
+
+ cap->cman_wred_packet_mode_supported = 0;
+ cap->cman_wred_byte_mode_supported = 0;
+ cap->cman_head_drop_supported = 0;
+ cap->cman_wred_context_n_max = 0;
+ cap->cman_wred_context_private_n_max = 0;
+ cap->cman_wred_context_shared_n_max = 0;
+ cap->cman_wred_context_shared_n_nodes_per_context_max = 0;
+ cap->cman_wred_context_shared_n_contexts_per_node_max = 0;
+
+ cap->dynamic_update_mask = 0;
+
+ cap->stats_mask = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_level_capabilities_get(struct rte_eth_dev *dev,
+ uint32_t level_id, struct rte_tm_level_capabilities *cap,
+ struct rte_tm_error *error)
+{
+ int32_t ret = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!cap || !error) {
+ PMD_LOG_ERR(DRV, "sxe2 get tm cap failed, cap or error is NULL.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (level_id >= adapter->tm_ctxt.tm_layers) {
+ ret = -EINVAL;
+ error->type = RTE_TM_ERROR_TYPE_LEVEL_ID;
+ error->message = "too deep level";
+ goto l_end;
+ }
+
+ cap->n_nodes_max = sxe2_tm_level_node_num_get(level_id);
+
+ cap->non_leaf_nodes_identical = true;
+
+ cap->leaf_nodes_identical = true;
+
+ if (level_id != adapter->tm_ctxt.tm_layers - 1) {
+ cap->n_nodes_nonleaf_max = cap->n_nodes_max;
+ cap->n_nodes_leaf_max = 0;
+
+ cap->nonleaf.shaper_private_supported = true;
+ cap->nonleaf.shaper_private_dual_rate_supported = true;
+ cap->nonleaf.shaper_private_rate_min = 0;
+
+ cap->nonleaf.shaper_private_rate_max = 12500000000ull;
+ cap->nonleaf.shaper_private_packet_mode_supported = 0;
+ cap->nonleaf.shaper_private_byte_mode_supported = 1;
+
+ cap->nonleaf.shaper_shared_n_max = 0;
+ cap->nonleaf.shaper_shared_packet_mode_supported = 0;
+ cap->nonleaf.shaper_shared_byte_mode_supported = 0;
+
+ cap->nonleaf.sched_n_children_max = SXE2_TM_MAX_CHILDREN_COUNT;
+ cap->nonleaf.sched_sp_n_priorities_max = 7;
+
+ cap->nonleaf.sched_wfq_n_children_per_group_max = 0;
+ cap->nonleaf.sched_wfq_n_groups_max = 0;
+ cap->nonleaf.sched_wfq_weight_max = 0;
+ cap->nonleaf.sched_wfq_packet_mode_supported = 0;
+ cap->nonleaf.sched_wfq_byte_mode_supported = 0;
+
+ cap->nonleaf.stats_mask = 0;
+ } else {
+ cap->n_nodes_nonleaf_max = 0;
+ cap->n_nodes_leaf_max = cap->n_nodes_max;
+
+ cap->leaf.shaper_private_supported = true;
+ cap->leaf.shaper_private_dual_rate_supported = true;
+ cap->leaf.shaper_private_rate_min = 0;
+ cap->leaf.shaper_private_rate_max = 12500000000ull;
+ cap->leaf.shaper_private_packet_mode_supported = 0;
+ cap->leaf.shaper_private_byte_mode_supported = 1;
+
+ cap->leaf.shaper_shared_n_max = 0;
+ cap->leaf.shaper_shared_packet_mode_supported = 0;
+ cap->leaf.shaper_shared_byte_mode_supported = 0;
+
+ cap->leaf.cman_head_drop_supported = false;
+ cap->leaf.cman_wred_context_private_supported = false;
+ cap->leaf.cman_wred_context_shared_n_max = 0;
+
+ cap->leaf.stats_mask = 0;
+ }
+
+l_end:
+ return ret;
+}
+
+static struct sxe2_tm_node *sxe2_tm_find_node(struct sxe2_tm_node *parent, uint32_t id)
+{
+ struct sxe2_tm_node *node = NULL;
+ uint32_t i;
+
+ if (parent == NULL || parent->id == id) {
+ node = parent;
+ goto l_end;
+ }
+
+ for (i = 0; i < parent->child_cnt; i++) {
+ node = sxe2_tm_find_node(parent->children[i], id);
+ if (node)
+ goto l_end;
+ }
+
+l_end:
+ return node;
+}
+
+static int32_t sxe2_node_capabilities_get(struct rte_eth_dev *dev, uint32_t node_id,
+ struct rte_tm_node_capabilities *cap,
+ struct rte_tm_error *error)
+{
+ int32_t ret = -EINVAL;
+ struct sxe2_tm_node *tm_node;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!cap || !error) {
+ PMD_LOG_ERR(DRV, "sxe2 get tm cap failed, cap or error is NULL.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "invalid node id");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ tm_node = sxe2_tm_find_node(adapter->tm_ctxt.root, node_id);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "no such node");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no such node";
+ goto l_end;
+ }
+
+ cap->shaper_private_supported = true;
+ cap->shaper_private_dual_rate_supported = true;
+ cap->shaper_private_rate_min = 0;
+ cap->shaper_private_rate_max = 12500000000ull;
+ cap->shaper_private_packet_mode_supported = 0;
+ cap->shaper_private_byte_mode_supported = 1;
+
+ cap->shaper_shared_n_max = 0;
+ cap->shaper_shared_packet_mode_supported = 0;
+ cap->shaper_shared_byte_mode_supported = 0;
+
+ if (tm_node->level == adapter->tm_ctxt.tm_layers - 1) {
+ cap->leaf.cman_head_drop_supported = false;
+ cap->leaf.cman_wred_context_private_supported = false;
+ cap->leaf.cman_wred_context_shared_n_max = 0;
+ } else {
+ cap->nonleaf.sched_n_children_max = SXE2_TM_MAX_CHILDREN_COUNT;
+ cap->nonleaf.sched_sp_n_priorities_max = 7;
+ cap->nonleaf.sched_wfq_n_children_per_group_max = 0;
+ cap->nonleaf.sched_wfq_n_groups_max = 0;
+ cap->nonleaf.sched_wfq_weight_max = 0;
+ cap->nonleaf.sched_wfq_packet_mode_supported = 0;
+ cap->nonleaf.sched_wfq_byte_mode_supported = 0;
+ }
+ cap->stats_mask = 0;
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_shaper_profile_param_check(const struct rte_tm_shaper_params *profile,
+ struct rte_tm_error *error)
+{
+ int32_t ret = 0;
+
+ if (profile->committed.size) {
+ PMD_LOG_ERR(DRV, "committed bucket size not supported.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE;
+ error->message = "committed bucket size not supported";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->peak.size) {
+ PMD_LOG_ERR(DRV, "peak bucket size not supported.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE;
+ error->message = "peak bucket size not supported";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->pkt_length_adjust) {
+ PMD_LOG_ERR(DRV, "packet length adjustment not supported.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN;
+ error->message = "packet length adjustment not supported";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->committed.rate > SXE2_HW_RATE_MAX ||
+ profile->committed.rate < SXE2_HW_RATE_MIN) {
+ PMD_LOG_ERR(DRV, "The committed rate limit value is required to be in "
+ "the range [%" PRIu64 ", %" PRIu64 "].",
+ (uint64_t)SXE2_HW_RATE_MIN, (uint64_t)SXE2_HW_RATE_MAX);
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
+ error->message = "invalid rate limit: value out of range.";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->peak.rate > SXE2_HW_RATE_MAX ||
+ profile->peak.rate < SXE2_HW_RATE_MIN) {
+ PMD_LOG_ERR(DRV, "The peak rate limit value is required to be in "
+ "the range [%" PRIu64 ", %" PRIu64 "].",
+ (uint64_t)SXE2_HW_RATE_MIN, (uint64_t)SXE2_HW_RATE_MAX);
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE;
+ error->message = "invalid rate limit: value out of range.";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (profile->committed.rate > profile->peak.rate) {
+ PMD_LOG_ERR(DRV, "committed rate can't be greater than peak rate.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
+ error->message = "committed rate can't be greater than peak rate.";
+ ret = -EINVAL;
+ goto l_end;
+ }
+l_end:
+ return ret;
+}
+
+static inline struct sxe2_tm_shaper_profile *
+sxe2_tm_shaper_profile_search(struct rte_eth_dev *dev, uint32_t id)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_shaper_profile_list *shaper_profile_list =
+ &adapter->tm_ctxt.profile_list;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+
+ TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) {
+ if (id == shaper_profile->id)
+ goto l_end;
+ }
+
+ shaper_profile = NULL;
+
+l_end:
+ return shaper_profile;
+}
+
+static int32_t sxe2_tm_shaper_profile_add(struct rte_eth_dev *dev, uint32_t shaper_profile_id,
+ const struct rte_tm_shaper_params *profile,
+ struct rte_tm_error *error)
+{
+ int32_t ret = -EINVAL;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!profile || !error) {
+ PMD_LOG_ERR(DRV, "Invalid input: profile:0x%p or error:0x%p is null.",
+ profile, error);
+ if (error) {
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Invalid input: profile or error is null.";
+ }
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ ret = sxe2_tm_shaper_profile_param_check(profile, error);
+ if (ret)
+ goto l_end;
+
+ shaper_profile = sxe2_tm_shaper_profile_search(dev, shaper_profile_id);
+ if (shaper_profile) {
+ PMD_LOG_ERR(DRV, "profile ID exist.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
+ error->message = "profile ID exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ shaper_profile = rte_zmalloc("sxe2_tm_shaper_profile",
+ sizeof(struct sxe2_tm_shaper_profile), 0);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "Alloc shaper_profile memory failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Alloc shaper_profile memory failed";
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ rte_memcpy(&shaper_profile->profile, profile,
+ sizeof(struct rte_tm_shaper_params));
+ shaper_profile->id = shaper_profile_id;
+
+ TAILQ_INSERT_TAIL(&adapter->tm_ctxt.profile_list, shaper_profile, node);
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_shaper_profile_del(struct rte_eth_dev *dev,
+ uint32_t id, struct rte_tm_error *error)
+{
+ int32_t ret = -EINVAL;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+
+ if (!error) {
+ PMD_LOG_ERR(DRV, "Error param is null.");
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ shaper_profile = sxe2_tm_shaper_profile_search(dev, id);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "profile ID not exist.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
+ error->message = "profile ID not exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (shaper_profile->ref_cnt) {
+ PMD_LOG_ERR(DRV, "profile in use.");
+ error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE;
+ error->message = "profile in use";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ TAILQ_REMOVE(&adapter->tm_ctxt.profile_list, shaper_profile, node);
+ rte_free(shaper_profile);
+
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_node_param_check(struct rte_eth_dev *dev,
+ uint32_t parent_node_type,
+ uint32_t node_id, uint32_t priority, uint32_t weight,
+ const struct rte_tm_node_params *params,
+ bool is_leaf, struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ int32_t ret = -EINVAL;
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "Invalid node id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ if (parent_node_type == SXE2_TM_NODE_TYPE_VSIG &&
+ priority >= tm_ctxt->prio_max) {
+ PMD_LOG_ERR(DRV, "Priority should be less than %u.", tm_ctxt->prio_max);
+ error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
+ error->message = "The priority is too high.";
+ goto l_end;
+ }
+
+ if (priority > SXE2_TM_PRIO_MAX) {
+ PMD_LOG_ERR(DRV, "Priority should be less than %u.", SXE2_TM_PRIO_MAX);
+ error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
+ error->message = "The priority is too high.";
+ goto l_end;
+ }
+
+ if (weight > SXE2_TM_WEIGHT_MAX || weight < SXE2_TM_WEIGHT_MIN) {
+ PMD_LOG_ERR(DRV, "Weight must be between 1 and 200.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT;
+ error->message = "weight must be between 1 and 200";
+ goto l_end;
+ }
+
+ if (params->shared_shaper_id) {
+ PMD_LOG_ERR(DRV, "Shared shaper not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID;
+ error->message = "shared shaper not supported";
+ goto l_end;
+ }
+ if (params->n_shared_shapers) {
+ PMD_LOG_ERR(DRV, "Shared shaper not supported..");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS;
+ error->message = "shared shaper not supported";
+ goto l_end;
+ }
+
+ if (!is_leaf) {
+ if (node_id <= dev->data->nb_tx_queues) {
+ PMD_LOG_ERR(DRV, "no leaf node id must bigger than queue id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no leaf node id must bigger than queue id.";
+ goto l_end;
+ }
+
+ if (params->nonleaf.wfq_weight_mode) {
+ PMD_LOG_ERR(DRV, "WFQ not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE;
+ error->message = "WFQ not supported";
+ goto l_end;
+ }
+
+ if (params->nonleaf.n_sp_priorities != 1) {
+ PMD_LOG_ERR(DRV, "SP priority not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES;
+ error->message = "SP priority not supported";
+ goto l_end;
+ }
+ } else {
+ if (node_id >= dev->data->nb_tx_queues) {
+ PMD_LOG_ERR(DRV, "leaf node id must be queue id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "leaf node id must be queue id.";
+ goto l_end;
+ }
+
+ if (params->leaf.cman) {
+ PMD_LOG_ERR(DRV, "Congestion management not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN;
+ error->message = "Congestion management not supported";
+ goto l_end;
+ }
+ if (params->leaf.wred.wred_profile_id != RTE_TM_WRED_PROFILE_ID_NONE) {
+ PMD_LOG_ERR(DRV, "WRED not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID;
+ error->message = "WRED not supported";
+ goto l_end;
+ }
+ if (params->leaf.wred.shared_wred_context_id) {
+ PMD_LOG_ERR(DRV, "WRED not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID;
+ error->message = "WRED not supported";
+ goto l_end;
+ }
+ if (params->leaf.wred.n_shared_wred_contexts) {
+ PMD_LOG_ERR(DRV, "WRED not supported.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS;
+ error->message = "WRED not supported";
+ goto l_end;
+ }
+ }
+ ret = 0;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_add_child(struct sxe2_tm_node *parent,
+ struct sxe2_tm_node *child)
+{
+ int32_t ret = -1;
+ uint32_t i;
+ for (i = 0; i < SXE2_TM_MAX_CHILDREN_COUNT; i++) {
+ if (parent->children[i] == NULL) {
+ parent->children[i] = child;
+ child->index_in_parent = i;
+ parent->child_cnt++;
+ ret = 0;
+ break;
+ }
+ }
+ return ret;
+}
+
+static int32_t sxe2_tm_node_add(struct rte_eth_dev *dev, uint32_t node_id, uint32_t parent_node_id,
+ uint32_t priority, uint32_t weight, uint32_t level_id,
+ const struct rte_tm_node_params *params,
+ struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_node *tm_node = NULL;
+ struct sxe2_tm_node *parent_node = NULL;
+ int32_t ret = -EINVAL;
+ bool is_leaf;
+
+ if (!params || !error) {
+ PMD_LOG_ERR(DRV, "Invalid input: params:0x%p or error:0x%p is null.",
+ params, error);
+ if (error) {
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Invalid input: params or error is null.";
+ }
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ shaper_profile = sxe2_tm_shaper_profile_search(dev, params->shaper_profile_id);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "Shaper profile does not exist.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID;
+ error->message = "shaper profile does not exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (parent_node_id == RTE_TM_NODE_ID_NULL) {
+ if (level_id != 0) {
+ PMD_LOG_ERR(DRV, "Wrong level, root node (NULL parent) must be at level 0.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "Wrong level, root node (NULL parent) must be at level 0";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (tm_ctxt->root) {
+ PMD_LOG_ERR(DRV, "Already have a root.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
+ error->message = "already have a root";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_tm_node_param_check(dev, SXE2_TM_NODE_TYPE_INVALID, node_id, priority,
+ weight, params, false, error);
+ if (ret)
+ goto l_end;
+
+ tm_node = rte_zmalloc("tm_node_root", sizeof(struct sxe2_tm_node), 0);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "Alloc tm_node memory failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Alloc tm_node memory failed";
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ tm_node->id = node_id;
+ tm_node->level = 0;
+ tm_node->parent = NULL;
+ tm_node->child_cnt = 0;
+ tm_node->weight = weight;
+ tm_node->hw_weight = SXE2_TM_WEIGHT_SUM;
+ tm_node->type = SXE2_TM_NODE_TYPE_VSIG;
+ tm_node->priority = priority;
+ tm_node->shaper_profile = shaper_profile;
+
+ tm_node->teid = tm_ctxt->root_teid;
+
+ shaper_profile->ref_cnt++;
+ tm_ctxt->root = tm_node;
+ ret = 0;
+ goto l_end;
+ }
+
+ parent_node = sxe2_tm_find_node(tm_ctxt->root, parent_node_id);
+ if (!parent_node) {
+ PMD_LOG_ERR(DRV, "Parent not exist.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
+ error->message = "parent not exist";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (parent_node->child_cnt >= SXE2_TM_MAX_CHILDREN_COUNT ||
+ (parent_node->type == SXE2_TM_NODE_TYPE_VSIG &&
+ parent_node->child_cnt >= tm_ctxt->root_max_children)) {
+ PMD_LOG_ERR(DRV, "Parent node is full.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "parent node is full";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (level_id == RTE_TM_NODE_LEVEL_ID_ANY) {
+ level_id = parent_node->level + 1;
+ } else if (level_id != parent_node->level + 1) {
+ PMD_LOG_ERR(DRV, "Wrong level.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "Wrong level";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (level_id >= tm_ctxt->tm_layers) {
+ PMD_LOG_ERR(DRV, "The maximum number of TM configuration levels is %d",
+ tm_ctxt->tm_layers);
+ error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS;
+ error->message = "TM level exceeds supported hardware limit";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (level_id + 1 == tm_ctxt->tm_layers)
+ is_leaf = true;
+ else
+ is_leaf = false;
+
+ ret = sxe2_tm_node_param_check(dev, parent_node->type, node_id, priority, weight,
+ params, is_leaf, error);
+ if (ret)
+ goto l_end;
+
+ if (sxe2_tm_find_node(tm_ctxt->root, node_id)) {
+ PMD_LOG_ERR(DRV, "Node id already used.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "node id already used";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ tm_node = rte_zmalloc("tm_node_no_root", sizeof(struct sxe2_tm_node), 0);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "Alloc tm_node memory failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Alloc tm_node memory failed";
+ ret = -ENOMEM;
+ goto l_end;
+ }
+
+ if (level_id + 1 != tm_ctxt->tm_layers)
+ tm_node->type = SXE2_TM_NODE_TYPE_MID;
+ else
+ tm_node->type = SXE2_TM_NODE_TYPE_QUEUE;
+ tm_node->id = node_id;
+ tm_node->level = level_id;
+ tm_node->parent = parent_node;
+ tm_node->child_cnt = 0;
+ tm_node->weight = weight;
+ tm_node->priority = priority;
+ tm_node->shaper_profile = shaper_profile;
+ shaper_profile->ref_cnt++;
+
+ ret = sxe2_tm_add_child(parent_node, tm_node);
+ if (ret) {
+ shaper_profile->ref_cnt--;
+ rte_free(tm_node);
+ }
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_tree_delete(struct sxe2_tm_node *tm_node)
+{
+ int32_t ret = 0;
+ uint32_t i, j;
+ struct sxe2_tm_node *parent = NULL;
+
+ if (!tm_node)
+ goto l_end;
+
+ parent = tm_node->parent;
+
+ if (tm_node->child_cnt != 0) {
+ for (i = SXE2_TM_MAX_CHILDREN_COUNT; i > 0; i--) {
+ if (tm_node->children[i - 1])
+ ret = sxe2_tm_tree_delete(tm_node->children[i - 1]);
+ }
+ }
+
+ if (tm_node->shaper_profile)
+ tm_node->shaper_profile->ref_cnt--;
+
+ if (tm_node->type != SXE2_TM_NODE_TYPE_VSIG && parent) {
+ for (i = 0; i < parent->child_cnt; i++) {
+ if (parent->children[i] == tm_node)
+ break;
+ }
+ for (j = i; j < parent->child_cnt - 1; j++)
+ parent->children[j] = parent->children[j + 1];
+
+ parent->children[parent->child_cnt - 1] = NULL;
+ parent->child_cnt--;
+ }
+ rte_free(tm_node);
+ tm_node = NULL;
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_node_delete(struct rte_eth_dev *dev, uint32_t node_id,
+ struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_node *tm_node = NULL;
+ int32_t ret = -EINVAL;
+
+ if (!error) {
+ PMD_LOG_ERR(DRV, "Invalid input: error is null.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "Invalid node id. node_id = %u", node_id);
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ tm_node = sxe2_tm_find_node(tm_ctxt->root, node_id);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "No such node.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no such node";
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ ret = sxe2_tm_tree_delete(tm_node);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Delete node failed.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "Delete node failed";
+ goto l_end;
+ }
+
+ if (tm_node == tm_ctxt->root)
+ tm_ctxt->root = NULL;
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_node_type_get(struct rte_eth_dev *dev, uint32_t node_id,
+ int32_t *is_leaf, struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_node *tm_node = NULL;
+ int32_t ret = -EINVAL;
+
+ if (!is_leaf || !error) {
+ PMD_LOG_ERR(DRV, "Invalid input: is_leaf:0x%p or error:0x%p is null.",
+ is_leaf, error);
+ if (error) {
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Invalid input: is_leaf or error is null";
+ }
+ goto l_end;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_end;
+ }
+
+ if (node_id == RTE_TM_NODE_ID_NULL) {
+ PMD_LOG_ERR(DRV, "Invalid node id.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "invalid node id";
+ goto l_end;
+ }
+
+ tm_node = sxe2_tm_find_node(tm_ctxt->root, node_id);
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "No such node.");
+ error->type = RTE_TM_ERROR_TYPE_NODE_ID;
+ error->message = "no such node";
+ goto l_end;
+ }
+
+ if (tm_node->level + 1 == tm_ctxt->tm_layers)
+ *is_leaf = true;
+ else
+ *is_leaf = false;
+ ret = 0;
+l_end:
+ return ret;
+}
+
+int32_t sxe2_tm_uninit(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+ int32_t ret = 0;
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ ret = 0;
+ goto l_end;
+ }
+
+ tm_ctxt->tm_layers = 0;
+ tm_ctxt->root_max_children = 0;
+ tm_ctxt->committed = false;
+
+ (void)sxe2_tm_tree_delete(tm_ctxt->root);
+
+ while ((shaper_profile = TAILQ_FIRST(&tm_ctxt->profile_list))) {
+ TAILQ_REMOVE(&tm_ctxt->profile_list, shaper_profile, node);
+ rte_free(shaper_profile);
+ }
+l_end:
+ return ret;
+}
+
+int32_t sxe2_tm_init(struct rte_eth_dev *dev)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ struct sxe2_sched_hw_cap *sched_ctxt = &adapter->sched_ctxt;
+ struct sxe2_tm_context *tm_ctxt = &adapter->tm_ctxt;
+ int32_t ret = 0;
+ struct sxe2_tm_shaper_profile *shaper_profile = NULL;
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0)
+ goto l_end;
+
+ tm_ctxt->tm_layers = sched_ctxt->tm_layers;
+ tm_ctxt->root_max_children = sched_ctxt->root_max_children;
+ tm_ctxt->prio_max = sched_ctxt->prio_max;
+ tm_ctxt->committed = false;
+ TAILQ_INIT(&tm_ctxt->profile_list);
+ tm_ctxt->root = NULL;
+
+ shaper_profile = rte_zmalloc("sxe2_tm_shaper_profile",
+ sizeof(struct sxe2_tm_shaper_profile), 0);
+ if (!shaper_profile) {
+ PMD_LOG_ERR(DRV, "Alloc shaper_profile memory failed.");
+ ret = -ENOMEM;
+ goto l_end;
+ }
+ shaper_profile->id = RTE_TM_SHAPER_PROFILE_ID_NONE;
+ shaper_profile->ref_cnt = 1;
+ shaper_profile->profile.committed.rate = UINT64_MAX;
+ shaper_profile->profile.peak.rate = UINT64_MAX;
+ TAILQ_INSERT_TAIL(&tm_ctxt->profile_list, shaper_profile, node);
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_chk_all_leaf(struct rte_eth_dev *dev)
+{
+ int32_t ret = 0;
+ uint32_t i = 0;
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ if (!sxe2_tm_find_node(adapter->tm_ctxt.root, i)) {
+ ret = -1;
+ break;
+ }
+ }
+ return ret;
+}
+
+static int32_t sxe2_tm_weight_calc(struct sxe2_tm_node *tm_node)
+{
+ int32_t ret = 0;
+ int32_t total_weight = 0;
+ int32_t total_weight2 = 0;
+ uint32_t i = 0;
+ uint32_t j = 0;
+ uint32_t k = 0;
+ uint32_t maxindex = 0;
+ uint32_t maxweight = 0;
+ struct sxe2_tm_node *cacl_node[SXE2_TM_MAX_CHILDREN_COUNT] = {NULL};
+
+ if (!tm_node) {
+ PMD_LOG_ERR(DRV, "Invalid input: tm_node is null.");
+ ret = -EINVAL;
+ goto l_end;
+ }
+
+ if (tm_node->child_cnt == 0)
+ goto l_end;
+
+ for (j = SXE2_TM_PRIO_MIN; j <= SXE2_TM_PRIO_MAX; j++) {
+ k = 0;
+ total_weight = 0;
+ total_weight2 = 0;
+ maxindex = 0;
+ maxweight = 0;
+
+ for (i = 0; i < tm_node->child_cnt; i++) {
+ if (tm_node->children[i]->priority == j)
+ cacl_node[k++] = tm_node->children[i];
+ }
+ if (k == 0)
+ continue;
+
+ for (i = 0; i < k; i++)
+ total_weight += cacl_node[i]->weight;
+
+ for (i = 0; i < k; i++) {
+ cacl_node[i]->hw_weight = cacl_node[i]->weight *
+ SXE2_TM_WEIGHT_SUM / total_weight;
+ total_weight2 += cacl_node[i]->hw_weight;
+ if (cacl_node[i]->hw_weight > maxweight) {
+ maxweight = cacl_node[i]->hw_weight;
+ maxindex = i;
+ }
+ }
+
+ cacl_node[maxindex]->hw_weight += SXE2_TM_WEIGHT_SUM - total_weight2;
+ }
+
+ for (i = 0; i < tm_node->child_cnt; i++) {
+ ret = sxe2_tm_weight_calc(tm_node->children[i]);
+ if (ret)
+ goto l_end;
+ }
+
+l_end:
+ return ret;
+}
+
+static int32_t sxe2_tm_hierarchy_commit(struct rte_eth_dev *dev,
+ int32_t clear_on_fail, struct rte_tm_error *error)
+{
+ struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
+ int32_t ret = -EINVAL;
+
+ if (!error) {
+ PMD_LOG_ERR(DRV, "Invalid input: error is null.");
+ ret = -EINVAL;
+ goto l_clear_on_fail;
+ }
+
+ if ((adapter->cap_flags & SXE2_DEV_CAPS_OFFLOAD_TM) == 0) {
+ PMD_LOG_ERR(DRV, "The TM capability is not supported.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The TM capability is not supported.";
+ ret = ENOTSUP;
+ goto l_clear_on_fail;
+ }
+
+ if (dev->data->dev_started) {
+ PMD_LOG_ERR(DRV, "Device failed to Stop.");
+ error->message = "Device failed to Stop";
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ ret = -EPERM;
+ goto l_clear_on_fail;
+ }
+
+ ret = sxe2_tm_chk_all_leaf(dev);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "All tx queues need config.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "All tx queues need config.";
+ goto l_clear_on_fail;
+ }
+
+ ret = sxe2_tm_weight_calc(adapter->tm_ctxt.root);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "The weight in tree is wrong.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "The weight in tree is wrong.";
+ goto l_clear_on_fail;
+ }
+
+ ret = sxe2_drv_tm_commit(adapter);
+ if (ret) {
+ PMD_LOG_ERR(DRV, "Commit tree to fw failed.");
+ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
+ error->message = "Commit tree to fw failed.";
+ goto l_clear_on_fail;
+ }
+
+ adapter->tm_ctxt.committed = true;
+ ret = 0;
+ goto l_end;
+
+l_clear_on_fail:
+ if (clear_on_fail) {
+ (void)sxe2_tm_uninit(dev);
+ (void)sxe2_tm_init(dev);
+ }
+
+l_end:
+ return ret;
+}
+
+static const struct rte_tm_ops sxe2_tm_ops = {
+ .capabilities_get = sxe2_tm_capabilities_get,
+ .level_capabilities_get = sxe2_level_capabilities_get,
+ .node_capabilities_get = sxe2_node_capabilities_get,
+ .shaper_profile_add = sxe2_tm_shaper_profile_add,
+ .shaper_profile_delete = sxe2_tm_shaper_profile_del,
+ .node_add = sxe2_tm_node_add,
+ .node_delete = sxe2_tm_node_delete,
+ .node_type_get = sxe2_tm_node_type_get,
+
+ .hierarchy_commit = sxe2_tm_hierarchy_commit,
+};
+
+int32_t sxe2_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
+{
+ int32_t ret = 0;
+
+ if (!arg) {
+ ret = -EINVAL;
+ PMD_LOG_ERR(DRV, "%s failed because arg is NULL", __func__);
+ goto l_end;
+ }
+ *(const void **)arg = &sxe2_tm_ops;
+l_end:
+ return ret;
+}
diff --git a/drivers/net/sxe2/sxe2_tm.h b/drivers/net/sxe2/sxe2_tm.h
new file mode 100644
index 0000000000..c4f8da6a8e
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_tm.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifndef __SXE2_TM_H__
+#define __SXE2_TM_H__
+#include <ethdev_driver.h>
+#include <rte_flow_driver.h>
+#include "sxe2_osal.h"
+
+#define SXE2_TM_MAX_LEVEL 7
+#define SXE2_TM_1L_NODE_NUM_MAX 1
+#define SXE2_TM_2L_NODE_NUM_MAX 8
+#define SXE2_TM_3L_NODE_NUM_MAX 64
+#define SXE2_TM_4L_NODE_NUM_MAX 256
+
+#define SXE2_TM_MAX_CHILDREN_COUNT 8
+
+#define SXE2_TM_WEIGHT_MAX (200)
+#define SXE2_TM_WEIGHT_MIN (1)
+#define SXE2_TM_WEIGHT_SUM (32768)
+
+#define SXE2_HW_RATE_MIN 62500ull
+#define SXE2_HW_RATE_MAX 12500000000ull
+
+#define SXE2_TM_PRIO_MAX (7)
+#define SXE2_TM_PRIO_MIN (0)
+
+enum sxe2_tm_node_type {
+ SXE2_TM_NODE_TYPE_VSIG = 0,
+ SXE2_TM_NODE_TYPE_MID,
+ SXE2_TM_NODE_TYPE_QUEUE,
+ SXE2_TM_NODE_TYPE_INVALID,
+};
+
+struct sxe2_tm_shaper_profile {
+ TAILQ_ENTRY(sxe2_tm_shaper_profile) node;
+ uint32_t id;
+ uint32_t ref_cnt;
+ struct rte_tm_shaper_params profile;
+};
+
+TAILQ_HEAD(sxe2_shaper_profile_list, sxe2_tm_shaper_profile);
+
+struct sxe2_tm_node {
+ uint16_t id;
+ uint16_t teid;
+ uint32_t level;
+ uint32_t child_cnt;
+ uint32_t type;
+ uint16_t hw_weight;
+ uint16_t weight;
+ uint8_t priority;
+ struct sxe2_tm_node *parent;
+ uint8_t index_in_parent;
+ struct sxe2_tm_node *children[SXE2_TM_MAX_CHILDREN_COUNT];
+ struct sxe2_tm_shaper_profile *shaper_profile;
+};
+
+struct sxe2_tm_context {
+ uint32_t tm_layers;
+ uint16_t root_teid;
+ uint8_t root_max_children;
+ uint8_t prio_max;
+ bool committed;
+ struct sxe2_tm_node *root;
+ struct sxe2_shaper_profile_list profile_list;
+};
+
+int32_t sxe2_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg);
+
+int32_t sxe2_tm_init(struct rte_eth_dev *dev);
+
+int32_t sxe2_tm_uninit(struct rte_eth_dev *dev);
+
+#endif /* __SXE2_TM_H__ */
diff --git a/drivers/net/sxe2/sxe2_tx.c b/drivers/net/sxe2/sxe2_tx.c
index a05beb8c7a..a280edc9c5 100644
--- a/drivers/net/sxe2/sxe2_tx.c
+++ b/drivers/net/sxe2/sxe2_tx.c
@@ -304,7 +304,6 @@ int32_t __rte_cold sxe2_tx_queue_setup(struct rte_eth_dev *dev,
}
offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
-
txq = sxe2_tx_queue_alloc(dev, queue_idx, nb_desc, socket_id);
if (txq == NULL) {
PMD_LOG_ERR(TX, "failed to alloc sxe2vf tx queue:%u resource", queue_idx);
--
2.31.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox