* [PATCH v2 1/5] bus/platform: match device by devicetree compatible string
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
@ 2026-08-07 6:48 ` Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 2/5] dma/imx_edma5: introduce eDMA5 dmadev skeleton Gagandeep Singh
` (5 subsequent siblings)
6 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-07 6:48 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh
The platform bus currently matches a DPDK driver to a device only by
comparing the kernel driver name bound to the device against the DPDK
driver name. Devices bound to the generic vfio-platform kernel driver
all report the same driver name, which carries no device identity, so a
specific DPDK driver cannot claim its device this way.
Add of_device_is_compatible(), which reads the device-tree "compatible"
strings exposed under the device's of_node in sysfs and compares them
against a requested string. platform_bus_match() now falls back to
matching the DPDK driver name or its alias against these compatible
strings, letting a driver bind to a device identified by its
device-tree "compatible" value.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
---
drivers/bus/platform/bus_platform_driver.h | 3 +-
drivers/bus/platform/platform.c | 67 +++++++++++++++++++++-
2 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/drivers/bus/platform/bus_platform_driver.h b/drivers/bus/platform/bus_platform_driver.h
index e4dcbacf5e..8eabe4a1fd 100644
--- a/drivers/bus/platform/bus_platform_driver.h
+++ b/drivers/bus/platform/bus_platform_driver.h
@@ -121,7 +121,8 @@ static const char *pdrvinit_ ## nm ## _alias; \
RTE_INIT(pdrvinitfn_ ##nm) \
{ \
(platform_drv).driver.name = RTE_STR(nm); \
- (platform_drv).driver.alias = pdrvinit_ ## nm ## _alias; \
+ if (pdrvinit_ ## nm ## _alias != NULL) \
+ (platform_drv).driver.alias = pdrvinit_ ## nm ## _alias; \
rte_platform_register(&(platform_drv)); \
} \
RTE_PMD_EXPORT_NAME(nm)
diff --git a/drivers/bus/platform/platform.c b/drivers/bus/platform/platform.c
index 90d865a8df..78ca896526 100644
--- a/drivers/bus/platform/platform.c
+++ b/drivers/bus/platform/platform.c
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2023 Marvell.
+ * Copyright 2026 NXP
*/
#include <uapi/linux/vfio.h>
@@ -230,6 +231,57 @@ of_resource_name(const char *dev_name, int index)
return NULL;
}
+/*
+ * Check whether any of the NUL-separated device-tree "compatible" strings
+ * exposed by the platform device matches the given string. A single pair of
+ * enclosing double quotes is stripped from the requested compatible so both
+ * quoted and unquoted spellings match the raw device-tree value.
+ */
+static bool
+of_device_is_compatible(const char *dev_name, const char *compat)
+{
+ char path[PATH_MAX], buf[BUFSIZ] = { };
+ char want[BUFSIZ];
+ const char *s;
+ size_t c_len;
+ FILE *f;
+ size_t len;
+
+ if (compat == NULL)
+ return false;
+
+ /* Copy the requested compatible, dropping a pair of enclosing quotes. */
+ c_len = strlen(compat);
+ if (c_len >= 2 && compat[0] == '"' && compat[c_len - 1] == '"') {
+ c_len -= 2;
+ if (c_len >= sizeof(want))
+ c_len = sizeof(want) - 1;
+ memcpy(want, compat + 1, c_len);
+ want[c_len] = '\0';
+ } else {
+ rte_strscpy(want, compat, sizeof(want));
+ }
+
+ snprintf(path, sizeof(path), PLATFORM_BUS_DEVICES_PATH "/%s/of_node/compatible", dev_name);
+ f = fopen(path, "r");
+ if (f == NULL)
+ return false;
+
+ /* Read the raw contents, preserving embedded NULL separators. */
+ len = fread(buf, 1, sizeof(buf) - 1, f);
+ fclose(f);
+ if (len == 0)
+ return false;
+
+ /* Bound the walk by the read length: some kernels omit the trailing NULL. */
+ for (s = buf; s < buf + len; s += strlen(s) + 1) {
+ if (!strcmp(s, want))
+ return true;
+ }
+
+ return false;
+}
+
static int
device_map_resources(struct rte_platform_device *pdev, unsigned int num)
{
@@ -386,8 +438,21 @@ platform_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
}
/* match by device name */
- if (!strcmp(pdev->name, pdrv->driver.name))
+ if (!strcmp(pdev->name, pdrv->driver.name)) {
match = true;
+ goto out;
+ }
+
+ /*
+ * The generic vfio-platform kernel driver name carries no device
+ * identity, so fall back to matching the device-tree "compatible"
+ * strings against the DPDK driver name or alias.
+ */
+ if (of_device_is_compatible(pdev->name, pdrv->driver.name) ||
+ of_device_is_compatible(pdev->name, pdrv->driver.alias)) {
+ match = true;
+ goto out;
+ }
out:
free(kdrv);
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v2 2/5] dma/imx_edma5: introduce eDMA5 dmadev skeleton
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 1/5] bus/platform: match device by devicetree compatible string Gagandeep Singh
@ 2026-08-07 6:48 ` Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 3/5] dma/imx_edma5: add device configuration Gagandeep Singh
` (4 subsequent siblings)
6 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-07 6:48 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add the skeleton of a dmadev PMD for the NXP i.MX95 eDMA5 (Enhanced
Direct Memory Access Type 5) controller. The eDMA5 exposes 64 channels
with a 64-bit TCD (TCD64) layout.
The controller is probed on the platform bus via the generic
vfio-platform kernel driver and claimed by its device-tree compatible
string fsl,imx95-edma5. This patch adds the register definitions, the
private data structures, probe and remove handling, device-tree
dma-channel-mask parsing to skip channels reserved for other bus
masters, global clock and arbitration setup, and the device information
query. Subsequent patches add configuration, the data path, and
statistics.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
MAINTAINERS | 5 +
doc/guides/dmadevs/imx_edma5.rst | 61 +++++++
doc/guides/dmadevs/index.rst | 1 +
doc/guides/rel_notes/release_26_11.rst | 6 +
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 219 ++++++++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_dmadev.h | 220 +++++++++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_hw.h | 157 ++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_logs.h | 16 ++
drivers/dma/imx_edma5/meson.build | 10 ++
drivers/dma/meson.build | 1 +
10 files changed, 696 insertions(+)
create mode 100644 doc/guides/dmadevs/imx_edma5.rst
create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.c
create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.h
create mode 100644 drivers/dma/imx_edma5/imx_edma5_hw.h
create mode 100644 drivers/dma/imx_edma5/imx_edma5_logs.h
create mode 100644 drivers/dma/imx_edma5/meson.build
diff --git a/MAINTAINERS b/MAINTAINERS
index e99a65d197..b2645ebe2a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1422,6 +1422,11 @@ M: Hemant Agrawal <hemant.agrawal@nxp.com>
F: drivers/dma/dpaa2/
F: doc/guides/dmadevs/dpaa2.rst
+NXP i.MX95 eDMA5
+M: Gagandeep Singh <g.singh@nxp.com>
+M: Prashant Gupta <prashant.gupta_3@nxp.com>
+F: drivers/dma/imx_edma5/
+F: doc/guides/dmadevs/imx_edma5.rst
RegEx Drivers
-------------
diff --git a/doc/guides/dmadevs/imx_edma5.rst b/doc/guides/dmadevs/imx_edma5.rst
new file mode 100644
index 0000000000..55b04dbd35
--- /dev/null
+++ b/doc/guides/dmadevs/imx_edma5.rst
@@ -0,0 +1,61 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2026 NXP
+
+NXP i.MX95 eDMA5 DMA Driver
+===========================
+
+The ``imx_edma5`` DMA driver is a poll-mode driver (PMD) for the NXP i.MX95
+Enhanced Direct Memory Access controller version 5 (eDMA5). It exposes each
+eDMA5 controller instance as a DPDK dmadev device and can be used through the
+generic DMA device (dmadev) API.
+
+The i.MX95 SoC integrates multiple eDMA instances. The eDMA5 instances provide
+64 hardware channels, 64-bit addressing and a 64-byte Transfer Control
+Descriptor (TCD64). This driver targets the eDMA5 instances only (device tree
+compatible ``fsl,imx95-edma5``).
+
+Supported Features
+------------------
+
+- Memory-to-memory copy (``RTE_DMA_DIR_MEM_TO_MEM``).
+- Single-operation copy (``rte_dma_copy``).
+- Scatter-gather copy (``rte_dma_copy_sg``) for equal-length source and
+ destination segment lists.
+- Per virtual channel statistics.
+
+Each configured virtual channel (vchan) is mapped one-to-one onto a hardware
+eDMA5 channel. Software-initiated (SWSTART) single-block transfers are
+programmed into the per-channel TCD and completion is detected by polling the
+TCD DONE status.
+
+Prerequisites
+-------------
+
+The eDMA5 register window is memory-mapped into the userspace process through
+the DPDK platform bus using the Linux ``vfio-platform`` mechanism. The device
+tree node targeted by this driver must be released from the kernel ``fsl-edma``
+driver (its status set to ``disabled`` or the node unbound) before it can be
+used by DPDK.
+
+Bind the platform device to ``vfio-platform``, for example::
+
+ echo vfio-platform > /sys/bus/platform/devices/<node>/driver_override
+ echo <node> > /sys/bus/platform/drivers/vfio-platform/bind
+
+where ``<node>`` is the platform device name of the eDMA5 instance (for
+example ``42000000.dma-controller``).
+
+Compilation
+-----------
+
+The driver is built as part of the standard DPDK meson build on Linux targets.
+No extra configuration option is required.
+
+Limitations
+-----------
+
+- Only the memory-to-memory transfer direction is supported.
+- The scatter-gather path programs one hardware transfer per equal-sized
+ source/destination segment pair; full TCD scatter-gather linking is not yet
+ implemented.
+- The driver operates in poll mode only; completion interrupts are not used.
diff --git a/doc/guides/dmadevs/index.rst b/doc/guides/dmadevs/index.rst
index 56beb1733f..47c52a9220 100644
--- a/doc/guides/dmadevs/index.rst
+++ b/doc/guides/dmadevs/index.rst
@@ -17,5 +17,6 @@ an application through DMA API.
hisi_acc
hisi_pciep
idxd
+ imx_edma5
ioat
odm
diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst
index c8cc86295d..8f634795cf 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -55,6 +55,12 @@ New Features
Also, make sure to start the actual text at the margin.
=======================================================
+* **Added NXP i.MX95 eDMA5 DMA driver.**
+
+ Added the ``imx_edma5`` DMA driver for the NXP i.MX95 eDMA5 controller.
+ The driver exposes each eDMA5 instance as a DPDK dmadev device and
+ supports memory-to-memory copy and scatter-gather copy operations.
+
Removed Items
-------------
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
new file mode 100644
index 0000000000..f1c65a474a
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -0,0 +1,219 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+/*
+ * NXP i.MX95 eDMA5 dmadev driver.
+ *
+ * Exposes an eDMA5 controller instance as a DPDK dmadev. Each configured
+ * virtual channel maps 1:1 onto a hardware eDMA5 channel. Memory-to-memory
+ * copy and scatter-gather copy are supported using software-initiated
+ * single-block transfers programmed through the per-channel 64-bit TCD.
+ *
+ * The device is bound to userspace through the platform bus (vfio-platform);
+ * the device tree node must be released from the kernel fsl-edma driver before
+ * it can be used here.
+ */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <bus_platform_driver.h>
+#include <rte_bitops.h>
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_dmadev_pmd.h>
+#include <rte_malloc.h>
+
+#include "imx_edma5_dmadev.h"
+
+#include "imx_edma5_logs.h"
+
+RTE_LOG_REGISTER_DEFAULT(imx_edma5_logtype, INFO);
+
+/* Compatible string for i.MX95 eDMA5 device tree nodes. */
+#define IMX_EDMA5_COMPAT "fsl,imx95-edma5"
+
+/* sysfs base path for platform devices' device-tree nodes. */
+#define IMX_EDMA5_SYSFS_DEVICES "/sys/bus/platform/devices"
+
+/*
+ * Read the device-tree "dma-channel-mask" property and return the combined
+ * 64-bit mask of reserved hardware channels. The property is a raw binary file
+ * of big-endian 32-bit cells: cell[0] covers channels 0-31, cell[1] channels
+ * 32-63. A set bit marks a channel owned by another bus master (e.g. SCMI
+ * firmware) that must not be accessed. Returns 0 if the property is absent.
+ */
+static uint64_t
+imx_edma5_read_channel_mask(const char *dev_name)
+{
+ char path[PATH_MAX];
+ uint32_t cells[2] = { 0, 0 };
+ uint64_t mask = 0;
+ size_t n;
+ FILE *f;
+
+ snprintf(path, sizeof(path),
+ IMX_EDMA5_SYSFS_DEVICES "/%s/of_node/dma-channel-mask",
+ dev_name);
+
+ f = fopen(path, "rb");
+ if (f == NULL)
+ return 0;
+
+ n = fread(cells, 1, sizeof(cells), f);
+ fclose(f);
+
+ /* Device-tree cells are big-endian regardless of CPU endianness. */
+ if (n >= sizeof(uint32_t))
+ mask |= rte_be_to_cpu_32(cells[0]);
+ if (n >= 2 * sizeof(uint32_t))
+ mask |= (uint64_t)rte_be_to_cpu_32(cells[1]) << 32;
+
+ return mask;
+}
+
+static int
+imx_edma5_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
+ uint32_t info_sz)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ RTE_SET_USED(info_sz);
+
+ dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
+ RTE_DMA_CAPA_OPS_COPY |
+ RTE_DMA_CAPA_OPS_COPY_SG;
+ dev_info->max_vchans = ed->max_vchans;
+ dev_info->max_desc = IMX_EDMA5_MAX_DESC;
+ dev_info->min_desc = IMX_EDMA5_MIN_DESC;
+ dev_info->max_sges = IMX_EDMA5_MAX_SGES;
+
+ return 0;
+}
+
+static const struct rte_dma_dev_ops imx_edma5_ops = {
+ .dev_info_get = imx_edma5_info_get,
+};
+
+static int
+imx_edma5_probe(struct rte_platform_device *pdev)
+{
+ struct rte_platform_resource *res;
+ struct imx_edma5_dev *ed;
+ struct rte_dma_dev *dev;
+ const char *name;
+
+ name = pdev->name;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ IMX_EDMA5_LOG(ERR, "Secondary process not supported for %s",
+ name);
+ return -ENOTSUP;
+ }
+
+ if (pdev->num_resource < 1 || pdev->resource == NULL) {
+ IMX_EDMA5_LOG(ERR, "No MMIO resource for %s", name);
+ return -EINVAL;
+ }
+ res = &pdev->resource[0];
+ if (res->mem.addr == NULL) {
+ IMX_EDMA5_LOG(ERR, "MMIO resource not mapped for %s", name);
+ return -EINVAL;
+ }
+
+ dev = rte_dma_pmd_allocate(name, rte_socket_id(),
+ sizeof(struct imx_edma5_dev));
+ if (dev == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to allocate dmadev for %s", name);
+ return -ENOMEM;
+ }
+
+ dev->device = &pdev->device;
+ dev->dev_ops = &imx_edma5_ops;
+
+ ed = dev->data->dev_private;
+ ed->reg_base = res->mem.addr;
+ ed->reg_size = res->mem.len;
+ ed->dev_id = dev->data->dev_id;
+
+ /*
+ * Build a map from usable vchan index to hardware channel index,
+ * skipping channels reserved for other bus masters by the device-tree
+ * "dma-channel-mask" (accessing them external-aborts).
+ */
+ ed->masked_channels = imx_edma5_read_channel_mask(name);
+ ed->nb_channels = 0;
+ {
+ uint16_t hw;
+
+ for (hw = 0; hw < IMX_EDMA5_MAX_CHANNELS; hw++) {
+ if (ed->masked_channels & (RTE_BIT64(hw)))
+ continue;
+ ed->chan_map[ed->nb_channels++] = hw;
+ }
+ }
+ ed->max_vchans = ed->nb_channels;
+
+ if (ed->nb_channels == 0) {
+ IMX_EDMA5_LOG(ERR,
+ "No usable eDMA5 channels for %s (mask 0x%" PRIx64 ")",
+ name, ed->masked_channels);
+ rte_dma_pmd_release(name);
+ return -ENODEV;
+ }
+
+ dev->state = RTE_DMA_DEV_READY;
+
+ /*
+ * Set MP_CSR.GCLC (Global Clock Control) before any per-channel register
+ * is touched: the per-channel windows are individually clock-gated and
+ * external-abort when accessed with GCLC clear. Also enable round-robin
+ * arbitration (ERCA). Use read-modify-write to preserve reset defaults.
+ */
+ {
+ uint32_t mp_csr = imx_edma5_read32(ed->reg_base, IMX_EDMA5_MP_CSR);
+
+ mp_csr |= IMX_EDMA5_MP_CSR_GCLC | IMX_EDMA5_MP_CSR_ERCA;
+ imx_edma5_write32(ed->reg_base, IMX_EDMA5_MP_CSR, mp_csr);
+ }
+
+ IMX_EDMA5_LOG(INFO, "Probed i.MX95 eDMA5 dmadev %s (%u channels)",
+ name, ed->nb_channels);
+
+ return 0;
+}
+
+static int
+imx_edma5_remove(struct rte_platform_device *pdev)
+{
+ const char *name = pdev->name;
+
+ return rte_dma_pmd_release(name);
+}
+
+static struct rte_platform_driver imx_edma5_pmd_drv = {
+ /*
+ * Set the DT compatible string as the driver alias. The platform bus
+ * match logic compares this against the device-tree "compatible" sysfs
+ * strings of each platform device when the kernel driver name (always
+ * "vfio-platform") does not uniquely identify the device.
+ */
+ .driver = {
+ .alias = IMX_EDMA5_COMPAT,
+ },
+ .probe = imx_edma5_probe,
+ .remove = imx_edma5_remove,
+ /*
+ * The eDMA5 is programmed with the IOVA of the buffers, so it works in
+ * both IOVA=VA and IOVA=PA modes; no IOVA-as-VA requirement is forced.
+ */
+ .drv_flags = 0,
+};
+
+RTE_PMD_REGISTER_PLATFORM(dma_imx_edma5, imx_edma5_pmd_drv);
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.h b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
new file mode 100644
index 0000000000..3da7957457
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
@@ -0,0 +1,220 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+#ifndef IMX_EDMA5_DMADEV_H
+#define IMX_EDMA5_DMADEV_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_dmadev.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+
+#include "imx_edma5_hw.h"
+
+/*
+ * CPU data-cache maintenance for the non-coherent eDMA5 master: clean the
+ * source before the transfer and clean+invalidate the destination after it.
+ * DC IVAC (invalidate-only) is EL1-only and faults from userspace, so DC CIVAC
+ * is used. No-ops on non-arm64 builds.
+ */
+static inline void
+imx_edma5_dcbf(void *p)
+{
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dc cvac, %0" : : "r"(p) : "memory");
+#else
+ RTE_SET_USED(p);
+#endif
+}
+
+static inline void
+imx_edma5_dccivac(void *p)
+{
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dc civac, %0" : : "r"(p) : "memory");
+#else
+ RTE_SET_USED(p);
+#endif
+}
+
+/* Clean a VA range to the Point of Coherency. */
+static inline void
+imx_edma5_cache_clean(void *addr, size_t len)
+{
+ uintptr_t p = (uintptr_t)addr & ~(uintptr_t)(RTE_CACHE_LINE_SIZE - 1);
+ uintptr_t end = (uintptr_t)addr + len;
+
+ for (; p < end; p += RTE_CACHE_LINE_SIZE)
+ imx_edma5_dcbf((void *)p);
+ /*
+ * DSB ensures all DC CVAC instructions have completed to the Point of
+ * Coherency before the eDMA5 is allowed to read the source data.
+ */
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dsb sy" ::: "memory");
+#endif
+}
+
+/* Clean+invalidate a VA range to the Point of Coherency. */
+static inline void
+imx_edma5_cache_inval(void *addr, size_t len)
+{
+ uintptr_t p = (uintptr_t)addr & ~(uintptr_t)(RTE_CACHE_LINE_SIZE - 1);
+ uintptr_t end = (uintptr_t)addr + len;
+
+ for (; p < end; p += RTE_CACHE_LINE_SIZE)
+ imx_edma5_dccivac((void *)p);
+ /*
+ * DSB ensures all DC CIVAC instructions have completed to the Point of
+ * Coherency before the CPU reads the destination data written by eDMA5.
+ */
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dsb sy" ::: "memory");
+#endif
+}
+
+/*
+ * NBYTES field maximum: bits 31:30 of TCD_NBYTES are SMLOE/DMLOE control bits
+ * when the minor loop offset feature is enabled. To avoid accidentally setting
+ * those bits, cap every single-block transfer at the 30-bit maximum count.
+ */
+#define IMX_EDMA5_MAX_NBYTES ((uint32_t)((1u << 30) - 1))
+
+/* Maximum scatter-gather segments per copy_sg request. */
+#define IMX_EDMA5_MAX_SGES 16
+
+/*
+ * In-memory TCD64 descriptors reserved per job slot for scatter-gather.
+ * Splitting an asymmetric src/dst segment list at the union of both sets of
+ * boundaries yields at most n_src + n_dst - 1 sub-transfers, so with up to
+ * IMX_EDMA5_MAX_SGES segments per side the worst case fits in this bound.
+ */
+#define IMX_EDMA5_SG_TCD_PER_JOB (2 * IMX_EDMA5_MAX_SGES)
+
+/* Software job ring size per virtual channel (power of two). */
+#define IMX_EDMA5_MAX_DESC 4096
+#define IMX_EDMA5_MIN_DESC 32
+
+/* Per in-flight job bookkeeping. */
+struct imx_edma5_job {
+ uint16_t ridx; /* ring index returned to application */
+ uint8_t submitted; /* job has been started on hardware */
+ uint8_t done; /* job completed */
+ uint8_t error; /* job completed with error */
+ /*
+ * Destination VA and byte count of the copy, used to invalidate the
+ * destination cache lines on completion (non-coherent eDMA master).
+ * NULL if the VA could not be resolved. Unused for SG jobs (nb_sg > 0).
+ */
+ void *dst_va;
+ uint32_t len;
+ /*
+ * Source/destination IOVAs of a plain single-block copy, recorded at
+ * enqueue time and used to program the TCD at submit time (the eDMA5 has
+ * a single register TCD shared by all jobs). Unused for SG jobs.
+ */
+ rte_iova_t src_iova;
+ rte_iova_t dst_iova;
+ /*
+ * Scatter-gather state. nb_sg is the segment count (0 for a plain copy);
+ * sg_tcd points at this job's slice of the vchan's in-memory TCD pool.
+ */
+ uint16_t nb_sg;
+ struct imx_edma5_hw_tcd64 *sg_tcd;
+};
+
+/* A virtual channel maps 1:1 onto a single eDMA5 hardware channel. */
+struct imx_edma5_vchan {
+ uint8_t *ch_regs; /* channel register window base */
+ uint8_t *tcd_regs; /* channel TCD base (ch_regs + TCD_OFF) */
+ uint32_t hw_chan; /* hardware channel index */
+
+ struct imx_edma5_job *jobs; /* software job ring */
+ uint16_t nb_desc; /* size of job ring (power of two) */
+ uint16_t desc_mask; /* nb_desc - 1 */
+
+ /*
+ * Pool of in-memory TCD64 descriptors for scatter-gather, sized
+ * nb_desc * IMX_EDMA5_SG_TCD_PER_JOB. Each job slot owns a contiguous
+ * slice of IMX_EDMA5_SG_TCD_PER_JOB descriptors. sg_tcd_iova is the
+ * pool base IOVA.
+ */
+ struct imx_edma5_hw_tcd64 *sg_tcd_pool;
+ rte_iova_t sg_tcd_iova;
+
+ uint16_t head; /* next slot to enqueue */
+ uint16_t tail; /* next slot to reap */
+ uint16_t nb_enqueued; /* outstanding jobs in ring (unreaped) */
+ uint16_t ridx; /* running ring index counter */
+ uint16_t last_idx; /* last completed ring index */
+
+ uint64_t submitted_count;
+ uint64_t completed_count;
+ uint64_t errors_count;
+
+ bool configured;
+};
+
+/* Per-device (per eDMA5 instance) private data. */
+struct imx_edma5_dev {
+ uint8_t *reg_base; /* mapped register window base */
+ uint64_t reg_size; /* mapped register window length */
+
+ uint16_t nb_channels; /* channels available on this instance */
+ uint16_t max_vchans; /* channels usable as dmadev vchans */
+
+ /*
+ * Bitmask of hardware channels reserved for other bus masters (from the
+ * device-tree "dma-channel-mask"). A set bit marks a channel this driver
+ * must not touch; accessing it faults with a bus external abort.
+ */
+ uint64_t masked_channels;
+ /* Map of usable dmadev vchan index -> hardware channel index. */
+ uint16_t chan_map[IMX_EDMA5_MAX_CHANNELS];
+
+ struct imx_edma5_vchan *vchans; /* array of vchan states */
+ uint16_t nb_vchans; /* number of configured vchans */
+
+ int16_t dev_id; /* dmadev id */
+};
+
+/* MMIO helpers (little-endian device). */
+static inline uint32_t
+imx_edma5_read32(const uint8_t *base, uint32_t off)
+{
+ return rte_le_to_cpu_32(rte_read32(base + off));
+}
+
+static inline void
+imx_edma5_write32(uint8_t *base, uint32_t off, uint32_t val)
+{
+ rte_write32(rte_cpu_to_le_32(val), base + off);
+}
+
+static inline uint16_t
+imx_edma5_read16(const uint8_t *base, uint32_t off)
+{
+ return rte_le_to_cpu_16(rte_read16(base + off));
+}
+
+static inline void
+imx_edma5_write16(uint8_t *base, uint32_t off, uint16_t val)
+{
+ rte_write16(rte_cpu_to_le_16(val), base + off);
+}
+
+static inline void
+imx_edma5_write64(uint8_t *base, uint32_t off, uint64_t val)
+{
+ /* Write the 64-bit field as two 32-bit accesses (order not significant). */
+ imx_edma5_write32(base, off, (uint32_t)(val & 0xFFFFFFFFu));
+ imx_edma5_write32(base, off + 4, (uint32_t)(val >> 32));
+}
+
+#endif /* IMX_EDMA5_DMADEV_H */
diff --git a/drivers/dma/imx_edma5/imx_edma5_hw.h b/drivers/dma/imx_edma5/imx_edma5_hw.h
new file mode 100644
index 0000000000..627cd495b7
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_hw.h
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+/*
+ * Hardware register definitions for the NXP i.MX95 eDMA5 controller.
+ * The block has a Management Page (MP) region followed by per-channel register
+ * windows (base offset 0x10000, stride 0x8000, 64 channels, 64-bit TCD64).
+ * All register accesses are little-endian.
+ */
+
+#ifndef IMX_EDMA5_HW_H
+#define IMX_EDMA5_HW_H
+
+#include <stdint.h>
+
+/* Number of DMA channels implemented on i.MX95 eDMA5. */
+#define IMX_EDMA5_MAX_CHANNELS 64
+
+/* Per-channel register window geometry. */
+#define IMX_EDMA5_CHAN_BASE_OFF 0x10000u
+#define IMX_EDMA5_CHAN_STRIDE 0x8000u
+
+/*
+ * Management Page (MP) registers (offsets from register window base).
+ * Only the fields used by this driver are documented here.
+ */
+#define IMX_EDMA5_MP_CSR 0x0000u /* Management control */
+#define IMX_EDMA5_MP_ES 0x0004u /* Management error status */
+#define IMX_EDMA5_MP_INT_LOW 0x0008u /* Interrupt request (chan 0-31) */
+#define IMX_EDMA5_MP_INT_HIGH 0x000Cu /* Interrupt request (chan 32-63) */
+#define IMX_EDMA5_MP_HRS_LOW 0x0010u /* Hardware request status low */
+#define IMX_EDMA5_MP_HRS_HIGH 0x0014u /* Hardware request status high */
+
+/* MP_CSR bit fields. */
+#define IMX_EDMA5_MP_CSR_EDBG (1u << 1) /* Enable debug */
+#define IMX_EDMA5_MP_CSR_ERCA (1u << 2) /* Enable round-robin arb */
+#define IMX_EDMA5_MP_CSR_HAE (1u << 4) /* Halt after error */
+#define IMX_EDMA5_MP_CSR_GCLC (1u << 6) /* Global clock control */
+#define IMX_EDMA5_MP_CSR_GMRC (1u << 7) /* Global master ID replic */
+
+/* MP_ES: valid bit indicates a logged error is present. */
+#define IMX_EDMA5_MP_ES_VLD (1u << 31)
+
+/*
+ * Per-channel control registers (offsets from a channel window base).
+ * Layout matches the eDMA4/eDMA5 fsl_edma3_ch_reg structure.
+ */
+#define IMX_EDMA5_CH_CSR 0x00u /* Channel control/status */
+#define IMX_EDMA5_CH_ES 0x04u /* Channel error status */
+#define IMX_EDMA5_CH_INT 0x08u /* Channel interrupt status */
+#define IMX_EDMA5_CH_SBR 0x0Cu /* System bus register */
+#define IMX_EDMA5_CH_PRI 0x10u /* Channel priority */
+#define IMX_EDMA5_CH_MUX 0x14u /* Channel multiplexor (source) */
+#define IMX_EDMA5_CH_MATTR 0x18u /* Memory attributes */
+
+/* Channel window offset of the TCD (Transfer Control Descriptor). */
+#define IMX_EDMA5_CH_TCD_OFF 0x20u
+
+/* CH_CSR bit fields. */
+#define IMX_EDMA5_CH_CSR_ERQ (1u << 0) /* Enable hardware request */
+#define IMX_EDMA5_CH_CSR_EARQ (1u << 1) /* Enable async hw request */
+#define IMX_EDMA5_CH_CSR_EEI (1u << 2) /* Enable error interrupt */
+#define IMX_EDMA5_CH_CSR_DONE (1u << 30) /* Channel done (w1c) */
+#define IMX_EDMA5_CH_CSR_ACTIVE (1u << 31) /* Channel active */
+
+/* CH_ES: valid bit indicates a logged channel error. */
+#define IMX_EDMA5_CH_ES_ERR (1u << 31)
+
+/* CH_INT: write 1 to clear the channel interrupt request. */
+#define IMX_EDMA5_CH_INT_INT (1u << 0)
+
+/* CH_SBR: read/write privileged/secure attributes for bus mastering. */
+#define IMX_EDMA5_CH_SBR_RD (1u << 22)
+#define IMX_EDMA5_CH_SBR_WR (1u << 21)
+
+/*
+ * CH_MATTR: AXI cache attributes and shareability domain for the transactions
+ * this channel issues. RCACHE/WCACHE are 4-bit cache-attribute fields;
+ * RDOMAINS/WDOMAINS select the shareability domain (2 = inner shareable).
+ */
+#define IMX_EDMA5_CH_MATTR_RCACHE (0xFu << 0)
+#define IMX_EDMA5_CH_MATTR_WCACHE (0xFu << 4)
+#define IMX_EDMA5_CH_MATTR_RDOMAINS(x) (((x) & 0x3u) << 8)
+#define IMX_EDMA5_CH_MATTR_WDOMAINS(x) (((x) & 0x3u) << 10)
+#define IMX_EDMA5_CH_MATTR_COHERENT (IMX_EDMA5_CH_MATTR_RCACHE | \
+ IMX_EDMA5_CH_MATTR_WCACHE | \
+ IMX_EDMA5_CH_MATTR_RDOMAINS(2) | \
+ IMX_EDMA5_CH_MATTR_WDOMAINS(2))
+
+/*
+ * TCD64 field offsets, relative to the channel TCD base
+ * (channel window base + IMX_EDMA5_CH_TCD_OFF).
+ */
+#define IMX_EDMA5_TCD_SADDR 0x00u
+#define IMX_EDMA5_TCD_SOFF 0x08u
+#define IMX_EDMA5_TCD_ATTR 0x0Au
+#define IMX_EDMA5_TCD_NBYTES 0x0Cu
+#define IMX_EDMA5_TCD_SLAST 0x10u
+#define IMX_EDMA5_TCD_DADDR 0x18u
+#define IMX_EDMA5_TCD_DLAST_SGA 0x20u
+#define IMX_EDMA5_TCD_DOFF 0x28u
+#define IMX_EDMA5_TCD_CITER 0x2Au
+#define IMX_EDMA5_TCD_CSR 0x2Cu
+#define IMX_EDMA5_TCD_BITER 0x2Eu
+
+/* TCD ATTR sub-fields: transfer size is encoded as log2(bytes). GET_* extract. */
+#define IMX_EDMA5_TCD_ATTR_DSIZE(x) (((x) & 0x7u))
+#define IMX_EDMA5_TCD_ATTR_SSIZE(x) (((x) & 0x7u) << 8)
+#define IMX_EDMA5_TCD_ATTR_GET_DSIZE(x) ((x) & 0x7u)
+#define IMX_EDMA5_TCD_ATTR_GET_SSIZE(x) (((x) >> 8) & 0x7u)
+
+/* Transfer size encodings for ATTR SSIZE/DSIZE (log2 of bytes). */
+#define IMX_EDMA5_TCD_SIZE_1B 0u
+#define IMX_EDMA5_TCD_SIZE_2B 1u
+#define IMX_EDMA5_TCD_SIZE_4B 2u
+#define IMX_EDMA5_TCD_SIZE_8B 3u
+#define IMX_EDMA5_TCD_SIZE_16B 4u
+#define IMX_EDMA5_TCD_SIZE_32B 5u
+#define IMX_EDMA5_TCD_SIZE_64B 6u
+
+/* Major iteration count field mask (15-bit CITER/BITER). */
+#define IMX_EDMA5_TCD_ITER_MASK 0x7FFFu
+
+/* TCD CSR bit fields. */
+#define IMX_EDMA5_TCD_CSR_START (1u << 0) /* Software start */
+#define IMX_EDMA5_TCD_CSR_INT_MAJOR (1u << 1) /* Interrupt on major done */
+#define IMX_EDMA5_TCD_CSR_INT_HALF (1u << 2) /* Interrupt on half done */
+#define IMX_EDMA5_TCD_CSR_D_REQ (1u << 3) /* Disable request on done */
+#define IMX_EDMA5_TCD_CSR_E_SG (1u << 4) /* Enable scatter-gather */
+#define IMX_EDMA5_TCD_CSR_E_LINK (1u << 5) /* Enable channel linking */
+#define IMX_EDMA5_TCD_CSR_ACTIVE (1u << 6) /* Channel active */
+#define IMX_EDMA5_TCD_CSR_DONE (1u << 7) /* Channel done */
+
+/*
+ * In-memory 64-bit Transfer Control Descriptor. The field order and offsets
+ * match the register TCD64 layout above; all fields are little-endian. The
+ * descriptor must be 32-byte aligned and, on this non-coherent SoC, cleaned
+ * from the CPU cache before the transfer is started.
+ */
+struct __rte_aligned(32) imx_edma5_hw_tcd64 {
+ uint64_t saddr; /* 0x00 source address */
+ uint16_t soff; /* 0x08 source offset */
+ uint16_t attr; /* 0x0A transfer attributes */
+ uint32_t nbytes; /* 0x0C minor loop byte count */
+ uint64_t slast; /* 0x10 last source adjustment */
+ uint64_t daddr; /* 0x18 destination address */
+ uint64_t dlast_sga; /* 0x20 next TCD address (scatter-gather) */
+ uint16_t doff; /* 0x28 destination offset */
+ uint16_t citer; /* 0x2A current major iteration count */
+ uint16_t csr; /* 0x2C control and status */
+ uint16_t biter; /* 0x2E starting major iteration count */
+ /* Pad to 64 bytes total; the type is 32-byte aligned for TCD fetches. */
+ uint8_t reserved[16];
+};
+
+#endif /* IMX_EDMA5_HW_H */
diff --git a/drivers/dma/imx_edma5/imx_edma5_logs.h b/drivers/dma/imx_edma5/imx_edma5_logs.h
new file mode 100644
index 0000000000..e73e496bb4
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_logs.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+#ifndef IMX_EDMA5_LOGS_H
+#define IMX_EDMA5_LOGS_H
+
+#include <rte_log.h>
+
+extern int imx_edma5_logtype;
+#define RTE_LOGTYPE_IMX_EDMA5 imx_edma5_logtype
+
+#define IMX_EDMA5_LOG(level, ...) \
+ RTE_LOG_LINE_PREFIX(level, IMX_EDMA5, "%s(): ", __func__, __VA_ARGS__)
+
+#endif /* IMX_EDMA5_LOGS_H */
diff --git a/drivers/dma/imx_edma5/meson.build b/drivers/dma/imx_edma5/meson.build
new file mode 100644
index 0000000000..e4f5d4e0bb
--- /dev/null
+++ b/drivers/dma/imx_edma5/meson.build
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2026 NXP
+
+if not is_linux
+ build = false
+ reason = 'only supported on linux'
+endif
+
+deps += ['dmadev', 'bus_platform']
+sources = files('imx_edma5_dmadev.c')
diff --git a/drivers/dma/meson.build b/drivers/dma/meson.build
index e0d94db967..dd00b1dae9 100644
--- a/drivers/dma/meson.build
+++ b/drivers/dma/meson.build
@@ -8,6 +8,7 @@ drivers = [
'hisi_acc',
'hisi_pciep',
'idxd',
+ 'imx_edma5',
'ioat',
'odm',
'skeleton',
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v2 3/5] dma/imx_edma5: add device configuration
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 1/5] bus/platform: match device by devicetree compatible string Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 2/5] dma/imx_edma5: introduce eDMA5 dmadev skeleton Gagandeep Singh
@ 2026-08-07 6:48 ` Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 4/5] dma/imx_edma5: add data path Gagandeep Singh
` (3 subsequent siblings)
6 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-07 6:48 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add the configuration and lifecycle operations for the i.MX95 eDMA5
dmadev: device configure, virtual channel setup, start, stop and close.
Each virtual channel maps 1:1 onto a usable hardware channel and owns a
software job ring plus a pool of in-memory TCD64 descriptors for
scatter-gather. Channel setup validates the requested direction and ring
size, allocates these rings, and resets the hardware channel to a known
idle state. Start re-arms global clock and arbitration and clears the
per-channel bookkeeping; stop resets every configured channel; close
frees the per-channel resources.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 246 +++++++++++++++++++++++
1 file changed, 246 insertions(+)
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
index f1c65a474a..fe5539b612 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -78,6 +78,14 @@ imx_edma5_read_channel_mask(const char *dev_name)
return mask;
}
+/* Return the register base of hardware channel n. */
+static inline uint8_t *
+imx_edma5_chan_base(struct imx_edma5_dev *ed, uint32_t chan)
+{
+ return ed->reg_base + IMX_EDMA5_CHAN_BASE_OFF +
+ (size_t)chan * IMX_EDMA5_CHAN_STRIDE;
+}
+
static int
imx_edma5_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
uint32_t info_sz)
@@ -97,8 +105,246 @@ imx_edma5_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
return 0;
}
+static void imx_edma5_reset_hw_chan(struct imx_edma5_vchan *vc);
+
+static int
+imx_edma5_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf,
+ uint32_t conf_sz)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ RTE_SET_USED(conf_sz);
+
+ if (conf->nb_vchans == 0 || conf->nb_vchans > ed->max_vchans) {
+ IMX_EDMA5_LOG(ERR, "Invalid nb_vchans %u (max %u)",
+ conf->nb_vchans, ed->max_vchans);
+ return -EINVAL;
+ }
+
+ if (ed->vchans == NULL) {
+ ed->vchans = rte_zmalloc_socket("imx_edma5_vchans",
+ ed->max_vchans * sizeof(struct imx_edma5_vchan),
+ RTE_CACHE_LINE_SIZE, dev->data->numa_node);
+ if (ed->vchans == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to alloc vchan array");
+ return -ENOMEM;
+ }
+ } else {
+ /* Reconfigure: reset and free every previously configured channel. */
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ if (!vc->configured)
+ continue;
+ imx_edma5_reset_hw_chan(vc);
+ rte_free(vc->jobs);
+ rte_free(vc->sg_tcd_pool);
+ memset(vc, 0, sizeof(*vc));
+ }
+ }
+
+ ed->nb_vchans = conf->nb_vchans;
+
+ return 0;
+}
+
+/* Reset a hardware channel to a known idle state. */
+static void
+imx_edma5_reset_hw_chan(struct imx_edma5_vchan *vc)
+{
+ uint8_t *ch = vc->ch_regs;
+ uint8_t *tcd = vc->tcd_regs;
+ uint32_t sbr;
+
+ /*
+ * Disable hardware request and clear latched completion state.
+ * CH_CSR.DONE is write-1-to-clear, so write the DONE bit to clear any
+ * stale completion (e.g. left by the bootloader/kernel driver) while
+ * leaving all other control bits disabled.
+ */
+ imx_edma5_write32(ch, IMX_EDMA5_CH_CSR, IMX_EDMA5_CH_CSR_DONE);
+ imx_edma5_write32(ch, IMX_EDMA5_CH_ES, IMX_EDMA5_CH_ES_ERR);
+ imx_edma5_write32(ch, IMX_EDMA5_CH_INT, IMX_EDMA5_CH_INT_INT);
+
+ /*
+ * Enable the read/write attribute bits in the System Bus Register with a
+ * read-modify-write. The security/privilege attribute bits carried here
+ * come up with a valid reset default that the bus fabric (XRDC) checks
+ * and that must be preserved; a blind write of just RD|WR would clear
+ * them and make the fabric reject the eDMA master transaction.
+ */
+ sbr = imx_edma5_read32(ch, IMX_EDMA5_CH_SBR);
+ sbr |= IMX_EDMA5_CH_SBR_RD | IMX_EDMA5_CH_SBR_WR;
+ imx_edma5_write32(ch, IMX_EDMA5_CH_SBR, sbr);
+
+ /*
+ * Leave CH_MATTR at its power-on reset value. The eDMA5 is a
+ * non-coherent bus master; cache coherency is maintained by the driver
+ * via explicit DC CVAC/CIVAC cache maintenance, not by AXI snooping.
+ */
+
+ /* Clear the TCD control/status so the channel is idle. */
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CSR, 0);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CITER, 0);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_BITER, 0);
+}
+
+static int
+imx_edma5_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
+ const struct rte_dma_vchan_conf *conf,
+ uint32_t conf_sz)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ struct imx_edma5_vchan *vc;
+
+ RTE_SET_USED(conf_sz);
+
+ if (vchan >= ed->nb_vchans) {
+ IMX_EDMA5_LOG(ERR, "vchan %u out of range", vchan);
+ return -EINVAL;
+ }
+
+ if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM) {
+ IMX_EDMA5_LOG(ERR, "Only mem-to-mem direction supported");
+ return -EINVAL;
+ }
+
+ if (!rte_is_power_of_2(conf->nb_desc) ||
+ conf->nb_desc < IMX_EDMA5_MIN_DESC ||
+ conf->nb_desc > IMX_EDMA5_MAX_DESC) {
+ IMX_EDMA5_LOG(ERR, "nb_desc must be power of 2 in [%u..%u]",
+ IMX_EDMA5_MIN_DESC, IMX_EDMA5_MAX_DESC);
+ return -EINVAL;
+ }
+
+ vc = &ed->vchans[vchan];
+
+ /* Free previous rings if this vchan is being reconfigured. */
+ rte_free(vc->jobs);
+ rte_free(vc->sg_tcd_pool);
+ memset(vc, 0, sizeof(*vc));
+
+ /*
+ * Map this vchan onto a usable hardware channel. chan_map[] skips
+ * channels reserved by "dma-channel-mask" (channels 0 and 1 on i.MX95).
+ */
+ vc->hw_chan = ed->chan_map[vchan];
+ vc->ch_regs = imx_edma5_chan_base(ed, vc->hw_chan);
+ vc->tcd_regs = vc->ch_regs + IMX_EDMA5_CH_TCD_OFF;
+ vc->nb_desc = conf->nb_desc;
+ vc->desc_mask = conf->nb_desc - 1;
+
+ vc->jobs = rte_zmalloc_socket("imx_edma5_jobs",
+ vc->nb_desc * sizeof(struct imx_edma5_job),
+ RTE_CACHE_LINE_SIZE, dev->data->numa_node);
+ if (vc->jobs == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to alloc job ring for vchan %u",
+ vchan);
+ return -ENOMEM;
+ }
+
+ /* One IMX_EDMA5_SG_TCD_PER_JOB descriptor slice per job ring slot. */
+ vc->sg_tcd_pool = rte_zmalloc_socket("imx_edma5_sgtcd",
+ (size_t)vc->nb_desc * IMX_EDMA5_SG_TCD_PER_JOB *
+ sizeof(struct imx_edma5_hw_tcd64),
+ RTE_CACHE_LINE_SIZE, dev->data->numa_node);
+ if (vc->sg_tcd_pool == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to alloc SG TCD pool for vchan %u",
+ vchan);
+ rte_free(vc->jobs);
+ vc->jobs = NULL;
+ return -ENOMEM;
+ }
+ vc->sg_tcd_iova = rte_malloc_virt2iova(vc->sg_tcd_pool);
+
+ imx_edma5_reset_hw_chan(vc);
+ vc->configured = true;
+
+ return 0;
+}
+
+static int
+imx_edma5_start(struct rte_dma_dev *dev)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint32_t mp_csr;
+ uint16_t i;
+
+ /*
+ * Enable round-robin arbitration with a read-modify-write so GCLC (set
+ * in probe) is preserved; clearing GCLC would re-gate the per-channel
+ * clocks and external-abort any subsequent channel access.
+ */
+ mp_csr = imx_edma5_read32(ed->reg_base, IMX_EDMA5_MP_CSR);
+ mp_csr |= IMX_EDMA5_MP_CSR_GCLC | IMX_EDMA5_MP_CSR_ERCA;
+ imx_edma5_write32(ed->reg_base, IMX_EDMA5_MP_CSR, mp_csr);
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ if (!vc->configured)
+ continue;
+ imx_edma5_reset_hw_chan(vc);
+ vc->head = 0;
+ vc->tail = 0;
+ vc->nb_enqueued = 0;
+ vc->ridx = 0;
+ /* Seed last_idx one step before the first cookie (0). */
+ vc->last_idx = UINT16_MAX;
+ vc->submitted_count = 0;
+ vc->completed_count = 0;
+ vc->errors_count = 0;
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_stop(struct rte_dma_dev *dev)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ if (vc->configured)
+ imx_edma5_reset_hw_chan(vc);
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_close(struct rte_dma_dev *dev)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint16_t i;
+
+ if (ed->vchans != NULL) {
+ for (i = 0; i < ed->max_vchans; i++) {
+ rte_free(ed->vchans[i].jobs);
+ rte_free(ed->vchans[i].sg_tcd_pool);
+ }
+ rte_free(ed->vchans);
+ ed->vchans = NULL;
+ }
+
+ ed->nb_vchans = 0;
+
+ return 0;
+}
+
static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_info_get = imx_edma5_info_get,
+ .dev_configure = imx_edma5_configure,
+ .dev_start = imx_edma5_start,
+ .dev_stop = imx_edma5_stop,
+ .dev_close = imx_edma5_close,
+
+ .vchan_setup = imx_edma5_vchan_setup,
};
static int
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v2 4/5] dma/imx_edma5: add data path
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
` (2 preceding siblings ...)
2026-08-07 6:48 ` [PATCH v2 3/5] dma/imx_edma5: add device configuration Gagandeep Singh
@ 2026-08-07 6:48 ` Gagandeep Singh
2026-08-07 6:48 ` [PATCH v2 5/5] dma/imx_edma5: add statistics and dump Gagandeep Singh
` (2 subsequent siblings)
6 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-07 6:48 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add the memory-to-memory data path for the eDMA5 dmadev.
The eDMA5 exposes a single transfer control descriptor and a single
completion flag per hardware channel, so jobs are serialised in software.
Each enqueued job is played out synchronously as one or more single-block
transfers: the per-channel TCD is programmed, the transfer is
software-started and the driver busy-waits for the DONE flag with a
wall-clock bounded timeout. Scatter-gather copies walk the source and
destination segment lists as two cursors, emitting one single-block
sub-transfer per iteration that fits both current segments.
As the eDMA5 is a non-coherent bus master, source and destination buffers
are cleaned from the CPU cache before a transfer and the destination is
invalidated after completion so the application observes the DMA result.
This adds copy, copy_sg, submit, completed, completed_status and
burst_capacity, wired through the device fast-path object.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 571 +++++++++++++++++++++++
1 file changed, 571 insertions(+)
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
index fe5539b612..e7f3bec528 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -337,6 +337,570 @@ imx_edma5_close(struct rte_dma_dev *dev)
return 0;
}
+/*
+ * Encode the largest natural transfer size (SSIZE/DSIZE) usable for a given
+ * source, destination and length. The address must be aligned to the transfer
+ * size and the byte count must be a multiple of it.
+ */
+static uint16_t
+imx_edma5_calc_attr(uint64_t src, uint64_t dst, uint32_t len)
+{
+ uint32_t sz = IMX_EDMA5_TCD_SIZE_1B;
+
+ if (((src | dst | len) & 0x1F) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_32B;
+ else if (((src | dst | len) & 0xF) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_16B;
+ else if (((src | dst | len) & 0x7) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_8B;
+ else if (((src | dst | len) & 0x3) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_4B;
+ else if (((src | dst | len) & 0x1) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_2B;
+
+ return IMX_EDMA5_TCD_ATTR_SSIZE(sz) | IMX_EDMA5_TCD_ATTR_DSIZE(sz);
+}
+
+/*
+ * Program the channel TCD for a single-block copy: one minor loop of "len"
+ * bytes with a major count of 1. Completion is polled via CH_CSR.DONE.
+ */
+static inline void
+imx_edma5_program_copy(struct imx_edma5_vchan *vc, uint64_t src, uint64_t dst,
+ uint32_t len)
+{
+ uint8_t *tcd = vc->tcd_regs;
+ uint16_t attr = imx_edma5_calc_attr(src, dst, len);
+
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_SADDR, src);
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_DADDR, dst);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_ATTR, attr);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_SOFF,
+ (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_SSIZE(attr)));
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_DOFF,
+ (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_DSIZE(attr)));
+ imx_edma5_write32(tcd, IMX_EDMA5_TCD_NBYTES, len);
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_SLAST, 0);
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_DLAST_SGA, 0);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CITER, 1);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_BITER, 1);
+ /* Clear CSR so no scatter-gather link is followed. */
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CSR, 0);
+}
+
+/* Fill an in-memory hardware TCD64 descriptor for one copy segment. */
+static inline void
+imx_edma5_fill_tcd(struct imx_edma5_hw_tcd64 *t, uint64_t src, uint64_t dst,
+ uint32_t len)
+{
+ uint16_t attr = imx_edma5_calc_attr(src, dst, len);
+ uint16_t soff = (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_SSIZE(attr));
+ uint16_t doff = (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_DSIZE(attr));
+
+ t->saddr = rte_cpu_to_le_64(src);
+ t->soff = rte_cpu_to_le_16(soff);
+ t->attr = rte_cpu_to_le_16(attr);
+ t->nbytes = rte_cpu_to_le_32(len);
+ t->slast = 0;
+ t->daddr = rte_cpu_to_le_64(dst);
+ t->dlast_sga = 0;
+ t->doff = rte_cpu_to_le_16(doff);
+ t->citer = rte_cpu_to_le_16(1);
+ t->csr = 0;
+ t->biter = rte_cpu_to_le_16(1);
+}
+
+static inline void
+imx_edma5_hw_start(struct imx_edma5_vchan *vc)
+{
+ uint16_t csr = imx_edma5_read16(vc->tcd_regs, IMX_EDMA5_TCD_CSR);
+
+ csr |= IMX_EDMA5_TCD_CSR_START;
+ imx_edma5_write16(vc->tcd_regs, IMX_EDMA5_TCD_CSR, csr);
+}
+
+/*
+ * Invalidate the CPU cache lines covering a completed job's destination(s) so
+ * the application reads the DMA result rather than stale cache. The lines were
+ * cleaned at enqueue time, so this clean+invalidate behaves as a pure
+ * invalidate.
+ */
+static inline void
+imx_edma5_job_invalidate_dst(struct imx_edma5_job *job)
+{
+ if (job->nb_sg > 0) {
+ uint16_t s;
+
+ for (s = 0; s < job->nb_sg; s++) {
+ rte_iova_t da = rte_le_to_cpu_64(job->sg_tcd[s].daddr);
+ uint32_t len = rte_le_to_cpu_32(job->sg_tcd[s].nbytes);
+ void *va = rte_mem_iova2virt(da);
+
+ if (va != NULL)
+ imx_edma5_cache_inval(va, len);
+ }
+ } else if (job->dst_va != NULL) {
+ imx_edma5_cache_inval(job->dst_va, job->len);
+ }
+}
+
+/*
+ * Upper bound on how long to poll for a single-block transfer to complete.
+ * A wall-clock deadline is used rather than a raw spin count so the bound is
+ * independent of CPU speed. Each single-block transfer programmed here is
+ * capped at IMX_EDMA5_MAX_NBYTES (1 GiB - 1); at an AXI bus rate of 2 GB/s
+ * that is 512 ms in the worst case, so 1000 ms is a safe ceiling while still
+ * bounding a wedged channel rather than hanging the CPU.
+ */
+#define IMX_EDMA5_WAIT_TIMEOUT_MS 1000
+
+/*
+ * Wait for the single register TCD transfer to finish and clear its latched
+ * status. Returns true on success, false on a logged channel error (CH_ES.ERR)
+ * or timeout. The per-transfer completion flag CH_CSR.DONE and the CH_ES error
+ * bit are both write-1-to-clear.
+ */
+static inline bool
+imx_edma5_wait_done(struct imx_edma5_vchan *vc)
+{
+ uint64_t deadline = rte_get_timer_cycles() +
+ (rte_get_timer_hz() * IMX_EDMA5_WAIT_TIMEOUT_MS) / 1000;
+
+ do {
+ uint32_t ch_es = imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES);
+ uint32_t ch_csr;
+
+ if (ch_es & IMX_EDMA5_CH_ES_ERR) {
+ imx_edma5_write32(vc->ch_regs, IMX_EDMA5_CH_ES,
+ IMX_EDMA5_CH_ES_ERR);
+ /* Reset the errored channel before the next job reuses it. */
+ imx_edma5_reset_hw_chan(vc);
+ return false;
+ }
+
+ ch_csr = imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR);
+ if (ch_csr & IMX_EDMA5_CH_CSR_DONE) {
+ imx_edma5_write32(vc->ch_regs, IMX_EDMA5_CH_CSR,
+ IMX_EDMA5_CH_CSR_DONE);
+ return true;
+ }
+ } while (rte_get_timer_cycles() < deadline);
+
+ IMX_EDMA5_LOG(ERR,
+ "channel %u timed out waiting for DONE (CH_CSR=0x%08x "
+ "CH_ES=0x%08x)",
+ vc->hw_chan,
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR),
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES));
+
+ /* Reset the possibly-still-active channel before the next job reuses it. */
+ imx_edma5_reset_hw_chan(vc);
+ return false;
+}
+
+/*
+ * Execute one job to completion on the channel's single register TCD.
+ *
+ * The eDMA5 exposes a single TCD and a single completion flag per channel, so
+ * jobs are serialised in software: a job is run synchronously here (program the
+ * TCD, software-start, busy-wait for DONE) and its completion recorded in
+ * job->done for the completion API to reap. Scatter-gather segments are played
+ * out one at a time as single-block transfers, since the eDMA5 does not
+ * auto-advance a hardware TCD chain for software-started mem-to-mem transfers.
+ * On success the destination cache lines are invalidated (non-coherent master).
+ */
+static inline void
+imx_edma5_run_job(struct imx_edma5_vchan *vc, struct imx_edma5_job *job)
+{
+ bool ok = true;
+
+ if (job->nb_sg > 0) {
+ uint16_t s;
+
+ for (s = 0; s < job->nb_sg; s++) {
+ uint64_t src = rte_le_to_cpu_64(job->sg_tcd[s].saddr);
+ uint64_t dst = rte_le_to_cpu_64(job->sg_tcd[s].daddr);
+ uint32_t len = rte_le_to_cpu_32(job->sg_tcd[s].nbytes);
+
+ imx_edma5_program_copy(vc, src, dst, len);
+ imx_edma5_hw_start(vc);
+ if (!imx_edma5_wait_done(vc)) {
+ ok = false;
+ break;
+ }
+ }
+ } else {
+ imx_edma5_program_copy(vc, job->src_iova, job->dst_iova,
+ job->len);
+ imx_edma5_hw_start(vc);
+ ok = imx_edma5_wait_done(vc);
+ }
+
+ if (ok)
+ imx_edma5_job_invalidate_dst(job);
+
+ job->error = ok ? 0 : 1;
+ job->done = 1;
+ vc->submitted_count++;
+}
+
+static int
+imx_edma5_copy(void *dev_private, uint16_t vchan, rte_iova_t src,
+ rte_iova_t dst, uint32_t length, uint64_t flags)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ struct imx_edma5_job *job;
+ uint16_t slot;
+
+ /* NBYTES = 0 is undefined on the eDMA5 and can wedge the channel. */
+ if (length == 0)
+ return -EINVAL;
+ /*
+ * TCD_NBYTES bits 31:30 are SMLOE/DMLOE when minor-loop offsets are
+ * enabled; writing a count larger than the 30-bit max would corrupt
+ * those control bits. Reject oversized requests.
+ */
+ if (length > IMX_EDMA5_MAX_NBYTES)
+ return -EINVAL;
+
+ /* Ring full? one slot is kept free to distinguish full from empty. */
+ if (vc->nb_enqueued >= (uint16_t)(vc->nb_desc - 1))
+ return -ENOSPC;
+
+ slot = vc->head;
+ job = &vc->jobs[slot];
+ job->ridx = vc->ridx;
+ job->submitted = 0;
+ job->done = 0;
+ job->error = 0;
+ job->nb_sg = 0;
+ job->len = length;
+ job->src_iova = src;
+ job->dst_iova = dst;
+ /*
+ * rte_mem_iova2virt() returns NULL when the IOVA is not in the
+ * memzone table (e.g. externally-allocated IOVA-contiguous memory not
+ * registered with DPDK). Cache maintenance is silently skipped for such
+ * addresses; callers are responsible for ensuring coherency in that case
+ * or for registering the memory so a VA mapping is available.
+ */
+ job->dst_va = rte_mem_iova2virt(dst);
+
+ /*
+ * Non-cache-coherent master: clean the source so the device reads the
+ * CPU's latest writes, and clean the destination so a prior dirty line
+ * cannot be written back over the DMA result (the destination is
+ * invalidated after completion).
+ */
+ {
+ void *src_va = rte_mem_iova2virt(src);
+
+ if (src_va != NULL)
+ imx_edma5_cache_clean(src_va, length);
+ if (job->dst_va != NULL)
+ imx_edma5_cache_clean(job->dst_va, length);
+ }
+
+ /*
+ * RTE_DMA_OP_FLAG_SUBMIT is equivalent to calling rte_dma_submit()
+ * after this enqueue: advance the head to include the new job and then
+ * run all pending (unsubmitted) jobs from tail to the new head in FIFO
+ * order, matching the behaviour of imx_edma5_submit().
+ */
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ uint16_t idx;
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ idx = vc->tail;
+ while (idx != vc->head) {
+ struct imx_edma5_job *j = &vc->jobs[idx];
+
+ if (!j->submitted) {
+ imx_edma5_run_job(vc, j);
+ j->submitted = 1;
+ }
+ idx = (idx + 1) & vc->desc_mask;
+ }
+
+ return vc->ridx++;
+ }
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ return vc->ridx++;
+}
+
+static int
+imx_edma5_copy_sg(void *dev_private, uint16_t vchan,
+ const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
+ uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ struct imx_edma5_hw_tcd64 *tcd;
+ struct imx_edma5_job *job;
+ uint16_t slot;
+ uint16_t s;
+ uint16_t si = 0, di = 0; /* current source/destination seg index */
+ uint32_t s_off = 0, d_off = 0; /* byte offset within current segment */
+ uint16_t nsg = 0; /* sub-transfers produced so far */
+ uint64_t src_total = 0, dst_total = 0;
+
+ /*
+ * The scatter-gather contract is a byte stream: the source and
+ * destination lists may be segmented independently but their
+ * concatenations are equal. The lists are walked as two cursors,
+ * emitting one single-block sub-transfer per run that fits in both the
+ * current source and destination segments. This yields at most
+ * nb_src + nb_dst - 1 sub-transfers, which fit in IMX_EDMA5_SG_TCD_PER_JOB.
+ */
+ if (nb_src == 0 || nb_dst == 0 ||
+ nb_src > IMX_EDMA5_MAX_SGES || nb_dst > IMX_EDMA5_MAX_SGES) {
+ IMX_EDMA5_LOG(ERR, "Unsupported SG shape src=%u dst=%u",
+ nb_src, nb_dst);
+ return -EINVAL;
+ }
+
+ /* Ring full? one slot is kept free to distinguish full from empty. */
+ if (vc->nb_enqueued >= (uint16_t)(vc->nb_desc - 1))
+ return -ENOSPC;
+
+ slot = vc->head;
+ job = &vc->jobs[slot];
+ job->ridx = vc->ridx;
+ job->submitted = 0;
+ job->done = 0;
+ job->error = 0;
+
+ /* This job's dedicated slice of the in-memory TCD pool. */
+ job->sg_tcd = &vc->sg_tcd_pool[(size_t)slot * IMX_EDMA5_SG_TCD_PER_JOB];
+ tcd = job->sg_tcd;
+
+ /*
+ * Clean every source and destination segment up front (non-coherent
+ * master): the device must read current source data, and dirty
+ * destination lines must be flushed before the transfer.
+ */
+ for (s = 0; s < nb_src; s++) {
+ void *va = rte_mem_iova2virt(src[s].addr);
+
+ src_total += src[s].length;
+ if (va != NULL)
+ imx_edma5_cache_clean(va, src[s].length);
+ }
+ for (s = 0; s < nb_dst; s++) {
+ void *va = rte_mem_iova2virt(dst[s].addr);
+
+ dst_total += dst[s].length;
+ if (va != NULL)
+ imx_edma5_cache_clean(va, dst[s].length);
+ }
+
+ /* copy_sg requires equal total bytes on both lists; reject misuse. */
+ if (src_total != dst_total) {
+ IMX_EDMA5_LOG(ERR,
+ "SG byte count mismatch src=%" PRIu64
+ " dst=%" PRIu64, src_total, dst_total);
+ return -EINVAL;
+ }
+
+ while (si < nb_src && di < nb_dst) {
+ uint32_t s_rem = src[si].length - s_off;
+ uint32_t d_rem = dst[di].length - d_off;
+ uint32_t len = RTE_MIN(s_rem, d_rem);
+
+ /* Skip zero-length segments without emitting a descriptor. */
+ if (len == 0) {
+ if (s_rem == 0) {
+ si++;
+ s_off = 0;
+ }
+ if (d_rem == 0) {
+ di++;
+ d_off = 0;
+ }
+ continue;
+ }
+
+ if (nsg >= IMX_EDMA5_SG_TCD_PER_JOB) {
+ IMX_EDMA5_LOG(ERR,
+ "SG produced too many sub-transfers "
+ "(src=%u dst=%u)", nb_src, nb_dst);
+ /* Defensive: clear nb_sg so the abandoned slot is not reused. */
+ job->nb_sg = 0;
+ return -EINVAL;
+ }
+
+ imx_edma5_fill_tcd(&tcd[nsg], src[si].addr + s_off,
+ dst[di].addr + d_off, len);
+ nsg++;
+
+ s_off += len;
+ d_off += len;
+ if (s_off == src[si].length) {
+ si++;
+ s_off = 0;
+ }
+ if (d_off == dst[di].length) {
+ di++;
+ d_off = 0;
+ }
+ }
+
+ /* No sub-transfer (all segments zero-length): NBYTES = 0 wedges eDMA5. */
+ if (nsg == 0) {
+ IMX_EDMA5_LOG(ERR, "SG produced zero sub-transfers");
+ return -EINVAL;
+ }
+
+ job->nb_sg = nsg;
+
+ /*
+ * RTE_DMA_OP_FLAG_SUBMIT is equivalent to calling rte_dma_submit()
+ * after this enqueue: advance the head to include the new job and then
+ * run all pending (unsubmitted) jobs from tail to the new head in FIFO
+ * order, matching the behaviour of imx_edma5_submit().
+ */
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ uint16_t idx;
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ idx = vc->tail;
+ while (idx != vc->head) {
+ struct imx_edma5_job *j = &vc->jobs[idx];
+
+ if (!j->submitted) {
+ imx_edma5_run_job(vc, j);
+ j->submitted = 1;
+ }
+ idx = (idx + 1) & vc->desc_mask;
+ }
+
+ return vc->ridx++;
+ }
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ return vc->ridx++;
+}
+
+static int
+imx_edma5_submit(void *dev_private, uint16_t vchan)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t idx = vc->tail;
+
+ /* Run every enqueued-but-not-yet-submitted job to completion (FIFO). */
+ while (idx != vc->head) {
+ struct imx_edma5_job *job = &vc->jobs[idx];
+
+ if (!job->submitted) {
+ imx_edma5_run_job(vc, job);
+ job->submitted = 1;
+ }
+ idx = (idx + 1) & vc->desc_mask;
+ }
+
+ return 0;
+}
+
+/*
+ * Reap completed jobs from the software ring in FIFO order. Jobs run
+ * synchronously, so a submitted job's result is already in job->done/error.
+ * An unsubmitted job stops the walk.
+ */
+static uint16_t
+imx_edma5_completed(void *dev_private, uint16_t vchan, const uint16_t nb_cpls,
+ uint16_t *last_idx, bool *has_error)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t count = 0;
+
+ *has_error = false;
+
+ while (count < nb_cpls && vc->tail != vc->head) {
+ struct imx_edma5_job *job = &vc->jobs[vc->tail];
+
+ if (!job->submitted || !job->done)
+ break;
+
+ /*
+ * Stop before an errored job: it is left in the ring for
+ * rte_dma_completed_status(), and last_idx stays at the last
+ * successful transfer.
+ */
+ if (job->error) {
+ *has_error = true;
+ break;
+ }
+
+ vc->last_idx = job->ridx;
+ vc->completed_count++;
+
+ vc->tail = (vc->tail + 1) & vc->desc_mask;
+ vc->nb_enqueued--;
+ count++;
+ }
+
+ *last_idx = vc->last_idx;
+
+ return count;
+}
+
+static uint16_t
+imx_edma5_completed_status(void *dev_private, uint16_t vchan,
+ const uint16_t nb_cpls, uint16_t *last_idx,
+ enum rte_dma_status_code *status)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t count = 0;
+
+ while (count < nb_cpls && vc->tail != vc->head) {
+ struct imx_edma5_job *job = &vc->jobs[vc->tail];
+
+ if (!job->submitted || !job->done)
+ break;
+
+ if (job->error) {
+ status[count] = RTE_DMA_STATUS_BUS_ERROR;
+ vc->errors_count++;
+ } else {
+ status[count] = RTE_DMA_STATUS_SUCCESSFUL;
+ }
+
+ vc->last_idx = job->ridx;
+ vc->completed_count++;
+
+ vc->tail = (vc->tail + 1) & vc->desc_mask;
+ vc->nb_enqueued--;
+ count++;
+ }
+
+ *last_idx = vc->last_idx;
+
+ return count;
+}
+
+static uint16_t
+imx_edma5_burst_capacity(const void *dev_private, uint16_t vchan)
+{
+ const struct imx_edma5_dev *ed = dev_private;
+ const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+
+ /* One slot is reserved to distinguish full from empty. */
+ return vc->nb_desc - 1 - vc->nb_enqueued;
+}
+
static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_info_get = imx_edma5_info_get,
.dev_configure = imx_edma5_configure,
@@ -382,6 +946,13 @@ imx_edma5_probe(struct rte_platform_device *pdev)
dev->device = &pdev->device;
dev->dev_ops = &imx_edma5_ops;
+ dev->fp_obj->dev_private = dev->data->dev_private;
+ dev->fp_obj->copy = imx_edma5_copy;
+ dev->fp_obj->copy_sg = imx_edma5_copy_sg;
+ dev->fp_obj->submit = imx_edma5_submit;
+ dev->fp_obj->completed = imx_edma5_completed;
+ dev->fp_obj->completed_status = imx_edma5_completed_status;
+ dev->fp_obj->burst_capacity = imx_edma5_burst_capacity;
ed = dev->data->dev_private;
ed->reg_base = res->mem.addr;
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v2 5/5] dma/imx_edma5: add statistics and dump
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
` (3 preceding siblings ...)
2026-08-07 6:48 ` [PATCH v2 4/5] dma/imx_edma5: add data path Gagandeep Singh
@ 2026-08-07 6:48 ` Gagandeep Singh
2026-08-10 15:37 ` [PATCH v2 0/5] dma/imx_edma5: introduce NXP i.MX95 eDMA5 driver Stephen Hemminger
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
6 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-07 6:48 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add per-vchan statistics, virtual channel status reporting and a device
dump callback to the eDMA5 dmadev.
stats_get and stats_reset expose the submitted, completed and error
counters maintained by the data path, supporting both a single vchan and
the RTE_DMA_ALL_VCHAN aggregate. vchan_status reports whether a channel is
idle, active or halted on an unreaped error by inspecting the outstanding
jobs in the software ring. dev_dump prints the device and per-vchan
software state to aid debugging.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 126 +++++++++++++++++++++++
1 file changed, 126 insertions(+)
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
index e7f3bec528..e9210a6793 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -901,6 +901,126 @@ imx_edma5_burst_capacity(const void *dev_private, uint16_t vchan)
return vc->nb_desc - 1 - vc->nb_enqueued;
}
+static int
+imx_edma5_vchan_status(const struct rte_dma_dev *dev, uint16_t vchan,
+ enum rte_dma_vchan_status *status)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+ const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t idx;
+
+ /*
+ * HALTED_ERROR if an errored job is still unreaped, ACTIVE while any job
+ * is outstanding (not done, or enqueued but not submitted), else IDLE.
+ */
+ *status = RTE_DMA_VCHAN_IDLE;
+
+ for (idx = vc->tail; idx != vc->head; idx = (idx + 1) & vc->desc_mask) {
+ const struct imx_edma5_job *job = &vc->jobs[idx];
+
+ if (job->submitted && job->done && job->error) {
+ *status = RTE_DMA_VCHAN_HALTED_ERROR;
+ break;
+ }
+
+ if (!job->submitted || !job->done) {
+ *status = RTE_DMA_VCHAN_ACTIVE;
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
+ struct rte_dma_stats *stats, uint32_t stats_sz)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ RTE_SET_USED(stats_sz);
+
+ stats->submitted = 0;
+ stats->completed = 0;
+ stats->errors = 0;
+
+ /* RTE_DMA_ALL_VCHAN requests the aggregate across every vchan. */
+ if (vchan == RTE_DMA_ALL_VCHAN) {
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ const struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ stats->submitted += vc->submitted_count;
+ stats->completed += vc->completed_count;
+ stats->errors += vc->errors_count;
+ }
+ } else if (vchan < ed->nb_vchans) {
+ const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+
+ stats->submitted = vc->submitted_count;
+ stats->completed = vc->completed_count;
+ stats->errors = vc->errors_count;
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ /* RTE_DMA_ALL_VCHAN requests a reset of every vchan. */
+ if (vchan == RTE_DMA_ALL_VCHAN) {
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ vc->submitted_count = 0;
+ vc->completed_count = 0;
+ vc->errors_count = 0;
+ }
+ } else if (vchan < ed->nb_vchans) {
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+
+ vc->submitted_count = 0;
+ vc->completed_count = 0;
+ vc->errors_count = 0;
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_dump(const struct rte_dma_dev *dev, FILE *f)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint16_t i;
+
+ (void)fprintf(f, " imx_edma5 nb_channels=%u nb_vchans=%u\n",
+ ed->nb_channels, ed->nb_vchans);
+ for (i = 0; i < ed->nb_vchans; i++) {
+ const struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ (void)fprintf(f,
+ " vchan %u: hw_chan=%u nb_desc=%u enqueued=%u "
+ "submitted=%" PRIu64 " completed=%" PRIu64
+ " errors=%" PRIu64 "\n",
+ i, vc->hw_chan, vc->nb_desc, vc->nb_enqueued,
+ vc->submitted_count, vc->completed_count,
+ vc->errors_count);
+ /* Hardware register snapshot for debug (CH_CSR, CH_ES). */
+ (void)fprintf(f,
+ " hw: CH_CSR=0x%08x CH_ES=0x%08x\n",
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR),
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES));
+ }
+
+ return 0;
+}
+
static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_info_get = imx_edma5_info_get,
.dev_configure = imx_edma5_configure,
@@ -909,6 +1029,12 @@ static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_close = imx_edma5_close,
.vchan_setup = imx_edma5_vchan_setup,
+ .vchan_status = imx_edma5_vchan_status,
+
+ .stats_get = imx_edma5_stats_get,
+ .stats_reset = imx_edma5_stats_reset,
+
+ .dev_dump = imx_edma5_dump,
};
static int
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH v2 0/5] dma/imx_edma5: introduce NXP i.MX95 eDMA5 driver
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
` (4 preceding siblings ...)
2026-08-07 6:48 ` [PATCH v2 5/5] dma/imx_edma5: add statistics and dump Gagandeep Singh
@ 2026-08-10 15:37 ` Stephen Hemminger
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
6 siblings, 0 replies; 22+ messages in thread
From: Stephen Hemminger @ 2026-08-10 15:37 UTC (permalink / raw)
To: Gagandeep Singh; +Cc: dev, hemant.agrawal
On Fri, 7 Aug 2026 12:18:48 +0530
Gagandeep Singh <g.singh@nxp.com> wrote:
> V2-changes:
> - Added dependent patch: bus/platform: match device by devicetree compatible string
> - Fixed multiple AI reported issues includes:
> - removed the alias from the driver, no need of it
> - removed imx_edma5_write32 from data-path, reported as coherency
> concern.
> - fix submit for previously enqueued jobs.
> - Added dsb in data path
> - Added a commnet for rte_mem_iova2virt() NULL behaviour.
> - Fixed: TCD NBYTES bits 31:30 are SMLOE/DMLOE — full 32-bit length corrupts them
> - Data path is fully synchronous: Acknowledged but intentionally deferred —
> the synchronous design is a deliberate simplification for the initial upstream submission.
> Noted in docs.
> - In-memory TCD64 pool is more complex than needed:
> Noted/deferred — the pool structure was retained for forward compatibility
> with hardware SG chaining.
> - Extended `imx_edma5_dump()` to read and print `CH_CSR` and `CH_ES` registers
> for every configured vchan.
> - Double and stary lines removed.
>
> V1:
> This patch series adds a new dmadev Poll-Mode Driver (PMD) for the NXP
> i.MX95 eDMA5 (Enhanced DMA Type 5) controller.
>
> Key features supported by this driver:
> - Memory-to-memory copy (RTE_DMA_OP_TYPE_MEMCPY)
> - Scatter-gather memory copy (RTE_DMA_OP_TYPE_SG)
> - 64-bit TCD (Transfer Control Descriptor) format
> - Non-coherent DMA with explicit cache clean/invalidate
> - Per-channel statistics and register dump for debug
>
> Patch breakdown:
> [1/4] Skeleton: bus probe/remove, dmadev registration, MAINTAINERS,
> doc index, and release notes for 26.11.
> [2/4] Device configuration: vchan setup, TCD ring allocation,
> start/stop, and capability reporting.
> [3/4] Data path: enqueue (copy and sg), doorbell, completion poll.
> [4/4] Statistics and dump: per-channel counters and register dump.
>
> Tested on NXP i.MX95 EVK with vfio-platform.
>
> Gagandeep Singh (5):
> bus/platform: match device by devicetree compatible string
> dma/imx_edma5: introduce eDMA5 dmadev skeleton
> dma/imx_edma5: add device configuration
> dma/imx_edma5: add data path
> dma/imx_edma5: add statistics and dump
>
> MAINTAINERS | 5 +
> doc/guides/dmadevs/imx_edma5.rst | 61 +
> doc/guides/dmadevs/index.rst | 1 +
> doc/guides/rel_notes/release_26_11.rst | 6 +
> drivers/bus/platform/bus_platform_driver.h | 3 +-
> drivers/bus/platform/platform.c | 67 +-
> drivers/dma/imx_edma5/imx_edma5_dmadev.c | 1162 ++++++++++++++++++++
> drivers/dma/imx_edma5/imx_edma5_dmadev.h | 220 ++++
> drivers/dma/imx_edma5/imx_edma5_hw.h | 157 +++
> drivers/dma/imx_edma5/imx_edma5_logs.h | 16 +
> drivers/dma/imx_edma5/meson.build | 10 +
> drivers/dma/meson.build | 1 +
> 12 files changed, 1707 insertions(+), 2 deletions(-)
> create mode 100644 doc/guides/dmadevs/imx_edma5.rst
> create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.c
> create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.h
> create mode 100644 drivers/dma/imx_edma5/imx_edma5_hw.h
> create mode 100644 drivers/dma/imx_edma5/imx_edma5_logs.h
> create mode 100644 drivers/dma/imx_edma5/meson.build
>
More AI feedback on V2
Review of [PATCH v2 0/5] dma/imx_edma5: introduce NXP i.MX95 eDMA5 driver
Verification: each of the five commits builds independently with
-Dwerror=true on x86_64 and aarch64 cross (gcc), full build including
documentation is clean, check-git-log reports 5/5 valid. Confirmed by
disassembly that the new DSB sequences are emitted on aarch64.
Thanks for the v2. The bus matching, the SUBMIT-flag semantics, the missing
DSB, the coherency contradiction and the style nits are all resolved. The
NBYTES cap is applied to copy() but not to copy_sg(), and a few smaller
items remain.
Patch 1/5 (bus/platform: match by devicetree compatible):
Warning: The quote-stripping in of_device_is_compatible() is unnecessary
and should be dropped. Now that imx_edma5 sets .driver.alias directly in
the driver struct, no in-tree caller passes a quoted string, so the branch
is dead code. It exists to paper over RTE_PMD_REGISTER_ALIAS, which
stringifies its argument via RTE_STR and therefore cannot express a
compatible string containing a comma. Silently accepting a mangled alias
in the bus hides that defect from the next driver that hits it. Either
leave the macro alone and drop the quote handling, or fix the macro (e.g.
a variant that takes the string unmodified) and drop it as well.
Info: The last match block ends with "goto out;" immediately above the
"out:" label; the goto is redundant.
Info: The commit changes matching behaviour for every platform-bus driver
and has no release notes entry. There is currently one in-tree consumer,
so this is a judgement call, but a line under New Features would be
reasonable.
Patch 2/5 (skeleton):
Warning: The driver documentation does not match the code. Both the
Supported Features and Limitations sections describe scatter-gather as
requiring equal-length or equal-sized source and destination segment
pairs, but imx_edma5_copy_sg() walks the two lists as independent cursors
and correctly handles unequal segmentation, requiring only equal totals.
The doc understates the driver. Please update both sections.
Info: IMX_EDMA5_CH_MATTR_COHERENT (and the RCACHE/WCACHE/RDOMAINS/WDOMAINS
macros it is built from) is now unused after the CH_MATTR write was
dropped in patch 3/5. Remove the definitions or note why they are kept.
Patch 4/5 (data path):
Error: The IMX_EDMA5_MAX_NBYTES cap is enforced in imx_edma5_copy() but
not in imx_edma5_copy_sg(). Sub-transfer lengths there come from
RTE_MIN(s_rem, d_rem) over rte_dma_sge.length, which is uint32_t, so a
single segment larger than 1 GiB - 1 reaches imx_edma5_program_copy() and
writes a count with bit 30 or 31 set into TCD_NBYTES - exactly the
SMLOE/DMLOE corruption the copy() check was added to prevent. Validate
each segment length (or each computed sub-transfer length) against
IMX_EDMA5_MAX_NBYTES and return -EINVAL.
Warning: An enqueue call can now block for a very long time. Each
sub-transfer waits up to IMX_EDMA5_WAIT_TIMEOUT_MS (1000 ms), and
imx_edma5_run_job() serialises SG sub-transfers, so a job with the maximum
32 sub-transfers can spin for up to 32 seconds inside rte_dma_copy_sg()
with RTE_DMA_OP_FLAG_SUBMIT, or inside rte_dma_submit(). Consider a
per-job deadline rather than a per-sub-transfer one.
Warning: imx_edma5_wait_done() still cannot stop a transfer it gave up on.
On timeout it calls imx_edma5_reset_hw_chan(), which writes CH_CSR.DONE,
CH_ES and the TCD control fields but does not cancel an in-flight transfer
(there is no MP_CSR.CX use in the driver), so a channel that is genuinely
stuck rather than merely slow will still be reprogrammed while ACTIVE and
the abandoned transfer keeps writing to the old destination. The length
cap makes this much harder to hit, so this is no longer an error, but a
cancel on the timeout path would close it properly.
Warning: The data path remains fully synchronous - enqueue or submit
programs the TCD, starts the channel and busy-waits for DONE - so the CPU
spins for the duration of every copy and the offload gains nothing over
memcpy. The hardware can run detached: program and START at submit time,
poll CH_CSR.DONE in completed()/completed_status(), and serialise only
when a second job needs the single register TCD. If this is deliberate for
the first submission, please state it in the Limitations section of the
driver doc rather than leaving it implicit.
Warning: When rte_mem_iova2virt() returns NULL, cache maintenance is
skipped and, on this non-coherent master, the transfer silently returns
wrong data. The v2 comment documents this, but a code comment is not
reachable by the application author. Either reject such addresses with
-EINVAL or state the restriction in the driver documentation.
Warning: rte_mem_iova2virt() is called per operation (source and
destination, and per segment in copy_sg) in the fast path; it walks the
memseg lists. In IOVA=VA mode the lookup is unnecessary and the value is
the address itself.
^ permalink raw reply [flat|nested] 22+ messages in thread* [PATCH v3 0/5] dma/imx_edma5: introduce NXP i.MX95 eDMA5 driver
2026-08-07 6:48 ` [PATCH v2 0/5] " Gagandeep Singh
` (5 preceding siblings ...)
2026-08-10 15:37 ` [PATCH v2 0/5] dma/imx_edma5: introduce NXP i.MX95 eDMA5 driver Stephen Hemminger
@ 2026-08-11 10:49 ` Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 1/5] bus/platform: match device by devicetree compatible string Gagandeep Singh
` (4 more replies)
6 siblings, 5 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-11 10:49 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=y, Size: 3706 bytes --]
V3-changes:
- Fixed below AI comments:
- Dead quote-stripping removed (`of_device_is_compatible`)
- Redundant `goto out` removed
- New Features entry added for bus platform changes.
- Scatter-gather doc corrected in both sections
- Unused CH_MATTR macros removed
- Fixed SG sub-transfer length cap (`imx_edma5_copy_sg`)
- Updated Per-job timeout logic.
- Updated `rte_mem_iova2virt` per-op in IOVA=VA mode logic
- Updated comments and documentation as per the AI comments.
V2-changes:
- Added dependent patch: bus/platform: match device by devicetree compatible string
- Fixed multiple AI reported issues includes:
- removed the alias from the driver, no need of it
- removed imx_edma5_write32 from data-path, reported as coherency
concern.
- fix submit for previously enqueued jobs.
- Added dsb in data path
- Added a commnet for rte_mem_iova2virt() NULL behaviour.
- Fixed: TCD NBYTES bits 31:30 are SMLOE/DMLOE — full 32-bit length corrupts them
- Data path is fully synchronous: Acknowledged but intentionally deferred —
the synchronous design is a deliberate simplification for the initial upstream submission.
Noted in docs.
- In-memory TCD64 pool is more complex than needed:
Noted/deferred — the pool structure was retained for forward compatibility
with hardware SG chaining.
- Extended `imx_edma5_dump()` to read and print `CH_CSR` and `CH_ES` registers
for every configured vchan.
- Double and stary lines removed.
V1:
This patch series adds a new dmadev Poll-Mode Driver (PMD) for the NXP
i.MX95 eDMA5 (Enhanced DMA Type 5) controller.
Key features supported by this driver:
- Memory-to-memory copy (RTE_DMA_OP_TYPE_MEMCPY)
- Scatter-gather memory copy (RTE_DMA_OP_TYPE_SG)
- 64-bit TCD (Transfer Control Descriptor) format
- Non-coherent DMA with explicit cache clean/invalidate
- Per-channel statistics and register dump for debug
Patch breakdown:
[1/4] Skeleton: bus probe/remove, dmadev registration, MAINTAINERS,
doc index, and release notes for 26.11.
[2/4] Device configuration: vchan setup, TCD ring allocation,
start/stop, and capability reporting.
[3/4] Data path: enqueue (copy and sg), doorbell, completion poll.
[4/4] Statistics and dump: per-channel counters and register dump.
Tested on NXP i.MX95 EVK with vfio-platform.
Gagandeep Singh (5):
bus/platform: match device by devicetree compatible string
dma/imx_edma5: introduce eDMA5 dmadev skeleton
dma/imx_edma5: add device configuration
dma/imx_edma5: add data path
dma/imx_edma5: add statistics and dump
MAINTAINERS | 5 +
doc/guides/dmadevs/imx_edma5.rst | 69 ++
doc/guides/dmadevs/index.rst | 1 +
doc/guides/rel_notes/release_26_11.rst | 15 +
drivers/bus/platform/bus_platform_driver.h | 3 +-
drivers/bus/platform/platform.c | 52 +-
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 1190 ++++++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_dmadev.h | 234 ++++
drivers/dma/imx_edma5/imx_edma5_hw.h | 144 +++
drivers/dma/imx_edma5/imx_edma5_logs.h | 16 +
drivers/dma/imx_edma5/meson.build | 10 +
drivers/dma/meson.build | 1 +
12 files changed, 1738 insertions(+), 2 deletions(-)
create mode 100644 doc/guides/dmadevs/imx_edma5.rst
create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.c
create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.h
create mode 100644 drivers/dma/imx_edma5/imx_edma5_hw.h
create mode 100644 drivers/dma/imx_edma5/imx_edma5_logs.h
create mode 100644 drivers/dma/imx_edma5/meson.build
--
2.25.1
^ permalink raw reply [flat|nested] 22+ messages in thread* [PATCH v3 1/5] bus/platform: match device by devicetree compatible string
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
@ 2026-08-11 10:49 ` Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 2/5] dma/imx_edma5: introduce eDMA5 dmadev skeleton Gagandeep Singh
` (3 subsequent siblings)
4 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-11 10:49 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh
The platform bus currently matches a DPDK driver to a device only by
comparing the kernel driver name bound to the device against the DPDK
driver name. Devices bound to the generic vfio-platform kernel driver
all report the same driver name, which carries no device identity, so a
specific DPDK driver cannot claim its device this way.
Add of_device_is_compatible(), which reads the device-tree "compatible"
strings exposed under the device's of_node in sysfs and compares them
against a requested string. platform_bus_match() now falls back to
matching the DPDK driver name or its alias against these compatible
strings, letting a driver bind to a device identified by its
device-tree "compatible" value.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
---
doc/guides/rel_notes/release_26_11.rst | 9 ++++
drivers/bus/platform/bus_platform_driver.h | 3 +-
drivers/bus/platform/platform.c | 52 +++++++++++++++++++++-
3 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst
index c8cc86295d..a2c0267818 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -55,6 +55,15 @@ New Features
Also, make sure to start the actual text at the margin.
=======================================================
+* **bus/platform: match devices by device-tree compatible string.**
+
+ The platform bus now matches DPDK drivers against the device-tree
+ ``compatible`` sysfs strings exposed by each platform device, in
+ addition to the existing kernel driver name and device name checks.
+ This allows drivers to be probed by DT compatible string when the
+ kernel driver is the generic ``vfio-platform``, which carries no
+ device identity in its name.
+
Removed Items
-------------
diff --git a/drivers/bus/platform/bus_platform_driver.h b/drivers/bus/platform/bus_platform_driver.h
index e4dcbacf5e..8eabe4a1fd 100644
--- a/drivers/bus/platform/bus_platform_driver.h
+++ b/drivers/bus/platform/bus_platform_driver.h
@@ -121,7 +121,8 @@ static const char *pdrvinit_ ## nm ## _alias; \
RTE_INIT(pdrvinitfn_ ##nm) \
{ \
(platform_drv).driver.name = RTE_STR(nm); \
- (platform_drv).driver.alias = pdrvinit_ ## nm ## _alias; \
+ if (pdrvinit_ ## nm ## _alias != NULL) \
+ (platform_drv).driver.alias = pdrvinit_ ## nm ## _alias; \
rte_platform_register(&(platform_drv)); \
} \
RTE_PMD_EXPORT_NAME(nm)
diff --git a/drivers/bus/platform/platform.c b/drivers/bus/platform/platform.c
index 90d865a8df..1c1341cc88 100644
--- a/drivers/bus/platform/platform.c
+++ b/drivers/bus/platform/platform.c
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(C) 2023 Marvell.
+ * Copyright 2026 NXP
*/
#include <uapi/linux/vfio.h>
@@ -230,6 +231,44 @@ of_resource_name(const char *dev_name, int index)
return NULL;
}
+/*
+ * Check whether any of the NUL-separated device-tree "compatible" strings
+ * exposed by the platform device matches the given string.
+ */
+static bool
+of_device_is_compatible(const char *dev_name, const char *compat)
+{
+ char path[PATH_MAX], buf[BUFSIZ] = { };
+ char want[BUFSIZ];
+ const char *s;
+ FILE *f;
+ size_t len;
+
+ if (compat == NULL)
+ return false;
+
+ rte_strscpy(want, compat, sizeof(want));
+
+ snprintf(path, sizeof(path), PLATFORM_BUS_DEVICES_PATH "/%s/of_node/compatible", dev_name);
+ f = fopen(path, "r");
+ if (f == NULL)
+ return false;
+
+ /* Read the raw contents, preserving embedded NULL separators. */
+ len = fread(buf, 1, sizeof(buf) - 1, f);
+ fclose(f);
+ if (len == 0)
+ return false;
+
+ /* Bound the walk by the read length: some kernels omit the trailing NULL. */
+ for (s = buf; s < buf + len; s += strlen(s) + 1) {
+ if (!strcmp(s, want))
+ return true;
+ }
+
+ return false;
+}
+
static int
device_map_resources(struct rte_platform_device *pdev, unsigned int num)
{
@@ -386,7 +425,18 @@ platform_bus_match(const struct rte_driver *drv, const struct rte_device *dev)
}
/* match by device name */
- if (!strcmp(pdev->name, pdrv->driver.name))
+ if (!strcmp(pdev->name, pdrv->driver.name)) {
+ match = true;
+ goto out;
+ }
+
+ /*
+ * The generic vfio-platform kernel driver name carries no device
+ * identity, so fall back to matching the device-tree "compatible"
+ * strings against the DPDK driver name or alias.
+ */
+ if (of_device_is_compatible(pdev->name, pdrv->driver.name) ||
+ of_device_is_compatible(pdev->name, pdrv->driver.alias))
match = true;
out:
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 2/5] dma/imx_edma5: introduce eDMA5 dmadev skeleton
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 1/5] bus/platform: match device by devicetree compatible string Gagandeep Singh
@ 2026-08-11 10:49 ` Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 3/5] dma/imx_edma5: add device configuration Gagandeep Singh
` (2 subsequent siblings)
4 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-11 10:49 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add the skeleton of a dmadev PMD for the NXP i.MX95 eDMA5 (Enhanced
Direct Memory Access Type 5) controller. The eDMA5 exposes 64 channels
with a 64-bit TCD (TCD64) layout.
The controller is probed on the platform bus via the generic
vfio-platform kernel driver and claimed by its device-tree compatible
string fsl,imx95-edma5. This patch adds the register definitions, the
private data structures, probe and remove handling, device-tree
dma-channel-mask parsing to skip channels reserved for other bus
masters, global clock and arbitration setup, and the device information
query. Subsequent patches add configuration, the data path, and
statistics.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
MAINTAINERS | 5 +
doc/guides/dmadevs/imx_edma5.rst | 60 +++++++
doc/guides/dmadevs/index.rst | 1 +
doc/guides/rel_notes/release_26_11.rst | 6 +
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 219 ++++++++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_dmadev.h | 220 +++++++++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_hw.h | 144 +++++++++++++++
drivers/dma/imx_edma5/imx_edma5_logs.h | 16 ++
drivers/dma/imx_edma5/meson.build | 10 ++
drivers/dma/meson.build | 1 +
10 files changed, 682 insertions(+)
create mode 100644 doc/guides/dmadevs/imx_edma5.rst
create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.c
create mode 100644 drivers/dma/imx_edma5/imx_edma5_dmadev.h
create mode 100644 drivers/dma/imx_edma5/imx_edma5_hw.h
create mode 100644 drivers/dma/imx_edma5/imx_edma5_logs.h
create mode 100644 drivers/dma/imx_edma5/meson.build
diff --git a/MAINTAINERS b/MAINTAINERS
index e99a65d197..b2645ebe2a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1422,6 +1422,11 @@ M: Hemant Agrawal <hemant.agrawal@nxp.com>
F: drivers/dma/dpaa2/
F: doc/guides/dmadevs/dpaa2.rst
+NXP i.MX95 eDMA5
+M: Gagandeep Singh <g.singh@nxp.com>
+M: Prashant Gupta <prashant.gupta_3@nxp.com>
+F: drivers/dma/imx_edma5/
+F: doc/guides/dmadevs/imx_edma5.rst
RegEx Drivers
-------------
diff --git a/doc/guides/dmadevs/imx_edma5.rst b/doc/guides/dmadevs/imx_edma5.rst
new file mode 100644
index 0000000000..f57881816e
--- /dev/null
+++ b/doc/guides/dmadevs/imx_edma5.rst
@@ -0,0 +1,60 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2026 NXP
+
+NXP i.MX95 eDMA5 DMA Driver
+===========================
+
+The ``imx_edma5`` DMA driver is a poll-mode driver (PMD) for the NXP i.MX95
+Enhanced Direct Memory Access controller version 5 (eDMA5). It exposes each
+eDMA5 controller instance as a DPDK dmadev device and can be used through the
+generic DMA device (dmadev) API.
+
+The i.MX95 SoC integrates multiple eDMA instances. The eDMA5 instances provide
+64 hardware channels, 64-bit addressing and a 64-byte Transfer Control
+Descriptor (TCD64). This driver targets the eDMA5 instances only (device tree
+compatible ``fsl,imx95-edma5``).
+
+Supported Features
+------------------
+
+- Memory-to-memory copy (``RTE_DMA_DIR_MEM_TO_MEM``).
+- Single-operation copy (``rte_dma_copy``).
+- Scatter-gather copy (``rte_dma_copy_sg``).
+- Per virtual channel statistics.
+
+Each configured virtual channel (vchan) is mapped one-to-one onto a hardware
+eDMA5 channel. Software-initiated (SWSTART) single-block transfers are
+programmed into the per-channel TCD and completion is detected by polling the
+TCD DONE status.
+
+Prerequisites
+-------------
+
+The eDMA5 register window is memory-mapped into the userspace process through
+the DPDK platform bus using the Linux ``vfio-platform`` mechanism. The device
+tree node targeted by this driver must be released from the kernel ``fsl-edma``
+driver (its status set to ``disabled`` or the node unbound) before it can be
+used by DPDK.
+
+Bind the platform device to ``vfio-platform``, for example::
+
+ echo vfio-platform > /sys/bus/platform/devices/<node>/driver_override
+ echo <node> > /sys/bus/platform/drivers/vfio-platform/bind
+
+where ``<node>`` is the platform device name of the eDMA5 instance (for
+example ``42000000.dma-controller``).
+
+Compilation
+-----------
+
+The driver is built as part of the standard DPDK meson build on Linux targets.
+No extra configuration option is required.
+
+Limitations
+-----------
+
+- Only the memory-to-memory transfer direction is supported.
+- The scatter-gather path iterates the source and destination segment lists
+ independently and programs one hardware transfer per consumed segment;
+ full TCD scatter-gather linking is not yet implemented.
+- The driver operates in poll mode only; completion interrupts are not used.
diff --git a/doc/guides/dmadevs/index.rst b/doc/guides/dmadevs/index.rst
index 56beb1733f..47c52a9220 100644
--- a/doc/guides/dmadevs/index.rst
+++ b/doc/guides/dmadevs/index.rst
@@ -17,5 +17,6 @@ an application through DMA API.
hisi_acc
hisi_pciep
idxd
+ imx_edma5
ioat
odm
diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst
index a2c0267818..638e92ad8d 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -64,6 +64,12 @@ New Features
kernel driver is the generic ``vfio-platform``, which carries no
device identity in its name.
+* **Added NXP i.MX95 eDMA5 DMA driver.**
+
+ Added the ``imx_edma5`` DMA driver for the NXP i.MX95 eDMA5 controller.
+ The driver exposes each eDMA5 instance as a DPDK dmadev device and
+ supports memory-to-memory copy and scatter-gather copy operations.
+
Removed Items
-------------
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
new file mode 100644
index 0000000000..f1c65a474a
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -0,0 +1,219 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+/*
+ * NXP i.MX95 eDMA5 dmadev driver.
+ *
+ * Exposes an eDMA5 controller instance as a DPDK dmadev. Each configured
+ * virtual channel maps 1:1 onto a hardware eDMA5 channel. Memory-to-memory
+ * copy and scatter-gather copy are supported using software-initiated
+ * single-block transfers programmed through the per-channel 64-bit TCD.
+ *
+ * The device is bound to userspace through the platform bus (vfio-platform);
+ * the device tree node must be released from the kernel fsl-edma driver before
+ * it can be used here.
+ */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <bus_platform_driver.h>
+#include <rte_bitops.h>
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_dmadev_pmd.h>
+#include <rte_malloc.h>
+
+#include "imx_edma5_dmadev.h"
+
+#include "imx_edma5_logs.h"
+
+RTE_LOG_REGISTER_DEFAULT(imx_edma5_logtype, INFO);
+
+/* Compatible string for i.MX95 eDMA5 device tree nodes. */
+#define IMX_EDMA5_COMPAT "fsl,imx95-edma5"
+
+/* sysfs base path for platform devices' device-tree nodes. */
+#define IMX_EDMA5_SYSFS_DEVICES "/sys/bus/platform/devices"
+
+/*
+ * Read the device-tree "dma-channel-mask" property and return the combined
+ * 64-bit mask of reserved hardware channels. The property is a raw binary file
+ * of big-endian 32-bit cells: cell[0] covers channels 0-31, cell[1] channels
+ * 32-63. A set bit marks a channel owned by another bus master (e.g. SCMI
+ * firmware) that must not be accessed. Returns 0 if the property is absent.
+ */
+static uint64_t
+imx_edma5_read_channel_mask(const char *dev_name)
+{
+ char path[PATH_MAX];
+ uint32_t cells[2] = { 0, 0 };
+ uint64_t mask = 0;
+ size_t n;
+ FILE *f;
+
+ snprintf(path, sizeof(path),
+ IMX_EDMA5_SYSFS_DEVICES "/%s/of_node/dma-channel-mask",
+ dev_name);
+
+ f = fopen(path, "rb");
+ if (f == NULL)
+ return 0;
+
+ n = fread(cells, 1, sizeof(cells), f);
+ fclose(f);
+
+ /* Device-tree cells are big-endian regardless of CPU endianness. */
+ if (n >= sizeof(uint32_t))
+ mask |= rte_be_to_cpu_32(cells[0]);
+ if (n >= 2 * sizeof(uint32_t))
+ mask |= (uint64_t)rte_be_to_cpu_32(cells[1]) << 32;
+
+ return mask;
+}
+
+static int
+imx_edma5_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
+ uint32_t info_sz)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ RTE_SET_USED(info_sz);
+
+ dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
+ RTE_DMA_CAPA_OPS_COPY |
+ RTE_DMA_CAPA_OPS_COPY_SG;
+ dev_info->max_vchans = ed->max_vchans;
+ dev_info->max_desc = IMX_EDMA5_MAX_DESC;
+ dev_info->min_desc = IMX_EDMA5_MIN_DESC;
+ dev_info->max_sges = IMX_EDMA5_MAX_SGES;
+
+ return 0;
+}
+
+static const struct rte_dma_dev_ops imx_edma5_ops = {
+ .dev_info_get = imx_edma5_info_get,
+};
+
+static int
+imx_edma5_probe(struct rte_platform_device *pdev)
+{
+ struct rte_platform_resource *res;
+ struct imx_edma5_dev *ed;
+ struct rte_dma_dev *dev;
+ const char *name;
+
+ name = pdev->name;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ IMX_EDMA5_LOG(ERR, "Secondary process not supported for %s",
+ name);
+ return -ENOTSUP;
+ }
+
+ if (pdev->num_resource < 1 || pdev->resource == NULL) {
+ IMX_EDMA5_LOG(ERR, "No MMIO resource for %s", name);
+ return -EINVAL;
+ }
+ res = &pdev->resource[0];
+ if (res->mem.addr == NULL) {
+ IMX_EDMA5_LOG(ERR, "MMIO resource not mapped for %s", name);
+ return -EINVAL;
+ }
+
+ dev = rte_dma_pmd_allocate(name, rte_socket_id(),
+ sizeof(struct imx_edma5_dev));
+ if (dev == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to allocate dmadev for %s", name);
+ return -ENOMEM;
+ }
+
+ dev->device = &pdev->device;
+ dev->dev_ops = &imx_edma5_ops;
+
+ ed = dev->data->dev_private;
+ ed->reg_base = res->mem.addr;
+ ed->reg_size = res->mem.len;
+ ed->dev_id = dev->data->dev_id;
+
+ /*
+ * Build a map from usable vchan index to hardware channel index,
+ * skipping channels reserved for other bus masters by the device-tree
+ * "dma-channel-mask" (accessing them external-aborts).
+ */
+ ed->masked_channels = imx_edma5_read_channel_mask(name);
+ ed->nb_channels = 0;
+ {
+ uint16_t hw;
+
+ for (hw = 0; hw < IMX_EDMA5_MAX_CHANNELS; hw++) {
+ if (ed->masked_channels & (RTE_BIT64(hw)))
+ continue;
+ ed->chan_map[ed->nb_channels++] = hw;
+ }
+ }
+ ed->max_vchans = ed->nb_channels;
+
+ if (ed->nb_channels == 0) {
+ IMX_EDMA5_LOG(ERR,
+ "No usable eDMA5 channels for %s (mask 0x%" PRIx64 ")",
+ name, ed->masked_channels);
+ rte_dma_pmd_release(name);
+ return -ENODEV;
+ }
+
+ dev->state = RTE_DMA_DEV_READY;
+
+ /*
+ * Set MP_CSR.GCLC (Global Clock Control) before any per-channel register
+ * is touched: the per-channel windows are individually clock-gated and
+ * external-abort when accessed with GCLC clear. Also enable round-robin
+ * arbitration (ERCA). Use read-modify-write to preserve reset defaults.
+ */
+ {
+ uint32_t mp_csr = imx_edma5_read32(ed->reg_base, IMX_EDMA5_MP_CSR);
+
+ mp_csr |= IMX_EDMA5_MP_CSR_GCLC | IMX_EDMA5_MP_CSR_ERCA;
+ imx_edma5_write32(ed->reg_base, IMX_EDMA5_MP_CSR, mp_csr);
+ }
+
+ IMX_EDMA5_LOG(INFO, "Probed i.MX95 eDMA5 dmadev %s (%u channels)",
+ name, ed->nb_channels);
+
+ return 0;
+}
+
+static int
+imx_edma5_remove(struct rte_platform_device *pdev)
+{
+ const char *name = pdev->name;
+
+ return rte_dma_pmd_release(name);
+}
+
+static struct rte_platform_driver imx_edma5_pmd_drv = {
+ /*
+ * Set the DT compatible string as the driver alias. The platform bus
+ * match logic compares this against the device-tree "compatible" sysfs
+ * strings of each platform device when the kernel driver name (always
+ * "vfio-platform") does not uniquely identify the device.
+ */
+ .driver = {
+ .alias = IMX_EDMA5_COMPAT,
+ },
+ .probe = imx_edma5_probe,
+ .remove = imx_edma5_remove,
+ /*
+ * The eDMA5 is programmed with the IOVA of the buffers, so it works in
+ * both IOVA=VA and IOVA=PA modes; no IOVA-as-VA requirement is forced.
+ */
+ .drv_flags = 0,
+};
+
+RTE_PMD_REGISTER_PLATFORM(dma_imx_edma5, imx_edma5_pmd_drv);
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.h b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
new file mode 100644
index 0000000000..3da7957457
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
@@ -0,0 +1,220 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+#ifndef IMX_EDMA5_DMADEV_H
+#define IMX_EDMA5_DMADEV_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_dmadev.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+
+#include "imx_edma5_hw.h"
+
+/*
+ * CPU data-cache maintenance for the non-coherent eDMA5 master: clean the
+ * source before the transfer and clean+invalidate the destination after it.
+ * DC IVAC (invalidate-only) is EL1-only and faults from userspace, so DC CIVAC
+ * is used. No-ops on non-arm64 builds.
+ */
+static inline void
+imx_edma5_dcbf(void *p)
+{
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dc cvac, %0" : : "r"(p) : "memory");
+#else
+ RTE_SET_USED(p);
+#endif
+}
+
+static inline void
+imx_edma5_dccivac(void *p)
+{
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dc civac, %0" : : "r"(p) : "memory");
+#else
+ RTE_SET_USED(p);
+#endif
+}
+
+/* Clean a VA range to the Point of Coherency. */
+static inline void
+imx_edma5_cache_clean(void *addr, size_t len)
+{
+ uintptr_t p = (uintptr_t)addr & ~(uintptr_t)(RTE_CACHE_LINE_SIZE - 1);
+ uintptr_t end = (uintptr_t)addr + len;
+
+ for (; p < end; p += RTE_CACHE_LINE_SIZE)
+ imx_edma5_dcbf((void *)p);
+ /*
+ * DSB ensures all DC CVAC instructions have completed to the Point of
+ * Coherency before the eDMA5 is allowed to read the source data.
+ */
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dsb sy" ::: "memory");
+#endif
+}
+
+/* Clean+invalidate a VA range to the Point of Coherency. */
+static inline void
+imx_edma5_cache_inval(void *addr, size_t len)
+{
+ uintptr_t p = (uintptr_t)addr & ~(uintptr_t)(RTE_CACHE_LINE_SIZE - 1);
+ uintptr_t end = (uintptr_t)addr + len;
+
+ for (; p < end; p += RTE_CACHE_LINE_SIZE)
+ imx_edma5_dccivac((void *)p);
+ /*
+ * DSB ensures all DC CIVAC instructions have completed to the Point of
+ * Coherency before the CPU reads the destination data written by eDMA5.
+ */
+#ifdef RTE_ARCH_ARM64
+ asm volatile("dsb sy" ::: "memory");
+#endif
+}
+
+/*
+ * NBYTES field maximum: bits 31:30 of TCD_NBYTES are SMLOE/DMLOE control bits
+ * when the minor loop offset feature is enabled. To avoid accidentally setting
+ * those bits, cap every single-block transfer at the 30-bit maximum count.
+ */
+#define IMX_EDMA5_MAX_NBYTES ((uint32_t)((1u << 30) - 1))
+
+/* Maximum scatter-gather segments per copy_sg request. */
+#define IMX_EDMA5_MAX_SGES 16
+
+/*
+ * In-memory TCD64 descriptors reserved per job slot for scatter-gather.
+ * Splitting an asymmetric src/dst segment list at the union of both sets of
+ * boundaries yields at most n_src + n_dst - 1 sub-transfers, so with up to
+ * IMX_EDMA5_MAX_SGES segments per side the worst case fits in this bound.
+ */
+#define IMX_EDMA5_SG_TCD_PER_JOB (2 * IMX_EDMA5_MAX_SGES)
+
+/* Software job ring size per virtual channel (power of two). */
+#define IMX_EDMA5_MAX_DESC 4096
+#define IMX_EDMA5_MIN_DESC 32
+
+/* Per in-flight job bookkeeping. */
+struct imx_edma5_job {
+ uint16_t ridx; /* ring index returned to application */
+ uint8_t submitted; /* job has been started on hardware */
+ uint8_t done; /* job completed */
+ uint8_t error; /* job completed with error */
+ /*
+ * Destination VA and byte count of the copy, used to invalidate the
+ * destination cache lines on completion (non-coherent eDMA master).
+ * NULL if the VA could not be resolved. Unused for SG jobs (nb_sg > 0).
+ */
+ void *dst_va;
+ uint32_t len;
+ /*
+ * Source/destination IOVAs of a plain single-block copy, recorded at
+ * enqueue time and used to program the TCD at submit time (the eDMA5 has
+ * a single register TCD shared by all jobs). Unused for SG jobs.
+ */
+ rte_iova_t src_iova;
+ rte_iova_t dst_iova;
+ /*
+ * Scatter-gather state. nb_sg is the segment count (0 for a plain copy);
+ * sg_tcd points at this job's slice of the vchan's in-memory TCD pool.
+ */
+ uint16_t nb_sg;
+ struct imx_edma5_hw_tcd64 *sg_tcd;
+};
+
+/* A virtual channel maps 1:1 onto a single eDMA5 hardware channel. */
+struct imx_edma5_vchan {
+ uint8_t *ch_regs; /* channel register window base */
+ uint8_t *tcd_regs; /* channel TCD base (ch_regs + TCD_OFF) */
+ uint32_t hw_chan; /* hardware channel index */
+
+ struct imx_edma5_job *jobs; /* software job ring */
+ uint16_t nb_desc; /* size of job ring (power of two) */
+ uint16_t desc_mask; /* nb_desc - 1 */
+
+ /*
+ * Pool of in-memory TCD64 descriptors for scatter-gather, sized
+ * nb_desc * IMX_EDMA5_SG_TCD_PER_JOB. Each job slot owns a contiguous
+ * slice of IMX_EDMA5_SG_TCD_PER_JOB descriptors. sg_tcd_iova is the
+ * pool base IOVA.
+ */
+ struct imx_edma5_hw_tcd64 *sg_tcd_pool;
+ rte_iova_t sg_tcd_iova;
+
+ uint16_t head; /* next slot to enqueue */
+ uint16_t tail; /* next slot to reap */
+ uint16_t nb_enqueued; /* outstanding jobs in ring (unreaped) */
+ uint16_t ridx; /* running ring index counter */
+ uint16_t last_idx; /* last completed ring index */
+
+ uint64_t submitted_count;
+ uint64_t completed_count;
+ uint64_t errors_count;
+
+ bool configured;
+};
+
+/* Per-device (per eDMA5 instance) private data. */
+struct imx_edma5_dev {
+ uint8_t *reg_base; /* mapped register window base */
+ uint64_t reg_size; /* mapped register window length */
+
+ uint16_t nb_channels; /* channels available on this instance */
+ uint16_t max_vchans; /* channels usable as dmadev vchans */
+
+ /*
+ * Bitmask of hardware channels reserved for other bus masters (from the
+ * device-tree "dma-channel-mask"). A set bit marks a channel this driver
+ * must not touch; accessing it faults with a bus external abort.
+ */
+ uint64_t masked_channels;
+ /* Map of usable dmadev vchan index -> hardware channel index. */
+ uint16_t chan_map[IMX_EDMA5_MAX_CHANNELS];
+
+ struct imx_edma5_vchan *vchans; /* array of vchan states */
+ uint16_t nb_vchans; /* number of configured vchans */
+
+ int16_t dev_id; /* dmadev id */
+};
+
+/* MMIO helpers (little-endian device). */
+static inline uint32_t
+imx_edma5_read32(const uint8_t *base, uint32_t off)
+{
+ return rte_le_to_cpu_32(rte_read32(base + off));
+}
+
+static inline void
+imx_edma5_write32(uint8_t *base, uint32_t off, uint32_t val)
+{
+ rte_write32(rte_cpu_to_le_32(val), base + off);
+}
+
+static inline uint16_t
+imx_edma5_read16(const uint8_t *base, uint32_t off)
+{
+ return rte_le_to_cpu_16(rte_read16(base + off));
+}
+
+static inline void
+imx_edma5_write16(uint8_t *base, uint32_t off, uint16_t val)
+{
+ rte_write16(rte_cpu_to_le_16(val), base + off);
+}
+
+static inline void
+imx_edma5_write64(uint8_t *base, uint32_t off, uint64_t val)
+{
+ /* Write the 64-bit field as two 32-bit accesses (order not significant). */
+ imx_edma5_write32(base, off, (uint32_t)(val & 0xFFFFFFFFu));
+ imx_edma5_write32(base, off + 4, (uint32_t)(val >> 32));
+}
+
+#endif /* IMX_EDMA5_DMADEV_H */
diff --git a/drivers/dma/imx_edma5/imx_edma5_hw.h b/drivers/dma/imx_edma5/imx_edma5_hw.h
new file mode 100644
index 0000000000..58e976ce4a
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_hw.h
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+/*
+ * Hardware register definitions for the NXP i.MX95 eDMA5 controller.
+ * The block has a Management Page (MP) region followed by per-channel register
+ * windows (base offset 0x10000, stride 0x8000, 64 channels, 64-bit TCD64).
+ * All register accesses are little-endian.
+ */
+
+#ifndef IMX_EDMA5_HW_H
+#define IMX_EDMA5_HW_H
+
+#include <stdint.h>
+
+/* Number of DMA channels implemented on i.MX95 eDMA5. */
+#define IMX_EDMA5_MAX_CHANNELS 64
+
+/* Per-channel register window geometry. */
+#define IMX_EDMA5_CHAN_BASE_OFF 0x10000u
+#define IMX_EDMA5_CHAN_STRIDE 0x8000u
+
+/*
+ * Management Page (MP) registers (offsets from register window base).
+ * Only the fields used by this driver are documented here.
+ */
+#define IMX_EDMA5_MP_CSR 0x0000u /* Management control */
+#define IMX_EDMA5_MP_ES 0x0004u /* Management error status */
+#define IMX_EDMA5_MP_INT_LOW 0x0008u /* Interrupt request (chan 0-31) */
+#define IMX_EDMA5_MP_INT_HIGH 0x000Cu /* Interrupt request (chan 32-63) */
+#define IMX_EDMA5_MP_HRS_LOW 0x0010u /* Hardware request status low */
+#define IMX_EDMA5_MP_HRS_HIGH 0x0014u /* Hardware request status high */
+
+/* MP_CSR bit fields. */
+#define IMX_EDMA5_MP_CSR_EDBG (1u << 1) /* Enable debug */
+#define IMX_EDMA5_MP_CSR_ERCA (1u << 2) /* Enable round-robin arb */
+#define IMX_EDMA5_MP_CSR_HAE (1u << 4) /* Halt after error */
+#define IMX_EDMA5_MP_CSR_GCLC (1u << 6) /* Global clock control */
+#define IMX_EDMA5_MP_CSR_GMRC (1u << 7) /* Global master ID replic */
+
+/* MP_ES: valid bit indicates a logged error is present. */
+#define IMX_EDMA5_MP_ES_VLD (1u << 31)
+
+/*
+ * Per-channel control registers (offsets from a channel window base).
+ * Layout matches the eDMA4/eDMA5 fsl_edma3_ch_reg structure.
+ */
+#define IMX_EDMA5_CH_CSR 0x00u /* Channel control/status */
+#define IMX_EDMA5_CH_ES 0x04u /* Channel error status */
+#define IMX_EDMA5_CH_INT 0x08u /* Channel interrupt status */
+#define IMX_EDMA5_CH_SBR 0x0Cu /* System bus register */
+#define IMX_EDMA5_CH_PRI 0x10u /* Channel priority */
+#define IMX_EDMA5_CH_MUX 0x14u /* Channel multiplexor (source) */
+#define IMX_EDMA5_CH_MATTR 0x18u /* Memory attributes */
+
+/* Channel window offset of the TCD (Transfer Control Descriptor). */
+#define IMX_EDMA5_CH_TCD_OFF 0x20u
+
+/* CH_CSR bit fields. */
+#define IMX_EDMA5_CH_CSR_ERQ (1u << 0) /* Enable hardware request */
+#define IMX_EDMA5_CH_CSR_EARQ (1u << 1) /* Enable async hw request */
+#define IMX_EDMA5_CH_CSR_EEI (1u << 2) /* Enable error interrupt */
+#define IMX_EDMA5_CH_CSR_DONE (1u << 30) /* Channel done (w1c) */
+#define IMX_EDMA5_CH_CSR_ACTIVE (1u << 31) /* Channel active */
+
+/* CH_ES: valid bit indicates a logged channel error. */
+#define IMX_EDMA5_CH_ES_ERR (1u << 31)
+
+/* CH_INT: write 1 to clear the channel interrupt request. */
+#define IMX_EDMA5_CH_INT_INT (1u << 0)
+
+/* CH_SBR: read/write privileged/secure attributes for bus mastering. */
+#define IMX_EDMA5_CH_SBR_RD (1u << 22)
+#define IMX_EDMA5_CH_SBR_WR (1u << 21)
+
+
+/*
+ * TCD64 field offsets, relative to the channel TCD base
+ * (channel window base + IMX_EDMA5_CH_TCD_OFF).
+ */
+#define IMX_EDMA5_TCD_SADDR 0x00u
+#define IMX_EDMA5_TCD_SOFF 0x08u
+#define IMX_EDMA5_TCD_ATTR 0x0Au
+#define IMX_EDMA5_TCD_NBYTES 0x0Cu
+#define IMX_EDMA5_TCD_SLAST 0x10u
+#define IMX_EDMA5_TCD_DADDR 0x18u
+#define IMX_EDMA5_TCD_DLAST_SGA 0x20u
+#define IMX_EDMA5_TCD_DOFF 0x28u
+#define IMX_EDMA5_TCD_CITER 0x2Au
+#define IMX_EDMA5_TCD_CSR 0x2Cu
+#define IMX_EDMA5_TCD_BITER 0x2Eu
+
+/* TCD ATTR sub-fields: transfer size is encoded as log2(bytes). GET_* extract. */
+#define IMX_EDMA5_TCD_ATTR_DSIZE(x) (((x) & 0x7u))
+#define IMX_EDMA5_TCD_ATTR_SSIZE(x) (((x) & 0x7u) << 8)
+#define IMX_EDMA5_TCD_ATTR_GET_DSIZE(x) ((x) & 0x7u)
+#define IMX_EDMA5_TCD_ATTR_GET_SSIZE(x) (((x) >> 8) & 0x7u)
+
+/* Transfer size encodings for ATTR SSIZE/DSIZE (log2 of bytes). */
+#define IMX_EDMA5_TCD_SIZE_1B 0u
+#define IMX_EDMA5_TCD_SIZE_2B 1u
+#define IMX_EDMA5_TCD_SIZE_4B 2u
+#define IMX_EDMA5_TCD_SIZE_8B 3u
+#define IMX_EDMA5_TCD_SIZE_16B 4u
+#define IMX_EDMA5_TCD_SIZE_32B 5u
+#define IMX_EDMA5_TCD_SIZE_64B 6u
+
+/* Major iteration count field mask (15-bit CITER/BITER). */
+#define IMX_EDMA5_TCD_ITER_MASK 0x7FFFu
+
+/* TCD CSR bit fields. */
+#define IMX_EDMA5_TCD_CSR_START (1u << 0) /* Software start */
+#define IMX_EDMA5_TCD_CSR_INT_MAJOR (1u << 1) /* Interrupt on major done */
+#define IMX_EDMA5_TCD_CSR_INT_HALF (1u << 2) /* Interrupt on half done */
+#define IMX_EDMA5_TCD_CSR_D_REQ (1u << 3) /* Disable request on done */
+#define IMX_EDMA5_TCD_CSR_E_SG (1u << 4) /* Enable scatter-gather */
+#define IMX_EDMA5_TCD_CSR_E_LINK (1u << 5) /* Enable channel linking */
+#define IMX_EDMA5_TCD_CSR_ACTIVE (1u << 6) /* Channel active */
+#define IMX_EDMA5_TCD_CSR_DONE (1u << 7) /* Channel done */
+
+/*
+ * In-memory 64-bit Transfer Control Descriptor. The field order and offsets
+ * match the register TCD64 layout above; all fields are little-endian. The
+ * descriptor must be 32-byte aligned and, on this non-coherent SoC, cleaned
+ * from the CPU cache before the transfer is started.
+ */
+struct __rte_aligned(32) imx_edma5_hw_tcd64 {
+ uint64_t saddr; /* 0x00 source address */
+ uint16_t soff; /* 0x08 source offset */
+ uint16_t attr; /* 0x0A transfer attributes */
+ uint32_t nbytes; /* 0x0C minor loop byte count */
+ uint64_t slast; /* 0x10 last source adjustment */
+ uint64_t daddr; /* 0x18 destination address */
+ uint64_t dlast_sga; /* 0x20 next TCD address (scatter-gather) */
+ uint16_t doff; /* 0x28 destination offset */
+ uint16_t citer; /* 0x2A current major iteration count */
+ uint16_t csr; /* 0x2C control and status */
+ uint16_t biter; /* 0x2E starting major iteration count */
+ /* Pad to 64 bytes total; the type is 32-byte aligned for TCD fetches. */
+ uint8_t reserved[16];
+};
+
+#endif /* IMX_EDMA5_HW_H */
diff --git a/drivers/dma/imx_edma5/imx_edma5_logs.h b/drivers/dma/imx_edma5/imx_edma5_logs.h
new file mode 100644
index 0000000000..e73e496bb4
--- /dev/null
+++ b/drivers/dma/imx_edma5/imx_edma5_logs.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2026 NXP
+ */
+
+#ifndef IMX_EDMA5_LOGS_H
+#define IMX_EDMA5_LOGS_H
+
+#include <rte_log.h>
+
+extern int imx_edma5_logtype;
+#define RTE_LOGTYPE_IMX_EDMA5 imx_edma5_logtype
+
+#define IMX_EDMA5_LOG(level, ...) \
+ RTE_LOG_LINE_PREFIX(level, IMX_EDMA5, "%s(): ", __func__, __VA_ARGS__)
+
+#endif /* IMX_EDMA5_LOGS_H */
diff --git a/drivers/dma/imx_edma5/meson.build b/drivers/dma/imx_edma5/meson.build
new file mode 100644
index 0000000000..e4f5d4e0bb
--- /dev/null
+++ b/drivers/dma/imx_edma5/meson.build
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2026 NXP
+
+if not is_linux
+ build = false
+ reason = 'only supported on linux'
+endif
+
+deps += ['dmadev', 'bus_platform']
+sources = files('imx_edma5_dmadev.c')
diff --git a/drivers/dma/meson.build b/drivers/dma/meson.build
index e0d94db967..dd00b1dae9 100644
--- a/drivers/dma/meson.build
+++ b/drivers/dma/meson.build
@@ -8,6 +8,7 @@ drivers = [
'hisi_acc',
'hisi_pciep',
'idxd',
+ 'imx_edma5',
'ioat',
'odm',
'skeleton',
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 3/5] dma/imx_edma5: add device configuration
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 1/5] bus/platform: match device by devicetree compatible string Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 2/5] dma/imx_edma5: introduce eDMA5 dmadev skeleton Gagandeep Singh
@ 2026-08-11 10:49 ` Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 4/5] dma/imx_edma5: add data path Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 5/5] dma/imx_edma5: add statistics and dump Gagandeep Singh
4 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-11 10:49 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add the configuration and lifecycle operations for the i.MX95 eDMA5
dmadev: device configure, virtual channel setup, start, stop and close.
Each virtual channel maps 1:1 onto a usable hardware channel and owns a
software job ring plus a pool of in-memory TCD64 descriptors for
scatter-gather. Channel setup validates the requested direction and ring
size, allocates these rings, and resets the hardware channel to a known
idle state. Start re-arms global clock and arbitration and clears the
per-channel bookkeeping; stop resets every configured channel; close
frees the per-channel resources.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 246 +++++++++++++++++++++++
1 file changed, 246 insertions(+)
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
index f1c65a474a..fe5539b612 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -78,6 +78,14 @@ imx_edma5_read_channel_mask(const char *dev_name)
return mask;
}
+/* Return the register base of hardware channel n. */
+static inline uint8_t *
+imx_edma5_chan_base(struct imx_edma5_dev *ed, uint32_t chan)
+{
+ return ed->reg_base + IMX_EDMA5_CHAN_BASE_OFF +
+ (size_t)chan * IMX_EDMA5_CHAN_STRIDE;
+}
+
static int
imx_edma5_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
uint32_t info_sz)
@@ -97,8 +105,246 @@ imx_edma5_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
return 0;
}
+static void imx_edma5_reset_hw_chan(struct imx_edma5_vchan *vc);
+
+static int
+imx_edma5_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf,
+ uint32_t conf_sz)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ RTE_SET_USED(conf_sz);
+
+ if (conf->nb_vchans == 0 || conf->nb_vchans > ed->max_vchans) {
+ IMX_EDMA5_LOG(ERR, "Invalid nb_vchans %u (max %u)",
+ conf->nb_vchans, ed->max_vchans);
+ return -EINVAL;
+ }
+
+ if (ed->vchans == NULL) {
+ ed->vchans = rte_zmalloc_socket("imx_edma5_vchans",
+ ed->max_vchans * sizeof(struct imx_edma5_vchan),
+ RTE_CACHE_LINE_SIZE, dev->data->numa_node);
+ if (ed->vchans == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to alloc vchan array");
+ return -ENOMEM;
+ }
+ } else {
+ /* Reconfigure: reset and free every previously configured channel. */
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ if (!vc->configured)
+ continue;
+ imx_edma5_reset_hw_chan(vc);
+ rte_free(vc->jobs);
+ rte_free(vc->sg_tcd_pool);
+ memset(vc, 0, sizeof(*vc));
+ }
+ }
+
+ ed->nb_vchans = conf->nb_vchans;
+
+ return 0;
+}
+
+/* Reset a hardware channel to a known idle state. */
+static void
+imx_edma5_reset_hw_chan(struct imx_edma5_vchan *vc)
+{
+ uint8_t *ch = vc->ch_regs;
+ uint8_t *tcd = vc->tcd_regs;
+ uint32_t sbr;
+
+ /*
+ * Disable hardware request and clear latched completion state.
+ * CH_CSR.DONE is write-1-to-clear, so write the DONE bit to clear any
+ * stale completion (e.g. left by the bootloader/kernel driver) while
+ * leaving all other control bits disabled.
+ */
+ imx_edma5_write32(ch, IMX_EDMA5_CH_CSR, IMX_EDMA5_CH_CSR_DONE);
+ imx_edma5_write32(ch, IMX_EDMA5_CH_ES, IMX_EDMA5_CH_ES_ERR);
+ imx_edma5_write32(ch, IMX_EDMA5_CH_INT, IMX_EDMA5_CH_INT_INT);
+
+ /*
+ * Enable the read/write attribute bits in the System Bus Register with a
+ * read-modify-write. The security/privilege attribute bits carried here
+ * come up with a valid reset default that the bus fabric (XRDC) checks
+ * and that must be preserved; a blind write of just RD|WR would clear
+ * them and make the fabric reject the eDMA master transaction.
+ */
+ sbr = imx_edma5_read32(ch, IMX_EDMA5_CH_SBR);
+ sbr |= IMX_EDMA5_CH_SBR_RD | IMX_EDMA5_CH_SBR_WR;
+ imx_edma5_write32(ch, IMX_EDMA5_CH_SBR, sbr);
+
+ /*
+ * Leave CH_MATTR at its power-on reset value. The eDMA5 is a
+ * non-coherent bus master; cache coherency is maintained by the driver
+ * via explicit DC CVAC/CIVAC cache maintenance, not by AXI snooping.
+ */
+
+ /* Clear the TCD control/status so the channel is idle. */
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CSR, 0);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CITER, 0);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_BITER, 0);
+}
+
+static int
+imx_edma5_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
+ const struct rte_dma_vchan_conf *conf,
+ uint32_t conf_sz)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ struct imx_edma5_vchan *vc;
+
+ RTE_SET_USED(conf_sz);
+
+ if (vchan >= ed->nb_vchans) {
+ IMX_EDMA5_LOG(ERR, "vchan %u out of range", vchan);
+ return -EINVAL;
+ }
+
+ if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM) {
+ IMX_EDMA5_LOG(ERR, "Only mem-to-mem direction supported");
+ return -EINVAL;
+ }
+
+ if (!rte_is_power_of_2(conf->nb_desc) ||
+ conf->nb_desc < IMX_EDMA5_MIN_DESC ||
+ conf->nb_desc > IMX_EDMA5_MAX_DESC) {
+ IMX_EDMA5_LOG(ERR, "nb_desc must be power of 2 in [%u..%u]",
+ IMX_EDMA5_MIN_DESC, IMX_EDMA5_MAX_DESC);
+ return -EINVAL;
+ }
+
+ vc = &ed->vchans[vchan];
+
+ /* Free previous rings if this vchan is being reconfigured. */
+ rte_free(vc->jobs);
+ rte_free(vc->sg_tcd_pool);
+ memset(vc, 0, sizeof(*vc));
+
+ /*
+ * Map this vchan onto a usable hardware channel. chan_map[] skips
+ * channels reserved by "dma-channel-mask" (channels 0 and 1 on i.MX95).
+ */
+ vc->hw_chan = ed->chan_map[vchan];
+ vc->ch_regs = imx_edma5_chan_base(ed, vc->hw_chan);
+ vc->tcd_regs = vc->ch_regs + IMX_EDMA5_CH_TCD_OFF;
+ vc->nb_desc = conf->nb_desc;
+ vc->desc_mask = conf->nb_desc - 1;
+
+ vc->jobs = rte_zmalloc_socket("imx_edma5_jobs",
+ vc->nb_desc * sizeof(struct imx_edma5_job),
+ RTE_CACHE_LINE_SIZE, dev->data->numa_node);
+ if (vc->jobs == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to alloc job ring for vchan %u",
+ vchan);
+ return -ENOMEM;
+ }
+
+ /* One IMX_EDMA5_SG_TCD_PER_JOB descriptor slice per job ring slot. */
+ vc->sg_tcd_pool = rte_zmalloc_socket("imx_edma5_sgtcd",
+ (size_t)vc->nb_desc * IMX_EDMA5_SG_TCD_PER_JOB *
+ sizeof(struct imx_edma5_hw_tcd64),
+ RTE_CACHE_LINE_SIZE, dev->data->numa_node);
+ if (vc->sg_tcd_pool == NULL) {
+ IMX_EDMA5_LOG(ERR, "Failed to alloc SG TCD pool for vchan %u",
+ vchan);
+ rte_free(vc->jobs);
+ vc->jobs = NULL;
+ return -ENOMEM;
+ }
+ vc->sg_tcd_iova = rte_malloc_virt2iova(vc->sg_tcd_pool);
+
+ imx_edma5_reset_hw_chan(vc);
+ vc->configured = true;
+
+ return 0;
+}
+
+static int
+imx_edma5_start(struct rte_dma_dev *dev)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint32_t mp_csr;
+ uint16_t i;
+
+ /*
+ * Enable round-robin arbitration with a read-modify-write so GCLC (set
+ * in probe) is preserved; clearing GCLC would re-gate the per-channel
+ * clocks and external-abort any subsequent channel access.
+ */
+ mp_csr = imx_edma5_read32(ed->reg_base, IMX_EDMA5_MP_CSR);
+ mp_csr |= IMX_EDMA5_MP_CSR_GCLC | IMX_EDMA5_MP_CSR_ERCA;
+ imx_edma5_write32(ed->reg_base, IMX_EDMA5_MP_CSR, mp_csr);
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ if (!vc->configured)
+ continue;
+ imx_edma5_reset_hw_chan(vc);
+ vc->head = 0;
+ vc->tail = 0;
+ vc->nb_enqueued = 0;
+ vc->ridx = 0;
+ /* Seed last_idx one step before the first cookie (0). */
+ vc->last_idx = UINT16_MAX;
+ vc->submitted_count = 0;
+ vc->completed_count = 0;
+ vc->errors_count = 0;
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_stop(struct rte_dma_dev *dev)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ if (vc->configured)
+ imx_edma5_reset_hw_chan(vc);
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_close(struct rte_dma_dev *dev)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint16_t i;
+
+ if (ed->vchans != NULL) {
+ for (i = 0; i < ed->max_vchans; i++) {
+ rte_free(ed->vchans[i].jobs);
+ rte_free(ed->vchans[i].sg_tcd_pool);
+ }
+ rte_free(ed->vchans);
+ ed->vchans = NULL;
+ }
+
+ ed->nb_vchans = 0;
+
+ return 0;
+}
+
static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_info_get = imx_edma5_info_get,
+ .dev_configure = imx_edma5_configure,
+ .dev_start = imx_edma5_start,
+ .dev_stop = imx_edma5_stop,
+ .dev_close = imx_edma5_close,
+
+ .vchan_setup = imx_edma5_vchan_setup,
};
static int
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 4/5] dma/imx_edma5: add data path
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
` (2 preceding siblings ...)
2026-08-11 10:49 ` [PATCH v3 3/5] dma/imx_edma5: add device configuration Gagandeep Singh
@ 2026-08-11 10:49 ` Gagandeep Singh
2026-08-11 10:49 ` [PATCH v3 5/5] dma/imx_edma5: add statistics and dump Gagandeep Singh
4 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-11 10:49 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add the memory-to-memory data path for the eDMA5 dmadev.
The eDMA5 exposes a single transfer control descriptor and a single
completion flag per hardware channel, so jobs are serialised in software.
Each enqueued job is played out synchronously as one or more single-block
transfers: the per-channel TCD is programmed, the transfer is
software-started and the driver busy-waits for the DONE flag with a
wall-clock bounded timeout. Scatter-gather copies walk the source and
destination segment lists as two cursors, emitting one single-block
sub-transfer per iteration that fits both current segments.
As the eDMA5 is a non-coherent bus master, source and destination buffers
are cleaned from the CPU cache before a transfer and the destination is
invalidated after completion so the application observes the DMA result.
This adds copy, copy_sg, submit, completed, completed_status and
burst_capacity, wired through the device fast-path object.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
doc/guides/dmadevs/imx_edma5.rst | 9 +
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 599 +++++++++++++++++++++++
drivers/dma/imx_edma5/imx_edma5_dmadev.h | 14 +
3 files changed, 622 insertions(+)
diff --git a/doc/guides/dmadevs/imx_edma5.rst b/doc/guides/dmadevs/imx_edma5.rst
index f57881816e..de7c49101e 100644
--- a/doc/guides/dmadevs/imx_edma5.rst
+++ b/doc/guides/dmadevs/imx_edma5.rst
@@ -58,3 +58,12 @@ Limitations
independently and programs one hardware transfer per consumed segment;
full TCD scatter-gather linking is not yet implemented.
- The driver operates in poll mode only; completion interrupts are not used.
+- Each operation executes synchronously: the driver programs the TCD,
+ starts the channel and busy-waits for completion inside the enqueue or
+ submit call. No transfer-offload benefit over memcpy is provided by this
+ first revision; the synchronous model is documented rather than implied.
+- Cache maintenance (source clean, destination clean+invalidate) is skipped
+ for any address that cannot be resolved to a CPU virtual address via
+ ``rte_mem_iova2virt()``. This affects externally-allocated memory not
+ registered with DPDK. Applications using such memory must ensure cache
+ coherency independently or register the memory with DPDK.
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
index fe5539b612..82eb1c87cb 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -337,6 +337,598 @@ imx_edma5_close(struct rte_dma_dev *dev)
return 0;
}
+/*
+ * Encode the largest natural transfer size (SSIZE/DSIZE) usable for a given
+ * source, destination and length. The address must be aligned to the transfer
+ * size and the byte count must be a multiple of it.
+ */
+static uint16_t
+imx_edma5_calc_attr(uint64_t src, uint64_t dst, uint32_t len)
+{
+ uint32_t sz = IMX_EDMA5_TCD_SIZE_1B;
+
+ if (((src | dst | len) & 0x1F) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_32B;
+ else if (((src | dst | len) & 0xF) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_16B;
+ else if (((src | dst | len) & 0x7) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_8B;
+ else if (((src | dst | len) & 0x3) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_4B;
+ else if (((src | dst | len) & 0x1) == 0)
+ sz = IMX_EDMA5_TCD_SIZE_2B;
+
+ return IMX_EDMA5_TCD_ATTR_SSIZE(sz) | IMX_EDMA5_TCD_ATTR_DSIZE(sz);
+}
+
+/*
+ * Program the channel TCD for a single-block copy: one minor loop of "len"
+ * bytes with a major count of 1. Completion is polled via CH_CSR.DONE.
+ */
+static inline void
+imx_edma5_program_copy(struct imx_edma5_vchan *vc, uint64_t src, uint64_t dst,
+ uint32_t len)
+{
+ uint8_t *tcd = vc->tcd_regs;
+ uint16_t attr = imx_edma5_calc_attr(src, dst, len);
+
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_SADDR, src);
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_DADDR, dst);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_ATTR, attr);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_SOFF,
+ (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_SSIZE(attr)));
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_DOFF,
+ (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_DSIZE(attr)));
+ imx_edma5_write32(tcd, IMX_EDMA5_TCD_NBYTES, len);
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_SLAST, 0);
+ imx_edma5_write64(tcd, IMX_EDMA5_TCD_DLAST_SGA, 0);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CITER, 1);
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_BITER, 1);
+ /* Clear CSR so no scatter-gather link is followed. */
+ imx_edma5_write16(tcd, IMX_EDMA5_TCD_CSR, 0);
+}
+
+/* Fill an in-memory hardware TCD64 descriptor for one copy segment. */
+static inline void
+imx_edma5_fill_tcd(struct imx_edma5_hw_tcd64 *t, uint64_t src, uint64_t dst,
+ uint32_t len)
+{
+ uint16_t attr = imx_edma5_calc_attr(src, dst, len);
+ uint16_t soff = (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_SSIZE(attr));
+ uint16_t doff = (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_DSIZE(attr));
+
+ t->saddr = rte_cpu_to_le_64(src);
+ t->soff = rte_cpu_to_le_16(soff);
+ t->attr = rte_cpu_to_le_16(attr);
+ t->nbytes = rte_cpu_to_le_32(len);
+ t->slast = 0;
+ t->daddr = rte_cpu_to_le_64(dst);
+ t->dlast_sga = 0;
+ t->doff = rte_cpu_to_le_16(doff);
+ t->citer = rte_cpu_to_le_16(1);
+ t->csr = 0;
+ t->biter = rte_cpu_to_le_16(1);
+}
+
+static inline void
+imx_edma5_hw_start(struct imx_edma5_vchan *vc)
+{
+ uint16_t csr = imx_edma5_read16(vc->tcd_regs, IMX_EDMA5_TCD_CSR);
+
+ csr |= IMX_EDMA5_TCD_CSR_START;
+ imx_edma5_write16(vc->tcd_regs, IMX_EDMA5_TCD_CSR, csr);
+}
+
+/*
+ * Invalidate the CPU cache lines covering a completed job's destination(s) so
+ * the application reads the DMA result rather than stale cache. The lines were
+ * cleaned at enqueue time, so this clean+invalidate behaves as a pure
+ * invalidate.
+ */
+static inline void
+imx_edma5_job_invalidate_dst(struct imx_edma5_job *job)
+{
+ if (job->nb_sg > 0) {
+ uint16_t s;
+
+ for (s = 0; s < job->nb_sg; s++) {
+ rte_iova_t da = rte_le_to_cpu_64(job->sg_tcd[s].daddr);
+ uint32_t len = rte_le_to_cpu_32(job->sg_tcd[s].nbytes);
+ void *va = imx_edma5_iova_to_virt(da);
+
+ if (va != NULL)
+ imx_edma5_cache_inval(va, len);
+ }
+ } else if (job->dst_va != NULL) {
+ imx_edma5_cache_inval(job->dst_va, job->len);
+ }
+}
+
+/*
+ * Upper bound on how long to poll for an entire job to complete.
+ * A wall-clock deadline is computed once per job in imx_edma5_run_job()
+ * and passed to every sub-transfer wait, so the bound covers the whole job
+ * rather than each sub-transfer individually.
+ * Each single-block transfer is capped at IMX_EDMA5_MAX_NBYTES (1 GiB - 1);
+ * at an AXI bus rate of 2 GB/s that is 512 ms in the worst case, so
+ * 1000 ms is a safe ceiling while still bounding a wedged channel.
+ */
+#define IMX_EDMA5_WAIT_TIMEOUT_MS 1000
+
+/*
+ * Wait for the single register TCD transfer to finish and clear its latched
+ * status. Returns true on success, false on a logged channel error (CH_ES.ERR)
+ * or timeout. The per-transfer completion flag CH_CSR.DONE and the CH_ES error
+ * bit are both write-1-to-clear.
+ */
+static inline bool
+imx_edma5_wait_done(struct imx_edma5_vchan *vc, uint64_t deadline)
+{
+ do {
+ uint32_t ch_es = imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES);
+ uint32_t ch_csr;
+
+ if (ch_es & IMX_EDMA5_CH_ES_ERR) {
+ imx_edma5_write32(vc->ch_regs, IMX_EDMA5_CH_ES,
+ IMX_EDMA5_CH_ES_ERR);
+ /* Reset the errored channel before the next job reuses it. */
+ imx_edma5_reset_hw_chan(vc);
+ return false;
+ }
+
+ ch_csr = imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR);
+ if (ch_csr & IMX_EDMA5_CH_CSR_DONE) {
+ imx_edma5_write32(vc->ch_regs, IMX_EDMA5_CH_CSR,
+ IMX_EDMA5_CH_CSR_DONE);
+ return true;
+ }
+ } while (rte_get_timer_cycles() < deadline);
+
+ IMX_EDMA5_LOG(ERR,
+ "channel %u timed out waiting for DONE (CH_CSR=0x%08x "
+ "CH_ES=0x%08x)",
+ vc->hw_chan,
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR),
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES));
+
+ /*
+ * Reset the channel registers to a known idle state. Note that this does
+ * not cancel an in-flight DMA transfer: the eDMA5 has no software-
+ * accessible abort/cancel bit (MP_CSR.CX does not exist on this
+ * variant). A channel that is genuinely stuck continues its bus
+ * transaction; reprogramming it here races with the active transfer.
+ */
+ imx_edma5_reset_hw_chan(vc);
+ return false;
+}
+
+/*
+ * Execute one job to completion on the channel's single register TCD.
+ *
+ * The eDMA5 exposes a single TCD and a single completion flag per channel, so
+ * jobs are serialised in software: a job is run synchronously here (program the
+ * TCD, software-start, busy-wait for DONE) and its completion recorded in
+ * job->done for the completion API to reap. Scatter-gather segments are played
+ * out one at a time as single-block transfers, since the eDMA5 does not
+ * auto-advance a hardware TCD chain for software-started mem-to-mem transfers.
+ * On success the destination cache lines are invalidated (non-coherent master).
+ */
+static inline void
+imx_edma5_run_job(struct imx_edma5_vchan *vc, struct imx_edma5_job *job)
+{
+ bool ok = true;
+ /*
+ * Compute a single deadline for the entire job before entering the
+ * sub-transfer loop. Passing it to every imx_edma5_wait_done() call
+ * means the timeout budget is shared across all sub-transfers rather
+ * than reset to a fresh 1000 ms per sub-transfer.
+ */
+ uint64_t deadline = rte_get_timer_cycles() +
+ (rte_get_timer_hz() * IMX_EDMA5_WAIT_TIMEOUT_MS) / 1000;
+
+ if (job->nb_sg > 0) {
+ uint16_t s;
+
+ for (s = 0; s < job->nb_sg; s++) {
+ uint64_t src = rte_le_to_cpu_64(job->sg_tcd[s].saddr);
+ uint64_t dst = rte_le_to_cpu_64(job->sg_tcd[s].daddr);
+ uint32_t len = rte_le_to_cpu_32(job->sg_tcd[s].nbytes);
+
+ imx_edma5_program_copy(vc, src, dst, len);
+ imx_edma5_hw_start(vc);
+ if (!imx_edma5_wait_done(vc, deadline)) {
+ ok = false;
+ break;
+ }
+ }
+ } else {
+ imx_edma5_program_copy(vc, job->src_iova, job->dst_iova,
+ job->len);
+ imx_edma5_hw_start(vc);
+ ok = imx_edma5_wait_done(vc, deadline);
+ }
+
+ if (ok)
+ imx_edma5_job_invalidate_dst(job);
+
+ job->error = ok ? 0 : 1;
+ job->done = 1;
+ vc->submitted_count++;
+}
+
+static int
+imx_edma5_copy(void *dev_private, uint16_t vchan, rte_iova_t src,
+ rte_iova_t dst, uint32_t length, uint64_t flags)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ struct imx_edma5_job *job;
+ uint16_t slot;
+
+ /* NBYTES = 0 is undefined on the eDMA5 and can wedge the channel. */
+ if (length == 0)
+ return -EINVAL;
+ /*
+ * TCD_NBYTES bits 31:30 are SMLOE/DMLOE when minor-loop offsets are
+ * enabled; writing a count larger than the 30-bit max would corrupt
+ * those control bits. Reject oversized requests.
+ */
+ if (length > IMX_EDMA5_MAX_NBYTES)
+ return -EINVAL;
+
+ /* Ring full? one slot is kept free to distinguish full from empty. */
+ if (vc->nb_enqueued >= (uint16_t)(vc->nb_desc - 1))
+ return -ENOSPC;
+
+ slot = vc->head;
+ job = &vc->jobs[slot];
+ job->ridx = vc->ridx;
+ job->submitted = 0;
+ job->done = 0;
+ job->error = 0;
+ job->nb_sg = 0;
+ job->len = length;
+ job->src_iova = src;
+ job->dst_iova = dst;
+ /*
+ * rte_mem_iova2virt() returns NULL when the IOVA is not in the
+ * memzone table (e.g. externally-allocated IOVA-contiguous memory not
+ * registered with DPDK). Cache maintenance is silently skipped for such
+ * addresses; callers are responsible for ensuring coherency in that case
+ * or for registering the memory so a VA mapping is available.
+ */
+ job->dst_va = imx_edma5_iova_to_virt(dst);
+
+ /*
+ * Non-cache-coherent master: clean the source so the device reads the
+ * CPU's latest writes, and clean the destination so a prior dirty line
+ * cannot be written back over the DMA result (the destination is
+ * invalidated after completion).
+ */
+ {
+ void *src_va = imx_edma5_iova_to_virt(src);
+
+ if (src_va != NULL)
+ imx_edma5_cache_clean(src_va, length);
+ if (job->dst_va != NULL)
+ imx_edma5_cache_clean(job->dst_va, length);
+ }
+
+ /*
+ * RTE_DMA_OP_FLAG_SUBMIT is equivalent to calling rte_dma_submit()
+ * after this enqueue: advance the head to include the new job and then
+ * run all pending (unsubmitted) jobs from tail to the new head in FIFO
+ * order, matching the behaviour of imx_edma5_submit().
+ */
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ uint16_t idx;
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ idx = vc->tail;
+ while (idx != vc->head) {
+ struct imx_edma5_job *j = &vc->jobs[idx];
+
+ if (!j->submitted) {
+ imx_edma5_run_job(vc, j);
+ j->submitted = 1;
+ }
+ idx = (idx + 1) & vc->desc_mask;
+ }
+
+ return vc->ridx++;
+ }
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ return vc->ridx++;
+}
+
+static int
+imx_edma5_copy_sg(void *dev_private, uint16_t vchan,
+ const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
+ uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ struct imx_edma5_hw_tcd64 *tcd;
+ struct imx_edma5_job *job;
+ uint16_t slot;
+ uint16_t s;
+ uint16_t si = 0, di = 0; /* current source/destination seg index */
+ uint32_t s_off = 0, d_off = 0; /* byte offset within current segment */
+ uint16_t nsg = 0; /* sub-transfers produced so far */
+ uint64_t src_total = 0, dst_total = 0;
+
+ /*
+ * The scatter-gather contract is a byte stream: the source and
+ * destination lists may be segmented independently but their
+ * concatenations are equal. The lists are walked as two cursors,
+ * emitting one single-block sub-transfer per run that fits in both the
+ * current source and destination segments. This yields at most
+ * nb_src + nb_dst - 1 sub-transfers, which fit in IMX_EDMA5_SG_TCD_PER_JOB.
+ */
+ if (nb_src == 0 || nb_dst == 0 ||
+ nb_src > IMX_EDMA5_MAX_SGES || nb_dst > IMX_EDMA5_MAX_SGES) {
+ IMX_EDMA5_LOG(ERR, "Unsupported SG shape src=%u dst=%u",
+ nb_src, nb_dst);
+ return -EINVAL;
+ }
+
+ /* Ring full? one slot is kept free to distinguish full from empty. */
+ if (vc->nb_enqueued >= (uint16_t)(vc->nb_desc - 1))
+ return -ENOSPC;
+
+ slot = vc->head;
+ job = &vc->jobs[slot];
+ job->ridx = vc->ridx;
+ job->submitted = 0;
+ job->done = 0;
+ job->error = 0;
+
+ /* This job's dedicated slice of the in-memory TCD pool. */
+ job->sg_tcd = &vc->sg_tcd_pool[(size_t)slot * IMX_EDMA5_SG_TCD_PER_JOB];
+ tcd = job->sg_tcd;
+
+ /*
+ * Clean every source and destination segment up front (non-coherent
+ * master): the device must read current source data, and dirty
+ * destination lines must be flushed before the transfer.
+ */
+ for (s = 0; s < nb_src; s++) {
+ void *va = imx_edma5_iova_to_virt(src[s].addr);
+
+ src_total += src[s].length;
+ if (va != NULL)
+ imx_edma5_cache_clean(va, src[s].length);
+ }
+ for (s = 0; s < nb_dst; s++) {
+ void *va = imx_edma5_iova_to_virt(dst[s].addr);
+
+ dst_total += dst[s].length;
+ if (va != NULL)
+ imx_edma5_cache_clean(va, dst[s].length);
+ }
+
+ /* copy_sg requires equal total bytes on both lists; reject misuse. */
+ if (src_total != dst_total) {
+ IMX_EDMA5_LOG(ERR,
+ "SG byte count mismatch src=%" PRIu64
+ " dst=%" PRIu64, src_total, dst_total);
+ return -EINVAL;
+ }
+
+ while (si < nb_src && di < nb_dst) {
+ uint32_t s_rem = src[si].length - s_off;
+ uint32_t d_rem = dst[di].length - d_off;
+ uint32_t len = RTE_MIN(s_rem, d_rem);
+
+ /*
+ * TCD_NBYTES bits 31:30 are SMLOE/DMLOE; a sub-transfer that
+ * spans more than IMX_EDMA5_MAX_NBYTES would corrupt those bits.
+ * Individual segment lengths up to UINT32_MAX are permitted by
+ * the dmadev API, so validate here rather than assume the caller
+ * has split them.
+ */
+ if (len > IMX_EDMA5_MAX_NBYTES) {
+ IMX_EDMA5_LOG(ERR,
+ "SG sub-transfer length %u exceeds "
+ "TCD_NBYTES max %u",
+ len, IMX_EDMA5_MAX_NBYTES);
+ job->nb_sg = 0;
+ return -EINVAL;
+ }
+
+ /* Skip zero-length segments without emitting a descriptor. */
+ if (len == 0) {
+ if (s_rem == 0) {
+ si++;
+ s_off = 0;
+ }
+ if (d_rem == 0) {
+ di++;
+ d_off = 0;
+ }
+ continue;
+ }
+
+ if (nsg >= IMX_EDMA5_SG_TCD_PER_JOB) {
+ IMX_EDMA5_LOG(ERR,
+ "SG produced too many sub-transfers "
+ "(src=%u dst=%u)", nb_src, nb_dst);
+ /* Defensive: clear nb_sg so the abandoned slot is not reused. */
+ job->nb_sg = 0;
+ return -EINVAL;
+ }
+
+ imx_edma5_fill_tcd(&tcd[nsg], src[si].addr + s_off,
+ dst[di].addr + d_off, len);
+ nsg++;
+
+ s_off += len;
+ d_off += len;
+ if (s_off == src[si].length) {
+ si++;
+ s_off = 0;
+ }
+ if (d_off == dst[di].length) {
+ di++;
+ d_off = 0;
+ }
+ }
+
+ /* No sub-transfer (all segments zero-length): NBYTES = 0 wedges eDMA5. */
+ if (nsg == 0) {
+ IMX_EDMA5_LOG(ERR, "SG produced zero sub-transfers");
+ return -EINVAL;
+ }
+
+ job->nb_sg = nsg;
+
+ /*
+ * RTE_DMA_OP_FLAG_SUBMIT is equivalent to calling rte_dma_submit()
+ * after this enqueue: advance the head to include the new job and then
+ * run all pending (unsubmitted) jobs from tail to the new head in FIFO
+ * order, matching the behaviour of imx_edma5_submit().
+ */
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ uint16_t idx;
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ idx = vc->tail;
+ while (idx != vc->head) {
+ struct imx_edma5_job *j = &vc->jobs[idx];
+
+ if (!j->submitted) {
+ imx_edma5_run_job(vc, j);
+ j->submitted = 1;
+ }
+ idx = (idx + 1) & vc->desc_mask;
+ }
+
+ return vc->ridx++;
+ }
+
+ vc->head = (vc->head + 1) & vc->desc_mask;
+ vc->nb_enqueued++;
+
+ return vc->ridx++;
+}
+
+static int
+imx_edma5_submit(void *dev_private, uint16_t vchan)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t idx = vc->tail;
+
+ /* Run every enqueued-but-not-yet-submitted job to completion (FIFO). */
+ while (idx != vc->head) {
+ struct imx_edma5_job *job = &vc->jobs[idx];
+
+ if (!job->submitted) {
+ imx_edma5_run_job(vc, job);
+ job->submitted = 1;
+ }
+ idx = (idx + 1) & vc->desc_mask;
+ }
+
+ return 0;
+}
+
+/*
+ * Reap completed jobs from the software ring in FIFO order. Jobs run
+ * synchronously, so a submitted job's result is already in job->done/error.
+ * An unsubmitted job stops the walk.
+ */
+static uint16_t
+imx_edma5_completed(void *dev_private, uint16_t vchan, const uint16_t nb_cpls,
+ uint16_t *last_idx, bool *has_error)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t count = 0;
+
+ *has_error = false;
+
+ while (count < nb_cpls && vc->tail != vc->head) {
+ struct imx_edma5_job *job = &vc->jobs[vc->tail];
+
+ if (!job->submitted || !job->done)
+ break;
+
+ /*
+ * Stop before an errored job: it is left in the ring for
+ * rte_dma_completed_status(), and last_idx stays at the last
+ * successful transfer.
+ */
+ if (job->error) {
+ *has_error = true;
+ break;
+ }
+
+ vc->last_idx = job->ridx;
+ vc->completed_count++;
+
+ vc->tail = (vc->tail + 1) & vc->desc_mask;
+ vc->nb_enqueued--;
+ count++;
+ }
+
+ *last_idx = vc->last_idx;
+
+ return count;
+}
+
+static uint16_t
+imx_edma5_completed_status(void *dev_private, uint16_t vchan,
+ const uint16_t nb_cpls, uint16_t *last_idx,
+ enum rte_dma_status_code *status)
+{
+ struct imx_edma5_dev *ed = dev_private;
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t count = 0;
+
+ while (count < nb_cpls && vc->tail != vc->head) {
+ struct imx_edma5_job *job = &vc->jobs[vc->tail];
+
+ if (!job->submitted || !job->done)
+ break;
+
+ if (job->error) {
+ status[count] = RTE_DMA_STATUS_BUS_ERROR;
+ vc->errors_count++;
+ } else {
+ status[count] = RTE_DMA_STATUS_SUCCESSFUL;
+ }
+
+ vc->last_idx = job->ridx;
+ vc->completed_count++;
+
+ vc->tail = (vc->tail + 1) & vc->desc_mask;
+ vc->nb_enqueued--;
+ count++;
+ }
+
+ *last_idx = vc->last_idx;
+
+ return count;
+}
+
+static uint16_t
+imx_edma5_burst_capacity(const void *dev_private, uint16_t vchan)
+{
+ const struct imx_edma5_dev *ed = dev_private;
+ const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+
+ /* One slot is reserved to distinguish full from empty. */
+ return vc->nb_desc - 1 - vc->nb_enqueued;
+}
+
static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_info_get = imx_edma5_info_get,
.dev_configure = imx_edma5_configure,
@@ -382,6 +974,13 @@ imx_edma5_probe(struct rte_platform_device *pdev)
dev->device = &pdev->device;
dev->dev_ops = &imx_edma5_ops;
+ dev->fp_obj->dev_private = dev->data->dev_private;
+ dev->fp_obj->copy = imx_edma5_copy;
+ dev->fp_obj->copy_sg = imx_edma5_copy_sg;
+ dev->fp_obj->submit = imx_edma5_submit;
+ dev->fp_obj->completed = imx_edma5_completed;
+ dev->fp_obj->completed_status = imx_edma5_completed_status;
+ dev->fp_obj->burst_capacity = imx_edma5_burst_capacity;
ed = dev->data->dev_private;
ed->reg_base = res->mem.addr;
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.h b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
index 3da7957457..c85ea4a726 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.h
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
@@ -12,6 +12,7 @@
#include <rte_byteorder.h>
#include <rte_common.h>
#include <rte_dmadev.h>
+#include <rte_eal.h>
#include <rte_io.h>
#include <rte_memory.h>
@@ -217,4 +218,17 @@ imx_edma5_write64(uint8_t *base, uint32_t off, uint64_t val)
imx_edma5_write32(base, off + 4, (uint32_t)(val >> 32));
}
+/*
+ * Resolve an IOVA to a CPU virtual address for cache maintenance.
+ * In IOVA=VA mode the IOVA is the VA directly; otherwise walk the memseg
+ * table. Avoids the memseg list walk on the common IOVA=VA fast path.
+ */
+static inline void *
+imx_edma5_iova_to_virt(rte_iova_t iova)
+{
+ if (rte_eal_iova_mode() == RTE_IOVA_VA)
+ return (void *)(uintptr_t)iova;
+ return rte_mem_iova2virt(iova);
+}
+
#endif /* IMX_EDMA5_DMADEV_H */
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 5/5] dma/imx_edma5: add statistics and dump
2026-08-11 10:49 ` [PATCH v3 " Gagandeep Singh
` (3 preceding siblings ...)
2026-08-11 10:49 ` [PATCH v3 4/5] dma/imx_edma5: add data path Gagandeep Singh
@ 2026-08-11 10:49 ` Gagandeep Singh
4 siblings, 0 replies; 22+ messages in thread
From: Gagandeep Singh @ 2026-08-11 10:49 UTC (permalink / raw)
To: dev; +Cc: hemant.agrawal, Gagandeep Singh, Prashant Gupta
Add per-vchan statistics, virtual channel status reporting and a device
dump callback to the eDMA5 dmadev.
stats_get and stats_reset expose the submitted, completed and error
counters maintained by the data path, supporting both a single vchan and
the RTE_DMA_ALL_VCHAN aggregate. vchan_status reports whether a channel is
idle, active or halted on an unreaped error by inspecting the outstanding
jobs in the software ring. dev_dump prints the device and per-vchan
software state to aid debugging.
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Signed-off-by: Prashant Gupta <prashant.gupta_3@nxp.com>
---
drivers/dma/imx_edma5/imx_edma5_dmadev.c | 126 +++++++++++++++++++++++
1 file changed, 126 insertions(+)
diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
index 82eb1c87cb..0793c9aa7a 100644
--- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
+++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
@@ -929,6 +929,126 @@ imx_edma5_burst_capacity(const void *dev_private, uint16_t vchan)
return vc->nb_desc - 1 - vc->nb_enqueued;
}
+static int
+imx_edma5_vchan_status(const struct rte_dma_dev *dev, uint16_t vchan,
+ enum rte_dma_vchan_status *status)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+ const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+ uint16_t idx;
+
+ /*
+ * HALTED_ERROR if an errored job is still unreaped, ACTIVE while any job
+ * is outstanding (not done, or enqueued but not submitted), else IDLE.
+ */
+ *status = RTE_DMA_VCHAN_IDLE;
+
+ for (idx = vc->tail; idx != vc->head; idx = (idx + 1) & vc->desc_mask) {
+ const struct imx_edma5_job *job = &vc->jobs[idx];
+
+ if (job->submitted && job->done && job->error) {
+ *status = RTE_DMA_VCHAN_HALTED_ERROR;
+ break;
+ }
+
+ if (!job->submitted || !job->done) {
+ *status = RTE_DMA_VCHAN_ACTIVE;
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
+ struct rte_dma_stats *stats, uint32_t stats_sz)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ RTE_SET_USED(stats_sz);
+
+ stats->submitted = 0;
+ stats->completed = 0;
+ stats->errors = 0;
+
+ /* RTE_DMA_ALL_VCHAN requests the aggregate across every vchan. */
+ if (vchan == RTE_DMA_ALL_VCHAN) {
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ const struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ stats->submitted += vc->submitted_count;
+ stats->completed += vc->completed_count;
+ stats->errors += vc->errors_count;
+ }
+ } else if (vchan < ed->nb_vchans) {
+ const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+
+ stats->submitted = vc->submitted_count;
+ stats->completed = vc->completed_count;
+ stats->errors = vc->errors_count;
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
+{
+ struct imx_edma5_dev *ed = dev->data->dev_private;
+
+ /* RTE_DMA_ALL_VCHAN requests a reset of every vchan. */
+ if (vchan == RTE_DMA_ALL_VCHAN) {
+ uint16_t i;
+
+ for (i = 0; i < ed->nb_vchans; i++) {
+ struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ vc->submitted_count = 0;
+ vc->completed_count = 0;
+ vc->errors_count = 0;
+ }
+ } else if (vchan < ed->nb_vchans) {
+ struct imx_edma5_vchan *vc = &ed->vchans[vchan];
+
+ vc->submitted_count = 0;
+ vc->completed_count = 0;
+ vc->errors_count = 0;
+ }
+
+ return 0;
+}
+
+static int
+imx_edma5_dump(const struct rte_dma_dev *dev, FILE *f)
+{
+ const struct imx_edma5_dev *ed = dev->data->dev_private;
+ uint16_t i;
+
+ fprintf(f, " imx_edma5 nb_channels=%u nb_vchans=%u\n",
+ ed->nb_channels, ed->nb_vchans);
+ for (i = 0; i < ed->nb_vchans; i++) {
+ const struct imx_edma5_vchan *vc = &ed->vchans[i];
+
+ fprintf(f,
+ " vchan %u: hw_chan=%u nb_desc=%u enqueued=%u "
+ "submitted=%" PRIu64 " completed=%" PRIu64
+ " errors=%" PRIu64 "\n",
+ i, vc->hw_chan, vc->nb_desc, vc->nb_enqueued,
+ vc->submitted_count, vc->completed_count,
+ vc->errors_count);
+ /* Hardware register snapshot for debug (CH_CSR, CH_ES). */
+ fprintf(f,
+ " hw: CH_CSR=0x%08x CH_ES=0x%08x\n",
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR),
+ imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES));
+ }
+
+ return 0;
+}
+
static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_info_get = imx_edma5_info_get,
.dev_configure = imx_edma5_configure,
@@ -937,6 +1057,12 @@ static const struct rte_dma_dev_ops imx_edma5_ops = {
.dev_close = imx_edma5_close,
.vchan_setup = imx_edma5_vchan_setup,
+ .vchan_status = imx_edma5_vchan_status,
+
+ .stats_get = imx_edma5_stats_get,
+ .stats_reset = imx_edma5_stats_reset,
+
+ .dev_dump = imx_edma5_dump,
};
static int
--
2.25.1
^ permalink raw reply related [flat|nested] 22+ messages in thread