* [PATCH v7 1/7] dmaengine: dw-edma: Add a channel transfer size operation
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:16 ` sashiko-bot
2026-11-04 16:12 ` [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status Koichiro Den
` (5 subsequent siblings)
6 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
Expose the channel transfer size through a core operation for eDMA and
HDMA. LL progress handling will use it to check for a stopped channel
with no payload left to transfer.
PCI Express DM Controller Databook 6.10a-lca06, section 7.3.4,
"Detecting the End of the HDMA Transfer", documents this polling method:
STATUS field of the HDMA Write/Read Channel Status register
(HDMA_STATUS_OFF_[WR|RD]CH_i) is Stopped, and the HDMA Transfer Size
register is 0x0.
The same condition applies to eDMA as well.
No functional change on its own.
Cc: stable@vger.kernel.org
Reviewed-by: Frank Li <Frank.Li@nxp.com>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- Add Frank's Reviewed-by.
- Add Cc:stable because patch 2 depends on this change.
- No code changes.
drivers/dma/dw-edma/dw-edma-core.h | 6 ++++++
drivers/dma/dw-edma/dw-edma-v0-core.c | 6 ++++++
drivers/dma/dw-edma/dw-hdma-v0-core.c | 6 ++++++
3 files changed, 18 insertions(+)
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 76fd5a41a707..84a811bdcb6d 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -163,6 +163,7 @@ struct dw_edma_core_ops {
int (*ch_quiesce)(struct dw_edma_chan *chan);
u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
enum dma_status (*ch_status)(struct dw_edma_chan *chan);
+ u32 (*ch_transfer_size)(struct dw_edma_chan *chan);
irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
dw_edma_handler_t handler);
void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child);
@@ -262,6 +263,11 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan)
return chan->dw->core->ch_status(chan);
}
+static inline u32 dw_edma_core_ch_transfer_size(struct dw_edma_chan *chan)
+{
+ return chan->dw->core->ch_transfer_size(chan);
+}
+
static inline irqreturn_t
dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
dw_edma_handler_t handler)
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index c053cb4c2844..fa8be85c7ac1 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -318,6 +318,11 @@ static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
return DMA_ERROR;
}
+static u32 dw_edma_v0_core_ch_transfer_size(struct dw_edma_chan *chan)
+{
+ return GET_CH_32(chan->dw, chan->dir, chan->id, transfer_size);
+}
+
static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
@@ -665,6 +670,7 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
.ch_quiesce = dw_edma_v0_core_ch_quiesce,
.ch_count = dw_edma_v0_core_ch_count,
.ch_status = dw_edma_v0_core_ch_status,
+ .ch_transfer_size = dw_edma_v0_core_ch_transfer_size,
.handle_int = dw_edma_v0_core_handle_int,
.ll_data = dw_edma_v0_core_ll_data,
.ll_link = dw_edma_v0_core_ll_link,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index d42cb4ad49a2..aa430219d0af 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -156,6 +156,11 @@ static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)
return DMA_ERROR;
}
+static u32 dw_hdma_v0_core_ch_transfer_size(struct dw_edma_chan *chan)
+{
+ return GET_CH_32(chan->dw, chan->dir, chan->id, transfer_size);
+}
+
static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
@@ -425,6 +430,7 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
.ch_quiesce = dw_hdma_v0_core_ch_quiesce,
.ch_count = dw_hdma_v0_core_ch_count,
.ch_status = dw_hdma_v0_core_ch_status,
+ .ch_transfer_size = dw_hdma_v0_core_ch_transfer_size,
.handle_int = dw_hdma_v0_core_handle_int,
.non_ll_start = dw_hdma_v0_core_non_ll_start,
.ll_data = dw_hdma_v0_core_ll_data,
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread* [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
2026-11-04 16:12 ` [PATCH v7 1/7] dmaengine: dw-edma: Add a channel transfer size operation Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:36 ` sashiko-bot
2026-11-04 17:21 ` Frank Li
2026-11-04 16:12 ` [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion Koichiro Den
` (4 subsequent siblings)
6 siblings, 2 replies; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
CS can still report STOPPED briefly after a doorbell write, before the
DMA engine starts consuming the published LL entries. Finishing
termination relying solely on that value can reset the ring while a
transfer is about to start, leaving hardware and software ring positions
out of sync and stalling subsequent transfers.
Require LLP to reach the published tail, on top of the standard criteria
(ie. CS=STOPPED and zero transfer size) before
dw_edma_finish_termination(). Non-LL mode behavior is unchanged.
Cc: stable@vger.kernel.org
Fixes: db79c4e264ef ("dmaengine: dw-edma: Make DMA link list work as a circular buffer")
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- New patch. Resolve issues found during terminate/pause test under
dma load.
drivers/dma/dw-edma/dw-edma-core.c | 24 ++++++++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index b08bb31657c9..fb38d51037c5 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -146,6 +146,23 @@ static bool dw_edma_ll_pending(struct dw_edma_chan *chan)
return chan->ll_head != chan->ll_done;
}
+static bool dw_edma_ll_stopped(struct dw_edma_chan *chan)
+{
+ return dw_edma_core_ch_status(chan) == DMA_COMPLETE &&
+ !dw_edma_core_ch_transfer_size(chan);
+}
+
+static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
+{
+ int idx = dw_edma_core_ll_cur_idx(chan);
+
+ if (idx == chan->ll_max)
+ idx = 0;
+
+ return idx >= 0 && (u32)idx == chan->ll_head &&
+ dw_edma_ll_stopped(chan);
+}
+
static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
{
struct dw_edma_chan *chan = desc->chan;
@@ -455,7 +472,9 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
dw_edma_finish_termination(chan);
} else if (chan->status == EDMA_ST_IDLE) {
dw_edma_finish_termination(chan);
- } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
+ } else if (chan->non_ll ?
+ dw_edma_core_ch_status(chan) == DMA_COMPLETE :
+ dw_edma_ll_drained(chan)) {
/*
* The channel is in a false BUSY state, probably didn't
* receive or lost an interrupt
@@ -790,7 +809,8 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
if (!vd)
break;
- dw_edma_finish_termination(chan);
+ if (chan->non_ll || dw_edma_ll_drained(chan))
+ dw_edma_finish_termination(chan);
break;
default:
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status
2026-11-04 16:12 ` [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status Koichiro Den
@ 2026-11-04 16:36 ` sashiko-bot
2026-11-04 17:21 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: sashiko-bot @ 2026-11-04 16:36 UTC (permalink / raw)
To: Koichiro Den; +Cc: linux-pci
> CS can still report STOPPED briefly after a doorbell write, before the
> DMA engine starts consuming the published LL entries. Finishing
> termination relying solely on that value can reset the ring while a
> transfer is about to start, leaving hardware and software ring positions
> out of sync and stalling subsequent transfers.
>
> Require LLP to reach the published tail, on top of the standard criteria
> (ie. CS=STOPPED and zero transfer size) before
> dw_edma_finish_termination(). Non-LL mode behavior is unchanged.
>
> Cc: stable@vger.kernel.org
> Fixes: db79c4e264ef ("dmaengine: dw-edma: Make DMA link list work as a circular buffer")
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
Sashiko has reviewed this patch and found no issues.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104161227.1742005-1-den@valinux.co.jp?part=2
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status
2026-11-04 16:12 ` [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status Koichiro Den
2026-11-04 16:36 ` sashiko-bot
@ 2026-11-04 17:21 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: Frank Li @ 2026-11-04 17:21 UTC (permalink / raw)
To: Koichiro Den
Cc: Vinod Koul, Frank Li, Manivannan Sadhasivam, Gustavo Pimentel,
Serge Semin, Cai Huoqing, Niklas Cassel, Devendra K Verma,
dmaengine, linux-kernel, linux-pci
On Thu, Nov 05, 2026 at 01:12:22AM +0900, Koichiro Den wrote:
> CS can still report STOPPED briefly after a doorbell write, before the
> DMA engine starts consuming the published LL entries. Finishing
> termination relying solely on that value can reset the ring while a
> transfer is about to start, leaving hardware and software ring positions
> out of sync and stalling subsequent transfers.
>
> Require LLP to reach the published tail, on top of the standard criteria
> (ie. CS=STOPPED and zero transfer size) before
> dw_edma_finish_termination(). Non-LL mode behavior is unchanged.
>
> Cc: stable@vger.kernel.org
> Fixes: db79c4e264ef ("dmaengine: dw-edma: Make DMA link list work as a circular buffer")
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
Reviewed-by: Frank Li <Frank.Li@nxp.com>
> Changes from v6:
> - New patch. Resolve issues found during terminate/pause test under
> dma load.
>
> drivers/dma/dw-edma/dw-edma-core.c | 24 ++++++++++++++++++++++--
> 1 file changed, 22 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
> index b08bb31657c9..fb38d51037c5 100644
> --- a/drivers/dma/dw-edma/dw-edma-core.c
> +++ b/drivers/dma/dw-edma/dw-edma-core.c
> @@ -146,6 +146,23 @@ static bool dw_edma_ll_pending(struct dw_edma_chan *chan)
> return chan->ll_head != chan->ll_done;
> }
>
> +static bool dw_edma_ll_stopped(struct dw_edma_chan *chan)
> +{
> + return dw_edma_core_ch_status(chan) == DMA_COMPLETE &&
> + !dw_edma_core_ch_transfer_size(chan);
> +}
> +
> +static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
> +{
> + int idx = dw_edma_core_ll_cur_idx(chan);
> +
> + if (idx == chan->ll_max)
> + idx = 0;
> +
> + return idx >= 0 && (u32)idx == chan->ll_head &&
> + dw_edma_ll_stopped(chan);
> +}
> +
> static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
> {
> struct dw_edma_chan *chan = desc->chan;
> @@ -455,7 +472,9 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
> dw_edma_finish_termination(chan);
> } else if (chan->status == EDMA_ST_IDLE) {
> dw_edma_finish_termination(chan);
> - } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
> + } else if (chan->non_ll ?
> + dw_edma_core_ch_status(chan) == DMA_COMPLETE :
> + dw_edma_ll_drained(chan)) {
> /*
> * The channel is in a false BUSY state, probably didn't
> * receive or lost an interrupt
> @@ -790,7 +809,8 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> if (!vd)
> break;
>
> - dw_edma_finish_termination(chan);
> + if (chan->non_ll || dw_edma_ll_drained(chan))
> + dw_edma_finish_termination(chan);
> break;
>
> default:
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
2026-11-04 16:12 ` [PATCH v7 1/7] dmaengine: dw-edma: Add a channel transfer size operation Koichiro Den
2026-11-04 16:12 ` [PATCH v7 2/7] dmaengine: dw-edma: Avoid resetting an LL on stale stopped status Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:15 ` sashiko-bot
2026-11-04 17:35 ` Frank Li
2026-11-04 16:12 ` [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress Koichiro Den
` (3 subsequent siblings)
6 siblings, 2 replies; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
Completing a descriptor takes three steps: record the callback result,
unlink it from the issued list and hand it to virt-dma.
Factor them into dw_edma_complete_vdesc() to avoid sprinkling
vchan_cookie_complete() calls throughout the driver.
No functional change.
Suggested-by: Frank Li <Frank.Li@nxp.com>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- New patch, per Frank's comment here:
https://lore.kernel.org/r/auTL2mHwCt20uhII@SMW015318/
"let vchan_cookie_complete() at one place only".
drivers/dma/dw-edma/dw-edma-core.c | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index fb38d51037c5..e2f7af5969c6 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -107,6 +107,15 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
res->residue = residue;
}
+/* Must be called with vc.lock held. */
+static void dw_edma_complete_vdesc(struct virt_dma_desc *vd,
+ enum dmaengine_tx_result result)
+{
+ dw_hdma_set_callback_result(vd, result);
+ list_del(&vd->node);
+ vchan_cookie_complete(vd);
+}
+
static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
{
u32 i;
@@ -788,9 +797,7 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
desc = vd2dw_edma_desc(vd);
if (desc->start_burst >= desc->nburst) {
- dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR);
- list_del(&vd->node);
- vchan_cookie_complete(vd);
+ dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
if (!chan->non_ll)
chan->ll_done = chan->ll_head;
}
@@ -838,9 +845,7 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
if (vd && chan->request == EDMA_REQ_STOP) {
dw_edma_terminate_all_descs(chan);
} else if (vd) {
- dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED);
- list_del(&vd->node);
- vchan_cookie_complete(vd);
+ dw_edma_complete_vdesc(vd, DMA_TRANS_ABORTED);
}
if (!chan->non_ll)
dw_edma_core_reset_ll(chan);
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion
2026-11-04 16:12 ` [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion Koichiro Den
@ 2026-11-04 16:15 ` sashiko-bot
2026-11-04 17:35 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: sashiko-bot @ 2026-11-04 16:15 UTC (permalink / raw)
To: Koichiro Den; +Cc: linux-pci
> Completing a descriptor takes three steps: record the callback result,
> unlink it from the issued list and hand it to virt-dma.
>
> Factor them into dw_edma_complete_vdesc() to avoid sprinkling
> vchan_cookie_complete() calls throughout the driver.
>
> No functional change.
>
> Suggested-by: Frank Li <Frank.Li@nxp.com>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
Sashiko has reviewed this patch and found no issues.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104161227.1742005-1-den@valinux.co.jp?part=3
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion
2026-11-04 16:12 ` [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion Koichiro Den
2026-11-04 16:15 ` sashiko-bot
@ 2026-11-04 17:35 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: Frank Li @ 2026-11-04 17:35 UTC (permalink / raw)
To: Koichiro Den
Cc: Vinod Koul, Frank Li, Manivannan Sadhasivam, Gustavo Pimentel,
Serge Semin, Cai Huoqing, Niklas Cassel, Devendra K Verma,
dmaengine, linux-kernel, linux-pci
On Thu, Nov 05, 2026 at 01:12:23AM +0900, Koichiro Den wrote:
> Completing a descriptor takes three steps: record the callback result,
> unlink it from the issued list and hand it to virt-dma.
>
> Factor them into dw_edma_complete_vdesc() to avoid sprinkling
> vchan_cookie_complete() calls throughout the driver.
>
> No functional change.
>
> Suggested-by: Frank Li <Frank.Li@nxp.com>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
Reviewed-by: Frank Li <Frank.Li@nxp.com>
> Changes from v6:
> - New patch, per Frank's comment here:
> https://lore.kernel.org/r/auTL2mHwCt20uhII@SMW015318/
> "let vchan_cookie_complete() at one place only".
>
> drivers/dma/dw-edma/dw-edma-core.c | 17 +++++++++++------
> 1 file changed, 11 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
> index fb38d51037c5..e2f7af5969c6 100644
> --- a/drivers/dma/dw-edma/dw-edma-core.c
> +++ b/drivers/dma/dw-edma/dw-edma-core.c
> @@ -107,6 +107,15 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
> res->residue = residue;
> }
>
> +/* Must be called with vc.lock held. */
> +static void dw_edma_complete_vdesc(struct virt_dma_desc *vd,
> + enum dmaengine_tx_result result)
> +{
> + dw_hdma_set_callback_result(vd, result);
> + list_del(&vd->node);
> + vchan_cookie_complete(vd);
> +}
> +
> static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
> {
> u32 i;
> @@ -788,9 +797,7 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
>
> desc = vd2dw_edma_desc(vd);
> if (desc->start_burst >= desc->nburst) {
> - dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR);
> - list_del(&vd->node);
> - vchan_cookie_complete(vd);
> + dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
> if (!chan->non_ll)
> chan->ll_done = chan->ll_head;
> }
> @@ -838,9 +845,7 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
> if (vd && chan->request == EDMA_REQ_STOP) {
> dw_edma_terminate_all_descs(chan);
> } else if (vd) {
> - dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED);
> - list_del(&vd->node);
> - vchan_cookie_complete(vd);
> + dw_edma_complete_vdesc(vd, DMA_TRANS_ABORTED);
> }
> if (!chan->non_ll)
> dw_edma_core_reset_ll(chan);
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
` (2 preceding siblings ...)
2026-11-04 16:12 ` [PATCH v7 3/7] dmaengine: dw-edma: Factor out descriptor completion Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:29 ` sashiko-bot
2026-11-04 17:29 ` Frank Li
2026-11-04 16:12 ` [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress Koichiro Den
` (2 subsequent siblings)
6 siblings, 2 replies; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
With dynamic appends, a DONE interrupt no longer means that all
published LL entries have completed. Prepare for this by using LLP to
advance ll_done and complete descriptors in issue order.
Reclaim completed entries and refill the ring as space becomes
available. Restart a stopped channel when published work remains. This
also allows descriptors larger than the ring to run in batches.
Hardware can keep running while the driver handles an interrupt, and
LLP does not always track payload completion. Use the following
exclusive completion boundaries, where idx is the raw LLP entry index
and prev(idx) is the previous data entry in the ring:
State/event Meaning of raw LLP Boundary
------------------- ------------------------------- ---------
eDMA/HDMA stopped Next entry to process idx
eDMA running Observed to advance before prev(idx)
payload completion
HDMA watermark Inclusive LLE recycling limit prev(idx)
A stopped sample requires CS=STOPPED and a zero transfer size.
While running, HDMA updates LLP only at watermark events. Its documented
recycling limit translates to an exclusive boundary of next(idx).
However, tests using that boundary for dmaengine completion reported
IOMMU faults on HDMA 6.30a SpacemiT K3. Use prev(idx) for both engines
while running to keep the handling simple and conservative. Entries held
back by this rule are eventually reclaimed on later progress or a
confirmed stop, where idx gives the exact boundary.
Handle termination and pause as follows:
- On termination, stop publishing new entries and drain those already
published, restarting the channel if needed. Terminate descriptors
without callbacks, as in the non-LL path.
- On pause, stop publishing new entries and enter the paused state once
the channel stops. Keep unfinished work for resume.
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- Handle STOP before consuming LL entries (Sashiko).
- Use modulo, assert vc.lock, and share descriptor completion code
(Frank).
- (*) Restart stopped channels during termination. Tests under DMA
load exposed the missing restart. In fact, v5 patch 19 had similar
handling and I mistakenly dropped that logic while doing major
v5->v6 reconstruction/simplification:
https://lore.kernel.org/r/20260812155721.2807506-20-den@valinux.co.jp/
- Substantially reword/polish the commit message.
- Link to v6:
https://lore.kernel.org/r/20261030150742.2586776-3-den@valinux.co.jp/
Note on (*): A channel can still report STOPPED just after a doorbell.
Another doorbell in this window may trigger the class B stall observed
on R-Car S4 [1]. That stall appears specific to S4 or its DWC IP
revision, and IMHO should not block this series any longer.
[1] https://lore.kernel.org/all/tkyu5rsxfa7i6rxxw6qxmmwke5qnlxptwx4j2s24fhftsmmf26@gamef2ec4wwm/
drivers/dma/dw-edma/dw-edma-core.c | 156 +++++++++++++++++++++++++----
1 file changed, 134 insertions(+), 22 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index e2f7af5969c6..e2f2a535d5ad 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -53,13 +53,6 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
{
struct dw_edma_desc *desc;
- /*
- * For now, a descriptor that does not fit would stall the channel
- * forever: reject it up front.
- */
- if (!chan->non_ll && nburst > chan->ll_max - 1)
- return NULL;
-
desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
if (unlikely(!desc))
return NULL;
@@ -161,17 +154,80 @@ static bool dw_edma_ll_stopped(struct dw_edma_chan *chan)
!dw_edma_core_ch_transfer_size(chan);
}
-static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
+/* Return the first entry we cannot reclaim yet. */
+static int dw_edma_ll_boundary(struct dw_edma_chan *chan, int idx,
+ bool stopped)
{
- int idx = dw_edma_core_ll_cur_idx(chan);
+ if (idx < 0 || (u32)idx > chan->ll_max)
+ return -EINVAL;
+
+ /* After a stop, LLP points past the completed entries. */
+ if (stopped)
+ return idx == chan->ll_max ? 0 : idx;
- if (idx == chan->ll_max)
- idx = 0;
+ /*
+ * DMA_LLP can advance before payload completion. Keep the boundary
+ * one entry behind it, wrapping at the start of the ring.
+ */
+ return (idx + chan->ll_max - 1) % chan->ll_max;
+}
+
+static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
+{
+ int idx = dw_edma_ll_boundary(chan, dw_edma_core_ll_cur_idx(chan), true);
return idx >= 0 && (u32)idx == chan->ll_head &&
dw_edma_ll_stopped(chan);
}
+static void dw_edma_ll_consume(struct dw_edma_chan *chan, int idx,
+ bool stopped)
+{
+ struct virt_dma_desc *vd, *_vd;
+ u32 done, used;
+
+ lockdep_assert_held(&chan->vc.lock);
+
+ idx = dw_edma_ll_boundary(chan, idx, stopped);
+ if (idx < 0)
+ return;
+
+ done = dw_edma_core_get_ll_dist(chan, chan->ll_done, idx);
+ used = dw_edma_core_get_used_num(chan);
+
+ if (!done || done > used)
+ return;
+
+ /*
+ * LLP may still point to the first pending entry. Subtracting one
+ * can wrap the index and falsely mark all pending entries complete.
+ * Reclaim them all only after the channel has stopped.
+ */
+ if (!stopped && done == used)
+ return;
+
+ chan->ll_done = idx;
+
+ /* Match reclaimed entries to descriptors in issue order. */
+ list_for_each_entry_safe(vd, _vd, &chan->vc.desc_issued, node) {
+ struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+ u32 consumed = min_t(u32, done,
+ desc->start_burst - desc->done_burst);
+
+ if (!consumed)
+ break;
+
+ desc->done_burst += consumed;
+ done -= consumed;
+ if (desc->done_burst != desc->nburst)
+ break;
+
+ dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
+ }
+
+ WARN_ON_ONCE(done);
+}
+
static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
{
struct dw_edma_chan *chan = desc->chan;
@@ -200,7 +256,6 @@ static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
}
}
- desc->done_burst = desc->start_burst;
desc->start_burst = i;
}
@@ -309,6 +364,52 @@ static void dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
dw_edma_core_ch_doorbell(chan);
}
+/* Must be called with vc.lock held. */
+static void dw_edma_ll_advance(struct dw_edma_chan *chan, bool event)
+{
+ bool stopped;
+
+ lockdep_assert_held(&chan->vc.lock);
+
+ /* Terminate without scheduling completion callbacks. */
+ if (chan->request == EDMA_REQ_STOP) {
+ if (dw_edma_ll_drained(chan))
+ dw_edma_finish_termination(chan);
+ else if (dw_edma_ll_stopped(chan))
+ /* A newly appended tail may have missed the last stop. */
+ dw_edma_core_ch_doorbell(chan);
+ return;
+ }
+
+ stopped = dw_edma_ll_stopped(chan);
+ /*
+ * HDMA updates LLP at watermark events while running. Use running
+ * samples only when processing an LL interrupt.
+ */
+ if (stopped || event)
+ dw_edma_ll_consume(chan, dw_edma_core_ll_cur_idx(chan),
+ stopped);
+
+ if (chan->request == EDMA_REQ_PAUSE) {
+ if (!stopped)
+ return;
+
+ dw_edma_set_request(chan, EDMA_REQ_NONE);
+ chan->status = EDMA_ST_PAUSE;
+ return;
+ }
+
+ dw_edma_start_transfer(chan);
+ if (!dw_edma_ll_pending(chan)) {
+ chan->status = EDMA_ST_IDLE;
+ return;
+ }
+
+ chan->status = EDMA_ST_BUSY;
+ if (stopped)
+ dw_edma_core_ch_doorbell(chan);
+}
+
static void dw_edma_device_caps(struct dma_chan *dchan,
struct dma_slave_caps *caps)
{
@@ -493,6 +594,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
err = -EPERM;
} else {
dw_edma_set_request(chan, EDMA_REQ_STOP);
+ if (!chan->non_ll)
+ dw_edma_ll_advance(chan, false);
}
if (chan->status == EDMA_ST_IDLE)
dw_edma_set_request(chan, EDMA_REQ_NONE);
@@ -777,7 +880,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
}
-/* Must be called with vc.lock held. */
+/* Non-LL completion; vc.lock must be held. */
static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
{
struct dw_edma_desc *desc;
@@ -796,11 +899,8 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
break;
desc = vd2dw_edma_desc(vd);
- if (desc->start_burst >= desc->nburst) {
+ if (desc->start_burst >= desc->nburst)
dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
- if (!chan->non_ll)
- chan->ll_done = chan->ll_head;
- }
if (chan->request == EDMA_REQ_PAUSE) {
dw_edma_set_request(chan, EDMA_REQ_NONE);
@@ -816,14 +916,12 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
if (!vd)
break;
- if (chan->non_ll || dw_edma_ll_drained(chan))
- dw_edma_finish_termination(chan);
+ dw_edma_finish_termination(chan);
break;
default:
break;
}
- dw_edma_core_ch_maybe_doorbell(chan);
}
static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
@@ -835,6 +933,16 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
+static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
+{
+ guard(spinlock_irqsave)(&chan->vc.lock);
+
+ if (chan->status == EDMA_ST_PAUSE)
+ return;
+
+ dw_edma_ll_advance(chan, true);
+}
+
static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
{
struct virt_dma_desc *vd;
@@ -863,8 +971,12 @@ static void dw_edma_irq_work(struct work_struct *work)
do {
events = atomic_xchg(&chan->irq_pending, 0);
- if (events & DW_EDMA_DEFERRED_DONE)
- dw_edma_done_interrupt(chan);
+ if (events & DW_EDMA_DEFERRED_DONE) {
+ if (chan->non_ll)
+ dw_edma_done_interrupt(chan);
+ else
+ dw_edma_ll_interrupt(chan);
+ }
if (events & DW_EDMA_DEFERRED_ABORT)
dw_edma_abort_interrupt(chan);
} while (atomic_read(&chan->irq_pending));
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress
2026-11-04 16:12 ` [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress Koichiro Den
@ 2026-11-04 16:29 ` sashiko-bot
2026-11-04 17:29 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: sashiko-bot @ 2026-11-04 16:29 UTC (permalink / raw)
To: Koichiro Den; +Cc: linux-pci
> With dynamic appends, a DONE interrupt no longer means that all
> published LL entries have completed. Prepare for this by using LLP to
> advance ll_done and complete descriptors in issue order.
>
> Reclaim completed entries and refill the ring as space becomes
> available. Restart a stopped channel when published work remains. This
> also allows descriptors larger than the ring to run in batches.
>
> Hardware can keep running while the driver handles an interrupt, and
> LLP does not always track payload completion. Use the following
> exclusive completion boundaries, where idx is the raw LLP entry index
> and prev(idx) is the previous data entry in the ring:
>
> State/event Meaning of raw LLP Boundary
> ------------------- ------------------------------- ---------
> [ ... ]
> without callbacks, as in the non-LL path.
> - On pause, stop publishing new entries and enter the paused state once
> the channel stops. Keep unfinished work for resume.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
Sashiko has reviewed this patch and found no issues.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104161227.1742005-1-den@valinux.co.jp?part=4
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress
2026-11-04 16:12 ` [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress Koichiro Den
2026-11-04 16:29 ` sashiko-bot
@ 2026-11-04 17:29 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: Frank Li @ 2026-11-04 17:29 UTC (permalink / raw)
To: Koichiro Den
Cc: Vinod Koul, Frank Li, Manivannan Sadhasivam, Gustavo Pimentel,
Serge Semin, Cai Huoqing, Niklas Cassel, Devendra K Verma,
dmaengine, linux-kernel, linux-pci
On Thu, Nov 05, 2026 at 01:12:24AM +0900, Koichiro Den wrote:
> With dynamic appends, a DONE interrupt no longer means that all
> published LL entries have completed. Prepare for this by using LLP to
> advance ll_done and complete descriptors in issue order.
>
> Reclaim completed entries and refill the ring as space becomes
> available. Restart a stopped channel when published work remains. This
> also allows descriptors larger than the ring to run in batches.
>
> Hardware can keep running while the driver handles an interrupt, and
> LLP does not always track payload completion. Use the following
> exclusive completion boundaries, where idx is the raw LLP entry index
> and prev(idx) is the previous data entry in the ring:
>
> State/event Meaning of raw LLP Boundary
> ------------------- ------------------------------- ---------
> eDMA/HDMA stopped Next entry to process idx
>
> eDMA running Observed to advance before prev(idx)
> payload completion
>
> HDMA watermark Inclusive LLE recycling limit prev(idx)
>
> A stopped sample requires CS=STOPPED and a zero transfer size.
>
> While running, HDMA updates LLP only at watermark events. Its documented
> recycling limit translates to an exclusive boundary of next(idx).
> However, tests using that boundary for dmaengine completion reported
> IOMMU faults on HDMA 6.30a SpacemiT K3. Use prev(idx) for both engines
> while running to keep the handling simple and conservative. Entries held
> back by this rule are eventually reclaimed on later progress or a
> confirmed stop, where idx gives the exact boundary.
>
> Handle termination and pause as follows:
> - On termination, stop publishing new entries and drain those already
> published, restarting the channel if needed. Terminate descriptors
> without callbacks, as in the non-LL path.
> - On pause, stop publishing new entries and enter the paused state once
> the channel stops. Keep unfinished work for resume.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
Reviewed-by: Frank Li <Frank.Li@nxp.com>
> Changes from v6:
> - Handle STOP before consuming LL entries (Sashiko).
> - Use modulo, assert vc.lock, and share descriptor completion code
> (Frank).
> - (*) Restart stopped channels during termination. Tests under DMA
> load exposed the missing restart. In fact, v5 patch 19 had similar
> handling and I mistakenly dropped that logic while doing major
> v5->v6 reconstruction/simplification:
> https://lore.kernel.org/r/20260812155721.2807506-20-den@valinux.co.jp/
> - Substantially reword/polish the commit message.
> - Link to v6:
> https://lore.kernel.org/r/20261030150742.2586776-3-den@valinux.co.jp/
>
> Note on (*): A channel can still report STOPPED just after a doorbell.
> Another doorbell in this window may trigger the class B stall observed
> on R-Car S4 [1]. That stall appears specific to S4 or its DWC IP
> revision, and IMHO should not block this series any longer.
>
> [1] https://lore.kernel.org/all/tkyu5rsxfa7i6rxxw6qxmmwke5qnlxptwx4j2s24fhftsmmf26@gamef2ec4wwm/
>
> drivers/dma/dw-edma/dw-edma-core.c | 156 +++++++++++++++++++++++++----
> 1 file changed, 134 insertions(+), 22 deletions(-)
>
> diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
> index e2f7af5969c6..e2f2a535d5ad 100644
> --- a/drivers/dma/dw-edma/dw-edma-core.c
> +++ b/drivers/dma/dw-edma/dw-edma-core.c
> @@ -53,13 +53,6 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
> {
> struct dw_edma_desc *desc;
>
> - /*
> - * For now, a descriptor that does not fit would stall the channel
> - * forever: reject it up front.
> - */
> - if (!chan->non_ll && nburst > chan->ll_max - 1)
> - return NULL;
> -
> desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
> if (unlikely(!desc))
> return NULL;
> @@ -161,17 +154,80 @@ static bool dw_edma_ll_stopped(struct dw_edma_chan *chan)
> !dw_edma_core_ch_transfer_size(chan);
> }
>
> -static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
> +/* Return the first entry we cannot reclaim yet. */
> +static int dw_edma_ll_boundary(struct dw_edma_chan *chan, int idx,
> + bool stopped)
> {
> - int idx = dw_edma_core_ll_cur_idx(chan);
> + if (idx < 0 || (u32)idx > chan->ll_max)
> + return -EINVAL;
> +
> + /* After a stop, LLP points past the completed entries. */
> + if (stopped)
> + return idx == chan->ll_max ? 0 : idx;
>
> - if (idx == chan->ll_max)
> - idx = 0;
> + /*
> + * DMA_LLP can advance before payload completion. Keep the boundary
> + * one entry behind it, wrapping at the start of the ring.
> + */
> + return (idx + chan->ll_max - 1) % chan->ll_max;
> +}
> +
> +static bool dw_edma_ll_drained(struct dw_edma_chan *chan)
> +{
> + int idx = dw_edma_ll_boundary(chan, dw_edma_core_ll_cur_idx(chan), true);
>
> return idx >= 0 && (u32)idx == chan->ll_head &&
> dw_edma_ll_stopped(chan);
> }
>
> +static void dw_edma_ll_consume(struct dw_edma_chan *chan, int idx,
> + bool stopped)
> +{
> + struct virt_dma_desc *vd, *_vd;
> + u32 done, used;
> +
> + lockdep_assert_held(&chan->vc.lock);
> +
> + idx = dw_edma_ll_boundary(chan, idx, stopped);
> + if (idx < 0)
> + return;
> +
> + done = dw_edma_core_get_ll_dist(chan, chan->ll_done, idx);
> + used = dw_edma_core_get_used_num(chan);
> +
> + if (!done || done > used)
> + return;
> +
> + /*
> + * LLP may still point to the first pending entry. Subtracting one
> + * can wrap the index and falsely mark all pending entries complete.
> + * Reclaim them all only after the channel has stopped.
> + */
> + if (!stopped && done == used)
> + return;
> +
> + chan->ll_done = idx;
> +
> + /* Match reclaimed entries to descriptors in issue order. */
> + list_for_each_entry_safe(vd, _vd, &chan->vc.desc_issued, node) {
> + struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
> + u32 consumed = min_t(u32, done,
> + desc->start_burst - desc->done_burst);
> +
> + if (!consumed)
> + break;
> +
> + desc->done_burst += consumed;
> + done -= consumed;
> + if (desc->done_burst != desc->nburst)
> + break;
> +
> + dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
> + }
> +
> + WARN_ON_ONCE(done);
> +}
> +
> static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
> {
> struct dw_edma_chan *chan = desc->chan;
> @@ -200,7 +256,6 @@ static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
> }
> }
>
> - desc->done_burst = desc->start_burst;
> desc->start_burst = i;
> }
>
> @@ -309,6 +364,52 @@ static void dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
> dw_edma_core_ch_doorbell(chan);
> }
>
> +/* Must be called with vc.lock held. */
> +static void dw_edma_ll_advance(struct dw_edma_chan *chan, bool event)
> +{
> + bool stopped;
> +
> + lockdep_assert_held(&chan->vc.lock);
> +
> + /* Terminate without scheduling completion callbacks. */
> + if (chan->request == EDMA_REQ_STOP) {
> + if (dw_edma_ll_drained(chan))
> + dw_edma_finish_termination(chan);
> + else if (dw_edma_ll_stopped(chan))
> + /* A newly appended tail may have missed the last stop. */
> + dw_edma_core_ch_doorbell(chan);
> + return;
> + }
> +
> + stopped = dw_edma_ll_stopped(chan);
> + /*
> + * HDMA updates LLP at watermark events while running. Use running
> + * samples only when processing an LL interrupt.
> + */
> + if (stopped || event)
> + dw_edma_ll_consume(chan, dw_edma_core_ll_cur_idx(chan),
> + stopped);
> +
> + if (chan->request == EDMA_REQ_PAUSE) {
> + if (!stopped)
> + return;
> +
> + dw_edma_set_request(chan, EDMA_REQ_NONE);
> + chan->status = EDMA_ST_PAUSE;
> + return;
> + }
> +
> + dw_edma_start_transfer(chan);
> + if (!dw_edma_ll_pending(chan)) {
> + chan->status = EDMA_ST_IDLE;
> + return;
> + }
> +
> + chan->status = EDMA_ST_BUSY;
> + if (stopped)
> + dw_edma_core_ch_doorbell(chan);
> +}
> +
> static void dw_edma_device_caps(struct dma_chan *dchan,
> struct dma_slave_caps *caps)
> {
> @@ -493,6 +594,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
> err = -EPERM;
> } else {
> dw_edma_set_request(chan, EDMA_REQ_STOP);
> + if (!chan->non_ll)
> + dw_edma_ll_advance(chan, false);
> }
> if (chan->status == EDMA_ST_IDLE)
> dw_edma_set_request(chan, EDMA_REQ_NONE);
> @@ -777,7 +880,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
> return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
> }
>
> -/* Must be called with vc.lock held. */
> +/* Non-LL completion; vc.lock must be held. */
> static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> {
> struct dw_edma_desc *desc;
> @@ -796,11 +899,8 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> break;
>
> desc = vd2dw_edma_desc(vd);
> - if (desc->start_burst >= desc->nburst) {
> + if (desc->start_burst >= desc->nburst)
> dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
> - if (!chan->non_ll)
> - chan->ll_done = chan->ll_head;
> - }
>
> if (chan->request == EDMA_REQ_PAUSE) {
> dw_edma_set_request(chan, EDMA_REQ_NONE);
> @@ -816,14 +916,12 @@ static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
> if (!vd)
> break;
>
> - if (chan->non_ll || dw_edma_ll_drained(chan))
> - dw_edma_finish_termination(chan);
> + dw_edma_finish_termination(chan);
> break;
>
> default:
> break;
> }
> - dw_edma_core_ch_maybe_doorbell(chan);
> }
>
> static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
> @@ -835,6 +933,16 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
> spin_unlock_irqrestore(&chan->vc.lock, flags);
> }
>
> +static void dw_edma_ll_interrupt(struct dw_edma_chan *chan)
> +{
> + guard(spinlock_irqsave)(&chan->vc.lock);
> +
> + if (chan->status == EDMA_ST_PAUSE)
> + return;
> +
> + dw_edma_ll_advance(chan, true);
> +}
> +
> static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
> {
> struct virt_dma_desc *vd;
> @@ -863,8 +971,12 @@ static void dw_edma_irq_work(struct work_struct *work)
> do {
> events = atomic_xchg(&chan->irq_pending, 0);
>
> - if (events & DW_EDMA_DEFERRED_DONE)
> - dw_edma_done_interrupt(chan);
> + if (events & DW_EDMA_DEFERRED_DONE) {
> + if (chan->non_ll)
> + dw_edma_done_interrupt(chan);
> + else
> + dw_edma_ll_interrupt(chan);
> + }
> if (events & DW_EDMA_DEFERRED_ABORT)
> dw_edma_abort_interrupt(chan);
> } while (atomic_read(&chan->irq_pending));
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
` (3 preceding siblings ...)
2026-11-04 16:12 ` [PATCH v7 4/7] dmaengine: dw-edma: Reclaim LL entries from hardware progress Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:21 ` sashiko-bot
2026-11-04 17:31 ` Frank Li
2026-11-04 16:12 ` [PATCH v7 6/7] dmaengine: dw-edma: Dynamically append requests while running Koichiro Den
2026-11-04 16:12 ` [PATCH v7 7/7] dmaengine: dw-edma: Add trace support Koichiro Den
6 siblings, 2 replies; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
While HDMA is running, DMA_LLP is updated only at watermark events.
Enable watermark interrupts to track progress without waiting for the
channel to stop.
Set the watermark bits on entries that request an interrupt, configure
local or remote delivery, and pass events to the LL progress handler.
Unmask local watermark interrupts only for LL channels in local IRQ mode.
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- Keep watermark interrupts masked for non-LL channels (Sashiko).
- Link to v6: https://lore.kernel.org/r/20261030150742.2586776-4-den@valinux.co.jp/
Note: Since v6, IRQ placement has been limited to descriptor ends and
the last free entry to keep the series small. Further tuning can
be done in follow-up patches. I should've mentioned this in v6
when dropping v5 patch 14:
https://lore.kernel.org/r/20260812155721.2807506-15-den@valinux.co.jp/
drivers/dma/dw-edma/dw-hdma-v0-core.c | 62 ++++++++++++++++++++++-----
drivers/dma/dw-edma/dw-hdma-v0-regs.h | 1 +
2 files changed, 52 insertions(+), 11 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index aa430219d0af..0cccd0003a1b 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -58,7 +58,8 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
{
val &= ~(HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN |
HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_REMOTE_STOP_INT_EN |
- HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK);
+ HDMA_V0_ABORT_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_STOP_INT_MASK);
/*
* HDMA_INT_STATUS.STOP and .ABORT are latched only when LSIE and
@@ -68,6 +69,9 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
*/
val |= HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN;
+ if (chan->non_ll || chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
+ val |= HDMA_V0_WATERMARK_INT_MASK;
+
if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
val |= HDMA_V0_REMOTE_ABORT_INT_EN |
HDMA_V0_REMOTE_STOP_INT_EN |
@@ -81,10 +85,12 @@ static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir,
u16 id)
{
SET_CH_32(dw, dir, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
SET_CH_32(dw, dir, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
}
static void dw_hdma_v0_core_off(struct dw_edma *dw)
@@ -95,18 +101,26 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw)
dir = EDMA_DIR_WRITE;
for (id = 0; id < dw->wr_ch_cnt; id++) {
SET_CH_32(dw, dir, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
}
dir = EDMA_DIR_READ;
for (id = 0; id < dw->rd_ch_cnt; id++) {
SET_CH_32(dw, dir, id, int_setup,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, int_clear,
- HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+ HDMA_V0_STOP_INT_MASK |
+ HDMA_V0_WATERMARK_INT_MASK |
+ HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
}
}
@@ -168,6 +182,14 @@ static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
}
+static void dw_hdma_v0_core_clear_watermark_int(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->dw;
+
+ SET_CH_32(dw, chan->dir, chan->id, int_clear,
+ HDMA_V0_WATERMARK_INT_MASK);
+}
+
static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
@@ -208,17 +230,23 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
continue;
val = dw_hdma_v0_core_status_int(chan);
- if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
+
+ if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val))
dw_hdma_v0_core_clear_done_int(chan);
- handler(chan, DW_EDMA_IRQ_STOP);
+ if (FIELD_GET(HDMA_V0_WATERMARK_INT_MASK, val))
+ dw_hdma_v0_core_clear_watermark_int(chan);
+ if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
+ handler(chan, DW_EDMA_IRQ_STOP);
+ ret = IRQ_HANDLED;
+ } else if (FIELD_GET(HDMA_V0_WATERMARK_INT_MASK, val)) {
+ handler(chan, DW_EDMA_IRQ_PROGRESS);
ret = IRQ_HANDLED;
}
if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
dw_hdma_v0_core_clear_abort_int(chan);
handler(chan, DW_EDMA_IRQ_ABORT);
-
ret = IRQ_HANDLED;
}
}
@@ -275,7 +303,7 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
/* Enable engine */
SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
- /* Interrupt unmask - stop, abort */
+ /* Interrupt unmask - stop, watermark, abort */
tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
tmp = dw_hdma_v0_core_int_setup(chan, tmp);
SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
@@ -336,6 +364,11 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
/* MSI done addr - low, high */
SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
+ /* MSI watermark addr - low, high */
+ SET_CH_32(dw, chan->dir, chan->id, msi_watermark.lsb,
+ chan->msi.address_lo);
+ SET_CH_32(dw, chan->dir, chan->id, msi_watermark.msb,
+ chan->msi.address_hi);
/* MSI abort addr - low, high */
SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
@@ -355,6 +388,13 @@ dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
if (cb)
control |= DW_HDMA_V0_CB;
+ if (irq) {
+ control |= DW_HDMA_V0_LWIE;
+
+ if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
+ control |= DW_HDMA_V0_RWIE;
+ }
+
dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
burst->dar);
}
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
index b88226cdd791..e240ffb3dea7 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
+++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
@@ -18,6 +18,7 @@
#define HDMA_V0_LOCAL_STOP_INT_EN BIT(4)
#define HDMA_V0_REMOTE_STOP_INT_EN BIT(3)
#define HDMA_V0_ABORT_INT_MASK BIT(2)
+#define HDMA_V0_WATERMARK_INT_MASK BIT(1)
#define HDMA_V0_STOP_INT_MASK BIT(0)
#define HDMA_V0_LINKLIST_EN BIT(0)
#define HDMA_V0_CONSUMER_CYCLE_STAT BIT(1)
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress
2026-11-04 16:12 ` [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress Koichiro Den
@ 2026-11-04 16:21 ` sashiko-bot
2026-11-04 17:31 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: sashiko-bot @ 2026-11-04 16:21 UTC (permalink / raw)
To: Koichiro Den; +Cc: linux-pci
> While HDMA is running, DMA_LLP is updated only at watermark events.
> Enable watermark interrupts to track progress without waiting for the
> channel to stop.
>
> Set the watermark bits on entries that request an interrupt, configure
> local or remote delivery, and pass events to the LL progress handler.
> Unmask local watermark interrupts only for LL channels in local IRQ mode.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
Sashiko has reviewed this patch and found no issues.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104161227.1742005-1-den@valinux.co.jp?part=5
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress
2026-11-04 16:12 ` [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress Koichiro Den
2026-11-04 16:21 ` sashiko-bot
@ 2026-11-04 17:31 ` Frank Li
1 sibling, 0 replies; 19+ messages in thread
From: Frank Li @ 2026-11-04 17:31 UTC (permalink / raw)
To: Koichiro Den
Cc: Vinod Koul, Frank Li, Manivannan Sadhasivam, Gustavo Pimentel,
Serge Semin, Cai Huoqing, Niklas Cassel, Devendra K Verma,
dmaengine, linux-kernel, linux-pci
On Thu, Nov 05, 2026 at 01:12:25AM +0900, Koichiro Den wrote:
> While HDMA is running, DMA_LLP is updated only at watermark events.
> Enable watermark interrupts to track progress without waiting for the
> channel to stop.
>
> Set the watermark bits on entries that request an interrupt, configure
> local or remote delivery, and pass events to the LL progress handler.
> Unmask local watermark interrupts only for LL channels in local IRQ mode.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
Reviewed-by: Frank Li <Frank.Li@nxp.com>
> Changes from v6:
> - Keep watermark interrupts masked for non-LL channels (Sashiko).
> - Link to v6: https://lore.kernel.org/r/20261030150742.2586776-4-den@valinux.co.jp/
>
> Note: Since v6, IRQ placement has been limited to descriptor ends and
> the last free entry to keep the series small. Further tuning can
> be done in follow-up patches. I should've mentioned this in v6
> when dropping v5 patch 14:
> https://lore.kernel.org/r/20260812155721.2807506-15-den@valinux.co.jp/
>
> drivers/dma/dw-edma/dw-hdma-v0-core.c | 62 ++++++++++++++++++++++-----
> drivers/dma/dw-edma/dw-hdma-v0-regs.h | 1 +
> 2 files changed, 52 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
> index aa430219d0af..0cccd0003a1b 100644
> --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
> +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
> @@ -58,7 +58,8 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
> {
> val &= ~(HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN |
> HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_REMOTE_STOP_INT_EN |
> - HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK);
> + HDMA_V0_ABORT_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_STOP_INT_MASK);
>
> /*
> * HDMA_INT_STATUS.STOP and .ABORT are latched only when LSIE and
> @@ -68,6 +69,9 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
> */
> val |= HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN;
>
> + if (chan->non_ll || chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
> + val |= HDMA_V0_WATERMARK_INT_MASK;
> +
> if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
> val |= HDMA_V0_REMOTE_ABORT_INT_EN |
> HDMA_V0_REMOTE_STOP_INT_EN |
> @@ -81,10 +85,12 @@ static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir,
> u16 id)
> {
> SET_CH_32(dw, dir, id, int_setup,
> - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
> + HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_ABORT_INT_MASK);
> SET_CH_32(dw, dir, id, ch_en, 0);
> SET_CH_32(dw, dir, id, int_clear,
> - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
> + HDMA_V0_STOP_INT_MASK | HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_ABORT_INT_MASK);
> }
>
> static void dw_hdma_v0_core_off(struct dw_edma *dw)
> @@ -95,18 +101,26 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw)
> dir = EDMA_DIR_WRITE;
> for (id = 0; id < dw->wr_ch_cnt; id++) {
> SET_CH_32(dw, dir, id, int_setup,
> - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
> + HDMA_V0_STOP_INT_MASK |
> + HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_ABORT_INT_MASK);
> SET_CH_32(dw, dir, id, int_clear,
> - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
> + HDMA_V0_STOP_INT_MASK |
> + HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_ABORT_INT_MASK);
> SET_CH_32(dw, dir, id, ch_en, 0);
> }
>
> dir = EDMA_DIR_READ;
> for (id = 0; id < dw->rd_ch_cnt; id++) {
> SET_CH_32(dw, dir, id, int_setup,
> - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
> + HDMA_V0_STOP_INT_MASK |
> + HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_ABORT_INT_MASK);
> SET_CH_32(dw, dir, id, int_clear,
> - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
> + HDMA_V0_STOP_INT_MASK |
> + HDMA_V0_WATERMARK_INT_MASK |
> + HDMA_V0_ABORT_INT_MASK);
> SET_CH_32(dw, dir, id, ch_en, 0);
> }
> }
> @@ -168,6 +182,14 @@ static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
> SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
> }
>
> +static void dw_hdma_v0_core_clear_watermark_int(struct dw_edma_chan *chan)
> +{
> + struct dw_edma *dw = chan->dw;
> +
> + SET_CH_32(dw, chan->dir, chan->id, int_clear,
> + HDMA_V0_WATERMARK_INT_MASK);
> +}
> +
> static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
> {
> struct dw_edma *dw = chan->dw;
> @@ -208,17 +230,23 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
> continue;
>
> val = dw_hdma_v0_core_status_int(chan);
> - if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
> +
> + if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val))
> dw_hdma_v0_core_clear_done_int(chan);
> - handler(chan, DW_EDMA_IRQ_STOP);
> + if (FIELD_GET(HDMA_V0_WATERMARK_INT_MASK, val))
> + dw_hdma_v0_core_clear_watermark_int(chan);
>
> + if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
> + handler(chan, DW_EDMA_IRQ_STOP);
> + ret = IRQ_HANDLED;
> + } else if (FIELD_GET(HDMA_V0_WATERMARK_INT_MASK, val)) {
> + handler(chan, DW_EDMA_IRQ_PROGRESS);
> ret = IRQ_HANDLED;
> }
>
> if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
> dw_hdma_v0_core_clear_abort_int(chan);
> handler(chan, DW_EDMA_IRQ_ABORT);
> -
> ret = IRQ_HANDLED;
> }
> }
> @@ -275,7 +303,7 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
>
> /* Enable engine */
> SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
> - /* Interrupt unmask - stop, abort */
> + /* Interrupt unmask - stop, watermark, abort */
> tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
> tmp = dw_hdma_v0_core_int_setup(chan, tmp);
> SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
> @@ -336,6 +364,11 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
> /* MSI done addr - low, high */
> SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
> SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
> + /* MSI watermark addr - low, high */
> + SET_CH_32(dw, chan->dir, chan->id, msi_watermark.lsb,
> + chan->msi.address_lo);
> + SET_CH_32(dw, chan->dir, chan->id, msi_watermark.msb,
> + chan->msi.address_hi);
> /* MSI abort addr - low, high */
> SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
> SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
> @@ -355,6 +388,13 @@ dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
> if (cb)
> control |= DW_HDMA_V0_CB;
>
> + if (irq) {
> + control |= DW_HDMA_V0_LWIE;
> +
> + if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
> + control |= DW_HDMA_V0_RWIE;
> + }
> +
> dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
> burst->dar);
> }
> diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
> index b88226cdd791..e240ffb3dea7 100644
> --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
> +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
> @@ -18,6 +18,7 @@
> #define HDMA_V0_LOCAL_STOP_INT_EN BIT(4)
> #define HDMA_V0_REMOTE_STOP_INT_EN BIT(3)
> #define HDMA_V0_ABORT_INT_MASK BIT(2)
> +#define HDMA_V0_WATERMARK_INT_MASK BIT(1)
> #define HDMA_V0_STOP_INT_MASK BIT(0)
> #define HDMA_V0_LINKLIST_EN BIT(0)
> #define HDMA_V0_CONSUMER_CYCLE_STAT BIT(1)
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v7 6/7] dmaengine: dw-edma: Dynamically append requests while running
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
` (4 preceding siblings ...)
2026-11-04 16:12 ` [PATCH v7 5/7] dmaengine: dw-edma: Use HDMA watermark interrupts as LL progress Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:47 ` sashiko-bot
2026-11-04 16:12 ` [PATCH v7 7/7] dmaengine: dw-edma: Add trace support Koichiro Den
6 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
From: Frank Li <Frank.Li@nxp.com>
Allow issue_pending() to fill free LL entries while the channel is busy,
then check progress and restart it if it has stopped. Publish requests
in descriptor order.
Multiple descriptors can now have entries in the LL ring. When resetting
the ring after an abort, mark the other descriptors with pending entries
as aborted too, so they cannot be replayed.
Since dw_edma_start_transfer() can now return zero with LL entries still
pending, keep such channels BUSY on resume.
Signed-off-by: Frank Li <Frank.Li@nxp.com>
Co-developed-by: Koichiro Den <den@valinux.co.jp>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- Use the common descriptor completion helper when aborting pending
descriptors during an LL reset (Frank).
- Polish the commit message a bit.
- Link to v6: https://lore.kernel.org/r/20261030150742.2586776-5-den@valinux.co.jp/
drivers/dma/dw-edma/dw-edma-core.c | 65 ++++++++++++++++++++++--------
1 file changed, 49 insertions(+), 16 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index e2f2a535d5ad..69b5488282de 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -111,8 +111,22 @@ static void dw_edma_complete_vdesc(struct virt_dma_desc *vd,
static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
{
+ struct virt_dma_desc *vd, *tmp;
u32 i;
+ /*
+ * We cannot tell which pending entries completed before the reset.
+ * Abort their descriptors to avoid repeating a transfer.
+ */
+ list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
+ struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+
+ if (desc->start_burst == desc->done_burst)
+ continue;
+
+ dw_edma_complete_vdesc(vd, DMA_TRANS_ABORTED);
+ }
+
chan->ll_head = 0;
chan->ll_done = 0;
/* Drop stale CB bits before reusing the circular LL ring. */
@@ -278,21 +292,34 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
{
struct dw_edma_desc *desc;
struct virt_dma_desc *vd;
+ int ret = 0;
- vd = vchan_next_desc(&chan->vc);
- if (!vd)
- return 0;
+ if (chan->non_ll) {
+ vd = vchan_next_desc(&chan->vc);
+ if (!vd)
+ return 0;
- desc = vd2dw_edma_desc(vd);
- if (!desc)
- return 0;
+ dw_edma_core_start(vd2dw_edma_desc(vd));
+ return 1;
+ }
- if (!chan->non_ll && !chan->ll_valid)
- dw_edma_core_reset_ll(chan);
+ list_for_each_entry(vd, &chan->vc.desc_issued, node) {
+ if (!dw_edma_core_get_free_num(chan))
+ break;
+
+ desc = vd2dw_edma_desc(vd);
+ /* Fully published descriptors may still be in flight. */
+ if (desc->start_burst == desc->nburst)
+ continue;
- dw_edma_core_start(desc);
+ if (!chan->ll_valid)
+ dw_edma_core_reset_ll(chan);
- return 1;
+ dw_edma_core_start(desc);
+ ret = 1;
+ }
+
+ return ret;
}
static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd)
@@ -561,7 +588,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
err = -EPERM;
} else {
chan->status = EDMA_ST_BUSY;
- if (!dw_edma_start_transfer(chan))
+ if (!dw_edma_start_transfer(chan) &&
+ (chan->non_ll || !dw_edma_ll_pending(chan)))
chan->status = EDMA_ST_IDLE;
dw_edma_core_ch_maybe_doorbell(chan);
}
@@ -610,11 +638,16 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
spin_lock_irqsave(&chan->vc.lock, flags);
if (chan->configured && vchan_issue_pending(&chan->vc) &&
- chan->request == EDMA_REQ_NONE &&
- chan->status == EDMA_ST_IDLE) {
- chan->status = EDMA_ST_BUSY;
- dw_edma_start_transfer(chan);
- dw_edma_core_ch_maybe_doorbell(chan);
+ chan->request == EDMA_REQ_NONE) {
+ if (chan->status == EDMA_ST_IDLE) {
+ chan->status = EDMA_ST_BUSY;
+ dw_edma_start_transfer(chan);
+ dw_edma_core_ch_maybe_doorbell(chan);
+ } else if (!chan->non_ll &&
+ chan->status == EDMA_ST_BUSY) {
+ dw_edma_start_transfer(chan);
+ dw_edma_ll_advance(chan, false);
+ }
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH v7 6/7] dmaengine: dw-edma: Dynamically append requests while running
2026-11-04 16:12 ` [PATCH v7 6/7] dmaengine: dw-edma: Dynamically append requests while running Koichiro Den
@ 2026-11-04 16:47 ` sashiko-bot
0 siblings, 0 replies; 19+ messages in thread
From: sashiko-bot @ 2026-11-04 16:47 UTC (permalink / raw)
To: Koichiro Den; +Cc: dmaengine, linux-pci, vkoul, Frank.Li
Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [High] dmaengine: dw-edma: premature completion in dw_edma_ll_advance() on append
- [Medium] dmaengine: dw-edma: stranded or aborted queued descriptors on LL reset
--
commit 09d38b6dc815a996191d342c3fb4a8c29aef4401
Author: Frank Li <Frank.Li@nxp.com>
dmaengine: dw-edma: Dynamically append requests while running
This commit updates dw_edma_start_transfer() and
dw_edma_device_issue_pending() to append descriptors into free LL ring slots
while a channel is already busy, and updates dw_edma_core_reset_ll() and
dw_edma_device_resume() to handle multiple pending descriptors in the LL
ring.
> diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
> index e2f2a535d5ad1..69b5488282de3 100644
> --- a/drivers/dma/dw-edma/dw-edma-core.c
> +++ b/drivers/dma/dw-edma/dw-edma-core.c
> @@ -111,8 +111,22 @@ static void dw_edma_complete_vdesc(struct virt_dma_desc *vd,
>
> static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
> {
> + struct virt_dma_desc *vd, *tmp;
> u32 i;
>
> + /*
> + * We cannot tell which pending entries completed before the reset.
> + * Abort their descriptors to avoid repeating a transfer.
> + */
> + list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
> + struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
> +
> + if (desc->start_burst == desc->done_burst)
> + continue;
> +
> + dw_edma_complete_vdesc(vd, DMA_TRANS_ABORTED);
> + }
[Severity: Medium]
What happens to unstarted descriptors that remain in chan->vc.desc_issued
when a hardware abort occurs?
When dw_edma_device_issue_pending() moves submitted descriptors to
chan->vc.desc_issued and dw_edma_start_transfer() fills the LL ring, any
trailing descriptors stay in chan->vc.desc_issued with start_burst == 0 and
done_burst == 0.
If an abort occurs on an earlier in-flight descriptor,
dw_edma_core_reset_ll() skips those unstarted descriptors because
desc->start_burst == desc->done_burst, leaving them in chan->vc.desc_issued.
However, dw_edma_abort_interrupt() then transitions the channel straight to
EDMA_ST_IDLE without starting them:
drivers/dma/dw-edma/dw-edma-core.c:dw_edma_abort_interrupt() {
...
if (!chan->non_ll)
dw_edma_core_reset_ll(chan);
dw_edma_set_request(chan, EDMA_REQ_NONE);
chan->status = EDMA_ST_IDLE;
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
Won't those remaining descriptors stay stranded in chan->vc.desc_issued on an
idle channel unless the client later issues another transfer?
Also, if an earlier descriptor in the ring raised DW_EDMA_DEFERRED_DONE
before a subsequent entry raised DW_EDMA_DEFERRED_ABORT, dw_edma_irq_work()
processes DW_EDMA_DEFERRED_DONE first:
drivers/dma/dw-edma/dw-edma-core.c:dw_edma_irq_work() {
if (events & DW_EDMA_DEFERRED_DONE) {
if (chan->non_ll)
dw_edma_done_interrupt(chan);
else
dw_edma_ll_interrupt(chan);
}
if (events & DW_EDMA_DEFERRED_ABORT)
dw_edma_abort_interrupt(chan);
}
In that case, dw_edma_ll_interrupt() -> dw_edma_ll_advance() reclaims the
completed slots and calls dw_edma_start_transfer(), publishing the waiting
unstarted descriptors into the LL ring (setting start_burst > 0), only for
dw_edma_abort_interrupt() -> dw_edma_core_reset_ll() to immediately abort
them in the same work pass.
[ ... ]
> @@ -610,11 +638,16 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
>
> spin_lock_irqsave(&chan->vc.lock, flags);
> if (chan->configured && vchan_issue_pending(&chan->vc) &&
> - chan->request == EDMA_REQ_NONE &&
> - chan->status == EDMA_ST_IDLE) {
> - chan->status = EDMA_ST_BUSY;
> - dw_edma_start_transfer(chan);
> - dw_edma_core_ch_maybe_doorbell(chan);
> + chan->request == EDMA_REQ_NONE) {
> + if (chan->status == EDMA_ST_IDLE) {
> + chan->status = EDMA_ST_BUSY;
> + dw_edma_start_transfer(chan);
> + dw_edma_core_ch_maybe_doorbell(chan);
> + } else if (!chan->non_ll &&
> + chan->status == EDMA_ST_BUSY) {
> + dw_edma_start_transfer(chan);
> + dw_edma_ll_advance(chan, false);
> + }
> }
> spin_unlock_irqrestore(&chan->vc.lock, flags);
> }
[Severity: High]
Could calling dw_edma_ll_advance(chan, false) immediately after appending new
LL entries here cause an in-flight descriptor to be completed prematurely?
When a first dw_edma_device_issue_pending() call transitions the channel from
EDMA_ST_IDLE to EDMA_ST_BUSY and rings the doorbell, the hardware channel
status can briefly continue to report DMA_COMPLETE with transfer_size == 0
before the engine begins fetching the entry at ll_done.
If a second dw_edma_device_issue_pending() call runs during that window,
dw_edma_start_transfer() appends a second entry (making used >= 2) and calls
dw_edma_ll_advance(chan, false):
drivers/dma/dw-edma/dw-edma-core.c:dw_edma_ll_advance() {
...
stopped = dw_edma_ll_stopped(chan);
/*
* HDMA updates LLP at watermark events while running. Use running
* samples only when processing an LL interrupt.
*/
if (stopped || event)
dw_edma_ll_consume(chan, dw_edma_core_ll_cur_idx(chan),
stopped);
}
Because dw_edma_ll_stopped(chan) and dw_edma_core_ll_cur_idx(chan) are
separate MMIO reads, dw_edma_ll_stopped(chan) can observe the stale
post-doorbell stopped state (stopped = true) just before the engine starts
processing the first entry at ll_done and advances DMA_LLP to ll_done + 1
while the payload transfer is still in flight.
Passing idx = ll_done + 1 and stopped = true to dw_edma_ll_consume() then
uses the stopped boundary (idx instead of idx - 1) and bypasses the
(!stopped && done == used) check:
drivers/dma/dw-edma/dw-edma-core.c:dw_edma_ll_boundary() {
...
/* After a stop, LLP points past the completed entries. */
if (stopped)
return idx == chan->ll_max ? 0 : idx;
/*
* DMA_LLP can advance before payload completion. Keep the boundary
* one entry behind it, wrapping at the start of the ring.
*/
return (idx + chan->ll_max - 1) % chan->ll_max;
}
Wouldn't that advance chan->ll_done past the first entry and complete the
first descriptor with DMA_TRANS_NOERROR while hardware is still transferring
its payload, as well as ring a duplicate doorbell at the end of
dw_edma_ll_advance()?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261104161227.1742005-1-den@valinux.co.jp?part=6
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v7 7/7] dmaengine: dw-edma: Add trace support
2026-11-04 16:12 [PATCH v7 0/7] dmaengine: dw-edma: Support dynamic LL appends Koichiro Den
` (5 preceding siblings ...)
2026-11-04 16:12 ` [PATCH v7 6/7] dmaengine: dw-edma: Dynamically append requests while running Koichiro Den
@ 2026-11-04 16:12 ` Koichiro Den
2026-11-04 16:21 ` sashiko-bot
6 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2026-11-04 16:12 UTC (permalink / raw)
To: Vinod Koul, Frank Li, Manivannan Sadhasivam
Cc: Gustavo Pimentel, Serge Semin, Cai Huoqing, Niklas Cassel,
Devendra K Verma, dmaengine, linux-kernel, linux-pci
From: Frank Li <Frank.Li@nxp.com>
Add tracepoints for LL publication, descriptor append and completion,
and doorbells. Record channel and ring indices to help diagnose stalls.
Signed-off-by: Frank Li <Frank.Li@nxp.com>
Co-developed-by: Koichiro Den <den@valinux.co.jp>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
- Use the dw_edma_ prefix for the event class and tracepoints (Sashiko).
- Link to v6: https://lore.kernel.org/r/20261030150742.2586776-6-den@valinux.co.jp/
drivers/dma/dw-edma/Makefile | 2 +
drivers/dma/dw-edma/dw-edma-core.c | 9 +++
drivers/dma/dw-edma/dw-edma-core.h | 2 +
drivers/dma/dw-edma/dw-edma-trace.c | 4 +
drivers/dma/dw-edma/dw-edma-trace.h | 117 ++++++++++++++++++++++++++++
5 files changed, 134 insertions(+)
create mode 100644 drivers/dma/dw-edma/dw-edma-trace.c
create mode 100644 drivers/dma/dw-edma/dw-edma-trace.h
diff --git a/drivers/dma/dw-edma/Makefile b/drivers/dma/dw-edma/Makefile
index 83ab58f87760..5f395fa1ede4 100644
--- a/drivers/dma/dw-edma/Makefile
+++ b/drivers/dma/dw-edma/Makefile
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: GPL-2.0
+CFLAGS_dw-edma-trace.o := -I$(src)
obj-$(CONFIG_DW_EDMA) += dw-edma.o
dw-edma-$(CONFIG_DEBUG_FS) := dw-edma-v0-debugfs.o \
dw-hdma-v0-debugfs.o
+dw-edma-$(CONFIG_TRACING) += dw-edma-trace.o
dw-edma-objs := dw-edma-core.o \
dw-edma-v0-core.o \
dw-hdma-v0-core.o $(dw-edma-y)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 69b5488282de..db850ceea82e 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -236,6 +236,7 @@ static void dw_edma_ll_consume(struct dw_edma_chan *chan, int idx,
if (desc->done_burst != desc->nburst)
break;
+ trace_dw_edma_complete_desc(desc);
dw_edma_complete_vdesc(vd, DMA_TRANS_NOERROR);
}
@@ -262,6 +263,12 @@ static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
chan->ll_head, chan->cb,
i == desc->nburst - 1 || free == 1);
+ trace_dw_edma_fill_ll(chan, chan->ll_head,
+ desc->vd.tx.cookie,
+ desc->burst[i].sar,
+ desc->burst[i].dar, desc->burst[i].sz,
+ chan->cb);
+
chan->ll_head++;
if (chan->ll_head == chan->ll_max) {
@@ -315,6 +322,7 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
if (!chan->ll_valid)
dw_edma_core_reset_ll(chan);
+ trace_dw_edma_append_desc(desc);
dw_edma_core_start(desc);
ret = 1;
}
@@ -378,6 +386,7 @@ static void dw_edma_core_ll_sync(struct dw_edma_chan *chan)
static void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
{
dw_edma_core_ll_sync(chan);
+ trace_dw_edma_doorbell(chan);
dw_edma_core_do_ch_doorbell(chan);
}
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 84a811bdcb6d..769c0c08a711 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -154,6 +154,8 @@ struct dw_edma {
const struct dw_edma_core_ops *core;
};
+#include "dw-edma-trace.h"
+
typedef void (*dw_edma_handler_t)(struct dw_edma_chan *chan,
unsigned int events);
diff --git a/drivers/dma/dw-edma/dw-edma-trace.c b/drivers/dma/dw-edma/dw-edma-trace.c
new file mode 100644
index 000000000000..2620ad61a943
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-trace.c
@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define CREATE_TRACE_POINTS
+#include "dw-edma-core.h"
diff --git a/drivers/dma/dw-edma/dw-edma-trace.h b/drivers/dma/dw-edma/dw-edma-trace.h
new file mode 100644
index 000000000000..0b199ab30b9a
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-trace.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2023 NXP.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM dw_edma
+
+#if !defined(__LINUX_DW_EDMA_TRACE) || defined(TRACE_HEADER_MULTI_READ)
+#define __LINUX_DW_EDMA_TRACE
+
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(dw_edma_desc_info,
+ TP_PROTO(struct dw_edma_desc *desc),
+ TP_ARGS(desc),
+ TP_STRUCT__entry(
+ __field(size_t, nburst)
+ __field(size_t, start_burst)
+ __field(dma_cookie_t, cookie)
+ __field(u32, id)
+ __field(u8, dir)
+ ),
+ TP_fast_assign(
+ __entry->nburst = desc->nburst;
+ __entry->start_burst = desc->start_burst;
+ __entry->id = desc->chan->id;
+ __entry->dir = desc->chan->dir;
+ __entry->cookie = desc->vd.tx.cookie;
+ ),
+ TP_printk("chan %u%c desc %d nburst %zu start_burst %zu",
+ __entry->id,
+ __entry->dir ? 'R' : 'W',
+ __entry->cookie,
+ __entry->nburst,
+ __entry->start_burst)
+);
+
+DEFINE_EVENT(dw_edma_desc_info, dw_edma_append_desc,
+ TP_PROTO(struct dw_edma_desc *desc),
+ TP_ARGS(desc)
+);
+
+DEFINE_EVENT(dw_edma_desc_info, dw_edma_complete_desc,
+ TP_PROTO(struct dw_edma_desc *desc),
+ TP_ARGS(desc)
+);
+
+TRACE_EVENT(dw_edma_doorbell,
+ TP_PROTO(struct dw_edma_chan *chan),
+ TP_ARGS(chan),
+ TP_STRUCT__entry(
+ __field(u32, id)
+ __field(u8, dir)
+ __field(u32, done)
+ __field(u32, head)
+ ),
+ TP_fast_assign(
+ __entry->id = chan->id;
+ __entry->dir = chan->dir;
+ __entry->done = chan->ll_done;
+ __entry->head = chan->ll_head;
+ ),
+ TP_printk("chan %u%c done %u head %u",
+ __entry->id,
+ __entry->dir ? 'R' : 'W',
+ __entry->done,
+ __entry->head)
+);
+
+TRACE_EVENT(dw_edma_fill_ll,
+ TP_PROTO(struct dw_edma_chan *chan, u32 idx, dma_cookie_t cookie, u64 src,
+ u64 dest, u32 sz, bool flag),
+ TP_ARGS(chan, idx, cookie, src, dest, sz, flag),
+ TP_STRUCT__entry(
+ __field(u32, idx)
+ __field(u64, src)
+ __field(u64, dest)
+ __field(u32, sz)
+ __field(u32, id)
+ __field(dma_cookie_t, cookie)
+ __field(bool, flag)
+ __field(u8, dir)
+ ),
+ TP_fast_assign(
+ __entry->idx = idx;
+ __entry->src = src;
+ __entry->dest = dest;
+ __entry->sz = sz;
+ __entry->id = chan->id;
+ __entry->dir = chan->dir;
+ __entry->cookie = cookie;
+ __entry->flag = flag;
+ ),
+ TP_printk("chan %u%c %d [%u] %c src: %08llx dest: %08llx sz: %04x",
+ __entry->id,
+ __entry->dir ? 'R' : 'W',
+ __entry->cookie,
+ __entry->idx,
+ __entry->flag ? 'C' : 'c',
+ __entry->src,
+ __entry->dest,
+ __entry->sz)
+);
+
+#endif
+
+/* This part must be outside the header guard. */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE dw-edma-trace
+
+#include <trace/define_trace.h>
--
2.54.0
^ permalink raw reply related [flat|nested] 19+ messages in thread