[PATCH] drm/etnaviv: Keep busy jobs alive in the hangcheck

[PATCH] drm/etnaviv: Keep busy jobs alive in the hangcheck

[Christian Gmeiner]

From: Christian Gmeiner <cgmeiner@igalia.com>

A compute dispatch consists of a single FE command. The FE DMA address
does not change while the kernel is running, so the hangcheck kills any
dispatch that takes longer than about one second. Long-running OpenCL
kernels never finish.

Read VIVS_HI_IDLE_STATE at each timeout tick. If any unit other than FE
and MC is busy, the job is still executing, so give it more time.
If all units are idle for two consecutive ticks, the FE is waiting for
something that will never come, and the job is killed as before.

A job that stays busy without FE progress for longer than the new
hang_max_ms module parameter (default 30 seconds) is killed anyway.
This also catches kernels that loop forever. Setting it to 0 restores
the old behaviour.

Signed-off-by: Christian Gmeiner <cgmeiner@igalia.com>
---
 drivers/gpu/drm/etnaviv/etnaviv_gpu.h   |  3 +++
 drivers/gpu/drm/etnaviv/etnaviv_sched.c | 39 ++++++++++++++++++++++++++++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index e89d591d7404..f871ab7899fd 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -147,6 +147,9 @@ struct etnaviv_gpu {
 	u32 hangcheck_dma_addr;
 	u32 hangcheck_primid;
 	u32 hangcheck_fence;
+	bool hangcheck_idle;
+	bool hangcheck_busy;
+	unsigned long hangcheck_busy_since;
 
 	void __iomem *mmio;
 	int irq;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 3cc50d697c89..cef1ed33ff91 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -17,6 +17,9 @@ static int etnaviv_job_hang_limit = 0;
 module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444);
 static int etnaviv_hw_jobs_limit = 4;
 module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444);
+static unsigned int etnaviv_hang_max_ms = 30000;
+module_param_named(hang_max_ms, etnaviv_hang_max_ms, uint, 0644);
+MODULE_PARM_DESC(hang_max_ms, "Time a busy job may run without front-end progress (ms, 0 = no extension)");
 
 static struct dma_fence *etnaviv_sched_run_job(struct drm_sched_job *sched_job)
 {
@@ -36,7 +39,8 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
 {
 	struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);
 	struct etnaviv_gpu *gpu = submit->gpu;
-	u32 dma_addr, primid = 0;
+	u32 dma_addr, primid = 0, mask;
+	bool idle, was_idle;
 	int change;
 
 	/*
@@ -62,6 +66,12 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
 		primid = gpu_read(gpu, VIVS_MC_PROFILE_FE_READ);
 		mutex_unlock(&gpu->lock);
 	}
+
+	mask = gpu->idle_mask & ~(VIVS_HI_IDLE_STATE_FE | VIVS_HI_IDLE_STATE_MC);
+	idle = (gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask) == mask;
+	was_idle = gpu->hangcheck_idle;
+	gpu->hangcheck_idle = idle;
+
 	if (gpu->state == ETNA_GPU_STATE_RUNNING &&
 	    (gpu->completed_fence != gpu->hangcheck_fence ||
 	     change < 0 || change > 16 ||
@@ -70,9 +80,36 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
 		gpu->hangcheck_dma_addr = dma_addr;
 		gpu->hangcheck_primid = primid;
 		gpu->hangcheck_fence = gpu->completed_fence;
+		gpu->hangcheck_busy = false;
 		return DRM_GPU_SCHED_STAT_NO_HANG;
 	}
 
+	/*
+	 * A compute dispatch is a single FE command, so the FE does not move
+	 * while a long kernel runs. Keep the job alive as long as some unit
+	 * other than FE and MC is busy, up to hang_max_ms. All of them idle at
+	 * two ticks in a row means the FE waits for something that never
+	 * comes.
+	 */
+	if (gpu->state == ETNA_GPU_STATE_RUNNING && !(idle && was_idle)) {
+		unsigned long max = msecs_to_jiffies(READ_ONCE(etnaviv_hang_max_ms));
+
+		if (!gpu->hangcheck_busy) {
+			gpu->hangcheck_busy = true;
+			gpu->hangcheck_busy_since = jiffies;
+		}
+
+		if (time_before(jiffies, gpu->hangcheck_busy_since + max))
+			return DRM_GPU_SCHED_STAT_NO_HANG;
+
+		if (max)
+			dev_err(gpu->dev, "job busy for more than %u ms without FE progress\n",
+				jiffies_to_msecs(max));
+	}
+
+	gpu->hangcheck_idle = false;
+	gpu->hangcheck_busy = false;
+
 	/* block scheduler */
 	drm_sched_stop(&gpu->sched, sched_job);
 

---
base-commit: 6628e7d4c6b25c8d18f1345e3a20d21829aae7e4
change-id: 20261102-etnaviv-compute-hangcheck-cb8d1e7af68f

Best regards,
-- 
Christian Gmeiner <cgmeiner@igalia.com>

Re: [PATCH] drm/etnaviv: Keep busy jobs alive in the hangcheck

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Low] drm/etnaviv: missing kernel-parameters.txt entry for hang_max_ms parameter
--

commit d016b8a43dc381c8473158580794cfa7c242820a
Author: Christian Gmeiner <cgmeiner@igalia.com>

drm/etnaviv: Keep busy jobs alive in the hangcheck

This commit checks VIVS_HI_IDLE_STATE in the scheduler timeout handler to
keep jobs alive while non-FE/MC hardware units are busy, up to a new
hang_max_ms module parameter limit.

> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> index 3cc50d697c891..cef1ed33ff919 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> @@ -17,6 +17,9 @@ static int etnaviv_job_hang_limit = 0;
>  module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444);
>  static int etnaviv_hw_jobs_limit = 4;
>  module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444);
> +static unsigned int etnaviv_hang_max_ms = 30000;
> +module_param_named(hang_max_ms, etnaviv_hang_max_ms, uint, 0644);
> +MODULE_PARM_DESC(hang_max_ms, "Time a busy job may run without front-end progress (ms, 0 = no extension)");

[Severity: Low]
Should this new hang_max_ms module parameter (etnaviv_hang_max_ms in
drivers/gpu/drm/etnaviv/etnaviv_sched.c), which can also be set on the kernel
command line at boot as etnaviv.hang_max_ms=, be documented in
Documentation/admin-guide/kernel-parameters.txt?

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261102-etnaviv-compute-hangcheck-v1-1-1f81dd1806a2@igalia.com?part=1

Re: [PATCH] drm/etnaviv: Keep busy jobs alive in the hangcheck

[Lucas Stach]

Hi Christian,

Am Montag, dem 02.11.2026 um 12:41 +0100 schrieb Christian Gmeiner:
> From: Christian Gmeiner <cgmeiner@igalia.com>
> 
> A compute dispatch consists of a single FE command. The FE DMA address
> does not change while the kernel is running, so the hangcheck kills any
> dispatch that takes longer than about one second. Long-running OpenCL
> kernels never finish.
> 
> Read VIVS_HI_IDLE_STATE at each timeout tick. If any unit other than FE
> and MC is busy, the job is still executing, so give it more time.
> If all units are idle for two consecutive ticks, the FE is waiting for
> something that will never come, and the job is killed as before.
> 
> A job that stays busy without FE progress for longer than the new
> hang_max_ms module parameter (default 30 seconds) is killed anyway.
> This also catches kernels that loop forever. Setting it to 0 restores
> the old behaviour.
> 
This will regress recovery time for a lot of actual GPU hang scenarios.
Most of the real GPU hangs I've seen lead to a stall somewhere in the
engines, which keeps them from going idle, so various busy flags are
set when real GPU hangs occur.

Do we have any way to cover the CL cases in a less invasive way by
checking for some kind of forward progress on the shader core?
Something like a current instruction pointer or retired instruction
performance counter we can read from the hangcheck handler? This will
still not cover the infinite loop case, but it would allow to recover
from actual GPU hangs faster than the 30s timeout.

Regards,
Lucas

> Signed-off-by: Christian Gmeiner <cgmeiner@igalia.com>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.h   |  3 +++
>  drivers/gpu/drm/etnaviv/etnaviv_sched.c | 39 ++++++++++++++++++++++++++++++++-
>  2 files changed, 41 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> index e89d591d7404..f871ab7899fd 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> @@ -147,6 +147,9 @@ struct etnaviv_gpu {
>  	u32 hangcheck_dma_addr;
>  	u32 hangcheck_primid;
>  	u32 hangcheck_fence;
> +	bool hangcheck_idle;
> +	bool hangcheck_busy;
> +	unsigned long hangcheck_busy_since;
>  
>  	void __iomem *mmio;
>  	int irq;
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> index 3cc50d697c89..cef1ed33ff91 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> @@ -17,6 +17,9 @@ static int etnaviv_job_hang_limit = 0;
>  module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444);
>  static int etnaviv_hw_jobs_limit = 4;
>  module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444);
> +static unsigned int etnaviv_hang_max_ms = 30000;
> +module_param_named(hang_max_ms, etnaviv_hang_max_ms, uint, 0644);
> +MODULE_PARM_DESC(hang_max_ms, "Time a busy job may run without front-end progress (ms, 0 = no extension)");
>  
>  static struct dma_fence *etnaviv_sched_run_job(struct drm_sched_job *sched_job)
>  {
> @@ -36,7 +39,8 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
>  {
>  	struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);
>  	struct etnaviv_gpu *gpu = submit->gpu;
> -	u32 dma_addr, primid = 0;
> +	u32 dma_addr, primid = 0, mask;
> +	bool idle, was_idle;
>  	int change;
>  
>  	/*
> @@ -62,6 +66,12 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
>  		primid = gpu_read(gpu, VIVS_MC_PROFILE_FE_READ);
>  		mutex_unlock(&gpu->lock);
>  	}
> +
> +	mask = gpu->idle_mask & ~(VIVS_HI_IDLE_STATE_FE | VIVS_HI_IDLE_STATE_MC);
> +	idle = (gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask) == mask;
> +	was_idle = gpu->hangcheck_idle;
> +	gpu->hangcheck_idle = idle;
> +
>  	if (gpu->state == ETNA_GPU_STATE_RUNNING &&
>  	    (gpu->completed_fence != gpu->hangcheck_fence ||
>  	     change < 0 || change > 16 ||
> @@ -70,9 +80,36 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
>  		gpu->hangcheck_dma_addr = dma_addr;
>  		gpu->hangcheck_primid = primid;
>  		gpu->hangcheck_fence = gpu->completed_fence;
> +		gpu->hangcheck_busy = false;
>  		return DRM_GPU_SCHED_STAT_NO_HANG;
>  	}
>  
> +	/*
> +	 * A compute dispatch is a single FE command, so the FE does not move
> +	 * while a long kernel runs. Keep the job alive as long as some unit
> +	 * other than FE and MC is busy, up to hang_max_ms. All of them idle at
> +	 * two ticks in a row means the FE waits for something that never
> +	 * comes.
> +	 */
> +	if (gpu->state == ETNA_GPU_STATE_RUNNING && !(idle && was_idle)) {
> +		unsigned long max = msecs_to_jiffies(READ_ONCE(etnaviv_hang_max_ms));
> +
> +		if (!gpu->hangcheck_busy) {
> +			gpu->hangcheck_busy = true;
> +			gpu->hangcheck_busy_since = jiffies;
> +		}
> +
> +		if (time_before(jiffies, gpu->hangcheck_busy_since + max))
> +			return DRM_GPU_SCHED_STAT_NO_HANG;
> +
> +		if (max)
> +			dev_err(gpu->dev, "job busy for more than %u ms without FE progress\n",
> +				jiffies_to_msecs(max));
> +	}
> +
> +	gpu->hangcheck_idle = false;
> +	gpu->hangcheck_busy = false;
> +
>  	/* block scheduler */
>  	drm_sched_stop(&gpu->sched, sched_job);
>  
> 
> ---
> base-commit: 6628e7d4c6b25c8d18f1345e3a20d21829aae7e4
> change-id: 20261102-etnaviv-compute-hangcheck-cb8d1e7af68f
> 
> Best regards,