All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] CI_ONLY: Defer TDR execution while submission is suspended
@ 2026-11-18 10:15 Arvind Yadav
  2026-11-18 10:46 ` sashiko-bot
  0 siblings, 1 reply; 2+ messages in thread
From: Arvind Yadav @ 2026-11-18 10:15 UTC (permalink / raw)
  To: intel-xe

CI ONLY...
guc_exec_queue_stop() can ban a queue with an incomplete job and request
immediate TDR cleanup. During suspend this runs from the PM thread,
while the timeout worker runs on gt->ordered_wq. The timeout worker can
therefore capture engine registers and restart scheduler submission
while GuC submission is being stopped and the device is powering down.

Disable each queue's timeout work before stopping submission. Do this
from gt->ordered_wq so any running drm_sched_job_timedout() has finished,
including the pending-list updates outside the Xe timeout callback.
Keep the GuC stop and suspend operations on the PM caller's thread.

Keep timeout work disabled until all queues have been restarted. Also
disable timeout work on queues created while submission is suspended.
Re-enable it on suspend failure and teardown abort paths so the new
disable count cannot prevent cleanup indefinitely.

Leave the scheduler timeout value unchanged. An immediate cleanup
request made while timeout work is disabled still sets it to zero,
so restarting submission re-arms that cleanup without an extra delay.

Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs")
Link: https://sashiko.dev/#/bug/linux-449968a5-0b4a-4c5d-acd3-7ced29f0129b
Signed-off-by: Arvind Yadav <arvind.yadav@intel.com>
---
 drivers/gpu/drm/xe/xe_device.c               |  7 +-
 drivers/gpu/drm/xe/xe_guc_exec_queue_types.h |  2 +
 drivers/gpu/drm/xe/xe_guc_submit.c           | 71 ++++++++++++++++++++
 drivers/gpu/drm/xe/xe_guc_submit.h           |  2 +
 drivers/gpu/drm/xe/xe_guc_types.h            |  2 +
 drivers/gpu/drm/xe/xe_uc.c                   | 26 ++++++-
 6 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 6911c2ac39f8..2de67e411953 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -1509,8 +1509,13 @@ int xe_device_resume(struct xe_device *xe, bool flr)
 
 	for_each_gt(gt, xe, id) {
 		err = xe_gt_resume(gt);
-		if (err)
+		if (err) {
+			/* GuC boot failed before submission could restart. */
+			if (!flr && err == -ENOEXEC)
+				xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_REBIND |
+						 DRM_WEDGE_RECOVERY_BUS_RESET);
 			break;
+		}
 	}
 
 	if (flr) {
diff --git a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
index 74b711abe257..6f033c4984b8 100644
--- a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
@@ -40,6 +40,8 @@ struct xe_guc_exec_queue {
 	struct xe_sched_msg static_msgs[MAX_STATIC_MSG_TYPE];
 	/** @destroy_async: do final destroy async from this worker */
 	struct work_struct destroy_async;
+	/** @tdr_suspended: work_tdr disabled, protected by submission_state.lock */
+	bool tdr_suspended;
 	/** @resume_time: time of last resume */
 	u64 resume_time;
 	/** @state: GuC specific state for this xe_exec_queue */
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index d76f125f9c3a..e9bc43753a82 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -2120,6 +2120,18 @@ static const struct xe_sched_backend_ops xe_sched_ops = {
 	.process_msg = guc_exec_queue_process_msg,
 };
 
+static void guc_exec_queue_suspend_tdr(struct xe_exec_queue *q)
+{
+	lockdep_assert_held(&exec_queue_to_guc(q)->submission_state.lock);
+
+	if (q->guc->tdr_suspended)
+		return;
+
+	/* Caller runs on the TDR workqueue, or the queue is not published yet. */
+	disable_delayed_work(&q->guc->sched.base.work_tdr);
+	q->guc->tdr_suspended = true;
+}
+
 static int guc_exec_queue_init(struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched;
@@ -2189,6 +2201,8 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 	mutex_lock(&guc->submission_state.lock);
 	if (xe_guc_read_stopped(guc) || vf_recovery(guc))
 		xe_sched_stop(sched);
+	if (guc->submission_state.tdr_suspended)
+		guc_exec_queue_suspend_tdr(q);
 	publish_guc_id(guc, q);
 	mutex_unlock(&guc->submission_state.lock);
 
@@ -3119,6 +3133,59 @@ static void guc_exec_queue_start(struct xe_exec_queue *q)
 	xe_sched_submission_resume_tdr(sched);
 }
 
+/**
+ * xe_guc_submit_suspend_tdr - Disable queue timeout work before suspend
+ * @guc: GuC submission being suspended
+ *
+ * Must run on the GT ordered workqueue, after reset preparation. This
+ * serializes against the whole DRM timeout worker, including pending-list
+ * removal and reinsertion outside the Xe timeout callback.
+ */
+void xe_guc_submit_suspend_tdr(struct xe_guc *guc)
+{
+	struct xe_exec_queue *q;
+	unsigned long index;
+
+	guard(mutex)(&guc->submission_state.lock);
+	guc->submission_state.tdr_suspended = true;
+	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
+		if (q->guc->id != index)
+			continue;
+		guc_exec_queue_suspend_tdr(q);
+	}
+}
+
+static void guc_submit_resume_tdr(struct xe_guc *guc)
+{
+	struct xe_exec_queue *q;
+	unsigned long index;
+
+	lockdep_assert_held(&guc->submission_state.lock);
+	if (!guc->submission_state.tdr_suspended)
+		return;
+
+	guc->submission_state.tdr_suspended = false;
+	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
+		if (q->guc->id != index || !q->guc->tdr_suspended)
+			continue;
+
+		q->guc->tdr_suspended = false;
+		enable_delayed_work(&q->guc->sched.base.work_tdr);
+		/* Cleanup requests made while disabled leave base.timeout at zero. */
+		xe_sched_submission_resume_tdr(&q->guc->sched);
+	}
+}
+
+/**
+ * xe_guc_submit_resume_tdr - Re-enable queue timeout work after suspend abort
+ * @guc: GuC submission whose timeout work was disabled
+ */
+void xe_guc_submit_resume_tdr(struct xe_guc *guc)
+{
+	guard(mutex)(&guc->submission_state.lock);
+	guc_submit_resume_tdr(guc);
+}
+
 int xe_guc_submit_start(struct xe_guc *guc)
 {
 	struct xe_exec_queue *q;
@@ -3135,6 +3202,8 @@ int xe_guc_submit_start(struct xe_guc *guc)
 
 		guc_exec_queue_start(q);
 	}
+	/* All queues must be restarted before any deferred TDR can run. */
+	guc_submit_resume_tdr(guc);
 	mutex_unlock(&guc->submission_state.lock);
 
 	wake_up_all(&guc->ct.wq);
@@ -3313,6 +3382,8 @@ void xe_guc_submit_pause_abort(struct xe_guc *guc)
 		xe_sched_submission_start(sched);
 		guc_exec_queue_kill(q);
 	}
+	/* Abort can tear down a device whose submission will never restart. */
+	guc_submit_resume_tdr(guc);
 	mutex_unlock(&guc->submission_state.lock);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h
index 7a81d0aa19bb..a0108b2bc166 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit.h
@@ -20,6 +20,8 @@ void xe_guc_submit_disable(struct xe_guc *guc);
 int xe_guc_submit_reset_prepare(struct xe_guc *guc);
 void xe_guc_submit_reset_wait(struct xe_guc *guc);
 void xe_guc_submit_stop(struct xe_guc *guc);
+void xe_guc_submit_suspend_tdr(struct xe_guc *guc);
+void xe_guc_submit_resume_tdr(struct xe_guc *guc);
 int xe_guc_submit_start(struct xe_guc *guc);
 void xe_guc_submit_pause(struct xe_guc *guc);
 void xe_guc_submit_pause_abort(struct xe_guc *guc);
diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h
index 6da43691e54c..e80d3f228aa1 100644
--- a/drivers/gpu/drm/xe/xe_guc_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_types.h
@@ -93,6 +93,8 @@ struct xe_guc {
 		atomic_t reset_blocked;
 		/** @submission_state.lock: protects submission state */
 		struct mutex lock;
+		/** @submission_state.tdr_suspended: disable TDR for existing and new queues */
+		bool tdr_suspended;
 		/** @submission_state.enabled: submission is enabled */
 		bool enabled;
 		/**
diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c
index b32ba504041c..aa154800a881 100644
--- a/drivers/gpu/drm/xe/xe_uc.c
+++ b/drivers/gpu/drm/xe/xe_uc.c
@@ -362,17 +362,41 @@ void xe_uc_suspend_prepare(struct xe_uc *uc)
 	xe_guc_stop_prepare(&uc->guc);
 }
 
+struct xe_uc_suspend_work {
+	struct work_struct work;
+	struct xe_uc *uc;
+};
+
+static void uc_suspend_work(struct work_struct *w)
+{
+	struct xe_uc_suspend_work *suspend =
+		container_of(w, struct xe_uc_suspend_work, work);
+	xe_guc_submit_suspend_tdr(&suspend->uc->guc);
+}
+
 int xe_uc_suspend(struct xe_uc *uc)
 {
+	struct xe_uc_suspend_work suspend = { .uc = uc };
+	int err;
+
 	/* GuC submission not enabled, nothing to do */
 	if (!xe_device_uc_enabled(uc_to_xe(uc)))
 		return 0;
 
 	uc_reset_wait(uc);
 
+	/* Drain the complete timeout worker before examining queue state. */
+	INIT_WORK_ONSTACK(&suspend.work, uc_suspend_work);
+	queue_work(uc_to_gt(uc)->ordered_wq, &suspend.work);
+	flush_work(&suspend.work);
+	destroy_work_on_stack(&suspend.work);
+
 	xe_uc_stop(uc);
+	err = xe_guc_suspend(&uc->guc);
+	if (err)
+		xe_guc_submit_resume_tdr(&uc->guc);
 
-	return xe_guc_suspend(&uc->guc);
+	return err;
 }
 
 /**
-- 
2.43.0


^ permalink raw reply related	[flat|nested] 2+ messages in thread

* Re: [PATCH] CI_ONLY: Defer TDR execution while submission is suspended
  2026-11-18 10:15 [PATCH] CI_ONLY: Defer TDR execution while submission is suspended Arvind Yadav
@ 2026-11-18 10:46 ` sashiko-bot
  0 siblings, 0 replies; 2+ messages in thread
From: sashiko-bot @ 2026-11-18 10:46 UTC (permalink / raw)
  To: Arvind Yadav, intel-xe

Thank you for your contribution! Sashiko AI review found 4 potential
issues to consider:
- [Medium] drm/xe: broken TDR and submission recovery on xe_uc_suspend()
  failure
- [Medium] drm/xe: work_tdr left disabled on resume and wedge failure
  paths
- [Medium] drm/xe: infinite busy loop in uc_reset_wait() when suspending
  wedged device
- [Low] drm/xe: CI_ONLY commit markers and undocumented
  xe_device_resume() change
It also noticed 1 related pre-existing issue this patch does not
introduce:
- [High] drm/xe: infinite loop in uc_reset_wait() [1]
--

commit dc721c0cb578b8e8efd82320e9f07c85073891f4
Author: Arvind Yadav <arvind.yadav@intel.com>

> CI_ONLY: Defer TDR execution while submission is suspended
> 
> CI ONLY...
> guc_exec_queue_stop() can ban a queue with an incomplete job and request
> immediate TDR cleanup. During suspend this runs from the PM thread,
> while the timeout worker runs on gt->ordered_wq. The timeout worker can
> therefore capture engine registers and restart scheduler submission
> while GuC submission is being stopped and the device is powering down.
> 
> Disable each queue's timeout work before stopping submission. Do this
> from gt->ordered_wq so any running drm_sched_job_timedout() has finished,
> including the pending-list updates outside the Xe timeout callback.
> Keep the GuC stop and suspend operations on the PM caller's thread.
> 
> Keep timeout work disabled until all queues have been restarted. Also
> disable timeout work on queues created while submission is suspended.
> Re-enable it on suspend failure and teardown abort paths so the new
> disable count cannot prevent cleanup indefinitely.
> 
> Leave the scheduler timeout value unchanged. An immediate cleanup
> request made while timeout work is disabled still sets it to zero,
> so restarting submission re-arms that cleanup without an extra delay.
> 
> Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs")
> Link: https://sashiko.dev/#/bug/linux-449968a5-0b4a-4c5d-acd3-7ced29f0129b
> Signed-off-by: Arvind Yadav <arvind.yadav@intel.com>

[Severity: Low]
This isn't a bug, but the commit subject and body still contain temporary
"CI_ONLY" markers, and the changelog does not mention or explain the
xe_wedge_declare() change in xe_device_resume().

> diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
> index 6911c2ac39f82..2de67e4119539 100644
> --- a/drivers/gpu/drm/xe/xe_device.c
> +++ b/drivers/gpu/drm/xe/xe_device.c
> @@ -1509,8 +1509,13 @@ int xe_device_resume(struct xe_device *xe, bool flr)
>  
>  	for_each_gt(gt, xe, id) {
>  		err = xe_gt_resume(gt);
> -		if (err)
> +		if (err) {
> +			/* GuC boot failed before submission could restart. */
> +			if (!flr && err == -ENOEXEC)
> +				xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_REBIND |
> +						 DRM_WEDGE_RECOVERY_BUS_RESET);

[Severity: Medium]
Can declaring the device wedged here while guc->submission_state.stopped
remains 1 cause a later xe_uc_suspend() call to spin forever in
uc_reset_wait()?

When xe_device_resume() fails after xe_uc_suspend() has set
guc->submission_state.stopped to 1, xe_wedge_declare() marks the device
wedged without clearing stopped.  On a subsequent system suspend,
xe_uc_suspend() calls uc_reset_wait(), where xe_guc_submit_reset_wait()
returns immediately because xe_wedged() is true while xe_uc_reset_prepare()
keeps returning 1 because stopped is still set, looping indefinitely
without sleeping.

>  			break;
> +		}
>  	}

[Severity: Medium]
Does work_tdr remain permanently disabled if resume fails along any other
path before xe_guc_submit_start()?

Only !flr && err == -ENOEXEC in xe_device_resume() calls xe_wedge_declare().
If xe_pcode_ready() or xe_bo_restore_early() fails earlier, if
xe_gt_resume() fails with another error such as -ETIMEDOUT or -EIO (or skips
remaining GTs after break), if D3Cold runtime resume fails in
xe_pm_runtime_resume(), or if xe->wedged.mode is
XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET or XE_WEDGED_MODE_NEVER,
guc_submit_resume_tdr() is never reached.  Pending jobs on banned or killed
queues then never time out or signal their fences, hanging xe_vm_close() and
show_run_ticks().

[ ... ]

> diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c
> index b32ba504041c5..aa154800a8815 100644
> --- a/drivers/gpu/drm/xe/xe_uc.c
> +++ b/drivers/gpu/drm/xe/xe_uc.c
> @@ -362,17 +362,41 @@ void xe_uc_suspend_prepare(struct xe_uc *uc)
>  	xe_guc_stop_prepare(&uc->guc);
>  }
>  
> +struct xe_uc_suspend_work {
> +	struct work_struct work;
> +	struct xe_uc *uc;
> +};
> +
> +static void uc_suspend_work(struct work_struct *w)
> +{
> +	struct xe_uc_suspend_work *suspend =
> +		container_of(w, struct xe_uc_suspend_work, work);
> +	xe_guc_submit_suspend_tdr(&suspend->uc->guc);
> +}
> +
>  int xe_uc_suspend(struct xe_uc *uc)
>  {
> +	struct xe_uc_suspend_work suspend = { .uc = uc };
> +	int err;
> +
>  	/* GuC submission not enabled, nothing to do */
>  	if (!xe_device_uc_enabled(uc_to_xe(uc)))
>  		return 0;
>  
>  	uc_reset_wait(uc);

[Severity: High]
This problem wasn't introduced by this patch, but can uc_reset_wait() spin
in an infinite loop [1] when suspending a wedged device?

drivers/gpu/drm/xe/xe_uc.c:uc_reset_wait() {
	int ret;

again:
	xe_guc_reset_wait(&uc->guc);

	ret = xe_uc_reset_prepare(uc);
	if (ret)
		goto again;
}

If the device is already wedged with guc->submission_state.stopped set to 1,
xe_guc_submit_reset_wait() returns immediately because xe_wedged() is true,
while xe_uc_reset_prepare() atomically sets stopped and returns its previous
value of 1, jumping back to again in a tight non-sleeping loop.

> +
> +	/* Drain the complete timeout worker before examining queue state. */
> +	INIT_WORK_ONSTACK(&suspend.work, uc_suspend_work);
> +	queue_work(uc_to_gt(uc)->ordered_wq, &suspend.work);
> +	flush_work(&suspend.work);
> +	destroy_work_on_stack(&suspend.work);
>  
>  	xe_uc_stop(uc);
> +	err = xe_guc_suspend(&uc->guc);
> +	if (err)
> +		xe_guc_submit_resume_tdr(&uc->guc);
>  
> -	return xe_guc_suspend(&uc->guc);
> +	return err;
>  }

[Severity: Medium]
If xe_guc_suspend() fails here, is calling xe_guc_submit_resume_tdr() safe
after uc_reset_wait() and xe_uc_stop() have already stopped submission?

By the time xe_guc_suspend() runs, guc->submission_state.stopped is 1, CT is
stopped, and guc_exec_queue_stop() has stopped each queue's scheduler,
cleared EXEC_QUEUE_STATE_REGISTERED and EXEC_QUEUE_STATE_ENABLED, and banned
user queues with started jobs (setting base.timeout to 0).  If
xe_guc_softreset() fails, GuC firmware is still running, so resuming TDR
immediately runs guc_exec_queue_timedout_job() on banned queues: because
REGISTERED and ENABLED were cleared, it skips disabling scheduling in GuC,
signals fences early, restarts the scheduler against a stopped CT, and
frees the queue's guc_id and LRC locally while GuC is still active.

In addition, idle queues remain stopped so later guc_exec_queue_destroy()
calls never process STATIC_MSG_CLEANUP, guc->submission_state.stopped stays
1 so subsequent suspends or GT resets stall, and on multi-GT systems a
suspend failure on a later GT in xe_device_suspend() or
xe_pm_runtime_suspend() leaves work_tdr disabled on earlier GTs that already
completed xe_gt_suspend().

[1] https://sashiko.dev/#/bug/linux-62eea921-894e-4f73-bbf0-ac02cad95b25

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261118101543.2210570-1-arvind.yadav@intel.com?part=1

^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2026-11-18 10:46 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-11-18 10:15 [PATCH] CI_ONLY: Defer TDR execution while submission is suspended Arvind Yadav
2026-11-18 10:46 ` sashiko-bot

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.