[PATCH v2 08/15] drm/amdgpu/userq: add helpers to keep the user queues of one engine across a reset

[Jesse Zhang <Jesse.Zhang@amd.com>]

An engine reset, eg. of the gfx engine, only resets the queues of one
engine of the GFX IP, while the queues of the other engines keep
running. amdgpu_userq_pre_reset() and amdgpu_userq_post_reset() handle
all the user queues for a full reset. Add separate helpers for the
queues of one engine, which keep them in the state they had:

- amdgpu_userq_pre_engine_reset(), called after the MES suspend, saves
  the ring state of the mapped queues still on a HQD: the reset clears
  their HQD, and MES couldn't preempt them into their MQD. The queues
  MES preempted have their state in their MQD already.
- amdgpu_userq_post_engine_reset(), called after the reset but before
  MES resumes, unmaps those queues, puts the saved ring state in their
  MQD and maps them again: MES doesn't load a queue whose HQD the reset
  cleared on its own, and the unmap may save the cleared HQD into the
  MQD. The queues keep their state and their work, except the one the
  pipe executed: its work in flight depends on pipe state the reset
  cleared, it is dropped and its fences signaled. The queue that hung,
  if any, is unmapped and hung.
- amdgpu_userq_cancel_engine_reset() drops the saved ring state for the
  full reset that follows a failed engine reset.

The helpers walk the queues and unmap and map them as
amdgpu_userq_pre_reset() and amdgpu_userq_post_reset() do. The saved
ring state is kept in the gfx IP, per HQD, and matched to a queue by its
doorbell and MQD address.

Signed-off-by: Jesse Zhang <Jesse.Zhang@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h   |   3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 109 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h |   5 +
 3 files changed, 117 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index ba6dcea3f7db..5e9338563de3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -524,6 +524,9 @@ struct amdgpu_gfx {
 
 	/* reset mask */
 	uint32_t 			gfx_supported_reset;
+	/* ring state of the gfx user queues on a HQD, saved by an engine reset */
+	struct amdgpu_gfx_hqd_ptrs	engine_reset_hqds[AMDGPU_MAX_GFX_QUEUES];
+	u32				num_engine_reset_hqds;
 	uint32_t 			compute_supported_reset;
 
 	/* gfx off */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 1a2aa85c504e..2c18175a6041 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1597,3 +1597,112 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
 
 	return ret;
 }
+
+/**
+ * amdgpu_userq_pre_engine_reset - save the ring state of the user queues
+ * @adev: amdgpu device
+ * @queue_type: AMDGPU_HW_IP_* type of the queues on the reset engine
+ */
+void amdgpu_userq_pre_engine_reset(struct amdgpu_device *adev, u32 queue_type)
+{
+	struct amdgpu_gfx_hqd_ptrs ptrs;
+	struct amdgpu_usermode_queue *queue;
+	unsigned long queue_id;
+
+	adev->gfx.num_engine_reset_hqds = 0;
+	if (!adev->gfx.funcs->read_gfx_hqd_ptrs)
+		return;
+
+	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
+		if (adev->gfx.num_engine_reset_hqds >= AMDGPU_MAX_GFX_QUEUES)
+			break;
+		if (queue->queue_type != queue_type ||
+		    queue->state != AMDGPU_USERQ_STATE_MAPPED)
+			continue;
+		if (adev->gfx.funcs->read_gfx_hqd_ptrs(adev, queue->doorbell_index,
+						       &ptrs) && ptrs.on_hqd)
+			adev->gfx.engine_reset_hqds[adev->gfx.num_engine_reset_hqds++] = ptrs;
+	}
+}
+
+static struct amdgpu_gfx_hqd_ptrs *
+amdgpu_userq_engine_reset_hqd(struct amdgpu_device *adev,
+			      struct amdgpu_usermode_queue *queue)
+{
+	struct amdgpu_gfx_hqd_ptrs *ptrs;
+	u32 i;
+
+	for (i = 0; i < adev->gfx.num_engine_reset_hqds; i++) {
+		ptrs = &adev->gfx.engine_reset_hqds[i];
+		/* Check the MQD too: a new queue may reuse the doorbell. */
+		if (ptrs->doorbell_index == queue->doorbell_index &&
+		    ptrs->mqd_addr == queue->mqd.gpu_addr)
+			return ptrs;
+	}
+	return NULL;
+}
+
+/**
+ * amdgpu_userq_post_engine_reset - restore the user queues after an engine reset
+ * @adev: amdgpu device
+ * @queue_type: AMDGPU_HW_IP_* type of the queues on the reset engine
+ * @guilty: the queue that hung, or NULL
+ *
+ * MES doesn't load a queue whose HQD the reset cleared again: unmap it, which
+ * may save the cleared HQD into its MQD, restore its MQD, and map it again.
+ */
+void amdgpu_userq_post_engine_reset(struct amdgpu_device *adev, u32 queue_type,
+				    struct amdgpu_usermode_queue *guilty)
+{
+	const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue_type];
+	struct amdgpu_usermode_queue *queue;
+	struct amdgpu_gfx_hqd_ptrs *ptrs;
+	unsigned long queue_id;
+	int r;
+
+	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
+		if (queue->queue_type != queue_type ||
+		    queue->state != AMDGPU_USERQ_STATE_MAPPED)
+			continue;
+
+		if (queue == guilty) {
+			userq_funcs->unmap(queue);
+			amdgpu_userq_fence_driver_force_completion(queue);
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
+			queue->state = AMDGPU_USERQ_STATE_HUNG;
+			continue;
+		}
+
+		ptrs = amdgpu_userq_engine_reset_hqd(adev, queue);
+		if (!ptrs)
+			continue;
+
+		/* Unmapping may write the cleared HQD into the MQD. */
+		userq_funcs->unmap(queue);
+		if (adev->gfx.funcs->restore_gfx_mqd_ring(adev, queue->mqd.cpu_ptr,
+							  queue->userq_prop->queue_size,
+							  ptrs))
+			amdgpu_userq_fence_driver_force_completion_with_seqno(queue,
+									      ptrs->wptr);
+		r = userq_funcs->map(queue);
+		if (r) {
+			dev_err(adev->dev, "Failed to remap user queue %llu: %d\n",
+				queue->doorbell_index, r);
+			amdgpu_userq_fence_driver_force_completion(queue);
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
+			queue->state = AMDGPU_USERQ_STATE_HUNG;
+		}
+	}
+	adev->gfx.num_engine_reset_hqds = 0;
+}
+
+/**
+ * amdgpu_userq_cancel_engine_reset - give up an engine reset
+ * @adev: amdgpu device
+ * @queue_type: AMDGPU_HW_IP_* type of the queues on the reset engine
+ */
+void amdgpu_userq_cancel_engine_reset(struct amdgpu_device *adev,
+				      u32 queue_type)
+{
+	adev->gfx.num_engine_reset_hqds = 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 488dc21d7c81..e07c71cd5af5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -181,6 +181,11 @@ int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
 void amdgpu_userq_reset_work(struct work_struct *work);
 void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
+void amdgpu_userq_pre_engine_reset(struct amdgpu_device *adev, u32 queue_type);
+void amdgpu_userq_post_engine_reset(struct amdgpu_device *adev, u32 queue_type,
+				    struct amdgpu_usermode_queue *guilty);
+void amdgpu_userq_cancel_engine_reset(struct amdgpu_device *adev,
+				      u32 queue_type);
 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);
 
-- 
2.50.1