[PATCH 0/3] Support queue reset on non-SPX mode

[PATCH 0/3] Support queue reset on non-SPX mode

[Amber Lin]

This series of patches fix the queue reset failure on non-SPX mode by
1. Specify the master XCC when calling MES.
2. Adjust the hqd_info offset when retrieving the hung queue information
from MES.

Amber Lin (3):
  drm/amdgpu: Add debug messages for MES queue reset
  drm/amdgpu: Add xcp to reset_mes_compute function
  drm/amdgpu: Correct hung queue number for reset

 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c       | 29 ++++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h       |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c       | 55 +++++++++++++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c     |  2 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c        |  2 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c        |  2 +-
 .../drm/amd/amdkfd/kfd_device_queue_manager.c |  2 +-
 7 files changed, 66 insertions(+), 27 deletions(-)

-- 
2.43.0

[PATCH 1/3] drm/amdgpu: Add debug messages for MES queue reset

[Amber Lin]

This patch adds debug messages to collect some MES queue reset data.

Signed-off-by: Amber Lin <amber.lin@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 12 ++++++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 0ae6e990c310..0c1c8ae822d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -2495,15 +2495,21 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	 */
 	r = amdgpu_mes_suspend(adev, 0);
 	/* if suspend all success, it should no hang queue */
-	if (!r)
+	if (!r) {
+		dev_dbg(adev->dev, "Suspend all succeeds. xcc_id %d\n", xcc_id);
 		/* always reset the KCQ/userq since we need to signal the fence
 		 * and we could be stuck in a loop which is preemptable.
 		 */
 		goto fence_reset;
+	}
 	r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
 						    true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0);
-	if (r)
+	if (r) {
+		dev_dbg(adev->dev, "MES fails on detecting hung queues\n");
 		goto out;
+	}
+	dev_dbg(adev->dev, "Detected %d hung queues\n", num_hung);
+
 	if (hung_queue_count)
 		*hung_queue_count = num_hung;
 
@@ -2530,6 +2536,8 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 		pipe = hqd_info[i].pipe_index;
 		queue = hqd_info[i].queue_index;
 		queue_type = hqd_info[i].queue_type;
+		dev_dbg(adev->dev, "Hung queue: queue_id %d at pipe %d type %d\n",
+					queue, pipe, queue_type);
 
 		/* reset any KCQs */
 		r = amdgpu_gfx_reset_mes_kcq(adev, ring,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 2ff1816adfc4..58766b27e150 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -557,6 +557,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 	for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
 		/* Finding hung queues where db_array[i] is a valid doorbell */
 		if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
+			dev_dbg(adev->dev, "MES: hung queue doorbell 0x%x\n", db_array[i]);
 			hung_db_array[i] = db_array[i];
 			*hung_db_num += 1;
 		}
-- 
2.43.0

[PATCH 2/3] drm/amdgpu: Add xcp to reset_mes_compute function

[Amber Lin]

Add xcp parameter to amdgpu_gfx_reset_mes_compute() so on an ASIC that
supports compute partitioning, we can identify the partition's node ID and
its master XCC. Pass this master XCC ID to MES when calling MES for
suspend, detecting hung queues, and resume.

Signed-off-by: Amber Lin <amber.lin@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c         | 17 +++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h         |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c       |  2 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c          |  2 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c          |  2 +-
 .../drm/amd/amdkfd/kfd_device_queue_manager.c   |  2 +-
 6 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 0c1c8ae822d4..ef34f7b02d0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -2470,6 +2470,7 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
 }
 
 int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
+				 struct amdgpu_xcp *xcp,
 				 struct amdgpu_ring *ring,
 				 struct amdgpu_fence *guilty_fence,
 				 struct amdgpu_usermode_queue *uq,
@@ -2486,6 +2487,14 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	struct amdgpu_gfx_deferred_entry deferred_end[AMDGPU_MAX_COMPUTE_RINGS + 1];
 	int n_deferred = 0;
 	int ring_err;
+	uint32_t xcc_mask, xcc_id = 0, xcp_id = 0;
+
+	if (xcp) {
+		amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &xcc_mask);
+		/* master xcc in the partition */
+		xcc_id = ffs(xcc_mask) - 1;
+		xcp_id = xcp->id;
+	}
 
 	guard(mutex)(&adev->gfx.mec.reset_mutex);
 	/* stop the drm schedulers for all compute queues */
@@ -2493,7 +2502,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	/* suspend all will determine which queues are hung.
 	 * reset detect will return the array of bad queue doorbells
 	 */
-	r = amdgpu_mes_suspend(adev, 0);
+	r = amdgpu_mes_suspend(adev, xcc_id);
 	/* if suspend all success, it should no hang queue */
 	if (!r) {
 		dev_dbg(adev->dev, "Suspend all succeeds. xcc_id %d\n", xcc_id);
@@ -2503,7 +2512,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 		goto fence_reset;
 	}
 	r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
-						    true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0);
+						    true, &num_hung, adev->gfx.mec.mes_hung_db_array, xcc_id);
 	if (r) {
 		dev_dbg(adev->dev, "MES fails on detecting hung queues\n");
 		goto out;
@@ -2551,7 +2560,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 			n_deferred++;
 		}
 		/* reset any KFD queues */
-		r = amdgpu_amdkfd_reset_mes_queue(adev, 0, queue_type, pipe, queue,
+		r = amdgpu_amdkfd_reset_mes_queue(adev, xcp_id, queue_type, pipe, queue,
 						  adev->gfx.mec.mes_hung_db_array[i]);
 		if (r)
 			goto out;
@@ -2582,7 +2591,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 
 out:
 	/* resume all will enable the non-hung queues */
-	amdgpu_mes_resume(adev, 0);
+	amdgpu_mes_resume(adev, xcc_id);
 
 	/* Now CP is running again — replay backed-up commands and ring
 	 * doorbells on each reset queue.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 5939e1fb3b24..26bbec8f92e8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -664,6 +664,7 @@ bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id);
 int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev);
 void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev);
 int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
+				 struct amdgpu_xcp *xcp,
 				 struct amdgpu_ring *ring,
 				 struct amdgpu_fence *guilty_fence,
 				 struct amdgpu_usermode_queue *uq,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 1a2aa85c504e..7dfe6e83f8ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -153,7 +153,7 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 		int r;
 
 		if (queue->queue_type == AMDGPU_HW_IP_COMPUTE)
-			r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL,
+			r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL,
 							 queue, NULL, NULL);
 		else
 			r = userq_funcs->reset(queue);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 315dfd806d56..fdc0620ca67f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -6932,7 +6932,7 @@ static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
+	return amdgpu_gfx_reset_mes_compute(adev, NULL, ring, timedout_fence, NULL, NULL, NULL);
 }
 
 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index d334fb805020..ec5ec1fd8b9d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -5254,7 +5254,7 @@ static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
+	return amdgpu_gfx_reset_mes_compute(adev, NULL, ring, timedout_fence, NULL, NULL, NULL);
 }
 
 static void gfx_v12_0_ring_begin_use(struct amdgpu_ring *ring)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ce67e56efa8c..0eb4081a5740 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -506,7 +506,7 @@ static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q)
 	queue_input.remove_queue_after_reset = false;
 	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
 	/* pass the known bad queue info to the reset function */
-	r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input);
+	r = amdgpu_gfx_reset_mes_compute(adev, dqm->dev->xcp, NULL, NULL, NULL, &num_hung, &queue_input);
 	if (r)
 		goto fail;
 	pdd = kfd_get_process_device_data(q->device, q->process);
-- 
2.43.0

[PATCH 3/3] drm/amdgpu: Correct hung queue number for reset

[Amber Lin]

In an ASIC supporting multi-compute partition, the number of queues
varies with the compute partition mode. We need to adjust the hqd_info
offset to get the correct hung queues doorbell offset and hqd_info.

Signed-off-by: Amber Lin <amber.lin@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 54 +++++++++++++++++--------
 1 file changed, 37 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 58766b27e150..5bcab09b15c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -108,6 +108,24 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
 
 static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
 
+static int get_num_queues_per_xcp(struct amdgpu_device *adev, int num_xcps)
+{
+	int i, num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
+	u32 num_queues = 0;
+
+	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
+		num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
+
+	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
+		num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
+
+	num_xcc /= num_xcps;
+	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
+		num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
+
+	return num_queues;
+}
+
 int amdgpu_mes_init(struct amdgpu_device *adev)
 {
 	int i, r, num_pipes, num_queues = 0;
@@ -222,21 +240,18 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		goto error_doorbell;
 
 	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
-		/* When queue/pipe reset is done in MES instead of in the
-		 * driver, MES passes hung queues information to the driver in
-		 * hung_queue_hqd_info. Calculate required space to store this
-		 * information.
+		/* When queue/pipe reset is handled by MES instead of by the
+		 * driver, driver provides the buffer for MES to store the hung
+		 * queues information. Calculate required buffer size, which is
+		 * the number of maximum queues. This number varies with compute
+		 * partition mode. We allocate the biggest size, which is SPX,
+		 * num_xcps = 1.
 		 */
-		for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
-			num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
-
-		for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
-			num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
-
-		for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
-			num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
-
+		num_queues = get_num_queues_per_xcp(adev, 1);
 		adev->mes.hung_queue_hqd_info_offset = num_queues;
+		/* Double num_queues. The first half is to store doorbell offset
+		 * and the second half is to store hqd_info.
+		 */
 		adev->mes.hung_queue_db_array_size = num_queues * 2;
 	}
 
@@ -526,8 +541,9 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 					    uint32_t xcc_id)
 {
 	struct mes_detect_and_reset_queue_input input;
-	u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
-	int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i;
+	u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[0];
+	int hqd_info_offset, r, i;
+	int num_xcps = 1;
 
 	if (!hung_db_num || !hung_db_array)
 		return -EINVAL;
@@ -537,8 +553,12 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 	    (queue_type != AMDGPU_RING_TYPE_SDMA))
 		return -EINVAL;
 
+	if (adev->xcp_mgr)
+		num_xcps = adev->xcp_mgr->num_xcps;
+	hqd_info_offset = get_num_queues_per_xcp(adev, num_xcps);
+
 	/* Clear the doorbell array before detection */
-	memset(adev->mes.hung_queue_db_array_cpu_addr[xcc_id], AMDGPU_MES_INVALID_DB_OFFSET,
+	memset(adev->mes.hung_queue_db_array_cpu_addr[0], AMDGPU_MES_INVALID_DB_OFFSET,
 		adev->mes.hung_queue_db_array_size * sizeof(u32));
 	input.queue_type = queue_type;
 	input.detect_only = detect_only;
@@ -554,7 +574,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 
 	*hung_db_num = 0;
 	/* MES passes hung queues' doorbell to driver */
-	for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
+	for (i = 0; i < hqd_info_offset; i++) {
 		/* Finding hung queues where db_array[i] is a valid doorbell */
 		if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
 			dev_dbg(adev->dev, "MES: hung queue doorbell 0x%x\n", db_array[i]);
-- 
2.43.0

Re: [PATCH 1/3] drm/amdgpu: Add debug messages for MES queue reset

[Alex Deucher]

On Wed, Oct 21, 2026 at 9:56 PM Amber Lin <amber.lin@amd.com> wrote:
>
> This patch adds debug messages to collect some MES queue reset data.
>
> Signed-off-by: Amber Lin <amber.lin@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 12 ++++++++++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c |  1 +
>  2 files changed, 11 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index 0ae6e990c310..0c1c8ae822d4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -2495,15 +2495,21 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>          */
>         r = amdgpu_mes_suspend(adev, 0);
>         /* if suspend all success, it should no hang queue */
> -       if (!r)
> +       if (!r) {
> +               dev_dbg(adev->dev, "Suspend all succeeds. xcc_id %d\n", xcc_id);
>                 /* always reset the KCQ/userq since we need to signal the fence
>                  * and we could be stuck in a loop which is preemptable.
>                  */
>                 goto fence_reset;
> +       }
>         r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
>                                                     true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0);
> -       if (r)
> +       if (r) {
> +               dev_dbg(adev->dev, "MES fails on detecting hung queues\n");
>                 goto out;
> +       }
> +       dev_dbg(adev->dev, "Detected %d hung queues\n", num_hung);
> +
>         if (hung_queue_count)
>                 *hung_queue_count = num_hung;
>
> @@ -2530,6 +2536,8 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>                 pipe = hqd_info[i].pipe_index;
>                 queue = hqd_info[i].queue_index;
>                 queue_type = hqd_info[i].queue_type;
> +               dev_dbg(adev->dev, "Hung queue: queue_id %d at pipe %d type %d\n",
> +                                       queue, pipe, queue_type);
>
>                 /* reset any KCQs */
>                 r = amdgpu_gfx_reset_mes_kcq(adev, ring,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index 2ff1816adfc4..58766b27e150 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -557,6 +557,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>         for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
>                 /* Finding hung queues where db_array[i] is a valid doorbell */
>                 if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
> +                       dev_dbg(adev->dev, "MES: hung queue doorbell 0x%x\n", db_array[i]);
>                         hung_db_array[i] = db_array[i];
>                         *hung_db_num += 1;
>                 }
> --
> 2.43.0
>

Re: [PATCH 2/3] drm/amdgpu: Add xcp to reset_mes_compute function

[Alex Deucher]

On Wed, Oct 21, 2026 at 9:56 PM Amber Lin <amber.lin@amd.com> wrote:
>
> Add xcp parameter to amdgpu_gfx_reset_mes_compute() so on an ASIC that
> supports compute partitioning, we can identify the partition's node ID and
> its master XCC. Pass this master XCC ID to MES when calling MES for
> suspend, detecting hung queues, and resume.
>
> Signed-off-by: Amber Lin <amber.lin@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c         | 17 +++++++++++++----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h         |  1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c       |  2 +-
>  drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c          |  2 +-
>  drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c          |  2 +-
>  .../drm/amd/amdkfd/kfd_device_queue_manager.c   |  2 +-
>  6 files changed, 18 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> index 0c1c8ae822d4..ef34f7b02d0e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
> @@ -2470,6 +2470,7 @@ static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
>  }
>
>  int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
> +                                struct amdgpu_xcp *xcp,
>                                  struct amdgpu_ring *ring,
>                                  struct amdgpu_fence *guilty_fence,
>                                  struct amdgpu_usermode_queue *uq,
> @@ -2486,6 +2487,14 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>         struct amdgpu_gfx_deferred_entry deferred_end[AMDGPU_MAX_COMPUTE_RINGS + 1];
>         int n_deferred = 0;
>         int ring_err;
> +       uint32_t xcc_mask, xcc_id = 0, xcp_id = 0;
> +
> +       if (xcp) {
> +               amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &xcc_mask);
> +               /* master xcc in the partition */
> +               xcc_id = ffs(xcc_mask) - 1;
> +               xcp_id = xcp->id;
> +       }
>
>         guard(mutex)(&adev->gfx.mec.reset_mutex);
>         /* stop the drm schedulers for all compute queues */
> @@ -2493,7 +2502,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>         /* suspend all will determine which queues are hung.
>          * reset detect will return the array of bad queue doorbells
>          */
> -       r = amdgpu_mes_suspend(adev, 0);
> +       r = amdgpu_mes_suspend(adev, xcc_id);
>         /* if suspend all success, it should no hang queue */
>         if (!r) {
>                 dev_dbg(adev->dev, "Suspend all succeeds. xcc_id %d\n", xcc_id);
> @@ -2503,7 +2512,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>                 goto fence_reset;
>         }
>         r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
> -                                                   true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0);
> +                                                   true, &num_hung, adev->gfx.mec.mes_hung_db_array, xcc_id);
>         if (r) {
>                 dev_dbg(adev->dev, "MES fails on detecting hung queues\n");
>                 goto out;
> @@ -2551,7 +2560,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>                         n_deferred++;
>                 }
>                 /* reset any KFD queues */
> -               r = amdgpu_amdkfd_reset_mes_queue(adev, 0, queue_type, pipe, queue,
> +               r = amdgpu_amdkfd_reset_mes_queue(adev, xcp_id, queue_type, pipe, queue,
>                                                   adev->gfx.mec.mes_hung_db_array[i]);
>                 if (r)
>                         goto out;
> @@ -2582,7 +2591,7 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
>
>  out:
>         /* resume all will enable the non-hung queues */
> -       amdgpu_mes_resume(adev, 0);
> +       amdgpu_mes_resume(adev, xcc_id);
>
>         /* Now CP is running again — replay backed-up commands and ring
>          * doorbells on each reset queue.
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> index 5939e1fb3b24..26bbec8f92e8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
> @@ -664,6 +664,7 @@ bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id);
>  int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev);
>  void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev);
>  int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
> +                                struct amdgpu_xcp *xcp,
>                                  struct amdgpu_ring *ring,
>                                  struct amdgpu_fence *guilty_fence,
>                                  struct amdgpu_usermode_queue *uq,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> index 1a2aa85c504e..7dfe6e83f8ce 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
> @@ -153,7 +153,7 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
>                 int r;
>
>                 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE)
> -                       r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL,
> +                       r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL,
>                                                          queue, NULL, NULL);
>                 else
>                         r = userq_funcs->reset(queue);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> index 315dfd806d56..fdc0620ca67f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> @@ -6932,7 +6932,7 @@ static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
>  {
>         struct amdgpu_device *adev = ring->adev;
>
> -       return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
> +       return amdgpu_gfx_reset_mes_compute(adev, NULL, ring, timedout_fence, NULL, NULL, NULL);
>  }
>
>  static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> index d334fb805020..ec5ec1fd8b9d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
> @@ -5254,7 +5254,7 @@ static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
>  {
>         struct amdgpu_device *adev = ring->adev;
>
> -       return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
> +       return amdgpu_gfx_reset_mes_compute(adev, NULL, ring, timedout_fence, NULL, NULL, NULL);
>  }
>
>  static void gfx_v12_0_ring_begin_use(struct amdgpu_ring *ring)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index ce67e56efa8c..0eb4081a5740 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -506,7 +506,7 @@ static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q)
>         queue_input.remove_queue_after_reset = false;
>         queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
>         /* pass the known bad queue info to the reset function */
> -       r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input);
> +       r = amdgpu_gfx_reset_mes_compute(adev, dqm->dev->xcp, NULL, NULL, NULL, &num_hung, &queue_input);
>         if (r)
>                 goto fail;
>         pdd = kfd_get_process_device_data(q->device, q->process);
> --
> 2.43.0
>

Re: [PATCH 3/3] drm/amdgpu: Correct hung queue number for reset

[Alex Deucher]

On Wed, Oct 21, 2026 at 9:56 PM Amber Lin <amber.lin@amd.com> wrote:
>
> In an ASIC supporting multi-compute partition, the number of queues
> varies with the compute partition mode. We need to adjust the hqd_info
> offset to get the correct hung queues doorbell offset and hqd_info.
>
> Signed-off-by: Amber Lin <amber.lin@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 54 +++++++++++++++++--------
>  1 file changed, 37 insertions(+), 17 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> index 58766b27e150..5bcab09b15c6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> @@ -108,6 +108,24 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
>
>  static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
>
> +static int get_num_queues_per_xcp(struct amdgpu_device *adev, int num_xcps)
> +{
> +       int i, num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
> +       u32 num_queues = 0;
> +
> +       for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
> +               num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
> +
> +       for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
> +               num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
> +
> +       num_xcc /= num_xcps;
> +       for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
> +               num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
> +
> +       return num_queues;
> +}
> +
>  int amdgpu_mes_init(struct amdgpu_device *adev)
>  {
>         int i, r, num_pipes, num_queues = 0;
> @@ -222,21 +240,18 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
>                 goto error_doorbell;
>
>         if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
> -               /* When queue/pipe reset is done in MES instead of in the
> -                * driver, MES passes hung queues information to the driver in
> -                * hung_queue_hqd_info. Calculate required space to store this
> -                * information.
> +               /* When queue/pipe reset is handled by MES instead of by the
> +                * driver, driver provides the buffer for MES to store the hung
> +                * queues information. Calculate required buffer size, which is
> +                * the number of maximum queues. This number varies with compute
> +                * partition mode. We allocate the biggest size, which is SPX,
> +                * num_xcps = 1.
>                  */
> -               for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
> -                       num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
> -
> -               for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
> -                       num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
> -
> -               for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
> -                       num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
> -
> +               num_queues = get_num_queues_per_xcp(adev, 1);
>                 adev->mes.hung_queue_hqd_info_offset = num_queues;
> +               /* Double num_queues. The first half is to store doorbell offset
> +                * and the second half is to store hqd_info.
> +                */
>                 adev->mes.hung_queue_db_array_size = num_queues * 2;
>         }
>
> @@ -526,8 +541,9 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>                                             uint32_t xcc_id)
>  {
>         struct mes_detect_and_reset_queue_input input;
> -       u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
> -       int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i;
> +       u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[0];

Does mes.hung_queue_db_array_cpu_addr still need to be an array?

> +       int hqd_info_offset, r, i;
> +       int num_xcps = 1;
>
>         if (!hung_db_num || !hung_db_array)
>                 return -EINVAL;
> @@ -537,8 +553,12 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>             (queue_type != AMDGPU_RING_TYPE_SDMA))
>                 return -EINVAL;
>
> +       if (adev->xcp_mgr)
> +               num_xcps = adev->xcp_mgr->num_xcps;
> +       hqd_info_offset = get_num_queues_per_xcp(adev, num_xcps);
> +
>         /* Clear the doorbell array before detection */
> -       memset(adev->mes.hung_queue_db_array_cpu_addr[xcc_id], AMDGPU_MES_INVALID_DB_OFFSET,
> +       memset(adev->mes.hung_queue_db_array_cpu_addr[0], AMDGPU_MES_INVALID_DB_OFFSET,
>                 adev->mes.hung_queue_db_array_size * sizeof(u32));
>         input.queue_type = queue_type;
>         input.detect_only = detect_only;
> @@ -554,7 +574,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>
>         *hung_db_num = 0;
>         /* MES passes hung queues' doorbell to driver */
> -       for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {

Can mes.hung_queue_hqd_info_offset be removed after this change?

Alex

> +       for (i = 0; i < hqd_info_offset; i++) {
>                 /* Finding hung queues where db_array[i] is a valid doorbell */
>                 if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
>                         dev_dbg(adev->dev, "MES: hung queue doorbell 0x%x\n", db_array[i]);
> --
> 2.43.0
>

RE: [PATCH 3/3] drm/amdgpu: Correct hung queue number for reset

[Lin, Amber]

AMD General

Regards,
Amber

-----Original Message-----
From: Alex Deucher <alexdeucher@gmail.com>
Sent: Thursday, October 22, 2026 11:40 PM
To: Lin, Amber <Amber.Lin@amd.com>
Cc: amd-gfx@lists.freedesktop.org; Liu, Shaoyun <Shaoyun.Liu@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>
Subject: Re: [PATCH 3/3] drm/amdgpu: Correct hung queue number for reset

On Wed, Oct 21, 2026 at 9:56 PM Amber Lin <amber.lin@amd.com> wrote:
>>
>> In an ASIC supporting multi-compute partition, the number of queues
>> varies with the compute partition mode. We need to adjust the hqd_info
>> offset to get the correct hung queues doorbell offset and hqd_info.
>>
>> Signed-off-by: Amber Lin <amber.lin@amd.com>
>> ---
>>  drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 54
>> +++++++++++++++++--------
>>  1 file changed, 37 insertions(+), 17 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
>> index 58766b27e150..5bcab09b15c6 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
>> @@ -108,6 +108,24 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32
>> num_pipe,
>>
>>  static void amdgpu_mes_userq_notify_unmap_work_handler(struct
>> work_struct *work);
>>
>> +static int get_num_queues_per_xcp(struct amdgpu_device *adev, int
>> +num_xcps) {
>> +       int i, num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
>> +       u32 num_queues = 0;
>> +
>> +       for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
>> +               num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
>> +
>> +       for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
>> +               num_queues +=
>> + hweight32(adev->mes.compute_hqd_mask[i]);
>> +
>> +       num_xcc /= num_xcps;
>> +       for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
>> +               num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) *
>> + num_xcc;
>> +
>> +       return num_queues;
>> +}
>> +
>>  int amdgpu_mes_init(struct amdgpu_device *adev)  {
>>         int i, r, num_pipes, num_queues = 0; @@ -222,21 +240,18 @@ int
>> amdgpu_mes_init(struct amdgpu_device *adev)
>>                 goto error_doorbell;
>>
>>         if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
>> -               /* When queue/pipe reset is done in MES instead of in the
>> -                * driver, MES passes hung queues information to the driver in
>> -                * hung_queue_hqd_info. Calculate required space to store this
>> -                * information.
>> +               /* When queue/pipe reset is handled by MES instead of by the
>> +                * driver, driver provides the buffer for MES to store the hung
>> +                * queues information. Calculate required buffer size, which is
>> +                * the number of maximum queues. This number varies with compute
>> +                * partition mode. We allocate the biggest size, which is SPX,
>> +                * num_xcps = 1.
>>                  */
>> -               for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
>> -                       num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
>> -
>> -               for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
>> -                       num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
>> -
>> -               for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
>> -                       num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
>> -
>> +               num_queues = get_num_queues_per_xcp(adev, 1);
>>                 adev->mes.hung_queue_hqd_info_offset = num_queues;
>> +               /* Double num_queues. The first half is to store doorbell offset
>> +                * and the second half is to store hqd_info.
>> +                */
>>                 adev->mes.hung_queue_db_array_size = num_queues * 2;
>>         }
>>
>> @@ -526,8 +541,9 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>>                                             uint32_t xcc_id)  {
>>         struct mes_detect_and_reset_queue_input input;
>> -       u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
>> -       int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i;
>> +       u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[0];
>
> Does mes.hung_queue_db_array_cpu_addr still need to be an array?

No, it doesn't need to be an array. Since we guard the access in amdgpu_gfx_mes_reset_compute function and only one XCC can perform detect_and_reset at one time, one hung_queue_db_array is enough. My original series included reverting the patch that added xcc copies of hung_queue_db_array. I was thinking to submit that revert in the next series (clean up), but I can do it in this series too.
Since we are in this topic, I have a question here. Besides of the xcc copies, amdgpu_mes_init also allocates AMDGPU_MAX_MES_PIPES copies of hung_queue_db_array buffer. Is it required for gfx 10? In gfx11 and after, we only use MES pipe 0 for detect_and_reset so we shouldn't need AMDGPU_MAX_MES_PIPES copies.

>> +       int hqd_info_offset, r, i;
>> +       int num_xcps = 1;
>>
>>         if (!hung_db_num || !hung_db_array)
>>                 return -EINVAL;
>> @@ -537,8 +553,12 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>>             (queue_type != AMDGPU_RING_TYPE_SDMA))
>>                 return -EINVAL;
>>
>> +       if (adev->xcp_mgr)
>> +               num_xcps = adev->xcp_mgr->num_xcps;
>> +       hqd_info_offset = get_num_queues_per_xcp(adev, num_xcps);
>> +
>>         /* Clear the doorbell array before detection */
>> -       memset(adev->mes.hung_queue_db_array_cpu_addr[xcc_id], AMDGPU_MES_INVALID_DB_OFFSET,
>> +       memset(adev->mes.hung_queue_db_array_cpu_addr[0],
>> + AMDGPU_MES_INVALID_DB_OFFSET,
>>                 adev->mes.hung_queue_db_array_size * sizeof(u32));
>>         input.queue_type = queue_type;
>>         input.detect_only = detect_only; @@ -554,7 +574,7 @@ int
>> amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
>>
>>         *hung_db_num = 0;
>>         /* MES passes hung queues' doorbell to driver */
>> -       for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {

Can mes.hung_queue_hqd_info_offset be removed after this change?

Alex

> +       for (i = 0; i < hqd_info_offset; i++) {
>                 /* Finding hung queues where db_array[i] is a valid doorbell */
>                 if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
>                         dev_dbg(adev->dev, "MES: hung queue doorbell
> 0x%x\n", db_array[i]);
> --
> 2.43.0
>

Re: [PATCH 3/3] drm/amdgpu: Correct hung queue number for reset

[Alex Deucher]

On Thu, Oct 22, 2026 at 10:20 PM Lin, Amber <Amber.Lin@amd.com> wrote:
>
> AMD General
>
> Regards,
> Amber
>
> -----Original Message-----
> From: Alex Deucher <alexdeucher@gmail.com>
> Sent: Thursday, October 22, 2026 11:40 PM
> To: Lin, Amber <Amber.Lin@amd.com>
> Cc: amd-gfx@lists.freedesktop.org; Liu, Shaoyun <Shaoyun.Liu@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: Re: [PATCH 3/3] drm/amdgpu: Correct hung queue number for reset
>
> On Wed, Oct 21, 2026 at 9:56 PM Amber Lin <amber.lin@amd.com> wrote:
> >>
> >> In an ASIC supporting multi-compute partition, the number of queues
> >> varies with the compute partition mode. We need to adjust the hqd_info
> >> offset to get the correct hung queues doorbell offset and hqd_info.
> >>
> >> Signed-off-by: Amber Lin <amber.lin@amd.com>
> >> ---
> >>  drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 54
> >> +++++++++++++++++--------
> >>  1 file changed, 37 insertions(+), 17 deletions(-)
> >>
> >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> >> b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> >> index 58766b27e150..5bcab09b15c6 100644
> >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
> >> @@ -108,6 +108,24 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32
> >> num_pipe,
> >>
> >>  static void amdgpu_mes_userq_notify_unmap_work_handler(struct
> >> work_struct *work);
> >>
> >> +static int get_num_queues_per_xcp(struct amdgpu_device *adev, int
> >> +num_xcps) {
> >> +       int i, num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
> >> +       u32 num_queues = 0;
> >> +
> >> +       for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
> >> +               num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
> >> +
> >> +       for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
> >> +               num_queues +=
> >> + hweight32(adev->mes.compute_hqd_mask[i]);
> >> +
> >> +       num_xcc /= num_xcps;
> >> +       for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
> >> +               num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) *
> >> + num_xcc;
> >> +
> >> +       return num_queues;
> >> +}
> >> +
> >>  int amdgpu_mes_init(struct amdgpu_device *adev)  {
> >>         int i, r, num_pipes, num_queues = 0; @@ -222,21 +240,18 @@ int
> >> amdgpu_mes_init(struct amdgpu_device *adev)
> >>                 goto error_doorbell;
> >>
> >>         if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
> >> -               /* When queue/pipe reset is done in MES instead of in the
> >> -                * driver, MES passes hung queues information to the driver in
> >> -                * hung_queue_hqd_info. Calculate required space to store this
> >> -                * information.
> >> +               /* When queue/pipe reset is handled by MES instead of by the
> >> +                * driver, driver provides the buffer for MES to store the hung
> >> +                * queues information. Calculate required buffer size, which is
> >> +                * the number of maximum queues. This number varies with compute
> >> +                * partition mode. We allocate the biggest size, which is SPX,
> >> +                * num_xcps = 1.
> >>                  */
> >> -               for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
> >> -                       num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
> >> -
> >> -               for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
> >> -                       num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
> >> -
> >> -               for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
> >> -                       num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
> >> -
> >> +               num_queues = get_num_queues_per_xcp(adev, 1);
> >>                 adev->mes.hung_queue_hqd_info_offset = num_queues;
> >> +               /* Double num_queues. The first half is to store doorbell offset
> >> +                * and the second half is to store hqd_info.
> >> +                */
> >>                 adev->mes.hung_queue_db_array_size = num_queues * 2;
> >>         }
> >>
> >> @@ -526,8 +541,9 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
> >>                                             uint32_t xcc_id)  {
> >>         struct mes_detect_and_reset_queue_input input;
> >> -       u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
> >> -       int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i;
> >> +       u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[0];
> >
> > Does mes.hung_queue_db_array_cpu_addr still need to be an array?
>
> No, it doesn't need to be an array. Since we guard the access in amdgpu_gfx_mes_reset_compute function and only one XCC can perform detect_and_reset at one time, one hung_queue_db_array is enough. My original series included reverting the patch that added xcc copies of hung_queue_db_array. I was thinking to submit that revert in the next series (clean up), but I can do it in this series too.

Sure.  Either way.  Might be better to do it now so it doesn't slip
through the cracks.  Just wanted to make sure I wasn't
mis-understanding the change.

> Since we are in this topic, I have a question here. Besides of the xcc copies, amdgpu_mes_init also allocates AMDGPU_MAX_MES_PIPES copies of hung_queue_db_array buffer. Is it required for gfx 10? In gfx11 and after, we only use MES pipe 0 for detect_and_reset so we shouldn't need AMDGPU_MAX_MES_PIPES copies.

gfx10 doesn't use MES so it shouldn't be an issue.

Alex

>
> >> +       int hqd_info_offset, r, i;
> >> +       int num_xcps = 1;
> >>
> >>         if (!hung_db_num || !hung_db_array)
> >>                 return -EINVAL;
> >> @@ -537,8 +553,12 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
> >>             (queue_type != AMDGPU_RING_TYPE_SDMA))
> >>                 return -EINVAL;
> >>
> >> +       if (adev->xcp_mgr)
> >> +               num_xcps = adev->xcp_mgr->num_xcps;
> >> +       hqd_info_offset = get_num_queues_per_xcp(adev, num_xcps);
> >> +
> >>         /* Clear the doorbell array before detection */
> >> -       memset(adev->mes.hung_queue_db_array_cpu_addr[xcc_id], AMDGPU_MES_INVALID_DB_OFFSET,
> >> +       memset(adev->mes.hung_queue_db_array_cpu_addr[0],
> >> + AMDGPU_MES_INVALID_DB_OFFSET,
> >>                 adev->mes.hung_queue_db_array_size * sizeof(u32));
> >>         input.queue_type = queue_type;
> >>         input.detect_only = detect_only; @@ -554,7 +574,7 @@ int
> >> amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
> >>
> >>         *hung_db_num = 0;
> >>         /* MES passes hung queues' doorbell to driver */
> >> -       for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
>
> Can mes.hung_queue_hqd_info_offset be removed after this change?
>
> Alex
>
> > +       for (i = 0; i < hqd_info_offset; i++) {
> >                 /* Finding hung queues where db_array[i] is a valid doorbell */
> >                 if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
> >                         dev_dbg(adev->dev, "MES: hung queue doorbell
> > 0x%x\n", db_array[i]);
> > --
> > 2.43.0
> >