Re: [Re] GPU reset when running the ROCm hsa runtime tests on gfx12 and next-20260701

[Bert Karwatzki <spasswolf@web.de>]

Am Sonntag, dem 05.07.2026 um 02:35 +0200 schrieb Bert Karwatzki:
> I identified the problem in  
> f94bbd648bb4 ("drm/amdgpu: use a single entry point for mes compute reset")
> it's reset_queues_mes() being called unconditionally. f94bbd648bb4 can be fixed
> like this:
> 
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index 6054c8e216b8..ed1ffa8b1743 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -484,15 +484,18 @@ static int suspend_all_queues_mes(struct device_queue_manager *dqm)
>  	if (!down_read_trylock(&adev->reset_domain->sem))
>  		return -EIO;
>  
> +	r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
>  
> -	if (!reset_queues_mes(dqm)) {
> -		r = 0;
> -		goto out;
> -	}
> +	if (r) {
> +		if (!reset_queues_mes(dqm)) {
> +			r = 0;
> +			goto out;
> +		}
>  
> -	dev_err(adev->dev, "failed to suspend gangs from MES\n");
> -	dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
> -	kfd_hws_hang(dqm);
> +		dev_err(adev->dev, "failed to suspend gangs from MES\n");
> +		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
> +		kfd_hws_hang(dqm);
> +	}
>  out:
>  
>  	up_read(&adev->reset_domain->sem);
> 
> 
> But when I try to fix next-20260701 like this
> 
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index 5c9dfb0c424f..5a78b1504f8c 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -493,7 +493,10 @@ static int recover_bad_queue_mes(struct device_queue_manager *dqm, struct queue
>  	if (!down_read_trylock(&adev->reset_domain->sem))
>  		return -EIO;
>  
> -	r = reset_queues_mes(dqm, q);
> +	r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
> +
> +	if (r)
> +		r = reset_queues_mes(dqm, q);
>  
>  	up_read(&adev->reset_domain->sem);
>  	return r;
> 
> I still get GPUVM errors (but no GPU reset) and when running
> 
> $ /usr/libexec/rocm/libhsa-runtime64-tests/run-tests
> 
> [  146.577245] [    T418] amdgpu 0000:03:00.0: [gfxhub] page fault (src_id:0 ring:157 vmid:0 pasid:0)
> [  146.577247] [    T418] amdgpu 0000:03:00.0:   in page starting at address 0x00000000002ba000 from client 10
> [  146.577248] [    T418] amdgpu 0000:03:00.0: GCVM_L2_PROTECTION_FAULT_STATUS:0x00000B3A
> [  146.577249] [    T418] amdgpu 0000:03:00.0: 	 Faulty UTCL2 client ID: CPC (0x5)
> [  146.577249] [    T418] amdgpu 0000:03:00.0: 	 MORE_FAULTS: 0x0
> [  146.577250] [    T418] amdgpu 0000:03:00.0: 	 WALKER_ERROR: 0x5
> [  146.577250] [    T418] amdgpu 0000:03:00.0: 	 PERMISSION_FAULTS: 0x3
> [  146.577250] [    T418] amdgpu 0000:03:00.0: 	 MAPPING_ERROR: 0x1
> [  146.577251] [    T418] amdgpu 0000:03:00.0: 	 RW: 0x0
> [  146.577606] [    T418] amdgpu 0000:03:00.0: [gfxhub] page fault (src_id:0 ring:157 vmid:0 pasid:0)
> [  146.577608] [    T418] amdgpu 0000:03:00.0:   in page starting at address 0x00000000002ba000 from client 10
> [  146.577609] [    T418] amdgpu 0000:03:00.0: GCVM_L2_PROTECTION_FAULT_STATUS:0x00000B3A
> [  146.577609] [    T418] amdgpu 0000:03:00.0: 	 Faulty UTCL2 client ID: CPC (0x5)
> [  146.577610] [    T418] amdgpu 0000:03:00.0: 	 MORE_FAULTS: 0x0
> [  146.577611] [    T418] amdgpu 0000:03:00.0: 	 WALKER_ERROR: 0x5
> [  146.577611] [    T418] amdgpu 0000:03:00.0: 	 PERMISSION_FAULTS: 0x3
> [  146.577611] [    T418] amdgpu 0000:03:00.0: 	 MAPPING_ERROR: 0x1
> [  146.577612] [    T418] amdgpu 0000:03:00.0: 	 RW: 0x0
> 
> This means there's at least another error in these commits (reverting all these in
> next-20260701 fixes the issue)
> 
> b789664e3e30 ("drm/amdkfd: Clean up suspend_all and resume_all mes")
> a665d09b10af ("drm/amdkfd: Pass known bad queue info to reset")
> a4e4d945cba8 ("drm/amdgpu/gfx: defer per-queue helper_end until after MES resume")
> f401a2633e02 ("drm/amdgpu: Remove faulty queue before resume")
> f94bbd648bb4 ("drm/amdgpu: use a single entry point for mes compute reset")
> 
> Bert Karwatzki

These GPUVM error are introduce by 
a665d09b10af ("drm/amdkfd: Pass known bad queue info to reset")
and can be fixed by a partial revert. a665d09b10af can be fixed by

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 0d95dd941129..0c49fe95a75d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -491,15 +491,18 @@ static int suspend_all_queues_mes(struct device_queue_manager *dqm, struct queue
 	if (!down_read_trylock(&adev->reset_domain->sem))
 		return -EIO;
 
+	r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
 
-	if (!reset_queues_mes(dqm, q)) {
-		r = 0;
-		goto out;
-	}
+	if (r) {
+		if (!reset_queues_mes(dqm, q)) {
+			r = 0;
+			goto out;
+		}
 
-	dev_err(adev->dev, "failed to suspend gangs from MES\n");
-	dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
-	kfd_hws_hang(dqm);
+		dev_err(adev->dev, "failed to suspend gangs from MES\n");
+		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
+		kfd_hws_hang(dqm);
+	}
 out:
 
 	up_read(&adev->reset_domain->sem);
@@ -3239,6 +3242,7 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel
 	struct kfd_process_device *pdd = NULL;
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd);
 	struct device_queue_manager *dqm = knode->dqm;
+	struct device *dev = dqm->dev->adev->dev;
 	struct qcm_process_device *qpd;
 	struct queue *q = NULL;
 	int ret = 0;
@@ -3260,6 +3264,12 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel
 				q->properties.is_active = false;
 				decrement_queue_count(dqm, qpd, q);
 
+				/* this will remove the bad queue and sched a GPU reset if needed */
+				ret = remove_queue_mes(dqm, q, qpd);
+				if (ret)
+					dev_err(dev, "Removing bad queue failed");
+				/* resume the good queues */
+				resume_all_queues_mes(dqm);
 				break;
 			}
 		}

This does not work in next-20260701 as commit
b789664e3e30 ("drm/amdkfd: Clean up suspend_all and resume_all mes")
removes resume_all_queues_mes().

Bert Karwatzki