[PATCH v2] drm/amdkfd: Fix TCP XNACK scoreboard reset race

[PATCH v2] drm/amdkfd: Fix TCP XNACK scoreboard reset race

[Gang Ba]

Fix a hardware RTL bug in the TCP XNACK scoreboard reset path where a wave
receiving a no-retry FAULT from UTC and subsequently terminating may leave
its scoreboard entry uncleared when reset events arrive back-to-back.

This can cause a later, unrelated wave using the same slot to incorrectly
inherit the XNACK error state, resulting in its VMEM operations being
dropped even though the wave never faulted.

Signed-off-by: Gang Ba <Gang.Ba@amd.com>
---
 .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c  |  3 +-
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 10 ++-
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c       | 83 +++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h       |  1 +
 drivers/gpu/drm/amd/amdkfd/kfd_debug.c        | 19 +++++
 .../gpu/drm/amd/include/kgd_kfd_interface.h   |  1 +
 7 files changed, 115 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
index 7e9f7a280c1b..98cd0435ced5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
@@ -193,5 +193,6 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = {
 	.program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
 	.hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v9_hqd_reset,
-	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell
+	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell,
+	.hqd_gfx_clean_fault = kgd_gfx_v9_clean_fault
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index b30ad9701bce..fc253260ade5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -37,6 +37,7 @@
 #include "soc15.h"
 #include "soc15d.h"
 #include "gfx_v9_0.h"
+#include "gfx_v9_4_2.h"
 #include "amdgpu_amdkfd_gfx_v9.h"
 #include <uapi/linux/kfd_ioctl.h>
 
@@ -1226,11 +1227,15 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device *adev,
 
 uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 					  int engine, int queue)
-
 {
 	return 0;
 }
 
+void kgd_gfx_v9_clean_fault(struct amdgpu_device *adev)
+{
+	gfx_v9_4_2_clean_fault(adev);
+}
+
 const struct kfd2kgd_calls gfx_v9_kfd2kgd = {
 	.program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping,
@@ -1261,5 +1266,6 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = {
 	.program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
 	.hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v9_hqd_reset,
-	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell
+	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell,
+	.hqd_gfx_clean_fault = kgd_gfx_v9_clean_fault
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
index 9f43f16a1ace..dba7ace872de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
@@ -115,3 +115,4 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device *adev,
 			      unsigned int utimeout);
 uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 					  int engine, int queue);
+void kgd_gfx_v9_clean_fault(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
index 8b84bba58752..344156bf20ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
@@ -343,6 +343,28 @@ const struct soc15_reg_entry sgpr64_init_regs_aldebaran[] = {
 	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
 };
 
+static const u32 fault_cleaner_compute_shader_aldebaran[] = {
+	0xbe8000ff, 0x00001000, 0x80808100, 0x86000000, 0xbf85fffd, 0xbf810000
+};
+
+const struct soc15_reg_entry fault_cleaner_init_regs_aldebaran[] = {
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 1 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0xaf0000 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0xc8 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0xea4fac },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
+};
+
 static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev,
 				 struct amdgpu_ring *ring,
 				 const u32 *shader_ptr, u32 shader_size,
@@ -692,6 +714,67 @@ static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
 	return r;
 }
 
+static int gfx_v9_4_2_clean_shaders(struct amdgpu_device *adev)
+{
+	int r;
+	/* CU_ID: 0~15, SIMD_ID: 0~3, WAVE_ID: 0 ~ 9 */
+	int wb_size = adev->gfx.config.max_shader_engines *
+			 CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
+	struct amdgpu_ib wb_ib;
+	struct dma_fence *fence = NULL;
+	u32 pattern = 0xa;
+
+	/* bail if the compute ring is not ready */
+	if (!adev->gfx.compute_ring[0].sched.ready)
+		return 0;
+
+	/* allocate the write-back buffer from IB */
+	memset(&wb_ib, 0, sizeof(wb_ib));
+	r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t),
+			  AMDGPU_IB_POOL_DIRECT, &wb_ib);
+	if (r) {
+		dev_err(adev->dev, "failed to get ib (%d) for wb.\n", r);
+		return r;
+	}
+	memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
+
+	r = gfx_v9_4_2_run_shader(adev,
+			&adev->gfx.compute_ring[0],
+			fault_cleaner_compute_shader_aldebaran,
+			sizeof(fault_cleaner_compute_shader_aldebaran),
+			fault_cleaner_init_regs_aldebaran,
+			ARRAY_SIZE(fault_cleaner_init_regs_aldebaran),
+			adev->gfx.cu_info.number * 32,
+			wb_ib.gpu_addr, pattern, &fence);
+
+	if (r) {
+		dev_err(adev->dev, "failed to clear MI200\n");
+		goto pro_end;
+	}
+
+	/* wait for the GPU to finish processing the IB */
+	r = dma_fence_wait(fence, false);
+	if (r)
+		dev_err(adev->dev, "timeout to clear MI200\n");
+
+pro_end:
+	if (fence)
+		dma_fence_put(fence);
+	amdgpu_ib_free(&wb_ib, NULL);
+
+	if (r)
+		dev_dbg(adev->dev, "Clean MI200 Failed\n");
+	else
+		dev_dbg(adev->dev, "Clean MI200 Successfully\n");
+
+	return r;
+}
+
+void gfx_v9_4_2_clean_fault(struct amdgpu_device *adev)
+{
+	gfx_v9_4_2_clean_shaders(adev);
+}
+
 int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 {
 	/* only support when RAS is enabled */
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
index a603724c1dfc..1cbade5ed657 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
@@ -30,6 +30,7 @@ void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev,
 				      uint32_t die_id);
 void gfx_v9_4_2_init_sq(struct amdgpu_device *adev);
 void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev);
+void gfx_v9_4_2_clean_fault(struct amdgpu_device *adev);
 int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev);
 
 extern struct amdgpu_gfx_ras gfx_v9_4_2_ras;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
index 0dd1fd448059..c5b36f0cbe41 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
@@ -195,6 +195,20 @@ bool kfd_dbg_ev_raise(uint64_t event_mask,
 	return is_subscribed;
 }
 
+static void gfx942_clean_fault(struct kfd_process_device *pdd)
+{
+	struct device_queue_manager *dqm;
+
+	/* Run the GFX cleaner on the faulting device.
+	 * Only run once per page fault (evicted == 1 after first evict).
+	 */
+	dqm = pdd->dev->dqm;
+	dqm_lock(dqm);
+	if (pdd->qpd.evicted == 1 && dqm->dev->kfd2kgd->hqd_gfx_clean_fault)
+		dqm->dev->kfd2kgd->hqd_gfx_clean_fault(dqm->dev->adev);
+	dqm_unlock(dqm);
+}
+
 /* set pending event queue entry from ring entry  */
 bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
 				   unsigned int pasid,
@@ -240,6 +254,11 @@ bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
 			mutex_unlock(&p->mutex);
 		} else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 			kfd_evict_process_device(pdd);
+
+			/* GPR cleaner shader for MI2XX */
+			if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
+				gfx942_clean_fault(pdd);
+
 			kfd_signal_vm_fault_event(pdd, NULL, exception_data);
 
 			signaled_to_debugger_or_runtime = true;
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 01c2631bbdff..d749669f8e6d 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -343,6 +343,7 @@ struct kfd2kgd_calls {
 	int (*hqd_sdma_get_counter)(struct amdgpu_device *adev,
 				    void *mqd, uint32_t num_sdma_queues_per_eng,
 				    uint64_t *val);
+	void (*hqd_gfx_clean_fault)(struct amdgpu_device *adev);
 };
 
 #endif	/* KGD_KFD_INTERFACE_H_INCLUDED */
-- 
2.55.0

Re: [PATCH v2] drm/amdkfd: Fix TCP XNACK scoreboard reset race

[Jay Cornwall]

On 9/3/2026 16:08, Gang Ba wrote:

> @@ -240,6 +254,11 @@ bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
>   			mutex_unlock(&p->mutex);
>   		} else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
>   			kfd_evict_process_device(pdd);
> +
> +			/* GPR cleaner shader for MI2XX */
> +			if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
> +				gfx942_clean_fault(pdd);
> +
>   			kfd_signal_vm_fault_event(pdd, NULL, exception_data);
>   
>   			signaled_to_debugger_or_runtime = true;

This code runs conditionally on:

if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true,
                       exception_data, exception_data_size)) {

It returns true if the process or debugger (I'm unclear on the logic) 
receives the exception. This is not what we want: this code must run 
unconditionally.