[RFC PATCH 1/1] drm/amdgpu: fall back to MMIO TLB invalidation when KIQ is unresponsive

[Denis Pisarev <pisarevden@gmail.com>]

After resume from S4 (hibernation) on gmc_v9 parts with GFXOFF
(observed on Cezanne / Ryzen 7 PRO 5850U, kernel 7.1.8), KIQ-based TLB
flushes start failing at the moment of the thaw and keep failing for
hours of normal desktop use:

  amdgpu 0000:07:00.0: failed to write reg 28b4 wait reg 28c6
  amdgpu 0000:07:00.0: failed to write reg 1a6f4 wait reg 1a706

(80-140 errors/hour measured over 9+ hours; bugzilla 219492)

Two problems follow from the current code:

1. Every failed flush burns the full retry window
   (MAX_KIQ_REG_TRY * MAX_KIQ_REG_BAILOUT_INTERVAL = ~5 s) before
   erroring out, which makes the whole desktop sluggish.

2. The invalidation is then silently dropped - stale TLB entries are
   left in place - because gmc_v9_0_flush_gpu_tlb() returns as soon as
   amdgpu_gmc_fw_reg_write_reg_wait() finishes, whether it succeeded
   or not.

The KIQ ring is marked ready during resume after its ring test passes,
but on affected systems the ring subsequently stops completing
invalidation commands. sched.ready therefore does not reflect the
state of the hardware in this failure mode, and there is no path back
to the direct MMIO invalidation that already exists in
gmc_v9_0_flush_gpu_tlb() for the pre-KIQ stage.

Make the failure observable and self-healing:

- amdgpu_gmc_fw_reg_write_reg_wait() returns 0/-ETIME and counts
  consecutive failures in adev->gmc.kiq_flush_failures
  (dev_err_ratelimited instead of dev_err, since affected systems
  print this 80-140x/hour for hours)

- gmc_v9_0_flush_gpu_tlb() falls back to its existing MMIO path when
  the KIQ submit fails, so the invalidation is no longer dropped

- after AMDGPU_KIQ_FLUSH_MAX_FAIL (3) consecutive failures the KIQ
  path is skipped entirely until the counter is reset, so wedged
  systems stop paying the 5 s retry window per flush

- the counter is reset on every success and in gmc_v9_0_hw_fini(),
  i.e. every suspend/resume cycle re-arms the KIQ path; nothing is
  disabled proactively

gmc_v10/v11/v12 call sites are unchanged (statement calls compile
fine against the new int return; behavior identical). They can get
the same fallback once this approach is agreed for gmc_v9.

The sibling PASID path (amdgpu_gmc_flush_gpu_tlb_pasid) already has
an -ETIME/MMIO split; this brings the per-VMID path in line with it.

RFC questions for maintainers:

- Is a per-xcd-inst latch preferred over the global gmc one? (single
  inst on the affected hardware here)
- Should the latch also gate the KIQ branch of
  amdgpu_gmc_flush_gpu_tlb_pasid()? (no failures observed on that
  path on the affected system)
- Root cause: with GFXOFF disabled across the S4 cycle (debugfs
  amdgpu_gfxoff), zero errors occur across resume and 30 min of use
  vs ~70-140 in the control arm. The wedge forms in the S4 resume
  window while GFXOFF is allowed, consistent with the existing
  semaphore workaround comment about losing invalidate acknowledge
  state across power-gating cycles in this file.

Tested on Cezanne (Ryzen 7 PRO 5850U, Manjaro 7.1.8): S4 resume with
GFXOFF enabled reproduces the failure storm on stock; hibernate loop
testing of this patch pending maintainer feedback on the approach.

Signed-off-by: Denis Pisarev <pisarevden@gmail.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 15 +++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  4 +++-
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c   | 18 ++++++++++++++----
 4 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 7b09410d6..cd5d9e56e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -360,6 +360,8 @@ enum amdgpu_kiq_irq {
 #define MAX_KIQ_REG_WAIT       5000 /* in usecs, 5ms */
 #define MAX_KIQ_REG_BAILOUT_INTERVAL   5 /* in msecs, 5ms */
 #define MAX_KIQ_REG_TRY 1000
+/* consecutive KIQ TLB flush failures before falling back to MMIO */
+#define AMDGPU_KIQ_FLUSH_MAX_FAIL 3
 
 /*
  * BIOS.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 5d6149ba7..000a1d107 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -874,7 +874,7 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
 	return r;
 }
 
-void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
+int amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
 				      uint32_t reg0, uint32_t reg1,
 				      uint32_t ref, uint32_t mask,
 				      uint32_t xcc_inst)
@@ -888,7 +888,7 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
 	if (adev->mes.ring[MES_PIPE_INST(xcc_inst, 0)].sched.ready) {
 		amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1,
 					      ref, mask, xcc_inst);
-		return;
+		return 0;
 	}
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
@@ -919,13 +919,20 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
 	if (cnt > MAX_KIQ_REG_TRY)
 		goto failed_kiq;
 
-	return;
+	atomic_set(&adev->gmc.kiq_flush_failures, 0);
+	return 0;
 
 failed_undo:
 	amdgpu_ring_undo(ring);
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 failed_kiq:
-	dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1);
+	if (atomic_inc_return(&adev->gmc.kiq_flush_failures) ==
+			AMDGPU_KIQ_FLUSH_MAX_FAIL)
+		dev_warn(adev->dev,
+			 "KIQ reg access keeps failing, falling back to MMIO\n");
+	dev_err_ratelimited(adev->dev,
+			    "failed to write reg %x wait reg %x\n", reg0, reg1);
+	return -ETIME;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index ddb0d500e..3e5c152ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -366,6 +366,8 @@ struct amdgpu_gmc {
 	bool flush_tlb_needs_extra_type_0;
 	bool flush_tlb_needs_extra_type_2;
 	bool flush_pasid_uses_kiq;
+	/* consecutive KIQ TLB flush failures; MMIO fallback when latched */
+	atomic_t kiq_flush_failures;
 
 	bool override_pte;
 };
@@ -447,7 +449,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
 				   uint32_t flush_type, bool all_hub,
 				   uint32_t inst);
-void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
+int amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
 				      uint32_t reg0, uint32_t reg1,
 				      uint32_t ref, uint32_t mask,
 				      uint32_t xcc_inst);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 8a5c44810..a262df837 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -798,13 +798,18 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	 * properly under bare metal
 	 */
 	if (adev->gfx.kiq[inst].ring.sched.ready &&
-	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
+	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
+	    atomic_read(&adev->gmc.kiq_flush_failures) <
+			AMDGPU_KIQ_FLUSH_MAX_FAIL) {
 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
 
-		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
-						 1 << vmid, inst);
-		return;
+		if (!amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
+						      1 << vmid, inst))
+			return;
+		/* KIQ submit failed - fall through to the MMIO path below
+		 * so the invalidation is not silently dropped
+		 */
 	}
 
 	/* This path is needed before KIQ/MES/GFXOFF are set up */
@@ -2238,6 +2243,11 @@ static int gmc_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	/* KIQ is re-initialized on the next resume; give it a clean
+	 * start for the MMIO fallback latch
+	 */
+	atomic_set(&adev->gmc.kiq_flush_failures, 0);
+
 	gmc_v9_0_gart_disable(adev);
 
 	if (amdgpu_sriov_vf(adev)) {
-- 
2.55.0