[PATCH 1/1 v4] drm/xe: Wait for HW clearance before issuing the next TLB inval.

[fei.yang@intel.com]

From: Fei Yang <fei.yang@intel.com>

Hardware requires the software to poll the valid bit and make sure
it's cleared before issuing a new TLB invalidation request.
We also need to avoid racing against GuC on TLB invalidations. In
order to achieve that, check for pending TLB invalidation in tlb_inval
list.

Signed-off-by: Fei Yang <fei.yang@intel.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Stuart Summers <stuart.summers@intel.com>
Cc: Roper Matthew D <matthew.d.roper@intel.com>
---
 drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 70 +++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
index cf6d106e6036..815fbd059112 100644
--- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
@@ -8,6 +8,7 @@
 #include "xe_device.h"
 #include "xe_exec_queue.h"
 #include "xe_exec_queue_types.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_stats.h"
 #include "xe_gt_types.h"
 #include "xe_guc.h"
@@ -60,6 +61,12 @@ static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
 
 static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 {
+	/*
+	 * Timeout for MMIO-based TLB invalidation polling. Make it 250ms
+	 * based on hw_tlb_timeout = HZ/4 defined in tlb_inval_timeout_delay().
+	 */
+	const long hw_tlb_timeout = HZ / 4;
+
 	struct xe_guc *guc = tlb_inval->private;
 	struct xe_gt *gt = guc_to_gt(guc);
 	struct xe_device *xe = guc_to_xe(guc);
@@ -79,19 +86,82 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 		return send_tlb_inval(guc, action, ARRAY_SIZE(action));
 	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
 		struct xe_mmio *mmio = &gt->mmio;
+		int err;
 
 		if (IS_SRIOV_VF(xe))
 			return -ECANCELED;
 
+		/*
+		 * seqno_lock serializes MMIO-based TLB invalidations to prevent
+		 * races between concurrent requests issued through GuC CT and
+		 * direct MMIO writes.
+		 */
+		lockdep_assert_held(&tlb_inval->seqno_lock);
+
+		/*
+		 * If there are pending GuC TLB invalidation requests MMIO-based
+		 * requests should be avoided.
+		 */
+		if (!xe_tlb_inval_idle(tlb_inval)) {
+			xe_gt_dbg(gt, "MMIO TLB INV skipped due to pending GuC INv\n");
+			return -ECANCELED;
+		}
+
 		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
+			/*
+			 * HW requires the valid bit to be clear before writing a
+			 * new request.
+			 */
+			err = xe_mmio_wait32(mmio, PVC_GUC_TLB_INV_DESC0,
+					     PVC_GUC_TLB_INV_DESC0_VALID, 0,
+					     hw_tlb_timeout, NULL, false);
+			if (err) {
+				xe_gt_err(gt, "MMIO TLB INV pre-write poll timeout\n");
+				return -ECANCELED;
+			}
+
 			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
 					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
 			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0,
 					PVC_GUC_TLB_INV_DESC0_VALID);
+
+			/*
+			 * Poll for clearance of the valid bit after writing to
+			 * ensure the request has been consumed and to avoid races
+			 * between TLB invalidation through GuC and MMIO.
+			 */
+			err = xe_mmio_wait32(mmio, PVC_GUC_TLB_INV_DESC0,
+					     PVC_GUC_TLB_INV_DESC0_VALID, 0,
+					     hw_tlb_timeout, NULL, false);
+			if (err)
+				xe_gt_err(gt, "MMIO TLB INV post-write poll timeout\n");
 		} else {
+			/*
+			 * HW requires the valid bit to be clear before writing a
+			 * new request.
+			 */
+			err = xe_mmio_wait32(mmio, GUC_TLB_INV_CR,
+					     GUC_TLB_INV_CR_INVALIDATE, 0,
+					     hw_tlb_timeout, NULL, false);
+			if (err) {
+				xe_gt_err(gt, "MMIO TLB INV pre-write poll timeout\n");
+				return -ECANCELED;
+			}
+
 			xe_mmio_write32(mmio, GUC_TLB_INV_CR,
 					GUC_TLB_INV_CR_INVALIDATE);
+
+			/*
+			 * Poll for clearance of the valid bit after writing to
+			 * ensure the request has been consumed and to avoid races
+			 * between TLB invalidation through GuC and MMIO.
+			 */
+			err = xe_mmio_wait32(mmio, GUC_TLB_INV_CR,
+					     GUC_TLB_INV_CR_INVALIDATE, 0,
+					     hw_tlb_timeout, NULL, false);
+			if (err)
+				xe_gt_err(gt, "MMIO TLB INV post-write poll timeout\n");
 		}
 	}
 
-- 
2.43.0