[PATCH v2] drm/xe/cri: Set downstream access to L3 class 3 and write-back

[Daniel Charles <daniel.charles@intel.com>]

From: Fei Yang <fei.yang@intel.com>

Program DWNSTRM_ACCESS_MODE (0x144154) with 0xcc on graphics
version 3511. Compared with 0x3c, this selects L3 class of service
3 instead of 0 and write-back rather than uncached L3 policy.
The L4 caching policy and coherency mode remain unchanged.

v2:
- Create a dedicated MERT workaround section/table instead of placing
  this workaround in gt_was. MERT registers are device-level registers
  accessed through root-tile MMIO, outside GT forcewake and GT
  save/restore bookkeeping.
- Add an RTP helper for full-register device MMIO writes and apply the
  MERT table during probe and after pcode is ready on system or D3cold
  resume. Match Xe3p_XPC explicitly and skip devices without
  MERT and SR-IOV VFs.

BSpec: 73603
Signed-off-by: Fei Yang <fei.yang@intel.com>
Signed-off-by: Daniel Charles <daniel.charles@intel.com>
---
 drivers/gpu/drm/xe/regs/xe_mert_regs.h |  2 ++
 drivers/gpu/drm/xe/xe_device.c         |  4 +++
 drivers/gpu/drm/xe/xe_pm.c             |  8 ++++++
 drivers/gpu/drm/xe/xe_rtp.c            | 39 ++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_rtp.h            |  2 ++
 drivers/gpu/drm/xe/xe_wa.c             | 23 +++++++++++++++
 drivers/gpu/drm/xe/xe_wa.h             |  1 +
 7 files changed, 79 insertions(+)

diff --git a/drivers/gpu/drm/xe/regs/xe_mert_regs.h b/drivers/gpu/drm/xe/regs/xe_mert_regs.h
index 99e5a26da6572..1e6ce818fcee9 100644
--- a/drivers/gpu/drm/xe/regs/xe_mert_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_mert_regs.h
@@ -8,6 +8,8 @@
 
 #include "regs/xe_reg_defs.h"
 
+#define DWNSTRM_ACCESS_MODE_GAMREQ_MERTSS	XE_REG(0x144154)
+
 #define MERT_LMEM_CFG				XE_REG(0x1448b0)
 
 #define MERT_TLB_CT_INTR_ERR_ID_PORT		XE_REG(0x145190)
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 8392c9fad3c48..27cff3137a7fe 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -953,6 +953,10 @@ int xe_device_probe(struct xe_device *xe)
 			return err;
 	}
 
+	err = xe_wa_apply_mert_workarounds(xe);
+	if (err)
+		return err;
+
 	/*
 	 * Allow allocations only now to ensure xe_display_init_early()
 	 * is the first to allocate, always.
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index daa7ba2df317a..ba863df3fbf59 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -251,6 +251,10 @@ int xe_pm_resume(struct xe_device *xe)
 	if (err)
 		return err;
 
+	err = xe_wa_apply_mert_workarounds(xe);
+	if (err)
+		return err;
+
 	xe_display_pm_resume_early(xe);
 
 	/*
@@ -705,6 +709,10 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 		if (err)
 			goto out;
 
+		err = xe_wa_apply_mert_workarounds(xe);
+		if (err)
+			goto out;
+
 		xe_display_pm_resume_early(xe);
 
 		/*
diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c
index 6a8d6ea68f25d..539d06eaba6fc 100644
--- a/drivers/gpu/drm/xe/xe_rtp.c
+++ b/drivers/gpu/drm/xe/xe_rtp.c
@@ -13,6 +13,7 @@
 #include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_gt_topology.h"
+#include "xe_mmio.h"
 #include "xe_reg_sr.h"
 #include "xe_sriov.h"
 
@@ -413,6 +414,44 @@ void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process);
 
+/**
+ * xe_rtp_apply_device_mmio_writes - Apply full-register device MMIO writes
+ * @xe: Device to apply the actions to
+ * @table: RTP action table containing only full-register writes
+ *
+ * Device registers are accessed via the root tile without GT forcewake or
+ * GT save/restore bookkeeping.
+ *
+ * Return: 0 on success or -EINVAL for an unsupported action.
+ */
+int xe_rtp_apply_device_mmio_writes(struct xe_device *xe,
+				    const struct xe_rtp_table_sr *table)
+{
+	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
+
+	for (size_t i = 0; i < table->n_entries; i++) {
+		const struct xe_rtp_entry_sr *entry = &table->entries[i];
+
+		if (!rule_matches(xe, NULL, NULL, entry->rules, entry->n_rules))
+			continue;
+
+		for (size_t j = 0; j < entry->n_actions; j++) {
+			const struct xe_rtp_action *action = &entry->actions[j];
+
+			if (drm_WARN_ON(&xe->drm, action->clr_bits != ~0u ||
+					action->reg.mcr || action->reg.masked ||
+					action->flags || action->use_func))
+				return -EINVAL;
+
+			xe_mmio_write32(mmio, action->reg, action->set_bits);
+		}
+
+		drm_dbg(&xe->drm, "Applied device workaround %s\n", entry->name);
+	}
+
+	return 0;
+}
+
 bool xe_rtp_match_always(const struct xe_device *xe,
 			 const struct xe_gt *gt,
 			 const struct xe_hw_engine *hwe)
diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h
index 0032f68ea1873..7f45bb8552244 100644
--- a/drivers/gpu/drm/xe/xe_rtp.h
+++ b/drivers/gpu/drm/xe/xe_rtp.h
@@ -514,6 +514,8 @@ void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx,
 
 void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
 		    const struct xe_rtp_table *table);
+int xe_rtp_apply_device_mmio_writes(struct xe_device *xe,
+				    const struct xe_rtp_table_sr *table);
 
 /* Match functions to be used with XE_RTP_MATCH_FUNC */
 
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index c94fe7753f9a9..18f953ffbc1cf 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -16,7 +16,9 @@
 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_guc_regs.h"
+#include "regs/xe_mert_regs.h"
 #include "regs/xe_regs.h"
+#include "xe_device.h"
 #include "xe_device_types.h"
 #include "xe_force_wake.h"
 #include "xe_gt_types.h"
@@ -823,6 +825,13 @@ VISIBLE_IF_KUNIT __maybe_unused const struct xe_rtp_table device_oob_was = {
 };
 EXPORT_SYMBOL_IF_KUNIT(device_oob_was);
 
+static const struct xe_rtp_table_sr mert_was = XE_RTP_TABLE_SR(
+	{ XE_RTP_NAME("14028586678"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3511)),
+	  XE_RTP_ACTIONS(WR(DWNSTRM_ACCESS_MODE_GAMREQ_MERTSS, 0xcc))
+	},
+);
+
 __diag_pop();
 
 /**
@@ -1032,3 +1041,17 @@ void xe_wa_apply_tile_workarounds(struct xe_tile *tile)
 	if (XE_DEVICE_WA(tile->xe, 22010954014))
 		xe_mmio_rmw32(mmio, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
 }
+
+/**
+ * xe_wa_apply_mert_workarounds - Apply root-tile MERT workarounds
+ * @xe: Xe device
+ *
+ * Called during probe and after pcode is ready on system or D3cold resume.
+ */
+int xe_wa_apply_mert_workarounds(struct xe_device *xe)
+{
+	if (!xe_device_has_mert(xe) || IS_SRIOV_VF(xe))
+		return 0;
+
+	return xe_rtp_apply_device_mmio_writes(xe, &mert_was);
+}
diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h
index f4da2b2713963..28516812cb2bf 100644
--- a/drivers/gpu/drm/xe/xe_wa.h
+++ b/drivers/gpu/drm/xe/xe_wa.h
@@ -22,6 +22,7 @@ void xe_wa_process_gt(struct xe_gt *gt);
 void xe_wa_process_engine(struct xe_hw_engine *hwe);
 void xe_wa_process_lrc(struct xe_hw_engine *hwe);
 void xe_wa_apply_tile_workarounds(struct xe_tile *tile);
+int xe_wa_apply_mert_workarounds(struct xe_device *xe);
 void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p);
 int xe_wa_gt_dump(struct xe_gt *gt, struct drm_printer *p);
 
-- 
2.43.0