[PATCH v6 20/24] drm/xe: Add ULLS migration job support to ring ops

[Matthew Brost <matthew.brost@intel.com>]

Add preamble and postamble for ULLS migrations jobs. Preamble clears
current semaphore for reuse. Postamble waits on next semaphore which is
set upon next job submission, then advances the ring tail over that job
with an LRI to RING_TAIL, so submitting it costs the CPU nothing beyond
signalling the semaphore.

A job updates the tail on behalf of a successor which has not been
emitted yet, so it cannot know how much ring that successor will occupy.
Pad every ULLS job out to a fixed ULLS_JOB_SIZE_BYTES, which makes the
next tail derivable from where the current job starts. The pad also
supplies the NOPs which must follow an in-ring tail update.

The last ULLS migration job skips BB submission, the postamble and the
tail update (clear current semaphore, write seqno, exit ULLS), padding
the difference so that it still fills a job slot.

Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Link: https://patch.msgid.link/20260228013501.106680-23-matthew.brost@intel.com
Signed-off-by: Maarten Lankhorst <dev@lankhorst.se>
---
 drivers/gpu/drm/xe/xe_ring_ops.c       | 86 ++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_ring_ops_types.h | 24 +++++++
 2 files changed, 110 insertions(+)

diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c
index 39a670e91ba7..f05b4276c8d1 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops.c
+++ b/drivers/gpu/drm/xe/xe_ring_ops.c
@@ -494,6 +494,79 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
 }
 
+static int emit_ulls_preamble(struct xe_lrc *lrc, u32 *dw, int i, u32 seqno)
+{
+	u32 addr = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno);
+
+	return emit_store_imm_ggtt(addr, LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR,
+				   dw, i);
+}
+
+/*
+ * Advance the ring tail from within the ring, so submitting the next ULLS job
+ * needs nothing from the CPU beyond signalling the semaphore. All ULLS jobs
+ * occupy exactly ULLS_JOB_SIZE_BYTES, so the tail the next job ends at is two
+ * job slots on from where this job started, even though that job has not been
+ * emitted yet.
+ *
+ * The saved tail is written before the tail register so that a context save
+ * racing the pair can only restore a tail behind the register, never ahead of
+ * it. The register write is deliberately not MI_LRI_FORCE_POSTED: posted, the
+ * new tail can land after the command streamer has already drained this job,
+ * at which point it sees an empty ring, parks, and can be taken off the
+ * hardware with no H2G left to bring it back.
+ *
+ * The tail is published before the semaphore wait rather than after it, so
+ * that the non-posted write drains while the engine is parked anyway instead
+ * of adding a register round trip after the semaphore is signalled.
+ */
+static int emit_ulls_ring_tail(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw,
+			       int i, u32 head)
+{
+	u32 next_tail = (head + 2 * ULLS_JOB_SIZE_BYTES) & (lrc->ring.size - 1);
+
+	xe_gt_assert(gt, IS_ALIGNED(next_tail, 8));
+
+	i = emit_store_imm_ggtt(xe_lrc_ring_tail_ggtt_addr(lrc), next_tail,
+				dw, i);
+
+	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) |
+		MI_LRI_LRM_CS_MMIO;
+	dw[i++] = RING_TAIL(0).addr;
+	dw[i++] = next_tail;
+
+	return i;
+}
+
+/* Publish the next job's tail, then park the engine on its semaphore */
+static int emit_ulls_postamble(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw,
+			       int i, u32 seqno, u32 head)
+{
+	i = emit_ulls_ring_tail(gt, lrc, dw, i, head);
+
+	dw[i++] = MI_SEMAPHORE_WAIT |
+		MI_SEMW_GGTT |
+		MI_SEMW_POLL |
+		MI_SEMW_COMPARE(SAD_EQ_SDD);
+	dw[i++] = LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL;
+	dw[i++] = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno + 1);
+	dw[i++] = 0;
+	dw[i++] = 0;
+
+	return i;
+}
+
+/* Pad out to the fixed ULLS job size */
+static int emit_ulls_pad(struct xe_gt *gt, u32 *dw, int i)
+{
+	xe_gt_assert(gt, i <= ULLS_JOB_SIZE_DW);
+
+	while (i < ULLS_JOB_SIZE_DW)
+		dw[i++] = MI_NOOP;
+
+	return i;
+}
+
 static void emit_migration_job_gen12(struct xe_sched_job *job,
 				     struct xe_lrc *lrc, u32 *head,
 				     u32 seqno)
@@ -507,10 +580,16 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
 
 	xe_gt_assert(gt, !job->ring_ops_force_reset);
 
+	if (xe_sched_job_is_ulls(job))
+		i = emit_ulls_preamble(lrc, dw, i, seqno);
+
 	i = emit_copy_timestamp(xe, lrc, dw, i);
 
 	i = emit_store_imm_ggtt(saddr, seqno, dw, i);
 
+	if (!xe_sched_job_ulls_has_batch(job))
+		goto seqno_write;
+
 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
 
 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
@@ -521,12 +600,19 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
 
 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
 
+seqno_write:
 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno,
 				job->migrate_flush_flags,
 				dw, i);
 
 	i = emit_user_interrupt(dw, i);
 
+	if (xe_sched_job_ulls_parks(job))
+		i = emit_ulls_postamble(gt, lrc, dw, i, seqno, *head);
+
+	if (xe_sched_job_is_ulls(job))
+		i = emit_ulls_pad(gt, dw, i);
+
 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
 
 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
diff --git a/drivers/gpu/drm/xe/xe_ring_ops_types.h b/drivers/gpu/drm/xe/xe_ring_ops_types.h
index 52ff96bc4100..ea4af321dd7c 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops_types.h
+++ b/drivers/gpu/drm/xe/xe_ring_ops_types.h
@@ -6,6 +6,7 @@
 #ifndef _XE_RING_OPS_TYPES_H_
 #define _XE_RING_OPS_TYPES_H_
 
+#include <linux/build_bug.h>
 #include <linux/types.h>
 
 struct xe_gt;
@@ -14,6 +15,29 @@ struct xe_sched_job;
 #define MAX_JOB_SIZE_DW 74
 #define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4)
 
+/*
+ * ULLS migration jobs advance the ring tail from within the ring itself, so a
+ * job has to know where its successor will end before that successor has been
+ * emitted. Every ULLS job is therefore padded to a fixed size, letting the
+ * next tail be derived arithmetically.
+ *
+ * Sized for the largest such job, emitted by emit_migration_job_gen12():
+ * preamble (4), copy timestamp (8, its size on an SRIOV VF), start seqno
+ * store (4), arbitration off (1), batch buffer starts (2 * 3), pre-parser
+ * bracketed flush invalidate (6), seqno flush (4), user interrupt (3) and
+ * postamble (7 + 5).
+ */
+#define ULLS_JOB_SIZE_DW 48
+#define ULLS_JOB_SIZE_BYTES (ULLS_JOB_SIZE_DW * 4)
+
+/*
+ * RING_TAIL only encodes a qword aligned offset, and xe_lrc_write_ring()
+ * appends a NOP to anything shorter, either of which would desynchronise the
+ * ring from the tail a job predicts for its successor.
+ */
+static_assert(ULLS_JOB_SIZE_BYTES % 8 == 0);
+static_assert(ULLS_JOB_SIZE_DW <= MAX_JOB_SIZE_DW);
+
 /**
  * struct xe_ring_ops - Ring operations
  */
-- 
2.34.1