[PATCH v2 13/33] drm/xe: Flush L2 asynchronously in xe_bo_trigger_rebind()

[Matthew Brost <matthew.brost@intel.com>]

On L2-flush-optimized HW with a dma-resv (non-fault) VM, evicting a BO
only needs to flush the L2 cache before the migration copy runs; the
mappings themselves are torn down and rebuilt lazily via
drm_gpuvm_bo_evict() and a subsequent rebind. Today this flush is done
by waiting for the BO to go idle and then issuing a synchronous TLB
invalidation per mapping VMA from inside xe_bo_trigger_rebind(). Both
the idle wait and the synchronous invalidation stall the calling thread
while holding the BO dma-resv lock, serializing the move behind all
in-flight GPU work on the BO.

Replace this with an asynchronous flush. Add xe_vm_flush_vm_bo_tlb_async()
which, for each VMA mapping the BO on each present tile, queues a TLB
invalidation job on the tile migrate (kernel) exec queue. The jobs depend
on the BO's in-flight GPU work, captured once as a singleton over
DMA_RESV_USAGE_BOOKKEEP, so the flush only fires once the GPU is done with
the current mapping. Each job's completion fence is installed into the
BO's dma-resv as a DMA_RESV_USAGE_KERNEL fence, so the migration copy -
which waits on the resv - waits on the flush without stalling this thread.

No PTEs are zapped and vma->tile_invalidated is left untouched: the
mapping stays valid until the lazy rebind, and the only work performed
here is the L2 flush. On any failure the caller falls back to the
existing blocking wait-idle plus xe_vm_invalidate_vma() path.

Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Tejas Upadhyay <tejas.upadhyay@intel.com>
Assisted-by: GitHub_Copilot:claude-opus-4.8
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
 drivers/gpu/drm/xe/xe_bo.c |   7 +++
 drivers/gpu/drm/xe/xe_vm.c | 125 +++++++++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_vm.h |   5 ++
 3 files changed, 137 insertions(+)

diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 60e0e568aa31..cca617cf34d1 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -732,6 +732,13 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
 			 */
 			if (!xe_device_is_l2_flush_optimized(xe))
 				continue;
+
+			/*
+			 * Attempt to flush L2 async, fallback to sync flush on
+			 * failure
+			 */
+			if (!xe_vm_flush_vm_bo_tlb_async(vm, bo, vm_bo))
+				continue;
 		}
 
 		if (!idle) {
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 73ac031ffb04..4557a8a4d270 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -39,6 +39,7 @@
 #include "xe_sync.h"
 #include "xe_tile.h"
 #include "xe_tlb_inval.h"
+#include "xe_tlb_inval_job.h"
 #include "xe_trace_bo.h"
 #include "xe_vm_madvise.h"
 #include "xe_wa.h"
@@ -4401,6 +4402,130 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
 	return ret;
 }
 
+/*
+ * xe_vma_tlb_flush_client - Queue an async TLB flush for one VMA on one client
+ *
+ * Create and push a TLB invalidation job on the tile migrate (kernel) exec
+ * queue covering @vma's range, depending on @dep (the BO's in-flight GPU work)
+ * so the flush only fires once the GPU is done with the current mapping. The
+ * job's completion fence is installed into @resv as a KERNEL fence so the
+ * subsequent migration waits on the flush. No PTEs are zapped; this only
+ * flushes L2 via the TLB invalidation.
+ */
+static int xe_vma_tlb_flush_client(struct xe_vm *vm, struct xe_vma *vma,
+				   struct xe_tile *tile, struct xe_gt *gt,
+				   struct dma_resv *resv, struct dma_fence *dep,
+				   int type)
+{
+	struct xe_exec_queue *q = xe_migrate_exec_queue(tile->migrate);
+	struct xe_tlb_inval_job *job;
+	struct dma_fence *fence;
+	int err;
+
+	job = xe_tlb_inval_job_create(q, &gt->tlb_inval,
+				      q->tlb_inval[type].dep_scheduler, vm,
+				      xe_vma_start(vma), xe_vma_end(vma), type);
+	if (IS_ERR(job))
+		return PTR_ERR(job);
+
+	err = xe_tlb_inval_job_alloc_dep(job);
+	if (err)
+		goto out_put;
+
+	err = dma_resv_reserve_fences(resv, 1);
+	if (err)
+		goto out_put;
+
+	/* Cannot fail; consumes a ref on @dep and returns a referenced fence. */
+	fence = xe_tlb_inval_job_push(job, tile->migrate, dep);
+	dma_resv_add_fence(resv, fence, DMA_RESV_USAGE_KERNEL);
+	dma_fence_put(fence);
+
+out_put:
+	/* Drop the creation reference (destroys the job if it was not pushed). */
+	xe_tlb_inval_job_put(job);
+	return err;
+}
+
+/**
+ * xe_vm_flush_vm_bo_tlb_async - Asynchronously flush TLBs for a vm_bo's mappings
+ * @vm: The VM @vm_bo belongs to
+ * @bo: The buffer object being moved
+ * @vm_bo: The gpuvm_bo linking @bo into @vm
+ *
+ * On L2-flush-optimized HW a BO move only needs to flush L2 (via a TLB
+ * invalidation) for the BO's live mappings; the mappings themselves are torn
+ * down and rebuilt lazily via the eviction/rebind path, so no PTEs need to be
+ * zapped here. Rather than blocking the caller on a synchronous invalidation,
+ * issue a TLB invalidation job per VMA per TLB-invalidation client (per present
+ * tile, primary and media GT). Each job waits on the BO's in-flight GPU work
+ * (all dma-resv usages) and its completion fence is installed into the BO's
+ * dma-resv KERNEL slots, so the following migration waits on the flush without
+ * stalling this thread.
+ *
+ * The caller must hold the BO's dma-resv lock and @vm must not be in fault
+ * mode.
+ *
+ * Return: 0 on success, negative error code on failure. On failure the caller
+ * should fall back to the blocking xe_vm_invalidate_vma() path; any jobs
+ * already queued install harmless extra flush fences.
+ */
+int xe_vm_flush_vm_bo_tlb_async(struct xe_vm *vm, struct xe_bo *bo,
+				struct drm_gpuvm_bo *vm_bo)
+{
+	struct xe_device *xe = vm->xe;
+	struct dma_resv *resv = bo->ttm.base.resv;
+	struct dma_fence *dep = NULL;
+	struct drm_gpuva *gpuva;
+	int err;
+
+	dma_resv_assert_held(resv);
+	xe_assert(xe, !xe_vm_in_fault_mode(vm));
+
+	/*
+	 * Single fence capturing all in-flight GPU work on the BO; the TLB
+	 * invalidation jobs depend on it so the flush fires only once the GPU
+	 * is done with the current mapping.
+	 */
+	err = dma_resv_get_singleton(resv, DMA_RESV_USAGE_BOOKKEEP, &dep);
+	if (err)
+		return err;
+	if (!dep)
+		dep = dma_fence_get_stub();
+
+	drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
+		struct xe_vma *vma = gpuva_to_vma(gpuva);
+		struct xe_tile *tile;
+		u8 id;
+
+		if (xe_vma_is_null(vma) || xe_vma_is_cpu_addr_mirror(vma))
+			continue;
+
+		for_each_tile(tile, xe, id) {
+			if (!(vma->tile_present & BIT(id)))
+				continue;
+
+			err = xe_vma_tlb_flush_client(vm, vma, tile,
+						      tile->primary_gt, resv, dep,
+						      XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
+			if (err)
+				goto out;
+
+			if (tile->media_gt) {
+				err = xe_vma_tlb_flush_client(vm, vma, tile,
+							      tile->media_gt, resv, dep,
+							      XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT);
+				if (err)
+					goto out;
+			}
+		}
+	}
+
+out:
+	dma_fence_put(dep);
+	return err;
+}
+
 int xe_vm_validate_protected(struct xe_vm *vm)
 {
 	struct drm_gpuva *gpuva;
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index c5b900f38ded..dd5b070eaede 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -26,6 +26,8 @@ struct ttm_buffer_object;
 
 struct dma_fence;
 
+struct xe_bo;
+struct drm_gpuvm_bo;
 struct xe_exec_queue;
 struct xe_file;
 struct xe_pagefault;
@@ -254,6 +256,9 @@ int xe_vm_invalidate_vma(struct xe_vma *vma);
 
 int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct xe_tlb_inval_batch *batch);
 
+int xe_vm_flush_vm_bo_tlb_async(struct xe_vm *vm, struct xe_bo *bo,
+				struct drm_gpuvm_bo *vm_bo);
+
 int xe_vm_validate_protected(struct xe_vm *vm);
 
 static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm)
-- 
2.34.1