Re: [PATCH v3] drm/xe: Flush L2 asynchronously in xe_bo_trigger_rebind()

[Matthew Brost <matthew.brost@intel.com>]

On Thu, Aug 20, 2026 at 11:10:55AM +0200, Thomas Hellström wrote:
> On Fri, 2026-07-17 at 12:38 -0700, Matthew Brost wrote:
> > On L2-flush-optimized HW with a dma-resv (non-fault) VM, evicting a
> > BO
> > only needs to flush the L2 cache before the migration copy runs; the
> > mappings themselves are torn down and rebuilt lazily via
> > drm_gpuvm_bo_evict() and a subsequent rebind. Today this flush is
> > done
> > by waiting for the BO to go idle and then issuing a synchronous TLB
> > invalidation per mapping VMA from inside xe_bo_trigger_rebind(). Both
> > the idle wait and the synchronous invalidation stall the calling
> > thread
> > while holding the BO dma-resv lock, serializing the move behind all
> > in-flight GPU work on the BO.
> > 
> > Replace this with an asynchronous flush. Add
> > xe_vm_flush_vm_bo_tlb_async()
> > which, for each VMA mapping the BO on each present tile, queues a TLB
> > invalidation job on the tile migrate (kernel) exec queue. The jobs
> > depend
> > on the BO's in-flight GPU work, captured once as a singleton over
> > DMA_RESV_USAGE_BOOKKEEP, so the flush only fires once the GPU is done
> > with
> > the current mapping. Each job's completion fence is installed into
> > the
> > BO's dma-resv as a DMA_RESV_USAGE_KERNEL fence, so the migration copy
> > -
> > which waits on the resv - waits on the flush without stalling this
> > thread.
> > 
> > No PTEs are zapped and vma->tile_invalidated is left untouched: the
> > mapping stays valid until the lazy rebind, and the only work
> > performed
> > here is the L2 flush. On any failure the caller falls back to the
> > existing blocking wait-idle plus xe_vm_invalidate_vma() path.
> > 
> > Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> > Cc: Tejas Upadhyay <tejas.upadhyay@intel.com>
> > Assisted-by: GitHub_Copilot:claude-opus-4.8
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > 
> > ---
> > v2: Less verbose comment (Thomas)
> > v3: Honor no_gpu_wait (Sashiko)
> > ---
> > ---
> >  drivers/gpu/drm/xe/xe_bo.c |   8 +++
> >  drivers/gpu/drm/xe/xe_vm.c | 125
> > +++++++++++++++++++++++++++++++++++++
> >  drivers/gpu/drm/xe/xe_vm.h |   5 ++
> >  3 files changed, 138 insertions(+)
> > 
> > diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
> > index c266fa6bade1..bb9dbc592941 100644
> > --- a/drivers/gpu/drm/xe/xe_bo.c
> > +++ b/drivers/gpu/drm/xe/xe_bo.c
> > @@ -686,6 +686,14 @@ static int xe_bo_trigger_rebind(struct xe_device
> > *xe, struct xe_bo *bo,
> >  			 */
> >  			if (!xe_device_is_l2_flush_optimized(xe))
> >  				continue;
> > +
> > +			/*
> > +			 * Attempt to flush L2 async, fallback to
> > sync flush on
> > +			 * no_wait_gpu or faliure
> > +			 */
> > +			if (!ctx->no_wait_gpu &&
> > +			    !xe_vm_flush_vm_bo_tlb_async(vm, bo,
> > vm_bo))
> > +				continue;
> 
> To follow the no_wait_gpu semantics, Shouldn't this be
> 

After thinking about this, the original way I had this is correct.

> if (!xe_vm_flush_vm_bo_tlb_async(vm, bo, vm_bo))
> 	continue;

Just the above if statement.

> else if (ctx->no_wait_gpu)
> 	return -EBUSY;

Not needed because the if (!idle) statement will be popped into which
honors no_wait_gpu.

Let me revert to the prior version.

Matt

> 
> 
> >  		}
> >  
> >  		if (!idle) {
> > diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
> > index 57bcf0660eb8..28223e32fd26 100644
> > --- a/drivers/gpu/drm/xe/xe_vm.c
> > +++ b/drivers/gpu/drm/xe/xe_vm.c
> > @@ -39,6 +39,7 @@
> >  #include "xe_sync.h"
> >  #include "xe_tile.h"
> >  #include "xe_tlb_inval.h"
> > +#include "xe_tlb_inval_job.h"
> >  #include "xe_trace_bo.h"
> >  #include "xe_vm_madvise.h"
> >  #include "xe_wa.h"
> > @@ -4432,6 +4433,130 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
> >  	return ret;
> >  }
> >  
> > +/*
> > + * xe_vma_tlb_flush_client - Queue an async TLB flush for one VMA on
> > one client
> > + *
> > + * Create and push a TLB invalidation job on the tile migrate
> > (kernel) exec
> > + * queue covering @vma's range, depending on @dep (the BO's in-
> > flight GPU work)
> > + * so the flush only fires once the GPU is done with the current
> > mapping. The
> > + * job's completion fence is installed into @resv as a KERNEL fence
> > so the
> > + * subsequent migration waits on the flush. No PTEs are zapped; this
> > only
> > + * flushes L2 via the TLB invalidation.
> > + */
> > +static int xe_vma_tlb_flush_client(struct xe_vm *vm, struct xe_vma
> > *vma,
> > +				   struct xe_tile *tile, struct
> > xe_gt *gt,
> > +				   struct dma_resv *resv, struct
> > dma_fence *dep,
> > +				   int type)
> > +{
> > +	struct xe_exec_queue *q = xe_migrate_exec_queue(tile-
> > >migrate);
> > +	struct xe_tlb_inval_job *job;
> > +	struct dma_fence *fence;
> > +	int err;
> > +
> > +	job = xe_tlb_inval_job_create(q, &gt->tlb_inval,
> > +				      q-
> > >tlb_inval[type].dep_scheduler, vm,
> > +				      xe_vma_start(vma),
> > xe_vma_end(vma), type);
> > +	if (IS_ERR(job))
> > +		return PTR_ERR(job);
> > +
> > +	err = xe_tlb_inval_job_alloc_dep(job);
> > +	if (err)
> > +		goto out_put;
> > +
> > +	err = dma_resv_reserve_fences(resv, 1);
> > +	if (err)
> > +		goto out_put;
> > +
> > +	/* Cannot fail; consumes a ref on @dep and returns a
> > referenced fence. */
> > +	fence = xe_tlb_inval_job_push(job, tile->migrate, dep);
> > +	dma_resv_add_fence(resv, fence, DMA_RESV_USAGE_KERNEL);
> > +	dma_fence_put(fence);
> > +
> > +out_put:
> > +	/* Drop the creation reference (destroys the job if it was
> > not pushed). */
> > +	xe_tlb_inval_job_put(job);
> > +	return err;
> > +}
> > +
> > +/**
> > + * xe_vm_flush_vm_bo_tlb_async - Asynchronously flush TLBs for a
> > vm_bo's mappings
> > + * @vm: The VM @vm_bo belongs to
> > + * @bo: The buffer object being moved
> > + * @vm_bo: The gpuvm_bo linking @bo into @vm
> > + *
> > + * On L2-flush-optimized HW a BO move only needs to flush L2 (via a
> > TLB
> > + * invalidation) for the BO's live mappings; the mappings themselves
> > are torn
> > + * down and rebuilt lazily via the eviction/rebind path, so no PTEs
> > need to be
> > + * zapped here. Rather than blocking the caller on a synchronous
> > invalidation,
> > + * issue a TLB invalidation job per VMA per TLB-invalidation client
> > (per present
> > + * tile, primary and media GT). Each job waits on the BO's in-flight
> > GPU work
> > + * (all dma-resv usages) and its completion fence is installed into
> > the BO's
> > + * dma-resv KERNEL slots, so the following migration waits on the
> > flush without
> > + * stalling this thread.
> > + *
> > + * The caller must hold the BO's dma-resv lock and @vm must not be
> > in fault
> > + * mode.
> > + *
> > + * Return: 0 on success, negative error code on failure. On failure
> > the caller
> > + * should fall back to the blocking xe_vm_invalidate_vma() path; any
> > jobs
> > + * already queued install harmless extra flush fences.
> > + */
> > +int xe_vm_flush_vm_bo_tlb_async(struct xe_vm *vm, struct xe_bo *bo,
> > +				struct drm_gpuvm_bo *vm_bo)
> > +{
> > +	struct xe_device *xe = vm->xe;
> > +	struct dma_resv *resv = bo->ttm.base.resv;
> > +	struct dma_fence *dep = NULL;
> > +	struct drm_gpuva *gpuva;
> > +	int err;
> > +
> > +	dma_resv_assert_held(resv);
> > +	xe_assert(xe, !xe_vm_in_fault_mode(vm));
> > +
> > +	/*
> > +	 * Single fence capturing all in-flight GPU work on the BO;
> > the TLB
> > +	 * invalidation jobs depend on it so the flush fires only
> > once the GPU
> > +	 * is done with the current mapping.
> > +	 */
> > +	err = dma_resv_get_singleton(resv, DMA_RESV_USAGE_BOOKKEEP,
> > &dep);
> > +	if (err)
> > +		return err;
> > +	if (!dep)
> > +		dep = dma_fence_get_stub();
> > +
> > +	drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
> > +		struct xe_vma *vma = gpuva_to_vma(gpuva);
> > +		struct xe_tile *tile;
> > +		u8 id;
> > +
> > +		if (xe_vma_is_null(vma) ||
> > xe_vma_is_cpu_addr_mirror(vma))
> > +			continue;
> > +
> > +		for_each_tile(tile, xe, id) {
> > +			if (!(vma->tile_present & BIT(id)))
> > +				continue;
> > +
> > +			err = xe_vma_tlb_flush_client(vm, vma, tile,
> > +						      tile-
> > >primary_gt, resv, dep,
> > +						     
> > XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
> > +			if (err)
> > +				goto out;
> > +
> > +			if (tile->media_gt) {
> > +				err = xe_vma_tlb_flush_client(vm,
> > vma, tile,
> > +							      tile-
> > >media_gt, resv, dep,
> > +							     
> > XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT);
> > +				if (err)
> > +					goto out;
> > +			}
> > +		}
> > +	}
> > +
> > +out:
> > +	dma_fence_put(dep);
> > +	return err;
> > +}
> > +
> >  int xe_vm_validate_protected(struct xe_vm *vm)
> >  {
> >  	struct drm_gpuva *gpuva;
> > diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
> > index c5b900f38ded..dd5b070eaede 100644
> > --- a/drivers/gpu/drm/xe/xe_vm.h
> > +++ b/drivers/gpu/drm/xe/xe_vm.h
> > @@ -26,6 +26,8 @@ struct ttm_buffer_object;
> >  
> >  struct dma_fence;
> >  
> > +struct xe_bo;
> > +struct drm_gpuvm_bo;
> >  struct xe_exec_queue;
> >  struct xe_file;
> >  struct xe_pagefault;
> > @@ -254,6 +256,9 @@ int xe_vm_invalidate_vma(struct xe_vma *vma);
> >  
> >  int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct
> > xe_tlb_inval_batch *batch);
> >  
> > +int xe_vm_flush_vm_bo_tlb_async(struct xe_vm *vm, struct xe_bo *bo,
> > +				struct drm_gpuvm_bo *vm_bo);
> > +
> >  int xe_vm_validate_protected(struct xe_vm *vm);
> >  
> >  static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm)