Re: [PATCH v2 11/34] drm/xe/vf: Add xe_gt_sriov_vf_recovery_inprogress helper

[Matthew Brost <matthew.brost@intel.com>]

On Wed, Sep 24, 2025 at 12:14:28PM +0200, Michal Wajdeczko wrote:
> 
> 
> On 9/24/2025 3:15 AM, Matthew Brost wrote:
> > Add xe_gt_sriov_vf_recovery_inprogress helper.
> > 
> > This helper serves as the singular point to determine whether a VF
> 
> hmm, this "singular" looks like a GT-level only, not global
> 

Yes, it is GT scoped. I will adjust the commit message.

> > post-migration recovery is currently in progress. Expected callers
> > include the GuC CT layer and the GuC submission layer. Atomically
> > visable as soon as vCPU are unhalted until VF recovery completes.
> > 
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> >  drivers/gpu/drm/xe/xe_gt_sriov_vf.c       | 17 ++++++++
> >  drivers/gpu/drm/xe/xe_gt_sriov_vf.h       |  2 +
> >  drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h | 10 +++++
> >  drivers/gpu/drm/xe/xe_memirq.c            | 48 ++++++++++++++++++++++-
> >  drivers/gpu/drm/xe/xe_memirq.h            |  3 ++
> >  5 files changed, 79 insertions(+), 1 deletion(-)
> > 
> > diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> > index 016c867e5e2b..c9d0e32e7a15 100644
> > --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> > +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> > @@ -26,6 +26,7 @@
> >  #include "xe_guc_hxg_helpers.h"
> >  #include "xe_guc_relay.h"
> >  #include "xe_lrc.h"
> > +#include "xe_memirq.h"
> >  #include "xe_mmio.h"
> >  #include "xe_sriov.h"
> >  #include "xe_sriov_vf.h"
> > @@ -828,6 +829,7 @@ void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt)
> >  	struct xe_device *xe = gt_to_xe(gt);
> >  
> >  	xe_gt_assert(gt, IS_SRIOV_VF(xe));
> > +	xe_gt_assert(gt, xe_gt_sriov_vf_recovery_inprogress(gt));
> >  
> >  	set_bit(gt->info.id, &xe->sriov.vf.migration.gt_flags);
> >  	/*
> > @@ -1172,3 +1174,18 @@ void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
> >  	drm_printf(p, "\thandshake:\t%u.%u\n",
> >  		   pf_version->major, pf_version->minor);
> >  }
> > +
> > +/**
> > + * xe_gt_sriov_vf_recovery_inprogress() - VF post migration recovery in progress
> > + * @gt: the &xe_gt
> > + *
> > + * Return: True if VF post migration recovery in progress, False otherwise
> > + */
> > +bool xe_gt_sriov_vf_recovery_inprogress(struct xe_gt *gt)
> > +{
> > +	struct xe_memirq *memirq = &gt_to_tile(gt)->memirq;
> > +
> > +	return IS_SRIOV_VF(gt_to_xe(gt)) &&
> 
> this is xe_gt_sriov_vf function, so it is expected to be called only by
> the VF code, thus we should rather use xe_gt_assert here and the caller
> is responsible for the IS_SRIOV_VF check
> 

That is not how I have coded this. I blindly call this in various places
and I don't think it could at call site to determine if it is a VF as we
have if (VF) statements all over the driver. If perf is the concern, we
could move the IS_SRIOV_VF(gt_to_xe(gt)) part of function to static
inline and reset of the function in an exported function.

> > +		(xe_memirq_vf_recovery_irq_pending(memirq, &gt->uc.guc) ||
> > +		 READ_ONCE(gt->sriov.vf.migration.recovery_inprogress));
> > +}
> > diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> > index 0af1dc769fe0..bb5f8eace19b 100644
> > --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> > +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> > @@ -25,6 +25,8 @@ void xe_gt_sriov_vf_default_lrcs_hwsp_rebase(struct xe_gt *gt);
> >  int xe_gt_sriov_vf_notify_resfix_done(struct xe_gt *gt);
> >  void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt);
> >  
> > +bool xe_gt_sriov_vf_recovery_inprogress(struct xe_gt *gt);
> > +
> >  u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt);
> >  u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);
> >  u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt);
> > diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
> > index d95857bd789b..7b10b8e1e10e 100644
> > --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
> > +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
> > @@ -49,6 +49,14 @@ struct xe_gt_sriov_vf_runtime {
> >  	} *regs;
> >  };
> >  
> > +/**
> > + * xe_gt_sriov_vf_migration - VF migration data.
> > + */
> > +struct xe_gt_sriov_vf_migration {
> > +	/** @recovery_inprogress: VF post migration recovery in progress */
> > +	bool recovery_inprogress;
> > +};
> > +
> >  /**
> >   * struct xe_gt_sriov_vf - GT level VF virtualization data.
> >   */
> > @@ -61,6 +69,8 @@ struct xe_gt_sriov_vf {
> >  	struct xe_gt_sriov_vf_selfconfig self_config;
> >  	/** @runtime: runtime data retrieved from the PF. */
> >  	struct xe_gt_sriov_vf_runtime runtime;
> > +	/** @migration: migration data for the VF. */
> > +	struct xe_gt_sriov_vf_migration migration;
> >  };
> >  
> >  #endif
> > diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c
> > index 49c45ec3e83c..94d5d6859aab 100644
> > --- a/drivers/gpu/drm/xe/xe_memirq.c
> > +++ b/drivers/gpu/drm/xe/xe_memirq.c
> > @@ -398,6 +398,23 @@ void xe_memirq_postinstall(struct xe_memirq *memirq)
> >  		memirq_set_enable(memirq, true);
> >  }
> >  
> > +static bool memirq_received_noclear(struct xe_memirq *memirq,
> > +				    struct iosys_map *vector,
> > +				    u16 offset, const char *name)
> 
> maybe instead of duplicating code of memirq_received() in 90% just add there
> the "bool clear" flag?
> 

Sure.

> > +{
> > +	u8 value;
> > +
> > +	value = iosys_map_rd(vector, offset, u8);
> > +	if (value) {
> > +		if (value != 0xff)
> > +			memirq_err_ratelimited(memirq,
> > +					       "Unexpected memirq value %#x from %s at %u\n",
> > +					       value, name, offset);
> > +	}
> > +
> > +	return value;
> > +}
> > +
> >  static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,
> >  			    u16 offset, const char *name)
> >  {
> > @@ -434,8 +451,16 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat
> >  	if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
> >  		xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
> >  
> > -	if (memirq_received(memirq, status, ilog2(GUC_INTR_SW_INT_0), name))
> > +	/*
> > +	 * We must wait to perform the clear operation until after
> > +	 * xe_gt_sriov_vf_start_migration_recovery() runs, to avoid race
> > +	 * conditions where xe_gt_sriov_vf_recovery_inprogress() returns false.
> 
> but the VF recovery "inprogress" shall be already set in the top level
> 
> 	xe_sriov_vf_start_migration_recovery()
> 
> even before the GT-level recovery starts, where is this race ?
> 

If we clear the interrupt it here, this is before the IRQ handler is
called which flips the software bit for "inprogress". There would be
window where xe_gt_sriov_vf_recovery_inprogress could return false which
is problematic.

> > +	 */
> > +	if (memirq_received_noclear(memirq, status, ilog2(GUC_INTR_SW_INT_0),
> > +				    name)) {
> >  		xe_guc_irq_handler(guc, GUC_INTR_SW_INT_0);
> 
> what if new MEMIRQ will arrive just here
> 
> is it ok that we will clear it immediately?
> 
> the whole memirq flow is that we clear irq byte first and then process
> it, so if anything comes right after we finish processing will be noticed
> on next iteration
> 

I don't see why that matters in this case. In either case multiple IRQs
could happen and single IRQ handler runs.

> I assume any races due to double migration shall be handled on the VF2GUC
> communication level while sending RESFIX_START/DONE, not here

I don't think it should be possible to get multiple RESFIX_START IRQs
before DONE is complete. This existing code upstream seems to handle
these cases, so my series attempts to handle this too, but it seems like
something that shouldn't be possible.

> > +		iosys_map_wr(status, ilog2(GUC_INTR_SW_INT_0), u8, 0x00);
> > +	}
> >  }
> >  
> >  /**
> > @@ -460,6 +485,27 @@ void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
> >  	}
> >  }
> >  
> > +/**
> > + * xe_memirq_vf_recovery_irq_pending() - VF recovery IRQ is pending
> 
> this function isn't really using anything VF specific except that on the
> VF the SW_INT_0 means "migrated"
> 
> maybe we can drop the _vf from function name?
> 

Sure, so 'xe_memirq_recovery_irq_pending'?

Or 'xe_memirq_sw_int0_irq_pending'?

> 	xe_memirq_pending_guc(memirq, guc, bit)
> > + * @memirq: the &xe_memirq
> > + * @guc: the &xe_guc to check for IRQ
> > + *
> > + * Return: True if VF recovery IRQ is pending on @guc, False otherwise
> > + */
> > +bool xe_memirq_vf_recovery_irq_pending(struct xe_memirq *memirq,
> > +				       struct xe_guc *guc)
> > +{
> > +	struct xe_gt *gt = guc_to_gt(guc);
> > +	struct iosys_map map;
> > +
> > +	if (xe_gt_is_media_type(gt))
> > +		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);
> > +	else
> > +		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);
> 
> nit: maybe just calc offset conditionally?
> 
> 	u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
> 

Sure.

> > +
> > +	return iosys_map_rd(&map, ilog2(GUC_INTR_SW_INT_0), u8);
> 
> as we have helpers we should use them
> 
> 	return memirq_received_noclear(...)

Sure.

Matt

> 
> > +}
> > +
> >  /**
> >   * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.
> >   * @memirq: the &xe_memirq
> > diff --git a/drivers/gpu/drm/xe/xe_memirq.h b/drivers/gpu/drm/xe/xe_memirq.h
> > index 06130650e9d6..476b8cba179d 100644
> > --- a/drivers/gpu/drm/xe/xe_memirq.h
> > +++ b/drivers/gpu/drm/xe/xe_memirq.h
> > @@ -25,4 +25,7 @@ void xe_memirq_handler(struct xe_memirq *memirq);
> >  
> >  int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc);
> >  
> > +bool xe_memirq_vf_recovery_irq_pending(struct xe_memirq *memirq,
> > +				       struct xe_guc *guc);
> > +
> >  #endif
>