Re: [PATCH v2 11/34] drm/xe/vf: Add xe_gt_sriov_vf_recovery_inprogress helper

[Matthew Brost <matthew.brost@intel.com>]

On Wed, Sep 24, 2025 at 10:12:10PM +0200, Michal Wajdeczko wrote:
> 
> 
> On 9/24/2025 9:39 PM, Matthew Brost wrote:
> > On Wed, Sep 24, 2025 at 12:14:28PM +0200, Michal Wajdeczko wrote:
> >>
> >>
> >> On 9/24/2025 3:15 AM, Matthew Brost wrote:
> >>> Add xe_gt_sriov_vf_recovery_inprogress helper.
> >>>
> >>> This helper serves as the singular point to determine whether a VF
> >>
> >> hmm, this "singular" looks like a GT-level only, not global
> >>
> > 
> > Yes, it is GT scoped. I will adjust the commit message.
> > 
> >>> post-migration recovery is currently in progress. Expected callers
> >>> include the GuC CT layer and the GuC submission layer. Atomically
> >>> visable as soon as vCPU are unhalted until VF recovery completes.
> >>>
> >>> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> >>> ---
> >>>  drivers/gpu/drm/xe/xe_gt_sriov_vf.c       | 17 ++++++++
> >>>  drivers/gpu/drm/xe/xe_gt_sriov_vf.h       |  2 +
> >>>  drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h | 10 +++++
> >>>  drivers/gpu/drm/xe/xe_memirq.c            | 48 ++++++++++++++++++++++-
> >>>  drivers/gpu/drm/xe/xe_memirq.h            |  3 ++
> >>>  5 files changed, 79 insertions(+), 1 deletion(-)
> >>>
> >>> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> >>> index 016c867e5e2b..c9d0e32e7a15 100644
> >>> --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> >>> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> >>> @@ -26,6 +26,7 @@
> >>>  #include "xe_guc_hxg_helpers.h"
> >>>  #include "xe_guc_relay.h"
> >>>  #include "xe_lrc.h"
> >>> +#include "xe_memirq.h"
> >>>  #include "xe_mmio.h"
> >>>  #include "xe_sriov.h"
> >>>  #include "xe_sriov_vf.h"
> >>> @@ -828,6 +829,7 @@ void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt)
> >>>  	struct xe_device *xe = gt_to_xe(gt);
> >>>  
> >>>  	xe_gt_assert(gt, IS_SRIOV_VF(xe));
> >>> +	xe_gt_assert(gt, xe_gt_sriov_vf_recovery_inprogress(gt));
> >>>  
> >>>  	set_bit(gt->info.id, &xe->sriov.vf.migration.gt_flags);
> >>>  	/*
> >>> @@ -1172,3 +1174,18 @@ void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
> >>>  	drm_printf(p, "\thandshake:\t%u.%u\n",
> >>>  		   pf_version->major, pf_version->minor);
> >>>  }
> >>> +
> >>> +/**
> >>> + * xe_gt_sriov_vf_recovery_inprogress() - VF post migration recovery in progress
> >>> + * @gt: the &xe_gt
> >>> + *
> >>> + * Return: True if VF post migration recovery in progress, False otherwise
> >>> + */
> >>> +bool xe_gt_sriov_vf_recovery_inprogress(struct xe_gt *gt)
> >>> +{
> >>> +	struct xe_memirq *memirq = &gt_to_tile(gt)->memirq;
> >>> +
> >>> +	return IS_SRIOV_VF(gt_to_xe(gt)) &&
> >>
> >> this is xe_gt_sriov_vf function, so it is expected to be called only by
> >> the VF code, thus we should rather use xe_gt_assert here and the caller
> >> is responsible for the IS_SRIOV_VF check
> >>
> > 
> > That is not how I have coded this. I blindly call this in various places
> > and I don't think it could at call site to determine if it is a VF as we
> > have if (VF) statements all over the driver. If perf is the concern, we
> > could move the IS_SRIOV_VF(gt_to_xe(gt)) part of function to static
> > inline and reset of the function in an exported function.
> 
> inline will work, preferable not in xe_gt_sriov_vf.h ;)
> 
> static inline xe_gt_recovery_inprogress(gt)
> {
> 	return IS_SRIOV_VF(xe) && xe_gt_sriov_vf_recovery_inprogress(gt);
> }
> 

+1

> > 
> >>> +		(xe_memirq_vf_recovery_irq_pending(memirq, &gt->uc.guc) ||
> >>> +		 READ_ONCE(gt->sriov.vf.migration.recovery_inprogress));
> >>> +}
> >>> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> >>> index 0af1dc769fe0..bb5f8eace19b 100644
> >>> --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> >>> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> >>> @@ -25,6 +25,8 @@ void xe_gt_sriov_vf_default_lrcs_hwsp_rebase(struct xe_gt *gt);
> >>>  int xe_gt_sriov_vf_notify_resfix_done(struct xe_gt *gt);
> >>>  void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt);
> >>>  
> >>> +bool xe_gt_sriov_vf_recovery_inprogress(struct xe_gt *gt);
> >>> +
> >>>  u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt);
> >>>  u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);
> >>>  u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt);
> >>> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
> >>> index d95857bd789b..7b10b8e1e10e 100644
> >>> --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
> >>> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
> >>> @@ -49,6 +49,14 @@ struct xe_gt_sriov_vf_runtime {
> >>>  	} *regs;
> >>>  };
> >>>  
> >>> +/**
> >>> + * xe_gt_sriov_vf_migration - VF migration data.
> >>> + */
> >>> +struct xe_gt_sriov_vf_migration {
> >>> +	/** @recovery_inprogress: VF post migration recovery in progress */
> >>> +	bool recovery_inprogress;
> >>> +};
> >>> +
> >>>  /**
> >>>   * struct xe_gt_sriov_vf - GT level VF virtualization data.
> >>>   */
> >>> @@ -61,6 +69,8 @@ struct xe_gt_sriov_vf {
> >>>  	struct xe_gt_sriov_vf_selfconfig self_config;
> >>>  	/** @runtime: runtime data retrieved from the PF. */
> >>>  	struct xe_gt_sriov_vf_runtime runtime;
> >>> +	/** @migration: migration data for the VF. */
> >>> +	struct xe_gt_sriov_vf_migration migration;
> >>>  };
> >>>  
> >>>  #endif
> >>> diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c
> >>> index 49c45ec3e83c..94d5d6859aab 100644
> >>> --- a/drivers/gpu/drm/xe/xe_memirq.c
> >>> +++ b/drivers/gpu/drm/xe/xe_memirq.c
> >>> @@ -398,6 +398,23 @@ void xe_memirq_postinstall(struct xe_memirq *memirq)
> >>>  		memirq_set_enable(memirq, true);
> >>>  }
> >>>  
> >>> +static bool memirq_received_noclear(struct xe_memirq *memirq,
> >>> +				    struct iosys_map *vector,
> >>> +				    u16 offset, const char *name)
> >>
> >> maybe instead of duplicating code of memirq_received() in 90% just add there
> >> the "bool clear" flag?
> >>
> > 
> > Sure.
> > 
> >>> +{
> >>> +	u8 value;
> >>> +
> >>> +	value = iosys_map_rd(vector, offset, u8);
> >>> +	if (value) {
> >>> +		if (value != 0xff)
> >>> +			memirq_err_ratelimited(memirq,
> >>> +					       "Unexpected memirq value %#x from %s at %u\n",
> >>> +					       value, name, offset);
> >>> +	}
> >>> +
> >>> +	return value;
> >>> +}
> >>> +
> >>>  static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,
> >>>  			    u16 offset, const char *name)
> >>>  {
> >>> @@ -434,8 +451,16 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat
> >>>  	if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
> >>>  		xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
> >>>  
> >>> -	if (memirq_received(memirq, status, ilog2(GUC_INTR_SW_INT_0), name))
> >>> +	/*
> >>> +	 * We must wait to perform the clear operation until after
> >>> +	 * xe_gt_sriov_vf_start_migration_recovery() runs, to avoid race
> >>> +	 * conditions where xe_gt_sriov_vf_recovery_inprogress() returns false.
> >>
> >> but the VF recovery "inprogress" shall be already set in the top level
> >>
> >> 	xe_sriov_vf_start_migration_recovery()
> >>
> >> even before the GT-level recovery starts, where is this race ?
> >>
> > 
> > If we clear the interrupt it here, this is before the IRQ handler is
> > called which flips the software bit for "inprogress". There would be
> > window where xe_gt_sriov_vf_recovery_inprogress could return false which
> > is problematic.
> 
> but who calls that recovery_inprogress() and why should it be problematic?
> 
> note there might be still other threads that will finish some actions
> (including sending CTB) that will just be stuck there until we start
> the recovery, so even if recovery_inprogress() returns false for the
> small window that shouldn't change the picture
> 

No. Read 'Waiters during VF post migration recovery' in [1]. That should
explain the reason why this is required. Let me know if what is
documented there is unclear.

[1] https://patchwork.freedesktop.org/patch/676374/?series=154627&rev=2

> > 
> >>> +	 */
> >>> +	if (memirq_received_noclear(memirq, status, ilog2(GUC_INTR_SW_INT_0),
> >>> +				    name)) {
> >>>  		xe_guc_irq_handler(guc, GUC_INTR_SW_INT_0);
> >>
> >> what if new MEMIRQ will arrive just here
> >>
> >> is it ok that we will clear it immediately?
> >>
> >> the whole memirq flow is that we clear irq byte first and then process
> >> it, so if anything comes right after we finish processing will be noticed
> >> on next iteration
> >>
> > 
> > I don't see why that matters in this cabuse. In either case multiple IRQs
> > could happen and single IRQ handler runs.
> > 
> >> I assume any races due to double migration shall be handled on the VF2GUC
> >> communication level while sending RESFIX_START/DONE, not here
> > 
> > I don't think it should be possible to get multiple RESFIX_START IRQs
> 
> I assume the SW_INT_0 is set on every migration, even in case we didn't
> have a chance to read it and clear it, or we didn't start (by sending 
> RESFIX_START) nor finish (RESFIX_DONE)
> 

But can migration be triggered and then another one before the initial
migration completes? I just don't see how that is possible or how it
wouldn't break the world (i.e., The VF of GuC explodes somewhere).

> > before DONE is complete. This existing code upstream seems to handle
> > these cases, so my series attempts to handle this too, but it seems like
> > something that shouldn't be possible.
> > 
> >>> +		iosys_map_wr(status, ilog2(GUC_INTR_SW_INT_0), u8, 0x00);
> >>> +	}
> >>>  }
> >>>  
> >>>  /**
> >>> @@ -460,6 +485,27 @@ void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
> >>>  	}
> >>>  }
> >>>  
> >>> +/**
> >>> + * xe_memirq_vf_recovery_irq_pending() - VF recovery IRQ is pending
> >>
> >> this function isn't really using anything VF specific except that on the
> >> VF the SW_INT_0 means "migrated"
> >>
> >> maybe we can drop the _vf from function name?
> >>
> > 
> > Sure, so 'xe_memirq_recovery_irq_pending'?
> > 
> > Or 'xe_memirq_sw_int0_irq_pending'?
> 
> 
> > 
> >> 	xe_memirq_pending_guc(memirq, guc, bit)
> 
> I would make it more generic and provide bit param
> 
> note that xe_guc_irq_handler() also takes a bit
> 

Sure.

Matt

> 
> >>> + * @memirq: the &xe_memirq
> >>> + * @guc: the &xe_guc to check for IRQ
> >>> + *
> >>> + * Return: True if VF recovery IRQ is pending on @guc, False otherwise
> >>> + */
> >>> +bool xe_memirq_vf_recovery_irq_pending(struct xe_memirq *memirq,
> >>> +				       struct xe_guc *guc)
> >>> +{
> >>> +	struct xe_gt *gt = guc_to_gt(guc);
> >>> +	struct iosys_map map;
> >>> +
> >>> +	if (xe_gt_is_media_type(gt))
> >>> +		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);
> >>> +	else
> >>> +		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);
> >>
> >> nit: maybe just calc offset conditionally?
> >>
> >> 	u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
> >>
> > 
> > Sure.
> > 
> >>> +
> >>> +	return iosys_map_rd(&map, ilog2(GUC_INTR_SW_INT_0), u8);
> >>
> >> as we have helpers we should use them
> >>
> >> 	return memirq_received_noclear(...)
> > 
> > Sure.
> > 
> > Matt
> > 
> >>
> >>> +}
> >>> +
> >>>  /**
> >>>   * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.
> >>>   * @memirq: the &xe_memirq
> >>> diff --git a/drivers/gpu/drm/xe/xe_memirq.h b/drivers/gpu/drm/xe/xe_memirq.h
> >>> index 06130650e9d6..476b8cba179d 100644
> >>> --- a/drivers/gpu/drm/xe/xe_memirq.h
> >>> +++ b/drivers/gpu/drm/xe/xe_memirq.h
> >>> @@ -25,4 +25,7 @@ void xe_memirq_handler(struct xe_memirq *memirq);
> >>>  
> >>>  int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc);
> >>>  
> >>> +bool xe_memirq_vf_recovery_irq_pending(struct xe_memirq *memirq,
> >>> +				       struct xe_guc *guc);
> >>> +
> >>>  #endif
> >>
>