Re: [PATCH] drm/xe/pf: Move VFs reprovisioning to worker

[Matthew Brost <matthew.brost@intel.com>]

On Sat, Jan 25, 2025 at 10:55:05PM +0100, Michal Wajdeczko wrote:
> Since the GuC is reset during GT reset, we need to re-send the
> entire SR-IOV provisioning configuration to the GuC. But since
> this whole configuration is protected by the PF master mutex and
> we can't avoid making allocations under this mutex (like during
> LMEM provisioning), we can't do this reprovisioning from gt-reset
> path if we want to be reclaim-safe. Move VFs reprovisioning to a
> async worker that we will start from the gt-reset path.
> 
> Signed-off-by: Michal Wajdeczko <michal.wajdeczko@intel.com>
> Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> Cc: Matthew Brost <matthew.brost@intel.com>

Reviewed-by: Matthew Brost <matthew.brost@intel.com>

> ---
>  drivers/gpu/drm/xe/xe_gt_sriov_pf.c       | 53 ++++++++++++++++++++---
>  drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h | 10 +++++
>  2 files changed, 56 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
> index 6f906c8e8108..d66478deab98 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
> @@ -15,7 +15,11 @@
>  #include "xe_gt_sriov_pf_helpers.h"
>  #include "xe_gt_sriov_pf_migration.h"
>  #include "xe_gt_sriov_pf_service.h"
> +#include "xe_gt_sriov_printk.h"
>  #include "xe_mmio.h"
> +#include "xe_pm.h"
> +
> +static void pf_worker_restart_func(struct work_struct *w);
>  
>  /*
>   * VF's metadata is maintained in the flexible array where:
> @@ -41,6 +45,11 @@ static int pf_alloc_metadata(struct xe_gt *gt)
>  	return 0;
>  }
>  
> +static void pf_init_workers(struct xe_gt *gt)
> +{
> +	INIT_WORK(&gt->sriov.pf.workers.restart, pf_worker_restart_func);
> +}
> +
>  /**
>   * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF.
>   * @gt: the &xe_gt to initialize
> @@ -65,6 +74,8 @@ int xe_gt_sriov_pf_init_early(struct xe_gt *gt)
>  	if (err)
>  		return err;
>  
> +	pf_init_workers(gt);
> +
>  	return 0;
>  }
>  
> @@ -155,14 +166,42 @@ void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid)
>  	pf_clear_vf_scratch_regs(gt, vfid);
>  }
>  
> -/**
> - * xe_gt_sriov_pf_restart - Restart SR-IOV support after a GT reset.
> - * @gt: the &xe_gt
> - *
> - * This function can only be called on PF.
> - */
> -void xe_gt_sriov_pf_restart(struct xe_gt *gt)
> +static void pf_restart(struct xe_gt *gt)
>  {
> +	struct xe_device *xe = gt_to_xe(gt);
> +
> +	xe_pm_runtime_get(xe);
>  	xe_gt_sriov_pf_config_restart(gt);
>  	xe_gt_sriov_pf_control_restart(gt);
> +	xe_pm_runtime_put(xe);
> +
> +	xe_gt_sriov_dbg(gt, "restart completed\n");
> +}
> +
> +static void pf_worker_restart_func(struct work_struct *w)
> +{
> +	struct xe_gt *gt = container_of(w, typeof(*gt), sriov.pf.workers.restart);
> +
> +	pf_restart(gt);
> +}
> +
> +static void pf_queue_restart(struct xe_gt *gt)
> +{
> +	struct xe_device *xe = gt_to_xe(gt);
> +
> +	xe_gt_assert(gt, IS_SRIOV_PF(xe));
> +
> +	if (!queue_work(xe->sriov.wq, &gt->sriov.pf.workers.restart))
> +		xe_gt_sriov_dbg(gt, "restart already in queue!\n");
> +}
> +
> +/**
> + * xe_gt_sriov_pf_restart - Restart SR-IOV support after a GT reset.
> + * @gt: the &xe_gt
> + *
> + * This function can only be called on PF.
> + */
> +void xe_gt_sriov_pf_restart(struct xe_gt *gt)
> +{
> +	pf_queue_restart(gt);
>  }
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
> index 0426b1a77069..a64a6835ad65 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
> @@ -35,8 +35,17 @@ struct xe_gt_sriov_metadata {
>  	struct xe_gt_sriov_state_snapshot snapshot;
>  };
>  
> +/**
> + * struct xe_gt_sriov_pf_workers - GT level workers used by the PF.
> + */
> +struct xe_gt_sriov_pf_workers {
> +	/** @restart: worker that executes actions post GT reset */
> +	struct work_struct restart;
> +};
> +
>  /**
>   * struct xe_gt_sriov_pf - GT level PF virtualization data.
> + * @workers: workers data.
>   * @service: service data.
>   * @control: control data.
>   * @policy: policy data.
> @@ -45,6 +54,7 @@ struct xe_gt_sriov_metadata {
>   * @vfs: metadata for all VFs.
>   */
>  struct xe_gt_sriov_pf {
> +	struct xe_gt_sriov_pf_workers workers;
>  	struct xe_gt_sriov_pf_service service;
>  	struct xe_gt_sriov_pf_control control;
>  	struct xe_gt_sriov_pf_policy policy;
> -- 
> 2.47.1
>