Re: [Intel-xe] [PATCH v3 09/20] drm/xe/device: Introduce xe_device_probe_early

[Matthew Brost <matthew.brost@intel.com>]

On Tue, Nov 14, 2023 at 02:02:20PM +0100, Michał Winiarski wrote:
> SR-IOV VF doesn't have access to MMIO registers used to determine
> graphics/media ID. It can however communicate with GuC.
> Introduce xe_device_probe_early, which initializes enough HW to use
> MMIO GuC communication.
> This will allow both VF and PF/native driver to have unified probe
> ordering.
> 
> Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>

Reviewed-by: Matthew Brost <matthew.brost@intel.com>

> ---
> v1 -> v2:
> - Add comment to "early" variant (Lucas)
> 
>  drivers/gpu/drm/xe/xe_device.c | 22 ++++++++++++++++++----
>  drivers/gpu/drm/xe/xe_device.h |  5 +++++
>  drivers/gpu/drm/xe/xe_mmio.c   | 16 ++++++++++------
>  drivers/gpu/drm/xe/xe_mmio.h   |  1 +
>  drivers/gpu/drm/xe/xe_pci.c    | 22 +++++++++++++++++++---
>  5 files changed, 53 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
> index 3a98e7b502190..6fe669fdf1c0a 100644
> --- a/drivers/gpu/drm/xe/xe_device.c
> +++ b/drivers/gpu/drm/xe/xe_device.c
> @@ -364,6 +364,24 @@ static int xe_set_dma_info(struct xe_device *xe)
>  	return err;
>  }
>  
> +/*
> + * Initialize MMIO resources that don't require any knowledge about tile count.
> + */
> +int xe_device_probe_early(struct xe_device *xe)
> +{
> +	int err;
> +
> +	err = xe_mmio_init(xe);
> +	if (err)
> +		return err;
> +
> +	err = xe_mmio_root_tile_init(xe);
> +	if (err)
> +		return err;
> +
> +	return 0;
> +}
> +
>  int xe_device_probe(struct xe_device *xe)
>  {
>  	struct xe_tile *tile;
> @@ -382,10 +400,6 @@ int xe_device_probe(struct xe_device *xe)
>  	if (err)
>  		return err;
>  
> -	err = xe_mmio_init(xe);
> -	if (err)
> -		return err;
> -
>  	xe_mmio_probe_tiles(xe);
>  
>  	err = drmm_add_action_or_reset(&xe->drm, xe_driver_flr_fini, xe);
> diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
> index c4232de40ae08..b80230c631068 100644
> --- a/drivers/gpu/drm/xe/xe_device.h
> +++ b/drivers/gpu/drm/xe/xe_device.h
> @@ -37,6 +37,7 @@ static inline struct xe_device *ttm_to_xe_device(struct ttm_device *ttm)
>  
>  struct xe_device *xe_device_create(struct pci_dev *pdev,
>  				   const struct pci_device_id *ent);
> +int xe_device_probe_early(struct xe_device *xe);
>  int xe_device_probe(struct xe_device *xe);
>  void xe_device_remove(struct xe_device *xe);
>  void xe_device_shutdown(struct xe_device *xe);
> @@ -123,6 +124,10 @@ static inline bool xe_device_uc_enabled(struct xe_device *xe)
>  	for ((id__) = 0; (id__) < (xe__)->info.tile_count; (id__)++) \
>  		for_each_if((tile__) = &(xe__)->tiles[(id__)])
>  
> +#define for_each_remote_tile(tile__, xe__, id__) \
> +	for ((id__) = 1; (id__) < (xe__)->info.tile_count; (id__)++) \
> +		for_each_if((tile__) = &(xe__)->tiles[(id__)])
> +
>  /*
>   * FIXME: This only works for now since multi-tile and standalone media
>   * happen to be mutually exclusive.  Future platforms may change this...
> diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c
> index acf6dfca0bc46..8eb19a4bca9c6 100644
> --- a/drivers/gpu/drm/xe/xe_mmio.c
> +++ b/drivers/gpu/drm/xe/xe_mmio.c
> @@ -15,10 +15,12 @@
>  #include "regs/xe_regs.h"
>  #include "xe_bo.h"
>  #include "xe_device.h"
> +#include "xe_ggtt.h"
>  #include "xe_gt.h"
>  #include "xe_gt_mcr.h"
>  #include "xe_macros.h"
>  #include "xe_module.h"
> +#include "xe_tile.h"
>  
>  #define XEHP_MTCFG_ADDR		XE_REG(0x101800)
>  #define TILE_COUNT		REG_GENMASK(15, 8)
> @@ -375,10 +377,8 @@ static int xe_verify_lmem_ready(struct xe_device *xe)
>  
>  int xe_mmio_init(struct xe_device *xe)
>  {
> -	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
>  	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
>  	const int mmio_bar = 0;
> -	int err;
>  
>  	/*
>  	 * Map the entire BAR.
> @@ -392,12 +392,16 @@ int xe_mmio_init(struct xe_device *xe)
>  		return -EIO;
>  	}
>  
> -	err = drmm_add_action_or_reset(&xe->drm, mmio_fini, xe);
> -	if (err)
> -		return err;
> +	return drmm_add_action_or_reset(&xe->drm, mmio_fini, xe);
> +}
> +
> +int xe_mmio_root_tile_init(struct xe_device *xe)
> +{
> +	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
> +	int err;
>  
>  	/* Setup first tile; other tiles (if present) will be setup later. */
> -	root_tile->mmio.size = xe->mmio.size;
> +	root_tile->mmio.size = SZ_16M;
>  	root_tile->mmio.regs = xe->mmio.regs;
>  
>  	err = xe_verify_lmem_ready(xe);
> diff --git a/drivers/gpu/drm/xe/xe_mmio.h b/drivers/gpu/drm/xe/xe_mmio.h
> index 9f1a25cfd2cf0..74976af15ea60 100644
> --- a/drivers/gpu/drm/xe/xe_mmio.h
> +++ b/drivers/gpu/drm/xe/xe_mmio.h
> @@ -21,6 +21,7 @@ struct xe_device;
>  #define GEN12_LMEM_BAR		2
>  
>  int xe_mmio_init(struct xe_device *xe);
> +int xe_mmio_root_tile_init(struct xe_device *xe);
>  void xe_mmio_probe_tiles(struct xe_device *xe);
>  
>  static inline u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg)
> diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
> index c50cc3f7cbe06..a1026b7ee4727 100644
> --- a/drivers/gpu/drm/xe/xe_pci.c
> +++ b/drivers/gpu/drm/xe/xe_pci.c
> @@ -537,10 +537,12 @@ static void handle_gmdid(struct xe_device *xe,
>   * Initialize device info content that only depends on static driver_data passed to the driver at
>   * probe time from PCI ID table.
>   */
> -static void xe_info_init_early(struct xe_device *xe,
> +static int xe_info_init_early(struct xe_device *xe,
>  			       const struct xe_device_desc *desc,
>  			       const struct xe_subplatform_desc *subplatform_desc)
>  {
> +	int err;
> +
>  	xe->info.platform = desc->platform;
>  	xe->info.subplatform = subplatform_desc ?
>  		subplatform_desc->subplatform : XE_SUBPLATFORM_NONE;
> @@ -554,6 +556,12 @@ static void xe_info_init_early(struct xe_device *xe,
>  	xe->info.enable_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) &&
>  				  enable_display &&
>  				  desc->has_display;
> +
> +	err = xe_tile_init_early(xe_device_get_root_tile(xe), xe, 0);
> +	if (err)
> +		return err;
> +
> +	return 0;
>  }
>  
>  /*
> @@ -621,13 +629,15 @@ static int xe_info_init(struct xe_device *xe,
>  	 */
>  	xe->info.tile_count = 1 + graphics_desc->max_remote_tiles;
>  
> -	for_each_tile(tile, xe, id) {
> +	for_each_remote_tile(tile, xe, id) {
>  		int err;
>  
>  		err = xe_tile_init_early(tile, xe, id);
>  		if (err)
>  			return err;
> +	}
>  
> +	for_each_tile(tile, xe, id) {
>  		gt = tile->primary_gt;
>  		gt->info.id = xe->info.gt_count++;
>  		gt->info.type = XE_GT_TYPE_MAIN;
> @@ -727,7 +737,13 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>  	pci_set_master(pdev);
>  	devm_add_action(&pdev->dev, xe_pci_clear_master, pdev);
>  
> -	xe_info_init_early(xe, desc, subplatform_desc);
> +	err = xe_info_init_early(xe, desc, subplatform_desc);
> +	if (err)
> +		return err;
> +
> +	err = xe_device_probe_early(xe);
> +	if (err)
> +		return err;
>  
>  	err = xe_info_init(xe, desc, subplatform_desc);
>  	if (err)
> -- 
> 2.42.1
>