Re: [PATCH v2 9/9] drm/xe/pci: Introduce PCIe FLR

["Vivi, Rodrigo" <rodrigo.vivi@intel.com>]

On Fri, 2026-02-27 at 22:30 +0530, Raag Jadav wrote:
> With all the pieces in place, we can finally introduce PCIe Function
> Level
> Reset (FLR) handling which re-initializes hardware state without the
> need
> for reloading the driver from userspace. All VRAM contents are lost
> along
> with hardware state, so the driver takes care of recreating the
> required
> kernel bos as part of re-initialization, but user still needs to
> recreate
> user bos and reload context after PCIe FLR.
> 
> Signed-off-by: Raag Jadav <raag.jadav@intel.com>

Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Aravind Iddamsetty <aravind.iddamsetty@linux.intel.com>

Do we really have everything in place? I'm missing the handling of VF
here. Hence we would have the same deadlock pointed out by Michal when
Aravind attempted it last time, no?!

We probably need to use the guc relay communication here so PF can
inform VF of the FLR, then stop all the workload and wedge the device,
before the AER triggers a kill of the VM, no?!

> ---
> v2: Spell out Function Level Reset (Jani)
> ---
>  drivers/gpu/drm/xe/Makefile     |   1 +
>  drivers/gpu/drm/xe/xe_pci.c     |   1 +
>  drivers/gpu/drm/xe/xe_pci.h     |   2 +
>  drivers/gpu/drm/xe/xe_pci_err.c | 150
> ++++++++++++++++++++++++++++++++
>  4 files changed, 154 insertions(+)
>  create mode 100644 drivers/gpu/drm/xe/xe_pci_err.c
> 
> diff --git a/drivers/gpu/drm/xe/Makefile
> b/drivers/gpu/drm/xe/Makefile
> index 7fc67c320086..bc468a9afc48 100644
> --- a/drivers/gpu/drm/xe/Makefile
> +++ b/drivers/gpu/drm/xe/Makefile
> @@ -99,6 +99,7 @@ xe-y += xe_bb.o \
>  	xe_page_reclaim.o \
>  	xe_pat.o \
>  	xe_pci.o \
> +	xe_pci_err.o \
>  	xe_pci_rebar.o \
>  	xe_pcode.o \
>  	xe_pm.o \
> diff --git a/drivers/gpu/drm/xe/xe_pci.c
> b/drivers/gpu/drm/xe/xe_pci.c
> index 0a3bc5067a76..47a2f9de9d61 100644
> --- a/drivers/gpu/drm/xe/xe_pci.c
> +++ b/drivers/gpu/drm/xe/xe_pci.c
> @@ -1301,6 +1301,7 @@ static struct pci_driver xe_pci_driver = {
>  #ifdef CONFIG_PM_SLEEP
>  	.driver.pm = &xe_pm_ops,
>  #endif
> +	.err_handler = &xe_pci_err_handlers,
>  };
>  
>  /**
> diff --git a/drivers/gpu/drm/xe/xe_pci.h
> b/drivers/gpu/drm/xe/xe_pci.h
> index 11bcc5fe2c5b..85e85e8508c3 100644
> --- a/drivers/gpu/drm/xe/xe_pci.h
> +++ b/drivers/gpu/drm/xe/xe_pci.h
> @@ -8,6 +8,8 @@
>  
>  struct pci_dev;
>  
> +extern const struct pci_error_handlers xe_pci_err_handlers;
> +
>  int xe_register_pci_driver(void);
>  void xe_unregister_pci_driver(void);
>  struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev);
> diff --git a/drivers/gpu/drm/xe/xe_pci_err.c
> b/drivers/gpu/drm/xe/xe_pci_err.c
> new file mode 100644
> index 000000000000..16fc6a9f8289
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_pci_err.c
> @@ -0,0 +1,150 @@
> +// SPDX-License-Identifier: MIT
> +/*
> + * Copyright © 2026 Intel Corporation
> + */
> +
> +#include "xe_bo_evict.h"
> +#include "xe_device.h"
> +#include "xe_gt.h"
> +#include "xe_gt_idle.h"
> +#include "xe_i2c.h"
> +#include "xe_irq.h"
> +#include "xe_late_bind_fw.h"
> +#include "xe_pci.h"
> +#include "xe_pcode.h"
> +#include "xe_printk.h"
> +#include "xe_pxp.h"
> +#include "xe_wa.h"
> +
> +static int xe_flr_prepare(struct xe_device *xe)
> +{
> +	struct xe_gt *gt;
> +	int err;
> +	u8 id;
> +
> +	err = xe_pxp_pm_suspend(xe->pxp);
> +	if (err)
> +		return err;
> +
> +	xe_late_bind_wait_for_worker_completion(&xe->late_bind);
> +
> +	for_each_gt(gt, xe, id)
> +		xe_gt_flr_prepare(gt);
> +
> +	xe_irq_disable(xe);
> +
> +	// TODO: Drop all user bos
> +	xe_bo_pci_dev_remove_pinned(xe);
> +
> +	return 0;
> +}
> +
> +static int xe_flr_done(struct xe_device *xe)
> +{
> +	struct xe_tile *tile;
> +	struct xe_gt *gt;
> +	int err;
> +	u8 id;
> +
> +	for_each_gt(gt, xe, id)
> +		xe_gt_idle_disable_c6(gt);
> +
> +	for_each_tile(tile, xe, id)
> +		xe_wa_apply_tile_workarounds(tile);
> +
> +	err = xe_pcode_ready(xe, true);
> +	if (err)
> +		return err;
> +
> +	xe_device_assert_lmem_ready(xe);
> +
> +	err = xe_bo_restore_map(xe);
> +	if (err)
> +		return err;
> +
> +	for_each_gt(gt, xe, id) {
> +		err = xe_gt_flr_done(gt);
> +		if (err)
> +			return err;
> +	}
> +
> +	xe_i2c_pm_resume(xe, true);
> +
> +	xe_irq_resume(xe);
> +
> +	for_each_gt(gt, xe, id) {
> +		err = xe_gt_resume(gt);
> +		if (err)
> +			return err;
> +	}
> +
> +	xe_pxp_pm_resume(xe->pxp);
> +
> +	xe_late_bind_fw_load(&xe->late_bind);
> +
> +	return 0;
> +}
> +
> +static void xe_pci_reset_prepare(struct pci_dev *pdev)
> +{
> +	struct xe_device *xe = pdev_to_xe_device(pdev);
> +
> +	/* TODO: Extend support as a follow-up */
> +	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || pci_num_vf(pdev) ||
> xe->info.probe_display) {
> +		xe_err(xe, "PCIe FLR not supported\n");
> +		return;
> +	}
> +
> +	/* Wedge the device to prevent userspace access but don't
> send the event yet */
> +	atomic_set(&xe->wedged.flag, 1);
> +
> +	/*
> +	 * The hardware could be in corrupted state and access
> unreliable, but we try to
> +	 * update data structures and cleanup any pending work to
> avoid side effects during
> +	 * PCIe FLR. This will be similar to xe_pm_suspend() flow
> but without migration.
> +	 */
> +	if (xe_flr_prepare(xe)) {
> +		xe_err(xe, "Failed to prepare for PCIe FLR\n");
> +		return;
> +	}
> +
> +	xe_info(xe, "Prepared for PCIe FLR\n");
> +}
> +
> +static void xe_pci_reset_done(struct pci_dev *pdev)
> +{
> +	struct xe_device *xe = pdev_to_xe_device(pdev);
> +
> +	/* TODO: Extend support as a follow-up */
> +	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || pci_num_vf(pdev) ||
> xe->info.probe_display)
> +		return;
> +
> +	if (!xe_device_wedged(xe)) {
> +		xe_err(xe, "Device in unexpected state, re-
> initialization aborted\n");
> +		return;
> +	}
> +
> +	/*
> +	 * We already have the data structures intact, so try to re-
> initialize the device.
> +	 * This will be similar to xe_pm_resume() flow, except we'll
> also need to recreate
> +	 * all VRAM contents.
> +	 */
> +	if (xe_flr_done(xe)) {
> +		xe_err(xe, "Re-initialization failed\n");
> +		return;
> +	}
> +
> +	/* Unwedge to allow userspace access */
> +	atomic_set(&xe->wedged.flag, 0);
> +
> +	xe_info(xe, "Re-initialization success\n");
> +}
> +
> +/*
> + * PCIe Function Level Reset (FLR) support only.
> + * TODO: Add PCIe error handlers using similar flow.
> + */
> +const struct pci_error_handlers xe_pci_err_handlers = {
> +	.reset_prepare = xe_pci_reset_prepare,
> +	.reset_done = xe_pci_reset_done,
> +};