Re: [PATCH v2 9/9] drm/xe/pci: Introduce PCIe FLR

[Raag Jadav <raag.jadav@intel.com>]

On Fri, Feb 27, 2026 at 11:19:12PM +0530, Vivi, Rodrigo wrote:
> On Fri, 2026-02-27 at 22:30 +0530, Raag Jadav wrote:
> > With all the pieces in place, we can finally introduce PCIe Function
> > Level
> > Reset (FLR) handling which re-initializes hardware state without the
> > need
> > for reloading the driver from userspace. All VRAM contents are lost
> > along
> > with hardware state, so the driver takes care of recreating the
> > required
> > kernel bos as part of re-initialization, but user still needs to
> > recreate
> > user bos and reload context after PCIe FLR.
> > 
> > Signed-off-by: Raag Jadav <raag.jadav@intel.com>
> 
> Cc: Michał Winiarski <michal.winiarski@intel.com>
> Cc: Aravind Iddamsetty <aravind.iddamsetty@linux.intel.com>
> 
> Do we really have everything in place? I'm missing the handling of VF
> here. Hence we would have the same deadlock pointed out by Michal when
> Aravind attempted it last time, no?!

There's a whole lot of TODOs in this file, so perhaps poor choice of words.

> We probably need to use the guc relay communication here so PF can
> inform VF of the FLR, then stop all the workload and wedge the device,
> before the AER triggers a kill of the VM, no?!

I was hoping to have the base functionality in place and build stuff on
top, but I'll let you all make the final call here.

Raag

> > ---
> > v2: Spell out Function Level Reset (Jani)
> > ---
> >  drivers/gpu/drm/xe/Makefile     |   1 +
> >  drivers/gpu/drm/xe/xe_pci.c     |   1 +
> >  drivers/gpu/drm/xe/xe_pci.h     |   2 +
> >  drivers/gpu/drm/xe/xe_pci_err.c | 150
> > ++++++++++++++++++++++++++++++++
> >  4 files changed, 154 insertions(+)
> >  create mode 100644 drivers/gpu/drm/xe/xe_pci_err.c
> > 
> > diff --git a/drivers/gpu/drm/xe/Makefile
> > b/drivers/gpu/drm/xe/Makefile
> > index 7fc67c320086..bc468a9afc48 100644
> > --- a/drivers/gpu/drm/xe/Makefile
> > +++ b/drivers/gpu/drm/xe/Makefile
> > @@ -99,6 +99,7 @@ xe-y += xe_bb.o \
> >  	xe_page_reclaim.o \
> >  	xe_pat.o \
> >  	xe_pci.o \
> > +	xe_pci_err.o \
> >  	xe_pci_rebar.o \
> >  	xe_pcode.o \
> >  	xe_pm.o \
> > diff --git a/drivers/gpu/drm/xe/xe_pci.c
> > b/drivers/gpu/drm/xe/xe_pci.c
> > index 0a3bc5067a76..47a2f9de9d61 100644
> > --- a/drivers/gpu/drm/xe/xe_pci.c
> > +++ b/drivers/gpu/drm/xe/xe_pci.c
> > @@ -1301,6 +1301,7 @@ static struct pci_driver xe_pci_driver = {
> >  #ifdef CONFIG_PM_SLEEP
> >  	.driver.pm = &xe_pm_ops,
> >  #endif
> > +	.err_handler = &xe_pci_err_handlers,
> >  };
> >  
> >  /**
> > diff --git a/drivers/gpu/drm/xe/xe_pci.h
> > b/drivers/gpu/drm/xe/xe_pci.h
> > index 11bcc5fe2c5b..85e85e8508c3 100644
> > --- a/drivers/gpu/drm/xe/xe_pci.h
> > +++ b/drivers/gpu/drm/xe/xe_pci.h
> > @@ -8,6 +8,8 @@
> >  
> >  struct pci_dev;
> >  
> > +extern const struct pci_error_handlers xe_pci_err_handlers;
> > +
> >  int xe_register_pci_driver(void);
> >  void xe_unregister_pci_driver(void);
> >  struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev);
> > diff --git a/drivers/gpu/drm/xe/xe_pci_err.c
> > b/drivers/gpu/drm/xe/xe_pci_err.c
> > new file mode 100644
> > index 000000000000..16fc6a9f8289
> > --- /dev/null
> > +++ b/drivers/gpu/drm/xe/xe_pci_err.c
> > @@ -0,0 +1,150 @@
> > +// SPDX-License-Identifier: MIT
> > +/*
> > + * Copyright © 2026 Intel Corporation
> > + */
> > +
> > +#include "xe_bo_evict.h"
> > +#include "xe_device.h"
> > +#include "xe_gt.h"
> > +#include "xe_gt_idle.h"
> > +#include "xe_i2c.h"
> > +#include "xe_irq.h"
> > +#include "xe_late_bind_fw.h"
> > +#include "xe_pci.h"
> > +#include "xe_pcode.h"
> > +#include "xe_printk.h"
> > +#include "xe_pxp.h"
> > +#include "xe_wa.h"
> > +
> > +static int xe_flr_prepare(struct xe_device *xe)
> > +{
> > +	struct xe_gt *gt;
> > +	int err;
> > +	u8 id;
> > +
> > +	err = xe_pxp_pm_suspend(xe->pxp);
> > +	if (err)
> > +		return err;
> > +
> > +	xe_late_bind_wait_for_worker_completion(&xe->late_bind);
> > +
> > +	for_each_gt(gt, xe, id)
> > +		xe_gt_flr_prepare(gt);
> > +
> > +	xe_irq_disable(xe);
> > +
> > +	// TODO: Drop all user bos
> > +	xe_bo_pci_dev_remove_pinned(xe);
> > +
> > +	return 0;
> > +}
> > +
> > +static int xe_flr_done(struct xe_device *xe)
> > +{
> > +	struct xe_tile *tile;
> > +	struct xe_gt *gt;
> > +	int err;
> > +	u8 id;
> > +
> > +	for_each_gt(gt, xe, id)
> > +		xe_gt_idle_disable_c6(gt);
> > +
> > +	for_each_tile(tile, xe, id)
> > +		xe_wa_apply_tile_workarounds(tile);
> > +
> > +	err = xe_pcode_ready(xe, true);
> > +	if (err)
> > +		return err;
> > +
> > +	xe_device_assert_lmem_ready(xe);
> > +
> > +	err = xe_bo_restore_map(xe);
> > +	if (err)
> > +		return err;
> > +
> > +	for_each_gt(gt, xe, id) {
> > +		err = xe_gt_flr_done(gt);
> > +		if (err)
> > +			return err;
> > +	}
> > +
> > +	xe_i2c_pm_resume(xe, true);
> > +
> > +	xe_irq_resume(xe);
> > +
> > +	for_each_gt(gt, xe, id) {
> > +		err = xe_gt_resume(gt);
> > +		if (err)
> > +			return err;
> > +	}
> > +
> > +	xe_pxp_pm_resume(xe->pxp);
> > +
> > +	xe_late_bind_fw_load(&xe->late_bind);
> > +
> > +	return 0;
> > +}
> > +
> > +static void xe_pci_reset_prepare(struct pci_dev *pdev)
> > +{
> > +	struct xe_device *xe = pdev_to_xe_device(pdev);
> > +
> > +	/* TODO: Extend support as a follow-up */
> > +	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || pci_num_vf(pdev) ||
> > xe->info.probe_display) {
> > +		xe_err(xe, "PCIe FLR not supported\n");
> > +		return;
> > +	}
> > +
> > +	/* Wedge the device to prevent userspace access but don't
> > send the event yet */
> > +	atomic_set(&xe->wedged.flag, 1);
> > +
> > +	/*
> > +	 * The hardware could be in corrupted state and access
> > unreliable, but we try to
> > +	 * update data structures and cleanup any pending work to
> > avoid side effects during
> > +	 * PCIe FLR. This will be similar to xe_pm_suspend() flow
> > but without migration.
> > +	 */
> > +	if (xe_flr_prepare(xe)) {
> > +		xe_err(xe, "Failed to prepare for PCIe FLR\n");
> > +		return;
> > +	}
> > +
> > +	xe_info(xe, "Prepared for PCIe FLR\n");
> > +}
> > +
> > +static void xe_pci_reset_done(struct pci_dev *pdev)
> > +{
> > +	struct xe_device *xe = pdev_to_xe_device(pdev);
> > +
> > +	/* TODO: Extend support as a follow-up */
> > +	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || pci_num_vf(pdev) ||
> > xe->info.probe_display)
> > +		return;
> > +
> > +	if (!xe_device_wedged(xe)) {
> > +		xe_err(xe, "Device in unexpected state, re-
> > initialization aborted\n");
> > +		return;
> > +	}
> > +
> > +	/*
> > +	 * We already have the data structures intact, so try to re-
> > initialize the device.
> > +	 * This will be similar to xe_pm_resume() flow, except we'll
> > also need to recreate
> > +	 * all VRAM contents.
> > +	 */
> > +	if (xe_flr_done(xe)) {
> > +		xe_err(xe, "Re-initialization failed\n");
> > +		return;
> > +	}
> > +
> > +	/* Unwedge to allow userspace access */
> > +	atomic_set(&xe->wedged.flag, 0);
> > +
> > +	xe_info(xe, "Re-initialization success\n");
> > +}
> > +
> > +/*
> > + * PCIe Function Level Reset (FLR) support only.
> > + * TODO: Add PCIe error handlers using similar flow.
> > + */
> > +const struct pci_error_handlers xe_pci_err_handlers = {
> > +	.reset_prepare = xe_pci_reset_prepare,
> > +	.reset_done = xe_pci_reset_done,
> > +};