Re: [PATCH v2 9/9] drm/xe/pci: Introduce PCIe FLR

Rodrigo Vivi
2026-03-02T17:00:02Z

On Sat, Feb 28, 2026 at 06:24:36AM +0100, Raag Jadav wrote:
> On Fri, Feb 27, 2026 at 11:19:12PM +0530, Vivi, Rodrigo wrote:
> > On Fri, 2026-02-27 at 22:30 +0530, Raag Jadav wrote:
> > > With all the pieces in place, we can finally introduce PCIe Function
> > > Level
> > > Reset (FLR) handling which re-initializes hardware state without the
> > > need
> > > for reloading the driver from userspace. All VRAM contents are lost
> > > along
> > > with hardware state, so the driver takes care of recreating the
> > > required
> > > kernel bos as part of re-initialization, but user still needs to
> > > recreate
> > > user bos and reload context after PCIe FLR.
> > >
> > > Signed-off-by: Raag Jadav <raag.jadav@intel.com>
> >
> > Cc: Michał Winiarski <michal.winiarski@intel.com>
> > Cc: Aravind Iddamsetty <aravind.iddamsetty@linux.intel.com>
> >
> > Do we really have everything in place? I'm missing the handling of VF
> > here. Hence we would have the same deadlock pointed out by Michal when
> > Aravind attempted it last time, no?!
>
> There's a whole lot of TODOs in this file, so perhaps poor choice of words.
>
> > We probably need to use the guc relay communication here so PF can
> > inform VF of the FLR, then stop all the workload and wedge the device,
> > before the AER triggers a kill of the VM, no?!
>
> I was hoping to have the base functionality in place and build stuff on
> top, but I'll let you all make the final call here.

I definitely support this approach. Let's gets the basic in and build on top.

>
> Raag
>
> > > ---
> > > v2: Spell out Function Level Reset (Jani)
> > > ---
> > >  drivers/gpu/drm/xe/Makefile     |   1 +
> > >  drivers/gpu/drm/xe/xe_pci.c     |   1 +
> > >  drivers/gpu/drm/xe/xe_pci.h     |   2 +
> > >  drivers/gpu/drm/xe/xe_pci_err.c | 150
> > > ++++++++++++++++++++++++++++++++
> > >  4 files changed, 154 insertions(+)
> > >  create mode 100644 drivers/gpu/drm/xe/xe_pci_err.c
> > >
> > > diff --git a/drivers/gpu/drm/xe/Makefile
> > > b/drivers/gpu/drm/xe/Makefile
> > > index 7fc67c320086..bc468a9afc48 100644
> > > --- a/drivers/gpu/drm/xe/Makefile
> > > +++ b/drivers/gpu/drm/xe/Makefile
> > > @@ -99,6 +99,7 @@ xe-y += xe_bb.o \
> > >   xe_page_reclaim.o \
> > >   xe_pat.o \
> > >   xe_pci.o \
> > > + xe_pci_err.o \
> > >   xe_pci_rebar.o \
> > >   xe_pcode.o \
> > >   xe_pm.o \
> > > diff --git a/drivers/gpu/drm/xe/xe_pci.c
> > > b/drivers/gpu/drm/xe/xe_pci.c
> > > index 0a3bc5067a76..47a2f9de9d61 100644
> > > --- a/drivers/gpu/drm/xe/xe_pci.c
> > > +++ b/drivers/gpu/drm/xe/xe_pci.c
> > > @@ -1301,6 +1301,7 @@ static struct pci_driver xe_pci_driver = {
> > >  #ifdef CONFIG_PM_SLEEP
> > >   .driver.pm = &xe_pm_ops,
> > >  #endif
> > > + .err_handler = &xe_pci_err_handlers,
> > >  };
> > >
> > >  /**
> > > diff --git a/drivers/gpu/drm/xe/xe_pci.h
> > > b/drivers/gpu/drm/xe/xe_pci.h
> > > index 11bcc5fe2c5b..85e85e8508c3 100644
> > > --- a/drivers/gpu/drm/xe/xe_pci.h
> > > +++ b/drivers/gpu/drm/xe/xe_pci.h
> > > @@ -8,6 +8,8 @@
> > >
> > >  struct pci_dev;
> > >
> > > +extern const struct pci_error_handlers xe_pci_err_handlers;
> > > +
> > >  int xe_register_pci_driver(void);
> > >  void xe_unregister_pci_driver(void);
> > >  struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev);
> > > diff --git a/drivers/gpu/drm/xe/xe_pci_err.c
> > > b/drivers/gpu/drm/xe/xe_pci_err.c
> > > new file mode 100644
> > > index 000000000000..16fc6a9f8289
> > > --- /dev/null
> > > +++ b/drivers/gpu/drm/xe/xe_pci_err.c
> > > @@ -0,0 +1,150 @@
> > > +// SPDX-License-Identifier: MIT
> > > +/*
> > > + * Copyright © 2026 Intel Corporation
> > > + */
> > > +
> > > +#include "xe_bo_evict.h"
> > > +#include "xe_device.h"
> > > +#include "xe_gt.h"
> > > +#include "xe_gt_idle.h"
> > > +#include "xe_i2c.h"
> > > +#include "xe_irq.h"
> > > +#include "xe_late_bind_fw.h"
> > > +#include "xe_pci.h"
> > > +#include "xe_pcode.h"
> > > +#include "xe_printk.h"
> > > +#include "xe_pxp.h"
> > > +#include "xe_wa.h"
> > > +
> > > +static int xe_flr_prepare(struct xe_device *xe)
> > > +{
> > > + struct xe_gt *gt;
> > > + int err;
> > > + u8 id;
> > > +
> > > + err = xe_pxp_pm_suspend(xe->pxp);
> > > + if (err)
> > > + return err;
> > > +
> > > + xe_late_bind_wait_for_worker_completion(&xe->late_bind);
> > > +
> > > + for_each_gt(gt, xe, id)
> > > + xe_gt_flr_prepare(gt);
> > > +
> > > + xe_irq_disable(xe);
> > > +
> > > + // TODO: Drop all user bos
> > > + xe_bo_pci_dev_remove_pinned(xe);
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static int xe_flr_done(struct xe_device *xe)
> > > +{
> > > + struct xe_tile *tile;
> > > + struct xe_gt *gt;
> > > + int err;
> > > + u8 id;
> > > +
> > > + for_each_gt(gt, xe, id)
> > > + xe_gt_idle_disable_c6(gt);
> > > +
> > > + for_each_tile(tile, xe, id)
> > > + xe_wa_apply_tile_workarounds(tile);
> > > +
> > > + err = xe_pcode_ready(xe, true);
> > > + if (err)
> > > + return err;
> > > +
> > > + xe_device_assert_lmem_ready(xe);
> > > +
> > > + err = xe_bo_restore_map(xe);
> > > + if (err)
> > > + return err;
> > > +
> > > + for_each_gt(gt, xe, id) {
> > > + err = xe_gt_flr_done(gt);
> > > + if (err)
> > > + return err;
> > > + }
> > > +
> > > + xe_i2c_pm_resume(xe, true);
> > > +
> > > + xe_irq_resume(xe);
> > > +
> > > + for_each_gt(gt, xe, id) {
> > > + err = xe_gt_resume(gt);
> > > + if (err)
> > > + return err;
> > > + }
> > > +
> > > + xe_pxp_pm_resume(xe->pxp);
> > > +
> > > + xe_late_bind_fw_load(&xe->late_bind);
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static void xe_pci_reset_prepare(struct pci_dev *pdev)
> > > +{
> > > + struct xe_device *xe = pdev_to_xe_device(pdev);
> > > +
> > > + /* TODO: Extend support as a follow-up */
> > > + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || pci_num_vf(pdev) ||
> > > xe->info.probe_display) {
> > > + xe_err(xe, "PCIe FLR not supported\n");
> > > + return;
> > > + }
> > > +
> > > + /* Wedge the device to prevent userspace access but don't
> > > send the event yet */
> > > + atomic_set(&xe->wedged.flag, 1);
> > > +
> > > + /*
> > > + * The hardware could be in corrupted state and access
> > > unreliable, but we try to
> > > + * update data structures and cleanup any pending work to
> > > avoid side effects during
> > > + * PCIe FLR. This will be similar to xe_pm_suspend() flow
> > > but without migration.
> > > + */
> > > + if (xe_flr_prepare(xe)) {
> > > + xe_err(xe, "Failed to prepare for PCIe FLR\n");
> > > + return;
> > > + }
> > > +
> > > + xe_info(xe, "Prepared for PCIe FLR\n");
> > > +}
> > > +
> > > +static void xe_pci_reset_done(struct pci_dev *pdev)
> > > +{
> > > + struct xe_device *xe = pdev_to_xe_device(pdev);
> > > +
> > > + /* TODO: Extend support as a follow-up */
> > > + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || pci_num_vf(pdev) ||
> > > xe->info.probe_display)
> > > + return;
> > > +
> > > + if (!xe_device_wedged(xe)) {
> > > + xe_err(xe, "Device in unexpected state, re-
> > > initialization aborted\n");
> > > + return;
> > > + }
> > > +
> > > + /*
> > > + * We already have the data structures intact, so try to re-
> > > initialize the device.
> > > + * This will be similar to xe_pm_resume() flow, except we'll
> > > also need to recreate
> > > + * all VRAM contents.
> > > + */
> > > + if (xe_flr_done(xe)) {
> > > + xe_err(xe, "Re-initialization failed\n");
> > > + return;
> > > + }
> > > +
> > > + /* Unwedge to allow userspace access */
> > > + atomic_set(&xe->wedged.flag, 0);
> > > +
> > > + xe_info(xe, "Re-initialization success\n");
> > > +}
> > > +
> > > +/*
> > > + * PCIe Function Level Reset (FLR) support only.
> > > + * TODO: Add PCIe error handlers using similar flow.
> > > + */
> > > +const struct pci_error_handlers xe_pci_err_handlers = {
> > > + .reset_prepare = xe_pci_reset_prepare,
> > > + .reset_done = xe_pci_reset_done,
> > > +};