* [PATCH v2 1/2] ppc/spapr: Preserve VFIO MSI-X shadow across EEH reset
2026-10-27 16:02 [PATCH v2 0/2] ppc/spapr: Fix MSI-X shadow and BAR mmap handling during EEH PE reset Narayana Murty N
@ 2026-10-27 16:02 ` Narayana Murty N
2026-10-27 16:02 ` [PATCH v2 2/2] ppc/spapr: Disable VFIO BAR mmap during EEH PE reset Narayana Murty N
1 sibling, 0 replies; 5+ messages in thread
From: Narayana Murty N @ 2026-10-27 16:02 UTC (permalink / raw)
To: qemu-devel, qemu-ppc, sbhat, mahesh, sourabhjain, adityag,
npiggin, harshpb, alex, clg
Cc: amachhiw, hbathini, shivangu, anushree.mathur
On pSeries, the MSI-X table shadow in QEMU is not sourced from the
physical device. It is populated by sPAPR RTAS through the
ibm,change-msi call path:
ibm,change-msi
-> spapr_msi_setmsg()
-> msix_set_message() /* writes each shadow entry */
EEH recovery does not necessarily cause the guest to re-issue
ibm,change-msi. The guest VFIO PCI driver restores interrupt delivery
by writing MSI-X Enable = 1 directly via config space. QEMU then
dispatches through:
vfio_msix_enable()
-> vfio_msix_vector_do_use() /* re-arms KVM irqfd from shadow */
The existing EEH pre-reset helper calls msix_reset() after clearing
MSI-X Enable. msix_reset() zeroes every shadow entry. Without the
shadow, vfio_msix_vector_do_use() cannot reconstruct the KVM irqfd
routes and device interrupts do not recover after EEH.
Fix the pre-reset helper:
1. Remove the msix_reset() call. The shadow must survive the PE reset
so that interrupt delivery can be restored when the guest re-enables
MSI-X without repeating ibm,change-msi.
2. Read the MSI-X flags from the cached pdev->config shadow instead of
calling pci_host_config_read_common(). The PE may already be frozen
when EEH pre-reset runs; reading from physical config space could
fault.
3. Rename the helpers from clear_dev_msix / clear_bus_msix to
prepare_dev / prepare_bus to reflect that they no longer clear any
state, and fix the pre-existing tab-indented bodies to use spaces
consistently with the surrounding code.
4. Switch the VFIO device type check from the "vfio-pci" string literal
to TYPE_VFIO_PCI_DEVICE, matching the convention used by other
non-VFIO source files that access VFIOPCIDevice (e.g.
hw/s390x/s390-pci-vfio.c). Add the corresponding #include for
hw/vfio/pci.h.
The guest recovery sequence after this fix:
EEH pre-reset:
MSI-X Enable -> 0 (pci_host_config_write_common)
-> vfio_msix_disable()
-> vfio_msix_vector_release() for each vector
-> KVM irqfd routes removed
-> VFIO MSIX_IRQ_INDEX disabled
shadow entries unchanged
PE reset + PE configure
guest slot_reset() / PCI state restore:
MSI-X Enable -> 1 (config-space write)
-> vfio_msix_enable()
-> vfio_msix_vector_do_use() for each vector
-> shadow entries present -> KVM irqfd routes re-armed
interrupts recover
Signed-off-by: Narayana Murty N <nnmlinux@linux.ibm.com>
---
hw/ppc/spapr_pci_vfio.c | 56 ++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 23 deletions(-)
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index a748a0bf4c..1fa8847031 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -25,6 +25,7 @@
#include "hw/pci/msix.h"
#include "hw/pci/pci_device.h"
#include "hw/vfio/vfio-container-legacy.h"
+#include "hw/vfio/pci.h"
#include "qemu/error-report.h"
#include CONFIG_DEVICES /* CONFIG_VFIO_PCI */
@@ -233,47 +234,56 @@ int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state)
return RTAS_OUT_SUCCESS;
}
-static void spapr_phb_vfio_eeh_clear_dev_msix(PCIBus *bus,
- PCIDevice *pdev,
- void *opaque)
+/*
+ * Prepare a single VFIO PCI device for an EEH PE hot or fundamental reset.
+ *
+ * On pSeries the MSI-X table shadow is populated by ibm,change-msi via
+ * spapr_msi_setmsg() -> msix_set_message(). EEH recovery does not
+ * necessarily re-issue ibm,change-msi; the guest restores MSI-X Enable
+ * directly, causing QEMU to dispatch through vfio_msix_enable() ->
+ * vfio_msix_vector_do_use(), which re-arms the KVM irqfd routes from the
+ * existing shadow entries.
+ *
+ * Therefore, msix_reset() must NOT be called here. Calling it would wipe
+ * those shadow entries and prevent interrupt delivery after EEH recovery.
+ *
+ * Instead, clear MSI-X Enable using the cached pdev->config shadow (avoiding
+ * a read from potentially frozen device config space) and write through
+ * pci_host_config_write_common() so that the VFIO config-write handler calls
+ * vfio_msix_disable(), cleanly releasing vectors and KVM irqfd routes while
+ * leaving the shadow intact.
+ */
+static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
+ PCIDevice *pdev,
+ void *opaque)
{
- /* Check if the device is VFIO PCI device */
- if (!object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+ uint16_t flags;
+
+ if (!object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI_DEVICE)) {
return;
}
- /*
- * The MSIx table will be cleaned out by reset. We need
- * disable it so that it can be reenabled properly. Also,
- * the cached MSIx table should be cleared as it's not
- * reflecting the contents in hardware.
- */
if (msix_enabled(pdev)) {
- uint16_t flags;
-
- flags = pci_host_config_read_common(pdev,
- pdev->msix_cap + PCI_MSIX_FLAGS,
- pci_config_size(pdev), 2);
+ flags = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
flags &= ~PCI_MSIX_FLAGS_ENABLE;
pci_host_config_write_common(pdev,
pdev->msix_cap + PCI_MSIX_FLAGS,
pci_config_size(pdev), flags, 2);
}
-
- msix_reset(pdev);
}
-static void spapr_phb_vfio_eeh_clear_bus_msix(PCIBus *bus, void *opaque)
+static void spapr_phb_vfio_eeh_prepare_bus(PCIBus *bus, void *opaque)
{
- pci_for_each_device_under_bus(bus, spapr_phb_vfio_eeh_clear_dev_msix,
- NULL);
+ pci_for_each_device_under_bus(bus,
+ spapr_phb_vfio_eeh_prepare_dev,
+ NULL);
}
static void spapr_phb_vfio_eeh_pre_reset(SpaprPhbState *sphb)
{
- PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
+ PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
- pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
+ pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_prepare_bus, NULL);
}
int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
--
2.54.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH v2 2/2] ppc/spapr: Disable VFIO BAR mmap during EEH PE reset
2026-10-27 16:02 [PATCH v2 0/2] ppc/spapr: Fix MSI-X shadow and BAR mmap handling during EEH PE reset Narayana Murty N
2026-10-27 16:02 ` [PATCH v2 1/2] ppc/spapr: Preserve VFIO MSI-X shadow across EEH reset Narayana Murty N
@ 2026-10-27 16:02 ` Narayana Murty N
2026-10-27 16:08 ` Cédric Le Goater
1 sibling, 1 reply; 5+ messages in thread
From: Narayana Murty N @ 2026-10-27 16:02 UTC (permalink / raw)
To: qemu-devel, qemu-ppc, sbhat, mahesh, sourabhjain, adityag,
npiggin, harshpb, alex, clg
Cc: amachhiw, hbathini, shivangu, anushree.mathur
Direct mmap-backed BAR accesses are not reliable while the PE is
undergoing EEH reset. Temporarily disable the mmap-backed BAR
subregions so accesses fall back to the VFIO region read/write path
during the reset window.
After VFIO_EEH_PE_CONFIGURE succeeds, restore normal VFIO mmap
behavior.
To keep VFIOPCIDevice internals encapsulated and avoid direct
manipulation of vdev->intx.mmap_timer and vdev->bars[] from
hw/ppc/spapr_pci_vfio.c, introduce two VFIO PCI helpers in
hw/vfio/pci.c:
- vfio_pci_eeh_pre_reset(): cancels any pending INTx mmap re-enable
timer and disables BAR mmaps across all BAR regions.
- vfio_pci_eeh_post_configure(): restores normal VFIO mmap policy.
If INTx is still pending, the INTx mmap timer is re-armed so mmap
remains disabled until the quiet period completes; otherwise,
BAR mmaps are immediately re-enabled.
The pSeries EEH handlers in hw/ppc/spapr_pci_vfio.c call
vfio_pci_eeh_pre_reset() during the pre-reset device preparation
(after MSI-X teardown) and call vfio_pci_eeh_post_configure() only
after VFIO_EEH_PE_CONFIGURE succeeds. On configure failure, BAR mmaps
remain disabled. RTAS_SLOT_RESET_DEACTIVATE does not invoke either
helper.
Signed-off-by: Narayana Murty N <nnmlinux@linux.ibm.com>
---
hw/ppc/spapr_pci_vfio.c | 60 ++++++++++++++++++++++++++++++++++++++---
hw/vfio/pci.c | 32 ++++++++++++++++++++++
hw/vfio/pci.h | 2 ++
3 files changed, 90 insertions(+), 4 deletions(-)
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index 1fa8847031..ab78489fa5 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -254,8 +254,8 @@ int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state)
* leaving the shadow intact.
*/
static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
- PCIDevice *pdev,
- void *opaque)
+ PCIDevice *pdev,
+ void *opaque)
{
uint16_t flags;
@@ -263,13 +263,27 @@ static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
return;
}
+ /*
+ * Disable MSI-X through the VFIO config-write path so the
+ * active interrupt routes are released while preserving the
+ * pSeries RTAS-managed MSI-X software shadow.
+ */
if (msix_enabled(pdev)) {
- flags = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
+ flags = pci_get_word(pdev->config +
+ pdev->msix_cap + PCI_MSIX_FLAGS);
+
flags &= ~PCI_MSIX_FLAGS_ENABLE;
+
pci_host_config_write_common(pdev,
pdev->msix_cap + PCI_MSIX_FLAGS,
- pci_config_size(pdev), flags, 2);
+ pci_config_size(pdev),
+ flags, 2);
}
+
+ /*
+ * This must be the final preparation step before the PE reset.
+ */
+ vfio_pci_eeh_pre_reset(pdev);
}
static void spapr_phb_vfio_eeh_prepare_bus(PCIBus *bus, void *opaque)
@@ -286,6 +300,37 @@ static void spapr_phb_vfio_eeh_pre_reset(SpaprPhbState *sphb)
pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_prepare_bus, NULL);
}
+/*
+ * Restore normal VFIO mmap policy for a single VFIO PCI device after a
+ * successful EEH PE configure. Called only on the configure success path;
+ * on failure mmaps remain disabled until the next hot/fundamental reset
+ * attempt.
+ */
+static void spapr_phb_vfio_eeh_post_configure_dev(PCIBus *bus,
+ PCIDevice *pdev,
+ void *opaque)
+{
+ if (!object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI_DEVICE)) {
+ return;
+ }
+
+ vfio_pci_eeh_post_configure(pdev);
+}
+
+static void spapr_phb_vfio_eeh_post_configure_bus(PCIBus *bus, void *opaque)
+{
+ pci_for_each_device_under_bus(bus,
+ spapr_phb_vfio_eeh_post_configure_dev,
+ NULL);
+}
+
+static void spapr_phb_vfio_eeh_post_configure(SpaprPhbState *sphb)
+{
+ PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
+
+ pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_post_configure_bus, NULL);
+}
+
int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
{
uint32_t op;
@@ -293,6 +338,11 @@ int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
switch (option) {
case RTAS_SLOT_RESET_DEACTIVATE:
+ /*
+ * Deactivate does not perform a full PE reset; BAR mmaps were already
+ * disabled by the preceding HOT or FUNDAMENTAL reset call and must not
+ * be re-enabled here.
+ */
op = VFIO_EEH_PE_RESET_DEACTIVATE;
break;
case RTAS_SLOT_RESET_HOT:
@@ -324,6 +374,8 @@ int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb)
return RTAS_OUT_PARAM_ERROR;
}
+ spapr_phb_vfio_eeh_post_configure(sphb);
+
return RTAS_OUT_SUCCESS;
}
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 428ab2f069..931df22add 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -1949,6 +1949,38 @@ static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
}
}
+/*
+ * Prevent a pending INTx mmap timer from restoring BAR mmap while
+ * the PE is undergoing EEH reset. Keep BAR accesses on the VFIO
+ * region read/write path until PE configuration completes.
+ */
+void vfio_pci_eeh_pre_reset(PCIDevice *pdev)
+{
+ VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
+
+ if (vdev->intx.mmap_timer) {
+ timer_del(vdev->intx.mmap_timer);
+ }
+
+ vfio_mmap_set_enabled(vdev, false);
+}
+
+void vfio_pci_eeh_post_configure(PCIDevice *pdev)
+{
+ VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
+
+ if (vdev->interrupt == VFIO_INT_INTx && vdev->intx.pending) {
+ if (vdev->intx.mmap_timer && vdev->intx.mmap_timeout) {
+ timer_mod(vdev->intx.mmap_timer,
+ qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+ vdev->intx.mmap_timeout);
+ }
+ return;
+ }
+
+ vfio_mmap_set_enabled(vdev, true);
+}
+
static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
{
VFIOBAR *bar = &vdev->bars[nr];
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index c9ab949870..63fa378fbc 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -266,6 +266,8 @@ struct vfio_pci_hot_reset_info;
void vfio_pci_pre_reset(VFIOPCIDevice *vdev);
void vfio_pci_post_reset(VFIOPCIDevice *vdev);
+void vfio_pci_eeh_pre_reset(PCIDevice *pdev);
+void vfio_pci_eeh_post_configure(PCIDevice *pdev);
bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name);
int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
struct vfio_pci_hot_reset_info **info_p);
--
2.54.0
^ permalink raw reply related [flat|nested] 5+ messages in thread