[PATCH v2 1/2] ppc/spapr: Preserve VFIO MSI-X shadow across EEH reset

[Narayana Murty N <nnmlinux@linux.ibm.com>]

On pSeries, the MSI-X table shadow in QEMU is not sourced from the
physical device.  It is populated by sPAPR RTAS through the
ibm,change-msi call path:

  ibm,change-msi
    -> spapr_msi_setmsg()
      -> msix_set_message()      /* writes each shadow entry */

EEH recovery does not necessarily cause the guest to re-issue
ibm,change-msi.  The guest VFIO PCI driver restores interrupt delivery
by writing MSI-X Enable = 1 directly via config space.  QEMU then
dispatches through:

  vfio_msix_enable()
    -> vfio_msix_vector_do_use()  /* re-arms KVM irqfd from shadow */

The existing EEH pre-reset helper calls msix_reset() after clearing
MSI-X Enable.  msix_reset() zeroes every shadow entry.  Without the
shadow, vfio_msix_vector_do_use() cannot reconstruct the KVM irqfd
routes and device interrupts do not recover after EEH.

Fix the pre-reset helper:

1. Remove the msix_reset() call.  The shadow must survive the PE reset
   so that interrupt delivery can be restored when the guest re-enables
   MSI-X without repeating ibm,change-msi.

2. Read the MSI-X flags from the cached pdev->config shadow instead of
   calling pci_host_config_read_common().  The PE may already be frozen
   when EEH pre-reset runs; reading from physical config space could
   fault.

3. Rename the helpers from clear_dev_msix / clear_bus_msix to
   prepare_dev / prepare_bus to reflect that they no longer clear any
   state, and fix the pre-existing tab-indented bodies to use spaces
   consistently with the surrounding code.

4. Switch the VFIO device type check from the "vfio-pci" string literal
   to TYPE_VFIO_PCI_DEVICE, matching the convention used by other
   non-VFIO source files that access VFIOPCIDevice (e.g.
   hw/s390x/s390-pci-vfio.c).  Add the corresponding #include for
   hw/vfio/pci.h.

The guest recovery sequence after this fix:

  EEH pre-reset:
    MSI-X Enable -> 0 (pci_host_config_write_common)
      -> vfio_msix_disable()
         -> vfio_msix_vector_release() for each vector
         -> KVM irqfd routes removed
         -> VFIO MSIX_IRQ_INDEX disabled
    shadow entries unchanged

  PE reset + PE configure

  guest slot_reset() / PCI state restore:
    MSI-X Enable -> 1 (config-space write)
      -> vfio_msix_enable()
         -> vfio_msix_vector_do_use() for each vector
         -> shadow entries present -> KVM irqfd routes re-armed
    interrupts recover

Signed-off-by: Narayana Murty N <nnmlinux@linux.ibm.com>
---
 hw/ppc/spapr_pci_vfio.c | 56 ++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index a748a0bf4c..1fa8847031 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -25,6 +25,7 @@
 #include "hw/pci/msix.h"
 #include "hw/pci/pci_device.h"
 #include "hw/vfio/vfio-container-legacy.h"
+#include "hw/vfio/pci.h"
 #include "qemu/error-report.h"
 #include CONFIG_DEVICES /* CONFIG_VFIO_PCI */
 
@@ -233,47 +234,56 @@ int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state)
     return RTAS_OUT_SUCCESS;
 }
 
-static void spapr_phb_vfio_eeh_clear_dev_msix(PCIBus *bus,
-                                              PCIDevice *pdev,
-                                              void *opaque)
+/*
+ * Prepare a single VFIO PCI device for an EEH PE hot or fundamental reset.
+ *
+ * On pSeries the MSI-X table shadow is populated by ibm,change-msi via
+ * spapr_msi_setmsg() -> msix_set_message().  EEH recovery does not
+ * necessarily re-issue ibm,change-msi; the guest restores MSI-X Enable
+ * directly, causing QEMU to dispatch through vfio_msix_enable() ->
+ * vfio_msix_vector_do_use(), which re-arms the KVM irqfd routes from the
+ * existing shadow entries.
+ *
+ * Therefore, msix_reset() must NOT be called here.  Calling it would wipe
+ * those shadow entries and prevent interrupt delivery after EEH recovery.
+ *
+ * Instead, clear MSI-X Enable using the cached pdev->config shadow (avoiding
+ * a read from potentially frozen device config space) and write through
+ * pci_host_config_write_common() so that the VFIO config-write handler calls
+ * vfio_msix_disable(), cleanly releasing vectors and KVM irqfd routes while
+ * leaving the shadow intact.
+ */
+static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
+                                            PCIDevice *pdev,
+                                            void *opaque)
 {
-    /* Check if the device is VFIO PCI device */
-    if (!object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+    uint16_t flags;
+
+    if (!object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI_DEVICE)) {
         return;
     }
 
-    /*
-     * The MSIx table will be cleaned out by reset. We need
-     * disable it so that it can be reenabled properly. Also,
-     * the cached MSIx table should be cleared as it's not
-     * reflecting the contents in hardware.
-     */
     if (msix_enabled(pdev)) {
-        uint16_t flags;
-
-        flags = pci_host_config_read_common(pdev,
-                                            pdev->msix_cap + PCI_MSIX_FLAGS,
-                                            pci_config_size(pdev), 2);
+        flags = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
         flags &= ~PCI_MSIX_FLAGS_ENABLE;
         pci_host_config_write_common(pdev,
                                      pdev->msix_cap + PCI_MSIX_FLAGS,
                                      pci_config_size(pdev), flags, 2);
     }
-
-    msix_reset(pdev);
 }
 
-static void spapr_phb_vfio_eeh_clear_bus_msix(PCIBus *bus, void *opaque)
+static void spapr_phb_vfio_eeh_prepare_bus(PCIBus *bus, void *opaque)
 {
-       pci_for_each_device_under_bus(bus, spapr_phb_vfio_eeh_clear_dev_msix,
-                                     NULL);
+    pci_for_each_device_under_bus(bus,
+                                  spapr_phb_vfio_eeh_prepare_dev,
+                                  NULL);
 }
 
 static void spapr_phb_vfio_eeh_pre_reset(SpaprPhbState *sphb)
 {
-       PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
+    PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
 
-       pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
+    pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_prepare_bus, NULL);
 }
 
 int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
-- 
2.54.0