[PATCH v2 2/2] ppc/spapr: Disable VFIO BAR mmap during EEH PE reset

[Narayana Murty N <nnmlinux@linux.ibm.com>]

Direct mmap-backed BAR accesses are not reliable while the PE is
undergoing EEH reset.  Temporarily disable the mmap-backed BAR
subregions so accesses fall back to the VFIO region read/write path
during the reset window.

After VFIO_EEH_PE_CONFIGURE succeeds, restore normal VFIO mmap
behavior.

To keep VFIOPCIDevice internals encapsulated and avoid direct
manipulation of vdev->intx.mmap_timer and vdev->bars[] from
hw/ppc/spapr_pci_vfio.c, introduce two VFIO PCI helpers in
hw/vfio/pci.c:

 - vfio_pci_eeh_pre_reset(): cancels any pending INTx mmap re-enable
   timer and disables BAR mmaps across all BAR regions.

 - vfio_pci_eeh_post_configure(): restores normal VFIO mmap policy.
   If INTx is still pending, the INTx mmap timer is re-armed so mmap
   remains disabled until the quiet period completes; otherwise,
   BAR mmaps are immediately re-enabled.

The pSeries EEH handlers in hw/ppc/spapr_pci_vfio.c call
vfio_pci_eeh_pre_reset() during the pre-reset device preparation
(after MSI-X teardown) and call vfio_pci_eeh_post_configure() only
after VFIO_EEH_PE_CONFIGURE succeeds.  On configure failure, BAR mmaps
remain disabled.  RTAS_SLOT_RESET_DEACTIVATE does not invoke either
helper.

Signed-off-by: Narayana Murty N <nnmlinux@linux.ibm.com>
---
 hw/ppc/spapr_pci_vfio.c | 60 ++++++++++++++++++++++++++++++++++++++---
 hw/vfio/pci.c           | 32 ++++++++++++++++++++++
 hw/vfio/pci.h           |  2 ++
 3 files changed, 90 insertions(+), 4 deletions(-)

diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index 1fa8847031..ab78489fa5 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -254,8 +254,8 @@ int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state)
  * leaving the shadow intact.
  */
 static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
-                                            PCIDevice *pdev,
-                                            void *opaque)
+                                           PCIDevice *pdev,
+                                           void *opaque)
 {
     uint16_t flags;
 
@@ -263,13 +263,27 @@ static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
         return;
     }
 
+    /*
+     * Disable MSI-X through the VFIO config-write path so the
+     * active interrupt routes are released while preserving the
+     * pSeries RTAS-managed MSI-X software shadow.
+     */
     if (msix_enabled(pdev)) {
-        flags = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
+        flags = pci_get_word(pdev->config +
+                             pdev->msix_cap + PCI_MSIX_FLAGS);
+
         flags &= ~PCI_MSIX_FLAGS_ENABLE;
+
         pci_host_config_write_common(pdev,
                                      pdev->msix_cap + PCI_MSIX_FLAGS,
-                                     pci_config_size(pdev), flags, 2);
+                                     pci_config_size(pdev),
+                                     flags, 2);
     }
+
+    /*
+     * This must be the final preparation step before the PE reset.
+     */
+    vfio_pci_eeh_pre_reset(pdev);
 }
 
 static void spapr_phb_vfio_eeh_prepare_bus(PCIBus *bus, void *opaque)
@@ -286,6 +300,37 @@ static void spapr_phb_vfio_eeh_pre_reset(SpaprPhbState *sphb)
     pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_prepare_bus, NULL);
 }
 
+/*
+ * Restore normal VFIO mmap policy for a single VFIO PCI device after a
+ * successful EEH PE configure.  Called only on the configure success path;
+ * on failure mmaps remain disabled until the next hot/fundamental reset
+ * attempt.
+ */
+static void spapr_phb_vfio_eeh_post_configure_dev(PCIBus *bus,
+                                                  PCIDevice *pdev,
+                                                  void *opaque)
+{
+    if (!object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI_DEVICE)) {
+        return;
+    }
+
+    vfio_pci_eeh_post_configure(pdev);
+}
+
+static void spapr_phb_vfio_eeh_post_configure_bus(PCIBus *bus, void *opaque)
+{
+    pci_for_each_device_under_bus(bus,
+                                  spapr_phb_vfio_eeh_post_configure_dev,
+                                  NULL);
+}
+
+static void spapr_phb_vfio_eeh_post_configure(SpaprPhbState *sphb)
+{
+    PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
+
+    pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_post_configure_bus, NULL);
+}
+
 int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
 {
     uint32_t op;
@@ -293,6 +338,11 @@ int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
 
     switch (option) {
     case RTAS_SLOT_RESET_DEACTIVATE:
+        /*
+         * Deactivate does not perform a full PE reset; BAR mmaps were already
+         * disabled by the preceding HOT or FUNDAMENTAL reset call and must not
+         * be re-enabled here.
+         */
         op = VFIO_EEH_PE_RESET_DEACTIVATE;
         break;
     case RTAS_SLOT_RESET_HOT:
@@ -324,6 +374,8 @@ int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb)
         return RTAS_OUT_PARAM_ERROR;
     }
 
+    spapr_phb_vfio_eeh_post_configure(sphb);
+
     return RTAS_OUT_SUCCESS;
 }
 
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 428ab2f069..931df22add 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -1949,6 +1949,38 @@ static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
     }
 }
 
+/*
+ * Prevent a pending INTx mmap timer from restoring BAR mmap while
+ * the PE is undergoing EEH reset.  Keep BAR accesses on the VFIO
+ * region read/write path until PE configuration completes.
+ */
+void vfio_pci_eeh_pre_reset(PCIDevice *pdev)
+{
+    VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
+
+    if (vdev->intx.mmap_timer) {
+        timer_del(vdev->intx.mmap_timer);
+    }
+
+    vfio_mmap_set_enabled(vdev, false);
+}
+
+void vfio_pci_eeh_post_configure(PCIDevice *pdev)
+{
+    VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
+
+    if (vdev->interrupt == VFIO_INT_INTx && vdev->intx.pending) {
+        if (vdev->intx.mmap_timer && vdev->intx.mmap_timeout) {
+            timer_mod(vdev->intx.mmap_timer,
+                      qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+                      vdev->intx.mmap_timeout);
+        }
+        return;
+    }
+
+    vfio_mmap_set_enabled(vdev, true);
+}
+
 static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
 {
     VFIOBAR *bar = &vdev->bars[nr];
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index c9ab949870..63fa378fbc 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -266,6 +266,8 @@ struct vfio_pci_hot_reset_info;
 
 void vfio_pci_pre_reset(VFIOPCIDevice *vdev);
 void vfio_pci_post_reset(VFIOPCIDevice *vdev);
+void vfio_pci_eeh_pre_reset(PCIDevice *pdev);
+void vfio_pci_eeh_post_configure(PCIDevice *pdev);
 bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name);
 int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
                                     struct vfio_pci_hot_reset_info **info_p);
-- 
2.54.0