[PATCH 8/8] amd_iommu: Allow IOMMU to attachment on non-system (pcie.0) bus

[Sairaj Kodilkar <sarunkod@amd.com>]

Allow the AMD IOMMU to attach on the pcie-root-port using pci device's "bus"
property. This helps in isolating the set of devices by creating a pcie
hierarchy under the pci expansion bus (pxb-pci). The IOMMU always tries
to find the first root bus by traversing up the PCI hierarchy and serves
all the devices under that bus.

Example1, for below PCI configuration, IOMMU will serve all the devices
under pxb-pci (i.e. Device1 and Device2) since its a first root bus,
but will not serve Device3 and Device3, effectively isolating the
devices under pxb-pci bus.

pcie.0 -+- pxb-pci -+- pcie-root-port1 --- AMD IOMMU
        |           +- pcie-root-port2 --- Device1
        |           \- pcie-root-port3 --- Device2
        +- Device3
        \- Device4

Example2, for below PCI configuration, IOMMU will serve all the devices
Device[1-4], since first root bus in the parent hierarchy is pcie.0

pcie.0 -+- pcie-root-port1 --- AMD IOMMU
        +- pcie-root-port2 --- Device1
        +- pcie-root-port3 --- Device2
        +- Device3
        \- Device4

AMD IOMMU uses this root bus to register pci IOMMU ops and build the IVRS
table.

Signed-off-by: Sairaj Kodilkar <sarunkod@amd.com>
---
 hw/i386/amd_iommu.c | 55 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 4 deletions(-)

diff --git a/hw/i386/amd_iommu.c b/hw/i386/amd_iommu.c
index 5e4d5d1f1fcd..e0968127ec40 100644
--- a/hw/i386/amd_iommu.c
+++ b/hw/i386/amd_iommu.c
@@ -24,6 +24,7 @@
 #include "hw/i386/pc.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pci_bus.h"
+#include "hw/pci/pci_bridge.h"
 #include "migration/vmstate.h"
 #include "amd_iommu.h"
 #include "qapi/error.h"
@@ -2680,6 +2681,50 @@ static const VMStateDescription vmstate_amdvi_sysbus_migratable = {
      }
 };
 
+/* Go up in the PCI hierarchy and find the root bus pcie.0 or pxb-pci */
+static PCIBus *get_root_bus(struct PCIBus *bus)
+{
+    while (bus) {
+
+        if (!pci_bus_is_express(bus)) {
+            error_report("attached to non PCI express bus %s",
+                         bus->qbus.name);
+            goto err_out;
+        }
+
+        if (bus->iommu_ops) {
+            error_report("different IOMMU already serves bus %s",
+                         bus->qbus.name);
+            goto err_out;
+        }
+
+        if (pci_bus_is_root(bus) &&
+            object_dynamic_cast(OBJECT(bus)->parent, TYPE_PCI_HOST_BRIDGE)) {
+
+            if (bus->parent_dev &&
+                !object_dynamic_cast(OBJECT(bus), TYPE_PXB_PCIE_BUS)) {
+
+                    error_report("only supports PXB-PCI as extra root bus");
+                    goto err_out;
+            }
+            /* We found the valid root bus so return it to the user*/
+            goto out;
+        }
+
+        if (bus->parent_dev) {
+            bus = pci_get_bus(bus->parent_dev);
+        } else {
+            error_report("could not find valid PCIE bus");
+            goto err_out;
+        }
+    };
+
+err_out:
+    exit(EXIT_FAILURE);
+out:
+    return bus;
+}
+
 static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
 {
     DeviceClass *dc = (DeviceClass *) object_get_class(OBJECT(dev));
@@ -2687,7 +2732,7 @@ static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
     MachineState *ms = MACHINE(qdev_get_machine());
     PCMachineState *pcms = PC_MACHINE(ms);
     X86MachineState *x86ms = X86_MACHINE(ms);
-    PCIBus *bus = pcms->pcibus;
+    PCIBus *iommu_bus;
 
     if (s->pci_id) {
         PCIDevice *pdev = NULL;
@@ -2703,17 +2748,19 @@ static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
             return;
         }
 
+        iommu_bus = pci_get_bus(pdev);
         s->pci = AMD_IOMMU_PCI(pdev);
         dc->vmsd = &vmstate_amdvi_sysbus_migratable;
     } else {
+        iommu_bus = pcms->pcibus;
         s->pci = AMD_IOMMU_PCI(object_new(TYPE_AMD_IOMMU_PCI));
         /* This device should take care of IOMMU PCI properties */
-        if (!qdev_realize(DEVICE(s->pci), &bus->qbus, errp)) {
+        if (!qdev_realize(DEVICE(s->pci), &iommu_bus->qbus, errp)) {
             return;
         }
     }
 
-    s->root_bus = bus;
+    s->root_bus = get_root_bus(iommu_bus);
 
     s->iotlb = g_hash_table_new_full(amdvi_iotlb_hash,
                                      amdvi_iotlb_equal, g_free, g_free);
@@ -2767,7 +2814,7 @@ static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
         }
     }
 
-    pci_setup_iommu(bus, &amdvi_iommu_ops, s);
+    pci_setup_iommu(s->root_bus, &amdvi_iommu_ops, s);
     amdvi_init(s);
 }
 
-- 
2.34.1