[Qemu-devel] [PATCH 12/13] pseries: Implement IOMMU and DMA for PAPR PCI devices

[Benjamin Herrenschmidt <benh@kernel.crashing.org>]

From: David Gibson <david@gibson.dropbear.id.au>

Currently the pseries machine emulation does not support DMA for emulated
PCI devices, because the PAPR spec always requires a (guest visible,
paravirtualized) IOMMU which was not implemented.  Now that we have
infrastructure for IOMMU emulation, we can correct this and allow PCI DMA
for pseries.

With the existing PAPR IOMMU code used for VIO devices, this is almost
trivial. We use a single DMAContext for each (virtual) PCI host bridge,
which is the usual configuration on real PAPR machines (which often have
_many_ PCI host bridges).

Cc: Alex Graf <agraf@suse.de>

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---
 hw/spapr.h       |    1 +
 hw/spapr_iommu.c |   52 ++++++++++++++++++++++++++++------------------------
 hw/spapr_pci.c   |   15 +++++++++++++++
 hw/spapr_pci.h   |    1 +
 4 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/hw/spapr.h b/hw/spapr.h
index df3e8b1..7c497aa 100644
--- a/hw/spapr.h
+++ b/hw/spapr.h
@@ -328,6 +328,7 @@ typedef struct sPAPRTCE {
 } sPAPRTCE;
 
 #define SPAPR_VIO_BASE_LIOBN    0x00000000
+#define SPAPR_PCI_BASE_LIOBN    0x80000000
 
 void spapr_iommu_init(void);
 DMAContext *spapr_tce_new_dma_context(uint32_t liobn, size_t window_size);
diff --git a/hw/spapr_iommu.c b/hw/spapr_iommu.c
index 87ed09c..79c2a06 100644
--- a/hw/spapr_iommu.c
+++ b/hw/spapr_iommu.c
@@ -162,6 +162,28 @@ void spapr_tce_free(DMAContext *dma)
     }
 }
 
+static target_ulong put_tce_emu(sPAPRTCETable *tcet, target_ulong ioba,
+                                target_ulong tce)
+{
+    sPAPRTCE *tcep;
+    target_ulong oldtce;
+
+    if (ioba >= tcet->window_size) {
+        hcall_dprintf("spapr_vio_put_tce on out-of-boards IOBA 0x"
+                      TARGET_FMT_lx "\n", ioba);
+        return H_PARAMETER;
+    }
+
+    tcep = tcet->table + (ioba >> SPAPR_TCE_PAGE_SHIFT);
+    oldtce = tcep->tce;
+    tcep->tce = tce;
+
+    if (oldtce != 0) {
+        iommu_wait_for_invalidated_maps(&tcet->dma, ioba, SPAPR_TCE_PAGE_SIZE);
+    }
+
+    return H_SUCCESS;
+}
 
 static target_ulong h_put_tce(CPUPPCState *env, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
@@ -170,43 +192,25 @@ static target_ulong h_put_tce(CPUPPCState *env, sPAPREnvironment *spapr,
     target_ulong ioba = args[1];
     target_ulong tce = args[2];
     sPAPRTCETable *tcet = spapr_tce_find_by_liobn(liobn);
-    sPAPRTCE *tcep;
-    target_ulong oldtce;
 
     if (liobn & 0xFFFFFFFF00000000ULL) {
         hcall_dprintf("spapr_vio_put_tce on out-of-boundsw LIOBN "
                       TARGET_FMT_lx "\n", liobn);
         return H_PARAMETER;
     }
-    if (!tcet) {
-        hcall_dprintf("spapr_vio_put_tce on non-existent LIOBN "
-                      TARGET_FMT_lx "\n", liobn);
-        return H_PARAMETER;
-    }
 
     ioba &= ~(SPAPR_TCE_PAGE_SIZE - 1);
 
+    if (tcet) {
+        return put_tce_emu(tcet, ioba, tce);
+    }
 #ifdef DEBUG_TCE
-    fprintf(stderr, "spapr_vio_put_tce on liobn=" TARGET_FMT_lx /*%s*/
+    fprintf(stderr, "%s on liobn=" TARGET_FMT_lx /*%s*/
             "  ioba 0x" TARGET_FMT_lx "  TCE 0x" TARGET_FMT_lx "\n",
-            liobn, /*dev->qdev.id, */ioba, tce);
+            __func__, liobn, /*dev->qdev.id, */ioba, tce);
 #endif
 
-    if (ioba >= tcet->window_size) {
-        hcall_dprintf("spapr_vio_put_tce on out-of-boards IOBA 0x"
-                      TARGET_FMT_lx "\n", ioba);
-        return H_PARAMETER;
-    }
-
-    tcep = tcet->table + (ioba >> SPAPR_TCE_PAGE_SHIFT);
-    oldtce = tcep->tce;
-    tcep->tce = tce;
-
-    if (oldtce != 0) {
-        iommu_wait_for_invalidated_maps(&tcet->dma, ioba, SPAPR_TCE_PAGE_SIZE);
-    }
-
-    return H_SUCCESS;
+    return H_PARAMETER;
 }
 
 void spapr_iommu_init(void)
diff --git a/hw/spapr_pci.c b/hw/spapr_pci.c
index 25b400a..7b9973c 100644
--- a/hw/spapr_pci.c
+++ b/hw/spapr_pci.c
@@ -265,12 +265,21 @@ static const MemoryRegionOps spapr_io_ops = {
 /*
  * PHB PCI device
  */
+static DMAContext *spapr_pci_dma_context_fn(PCIBus *bus, void *opaque,
+                                            int devfn)
+{
+    sPAPRPHBState *phb = opaque;
+
+    return phb->dma;
+}
+
 static int spapr_phb_init(SysBusDevice *s)
 {
     sPAPRPHBState *phb = FROM_SYSBUS(sPAPRPHBState, s);
     char *namebuf;
     int i;
     PCIBus *bus;
+    uint32_t liobn;
 
     phb->dtbusname = g_strdup_printf("pci@%" PRIx64, phb->buid);
     namebuf = alloca(strlen(phb->dtbusname) + 32);
@@ -311,6 +320,10 @@ static int spapr_phb_init(SysBusDevice *s)
                            PCI_DEVFN(0, 0), PCI_NUM_PINS);
     phb->host_state.bus = bus;
 
+    liobn = SPAPR_PCI_BASE_LIOBN | (pci_find_domain(bus) << 16);
+    phb->dma = spapr_tce_new_dma_context(liobn, 0x40000000);
+    pci_setup_iommu(bus, spapr_pci_dma_context_fn, phb);
+
     QLIST_INSERT_HEAD(&spapr->phbs, phb, list);
 
     /* Initialize the LSI table */
@@ -471,6 +484,8 @@ int spapr_populate_pci_devices(sPAPRPHBState *phb,
     _FDT(fdt_setprop(fdt, bus_off, "interrupt-map", &interrupt_map,
                      sizeof(interrupt_map)));
 
+    spapr_dma_dt(fdt, bus_off, "ibm,dma-window", phb->dma);
+
     return 0;
 }
 
diff --git a/hw/spapr_pci.h b/hw/spapr_pci.h
index f54c2e8..d9e46e2 100644
--- a/hw/spapr_pci.h
+++ b/hw/spapr_pci.h
@@ -38,6 +38,7 @@ typedef struct sPAPRPHBState {
     MemoryRegion memspace, iospace;
     target_phys_addr_t mem_win_addr, mem_win_size, io_win_addr, io_win_size;
     MemoryRegion memwindow, iowindow;
+    DMAContext *dma;
 
     struct {
         uint32_t dt_irq;
-- 
1.7.9.5