[Qemu-devel] [PATCH v1 0/3] [RFC] get real IOMMU information from VFIO IOMMU

[Qemu-devel] [PATCH v1 0/3] [RFC] get real IOMMU information from VFIO IOMMU

[Pierre Morel]

In the Z-Architecture, the IOMMU information may change on a device
basis.
until this patch series we used hardcoded values to define the
IOMMU aperture we provide to the guest during PCI enumeration.

Let's ask the host about the IOMMU specific configuration using
VFIO capabilities.

NOTES:
1- You will need the according Linux patch to test this.

2- I am really not happy to add S390 dedicated code in the
   VFIO common code, as do SPAPR, but I did not find a better
   solution.
   Any idea?

Pierre Morel (3):
  vfio: Linux header placeholder
  vfio/pci: Get real IOMMU information from container
  s390x/pci: Reporting the host aperture to the guest

 hw/s390x/s390-pci-bus.c       |  2 +-
 hw/s390x/s390-pci-bus.h       |  3 ++
 hw/s390x/s390-pci-inst.c      | 20 ++++++++++--
 hw/vfio/common.c              | 75 +++++++++++++++++++++++++++++++++----------
 include/hw/vfio/vfio-common.h |  3 ++
 linux-headers/linux/vfio.h    | 65 +++++++++++++++++++++++++++++++++++--
 6 files changed, 144 insertions(+), 24 deletions(-)

-- 
2.7.4

[Qemu-devel] [PATCH v1 1/3] vfio: Linux header placeholder

[Pierre Morel]

This is a place holder for VFIO.h as changed by the Linux patch
associated with this QEMU series.

Signed-off-by: Pierre Morel <pmorel@linux.ibm.com>
---
 linux-headers/linux/vfio.h | 65 +++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 62 insertions(+), 3 deletions(-)

diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index ceb6453..54c4fcb 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -9,8 +9,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
-#ifndef VFIO_H
-#define VFIO_H
+#ifndef _UAPIVFIO_H
+#define _UAPIVFIO_H
 
 #include <linux/types.h>
 #include <linux/ioctl.h>
@@ -303,6 +303,56 @@ struct vfio_region_info_cap_type {
 #define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG	(2)
 #define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG	(3)
 
+#define VFIO_REGION_TYPE_GFX                    (1)
+#define VFIO_REGION_SUBTYPE_GFX_EDID            (1)
+
+/**
+ * struct vfio_region_gfx_edid - EDID region layout.
+ *
+ * Set display link state and EDID blob.
+ *
+ * The EDID blob has monitor information such as brand, name, serial
+ * number, physical size, supported video modes and more.
+ *
+ * This special region allows userspace (typically qemu) set a virtual
+ * EDID for the virtual monitor, which allows a flexible display
+ * configuration.
+ *
+ * For the edid blob spec look here:
+ *    https://en.wikipedia.org/wiki/Extended_Display_Identification_Data
+ *
+ * On linux systems you can find the EDID blob in sysfs:
+ *    /sys/class/drm/${card}/${connector}/edid
+ *
+ * You can use the edid-decode ulility (comes with xorg-x11-utils) to
+ * decode the EDID blob.
+ *
+ * @edid_offset: location of the edid blob, relative to the
+ *               start of the region (readonly).
+ * @edid_max_size: max size of the edid blob (readonly).
+ * @edid_size: actual edid size (read/write).
+ * @link_state: display link state (read/write).
+ * VFIO_DEVICE_GFX_LINK_STATE_UP: Monitor is turned on.
+ * VFIO_DEVICE_GFX_LINK_STATE_DOWN: Monitor is turned off.
+ * @max_xres: max display width (0 == no limitation, readonly).
+ * @max_yres: max display height (0 == no limitation, readonly).
+ *
+ * EDID update protocol:
+ *   (1) set link-state to down.
+ *   (2) update edid blob and size.
+ *   (3) set link-state to up.
+ */
+struct vfio_region_gfx_edid {
+	__u32 edid_offset;
+	__u32 edid_max_size;
+	__u32 edid_size;
+	__u32 max_xres;
+	__u32 max_yres;
+	__u32 link_state;
+#define VFIO_DEVICE_GFX_LINK_STATE_UP    1
+#define VFIO_DEVICE_GFX_LINK_STATE_DOWN  2
+};
+
 /*
  * The MSIX mappable capability informs that MSIX data of a BAR can be mmapped
  * which allows direct access to non-MSIX registers which happened to be within
@@ -619,6 +669,15 @@ struct vfio_iommu_type1_info {
 	__u32	flags;
 #define VFIO_IOMMU_INFO_PGSIZES (1 << 0)	/* supported page sizes info */
 	__u64	iova_pgsizes;		/* Bitmap of supported page sizes */
+#define VFIO_IOMMU_INFO_CAPABILITIES (1 << 1)  /* support capabilities info */
+	__u64   cap_offset;     /* Offset within info struct of first cap */
+};
+
+#define VFIO_IOMMU_INFO_CAP_DMA 1
+struct vfio_iommu_cap_dma {
+	struct vfio_info_cap_header header;
+	__u64   dma_start;
+	__u64   dma_end;
 };
 
 #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
@@ -818,4 +877,4 @@ struct vfio_iommu_spapr_tce_remove {
 
 /* ***************************************************************** */
 
-#endif /* VFIO_H */
+#endif /* _UAPIVFIO_H */
-- 
2.7.4

[Qemu-devel] [PATCH v1 2/3] vfio/pci: Get real IOMMU information from container

[Pierre Morel]

When the vfio_iommu_type1 supports the VFIO_IOMMU_INFO_CAPABILITIES
and the capability ID VFIO_IOMMU_INFO_CAP_DMA we can use an ioctl
to retrieve this information from the real IOMMU device.

Let's use this information to add the host window associated with
the container.

Signed-off-by: Pierre Morel <pmorel@linux.ibm.com>
---
 hw/vfio/common.c | 73 +++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 56 insertions(+), 17 deletions(-)

diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 7c185e5a..fd7e991 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -1036,6 +1036,61 @@ static void vfio_put_address_space(VFIOAddressSpace *space)
     }
 }
 
+static void vfio_iommu_type1_get_info(int fd, VFIOContainer *container)
+{
+    struct vfio_iommu_type1_info *info;
+    size_t argsz = sizeof(*info);
+    struct vfio_info_cap_header *hdr;
+    struct vfio_iommu_cap_dma *dma_info;
+    int ret = 0;
+
+    info = g_malloc0(argsz);
+
+retry:
+    info->argsz = argsz;
+    info->flags = VFIO_IOMMU_INFO_CAPABILITIES;
+
+    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
+    /* Ignore errors */
+    if (ret) {
+        goto out;
+    }
+    if (!(info->flags & VFIO_IOMMU_INFO_PGSIZES)) {
+        info->iova_pgsizes = 4096;
+    }
+
+    if (info->argsz > argsz) {
+        argsz = info->argsz;
+        info = g_realloc(info, argsz);
+
+        goto retry;
+    }
+    if (info->argsz != argsz) {
+        goto out;
+    }
+    /* Now we have the capabilities */
+    hdr = (struct vfio_info_cap_header *)((unsigned char *)info +
+                                          sizeof(struct vfio_iommu_type1_info) +
+                                          info->cap_offset);
+    do {
+        dma_info = (struct vfio_iommu_cap_dma *) (hdr);
+        if (hdr->id == VFIO_IOMMU_INFO_CAP_DMA) {
+            vfio_host_win_add(container, 0,
+                              dma_info->dma_end - dma_info->dma_start,
+                              info->iova_pgsizes);
+            container->pgsizes = info->iova_pgsizes;
+            return;
+        }
+        hdr = (struct vfio_info_cap_header *)((unsigned char *) dma_info +
+                                              hdr->next);
+    } while (hdr->next);
+out:
+    /* Assume 4k IOVA page size */
+    vfio_host_win_add(container, 0, (hwaddr)-1,  4096);
+    container->pgsizes = 4096;
+    return;
+}
+
 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
                                   Error **errp)
 {
@@ -1104,7 +1159,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
     if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ||
         ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) {
         bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU);
-        struct vfio_iommu_type1_info info;
 
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
@@ -1121,22 +1175,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
             goto free_container_exit;
         }
 
-        /*
-         * FIXME: This assumes that a Type1 IOMMU can map any 64-bit
-         * IOVA whatsoever.  That's not actually true, but the current
-         * kernel interface doesn't tell us what it can map, and the
-         * existing Type1 IOMMUs generally support any IOVA we're
-         * going to actually try in practice.
-         */
-        info.argsz = sizeof(info);
-        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info);
-        /* Ignore errors */
-        if (ret || !(info.flags & VFIO_IOMMU_INFO_PGSIZES)) {
-            /* Assume 4k IOVA page size */
-            info.iova_pgsizes = 4096;
-        }
-        vfio_host_win_add(container, 0, (hwaddr)-1, info.iova_pgsizes);
-        container->pgsizes = info.iova_pgsizes;
+        vfio_iommu_type1_get_info(fd, container);
     } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU) ||
                ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU)) {
         struct vfio_iommu_spapr_tce_info info;
-- 
2.7.4

[Qemu-devel] [PATCH v1 3/3] s390x/pci: Reporting the host aperture to the guest

[Pierre Morel]

The S390 PCI driver in the guest needs to know the IOMMU
aperture associated with the zPCI real device in the host
through Start DMA and End DMA value in the response to the
Get Function Group information query.

Let's report these SDMA and EDMA using the value we got from
the host through the real IOMMU associated with the VFIO container.

Signed-off-by: Pierre Morel <pmorel@linux.ibm.com>
---
 hw/s390x/s390-pci-bus.c       |  2 +-
 hw/s390x/s390-pci-bus.h       |  3 +++
 hw/s390x/s390-pci-inst.c      | 20 +++++++++++++++++---
 hw/vfio/common.c              |  4 +++-
 include/hw/vfio/vfio-common.h |  3 +++
 5 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 27963b2..29177ac 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -551,7 +551,7 @@ static void s390_pci_iommu_replay(IOMMUMemoryRegion *iommu,
     return;
 }
 
-static S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus,
+S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus,
                                         int devfn)
 {
     uint64_t key = (uintptr_t)bus;
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 1f7f9b5..951a3d8 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -276,6 +276,8 @@ typedef struct S390PCIIOMMU {
     IOMMUMemoryRegion iommu_mr;
     bool enabled;
     uint64_t g_iota;
+    uint64_t sdma;
+    uint64_t edma;
     uint64_t pba;
     uint64_t pal;
     GHashTable *iotlb;
@@ -344,5 +346,6 @@ S390PCIBusDevice *s390_pci_find_dev_by_target(S390pciState *s,
                                               const char *target);
 S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
                                                S390PCIBusDevice *pbdev);
+S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus, int devfn);
 
 #endif
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 7b61367..5125c1b 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -19,6 +19,7 @@
 #include "exec/memory-internal.h"
 #include "qemu/error-report.h"
 #include "sysemu/hw_accel.h"
+#include "hw/vfio/vfio-common.h"
 
 #ifndef DEBUG_S390PCI_INST
 #define DEBUG_S390PCI_INST  0
@@ -31,6 +32,16 @@
         }                                                          \
     } while (0)
 
+void vfio_s390_iommu_setup(VFIOContainer *container, uint64_t min,
+                           uint64_t max, uint64_t pgsize)
+{
+    S390PCIIOMMU *iommu;
+
+    iommu = container_of(container->space->as, S390PCIIOMMU, as);
+    iommu->sdma = min;
+    iommu->edma = max;
+}
+
 static void s390_set_status_code(CPUS390XState *env,
                                  uint8_t r, uint64_t status_code)
 {
@@ -153,6 +164,7 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
     uint8_t cc = 0;
     CPUS390XState *env = &cpu->env;
     S390pciState *s = s390_get_phb();
+    S390PCIIOMMU *iommu;
     int i;
 
     if (env->psw.mask & PSW_MASK_PSTATE) {
@@ -279,8 +291,10 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
                     resquery->bar_size[i]);
         }
 
-        stq_p(&resquery->sdma, ZPCI_SDMA_ADDR);
-        stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
+        iommu = s390_pci_get_iommu(s, pci_get_bus(pbdev->pdev),
+                                   PCI_FUNC(pbdev->pdev->devfn));
+        stq_p(&resquery->sdma, iommu->sdma);
+        stq_p(&resquery->edma, iommu->edma);
         stl_p(&resquery->fid, pbdev->fid);
         stw_p(&resquery->pchid, 0);
         stw_p(&resquery->ug, 1);
@@ -860,7 +874,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIIOMMU *iommu, ZpciFib fib,
 
     pba &= ~0xfff;
     pal |= 0xfff;
-    if (pba > pal || pba < ZPCI_SDMA_ADDR || pal > ZPCI_EDMA_ADDR) {
+    if (pba > pal || pba < iommu->sdma || pal > iommu->edma) {
         s390_program_interrupt(env, PGM_OPERAND, 6, ra);
         return -EINVAL;
     }
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index fd7e991..906b208 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -1045,7 +1045,6 @@ static void vfio_iommu_type1_get_info(int fd, VFIOContainer *container)
     int ret = 0;
 
     info = g_malloc0(argsz);
-
 retry:
     info->argsz = argsz;
     info->flags = VFIO_IOMMU_INFO_CAPABILITIES;
@@ -1079,6 +1078,8 @@ retry:
                               dma_info->dma_end - dma_info->dma_start,
                               info->iova_pgsizes);
             container->pgsizes = info->iova_pgsizes;
+            vfio_s390_iommu_setup(container, dma_info->dma_start,
+                                  dma_info->dma_end, info->iova_pgsizes);
             return;
         }
         hdr = (struct vfio_info_cap_header *)((unsigned char *) dma_info +
@@ -1088,6 +1089,7 @@ out:
     /* Assume 4k IOVA page size */
     vfio_host_win_add(container, 0, (hwaddr)-1,  4096);
     container->pgsizes = 4096;
+    vfio_s390_iommu_setup(container, 0,  (hwaddr)-1,  4096);
     return;
 }
 
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 1b434d0..2c8bea8 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -198,4 +198,7 @@ int vfio_spapr_create_window(VFIOContainer *container,
 int vfio_spapr_remove_window(VFIOContainer *container,
                              hwaddr offset_within_address_space);
 
+void vfio_s390_iommu_setup(VFIOContainer *container, uint64_t min,
+                           uint64_t max, uint64_t pgsize);
+
 #endif /* HW_VFIO_VFIO_COMMON_H */
-- 
2.7.4

Re: [Qemu-devel] [PATCH v1 0/3] [RFC] get real IOMMU information from VFIO IOMMU

[no-reply]

Patchew URL: https://patchew.org/QEMU/1547037680-21458-1-git-send-email-pmorel@linux.ibm.com/



Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
time make docker-test-quick@centos7 SHOW_ENV=1 J=8
=== TEST SCRIPT END ===




The full log is available at
http://patchew.org/logs/1547037680-21458-1-git-send-email-pmorel@linux.ibm.com/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [http://patchew.org/].
Please send your feedback to patchew-devel@redhat.com