[Qemu-devel] [RFC PATCH 0/3] vfio/pci: Add support for mmapping sub-page MMIO BARs and MSI-X table

[Qemu-devel] [RFC PATCH 0/3] vfio/pci: Add support for mmapping sub-page MMIO BARs and MSI-X table

[Yongji Xie]

This patch set adds support for two VFIO-PCI ioctl flags:
VFIO_DEVICE_FLAGS_PCI_PAGE_ALIGNED and VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP.
Note that the kernel bits of this two flags are not in upstream and
posted as <Allow to mmap sub-page MMIO BARs and MSI-X table on
PPC64 platform>.

VFIO_DEVICE_FLAGS_PCI_PAGE_ALIGNED indicates that platform support
all PCI MMIO BARs to be page aligned which means any BARs' mmio
page would not be shared with other BARs. So it's allowed to mmap
sub-page(size < PAGE_SIZE) MMIO BARs in QEMU.

VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP indicates that it's safe to mmap MSI-X
table in QEMU.

Besides supporting for these two mmapping cases, we also fix some issues
which would block KVM to create memory slot for these mmapped sub-page
MMIO BARs and mmapped MSI-X table.

Posted kernel patches link:
- https://www.mail-archive.com/kvm@vger.kernel.org/msg124031.html

Yongji Xie (3):
  linux-headers: Update VFIO headers from linux-next tag ToBeFilled
  vfio/pci: Add support for mmapped sub-page MMIO BARs
  vfio/pci: Add support for mmapping MSI-X table

 hw/vfio/pci.c              |   65 ++++++++++++++++++++++++++++++++++++++++----
 linux-headers/linux/vfio.h |    4 +++
 2 files changed, 64 insertions(+), 5 deletions(-)

-- 
1.7.9.5

[Qemu-devel] [RFC PATCH 1/3] linux-headers: Update VFIO headers from linux-next tag ToBeFilled

[Yongji Xie]

Syncup VFIO related linux headers from linux-next tree using
scripts/update-linux-headers.sh.

Integrate two VFIO-PCI ioctl flags:
- VFIO_DEVICE_FLAGS_PCI_PAGE_ALIGNED
- VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP

Signed-off-by: Yongji Xie <xyjxie@linux.vnet.ibm.com>
---
 linux-headers/linux/vfio.h |    4 ++++
 1 file changed, 4 insertions(+)

diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index aa276bc..3b79d63 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -164,6 +164,10 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
 #define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)	/* vfio-platform device */
 #define VFIO_DEVICE_FLAGS_AMBA  (1 << 3)	/* vfio-amba device */
+/* Platform support all PCI MMIO BARs to be page aligned */
+#define VFIO_DEVICE_FLAGS_PCI_PAGE_ALIGNED  (1 << 4)
+/* platform support mmapping PCI MSI-X vector table */
+#define VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP  (1 << 5)
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 };
-- 
1.7.9.5

[Qemu-devel] [RFC PATCH 2/3] vfio/pci: Add support for mmapped sub-page MMIO BARs

[Yongji Xie]

The VFIO-PCI ioctl flag VFIO_DEVICE_FLAGS_PCI_PAGE_ALIGNED indicates platform
support all PCI MMIO BARs to be page aligned and sub-page(size < PAGE_SIZE)
MMIO BARs can be mmapped.

But this has an issue for these mmapped sub-page MMIO BARs - KVM would not
allow to create memory slot for them via ioctl KVM_SET_USER_MEMORY_REGION
because these BARs' size are still less than PAGE_SIZE.

So this patch expands these sub-page MMIO BARs to page size in
vfio_pci_write_config(), so that the BARs could be passed to KVM
ioctl KVM_SET_USER_MEMORY_REGION with a valid size. And we also set the
priority of these BARs' memory regions to zero in case of overlap with
page unaligned BARs when guest doesn't support all PCI MMIO BARs to be
page aligned.

Signed-off-by: Yongji Xie <xyjxie@linux.vnet.ibm.com>
---
 hw/vfio/pci.c |   46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 1fb868c..a7c6171 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -1028,6 +1028,44 @@ static const MemoryRegionOps vfio_vga_ops = {
 };
 
 /*
+ * Expand sub-page(size < PAGE_SIZE) MMIO BARs to page size if host support
+ * all MMIO BARs to be page aligned. And we should set the priority of these
+ * BARs' memory regions to zero in case of overlap with page unaligned
+ * BARs when guest doesn't support all MMIO BARs to be page aligned.
+ */
+static void vfio_expand_bar_to_page_size(PCIDevice *pdev)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    MemoryRegion *mmap_mr;
+    MemoryRegion *mr;
+    PCIIORegion *r;
+    pcibus_t bar_addr;
+    int i;
+
+    memory_region_transaction_begin();
+    for (i = 0; i < PCI_NUM_REGIONS; i++) {
+        r = &pdev->io_regions[i];
+        if (r->size > 0 && r->size < qemu_real_host_page_size) {
+            bar_addr = r->addr;
+            if (bar_addr != PCI_BAR_UNMAPPED &&
+                !(bar_addr & ~qemu_real_host_page_mask)) {
+                mr = &vdev->bars[i].region.mem;
+                mmap_mr = &vdev->bars[i].region.mmap_mem;
+                if (memory_region_is_mapped(mr) && vdev->bars[i].region.mmap &&
+                    memory_region_size(mr) < qemu_real_host_page_size) {
+                    memory_region_del_subregion(r->address_space, mr);
+                    memory_region_set_size(mr, qemu_real_host_page_size);
+                    memory_region_set_size(mmap_mr, qemu_real_host_page_size);
+                    memory_region_add_subregion_overlap(r->address_space,
+                                                        bar_addr, mr, 0);
+                }
+            }
+        }
+    }
+    memory_region_transaction_commit();
+}
+
+/*
  * PCI config space
  */
 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
@@ -1112,6 +1150,14 @@ void vfio_pci_write_config(PCIDevice *pdev,
         } else if (was_enabled && !is_enabled) {
             vfio_msix_disable(vdev);
         }
+    } else if (vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_PAGE_ALIGNED &&
+        (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
+        ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) ||
+        ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) ||
+        range_covers_byte(addr, len, PCI_COMMAND))) {
+        pci_default_write_config(pdev, addr, val, len);
+
+        vfio_expand_bar_to_page_size(pdev);
     } else {
         /* Write everything to QEMU to keep emulated bits correct */
         pci_default_write_config(pdev, addr, val, len);
-- 
1.7.9.5

[Qemu-devel] [RFC PATCH 3/3] vfio/pci: Add support for mmapping MSI-X table

[Yongji Xie]

The VFIO-PCI ioctl VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP flag indicate that
platform support mmapping MSI-X table.

With this flag set, we skip some MSI-X table check so that QEMU can
mmap MSI-X table successfully. And we also raise the priority of mmap
memory region in case of overlap with MSI-X/PBA memory region which
is added in msix_init().

Signed-off-by: Yongji Xie <xyjxie@linux.vnet.ibm.com>
---
 hw/vfio/pci.c |   19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index a7c6171..bec301c 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -1339,7 +1339,8 @@ static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
         }
 
         memory_region_set_enabled(&bar->region.mmap_mem, enabled);
-        if (vdev->msix && vdev->msix->table_bar == i) {
+        if (vdev->msix && vdev->msix->table_bar == i &&
+            !(vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP)) {
             memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
         }
     }
@@ -1357,7 +1358,8 @@ static void vfio_unregister_bar(VFIOPCIDevice *vdev, int nr)
 
     memory_region_del_subregion(&bar->region.mem, &bar->region.mmap_mem);
 
-    if (vdev->msix && vdev->msix->table_bar == nr) {
+    if (vdev->msix && vdev->msix->table_bar == nr &&
+        !(vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP)) {
         memory_region_del_subregion(&bar->region.mem, &vdev->msix->mmap_mem);
     }
 }
@@ -1374,7 +1376,8 @@ static void vfio_unmap_bar(VFIOPCIDevice *vdev, int nr)
 
     munmap(bar->region.mmap, memory_region_size(&bar->region.mmap_mem));
 
-    if (vdev->msix && vdev->msix->table_bar == nr) {
+    if (vdev->msix && vdev->msix->table_bar == nr &&
+        !(vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP)) {
         munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
     }
 }
@@ -1420,7 +1423,8 @@ static void vfio_map_bar(VFIOPCIDevice *vdev, int nr)
      * We can't mmap areas overlapping the MSIX vector table, so we
      * potentially insert a direct-mapped subregion before and after it.
      */
-    if (vdev->msix && vdev->msix->table_bar == nr) {
+    if (vdev->msix && vdev->msix->table_bar == nr &&
+        !(vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP)) {
         size = vdev->msix->table_offset & qemu_real_host_page_mask;
     }
 
@@ -1429,9 +1433,14 @@ static void vfio_map_bar(VFIOPCIDevice *vdev, int nr)
                       &bar->region.mmap_mem, &bar->region.mmap,
                       size, 0, name)) {
         error_report("%s unsupported. Performance may be slow", name);
+    } else if (vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP) {
+        memory_region_del_subregion(&bar->region.mem, &bar->region.mmap_mem);
+        memory_region_add_subregion_overlap(&bar->region.mem, 0,
+                      &bar->region.mmap_mem, 1);
     }
 
-    if (vdev->msix && vdev->msix->table_bar == nr) {
+    if (vdev->msix && vdev->msix->table_bar == nr &&
+        !(vdev->vbasedev.flags & VFIO_DEVICE_FLAGS_PCI_MSIX_MMAP)) {
         uint64_t start;
 
         start = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
-- 
1.7.9.5