[Qemu-devel] [PATCH 0/2] vfio-pci: Enable KVM-VFIO device

[Qemu-devel] [PATCH 0/2] vfio-pci: Enable KVM-VFIO device

[Alex Williamson]

The KVM-VFIO device was just introduced into kvm.git/next with an
initial goal of managing whether guests are capable of non-coherent
DMA.  This is potentially important for any VFIO assigned device, but
particularly important to some graphics cards which are known to make
use of the NoSnoop capabilities of PCIe.  By registering VFIO groups
into the KVM-VFIO device, KVM is able to verify that a user has an
assigned device and (eventually) query VFIO directly for properties of
the IOMMU domain.  In the short term, KVM assumes any VFIO assigned
device is capable of non-coherent DMA.

This support can certainly be seen as a bug fix (devices are capable
of NoSnoop DMA today and qemu/kvm does not emulate coherency ops like
WBINVD), so I'd certainly like to get feedback on whether this is
acceptable QEMU 1.7 material.  The fact that it depends on a kernel
header update which is not yet in mainline may be a gating factor as
well.  We've taken linux-header updates from kvm.git in the past and
barring a patch faux pas from Gleb & Paolo it will get into mainline.
I've stated in the past that we should default to taking header
updates from mainline, but the impending v3.12 release and merge
window doesn't fit well with the QEMU 1.7 release timeframe.  I could
also strip the header update down to minimum required if that helps.

The other option is to wait for QEMU 1.8 to open at which point we
should be close to having a v3.13-rc1 tag to pull header updates from
and I can propose both the KVM and QEMU patches for stable.  Getting
it in for QEMU 1.7 would make a lot of people playing with VGA
passthrough very happy.  Thoughts?  Thanks,

Alex
---

Alex Williamson (2):
      linux-headers: Update from kvm.git 81e87e26796782e014fd1f2bb9cd8fb6ce4021a8
      vfio-pci: Make use of new KVM-VFIO device


 hw/misc/vfio.c                           |   67 ++++++++++++++++++++++++++++++
 linux-headers/asm-arm/kvm.h              |    3 +
 linux-headers/asm-powerpc/epapr_hcalls.h |    4 +-
 linux-headers/asm-x86/kvm.h              |    6 +--
 linux-headers/linux/kvm.h                |    7 +++
 5 files changed, 81 insertions(+), 6 deletions(-)

[Qemu-devel] [PATCH 1/2] linux-headers: Update from kvm.git 81e87e26796782e014fd1f2bb9cd8fb6ce4021a8

[Alex Williamson]

Current next branch.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---
 linux-headers/asm-arm/kvm.h              |    3 ++-
 linux-headers/asm-powerpc/epapr_hcalls.h |    4 ++--
 linux-headers/asm-x86/kvm.h              |    6 +++---
 linux-headers/linux/kvm.h                |    7 +++++++
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/linux-headers/asm-arm/kvm.h b/linux-headers/asm-arm/kvm.h
index c1ee007..c498b60 100644
--- a/linux-headers/asm-arm/kvm.h
+++ b/linux-headers/asm-arm/kvm.h
@@ -63,7 +63,8 @@ struct kvm_regs {
 
 /* Supported Processor Types */
 #define KVM_ARM_TARGET_CORTEX_A15	0
-#define KVM_ARM_NUM_TARGETS		1
+#define KVM_ARM_TARGET_CORTEX_A7	1
+#define KVM_ARM_NUM_TARGETS		2
 
 /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
 #define KVM_ARM_DEVICE_TYPE_SHIFT	0
diff --git a/linux-headers/asm-powerpc/epapr_hcalls.h b/linux-headers/asm-powerpc/epapr_hcalls.h
index 33b3f89..06f7247 100644
--- a/linux-headers/asm-powerpc/epapr_hcalls.h
+++ b/linux-headers/asm-powerpc/epapr_hcalls.h
@@ -78,7 +78,7 @@
 #define EV_SUCCESS		0
 #define EV_EPERM		1	/* Operation not permitted */
 #define EV_ENOENT		2	/*  Entry Not Found */
-#define EV_EIO			3	/* I/O error occurred */
+#define EV_EIO			3	/* I/O error occured */
 #define EV_EAGAIN		4	/* The operation had insufficient
 					 * resources to complete and should be
 					 * retried
@@ -89,7 +89,7 @@
 #define EV_ENODEV		7	/* No such device */
 #define EV_EINVAL		8	/* An argument supplied to the hcall
 					   was out of range or invalid */
-#define EV_INTERNAL		9	/* An internal error occurred */
+#define EV_INTERNAL		9	/* An internal error occured */
 #define EV_CONFIG		10	/* A configuration error was detected */
 #define EV_INVALID_STATE	11	/* The object is in an invalid state */
 #define EV_UNIMPLEMENTED	12	/* Unimplemented hypercall */
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index 5d9a303..d3a8778 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -211,9 +211,9 @@ struct kvm_cpuid_entry2 {
 	__u32 padding[3];
 };
 
-#define KVM_CPUID_FLAG_SIGNIFCANT_INDEX 1
-#define KVM_CPUID_FLAG_STATEFUL_FUNC    2
-#define KVM_CPUID_FLAG_STATE_READ_NEXT  4
+#define KVM_CPUID_FLAG_SIGNIFCANT_INDEX		BIT(0)
+#define KVM_CPUID_FLAG_STATEFUL_FUNC		BIT(1)
+#define KVM_CPUID_FLAG_STATE_READ_NEXT		BIT(2)
 
 /* for KVM_SET_CPUID2 */
 struct kvm_cpuid2 {
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 13e890c..8544336 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -541,6 +541,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_TRACE_ENABLE          __KVM_DEPRECATED_MAIN_W_0x06
 #define KVM_TRACE_PAUSE           __KVM_DEPRECATED_MAIN_0x07
 #define KVM_TRACE_DISABLE         __KVM_DEPRECATED_MAIN_0x08
+#define KVM_GET_EMULATED_CPUID	  _IOWR(KVMIO, 0x09, struct kvm_cpuid2)
 
 /*
  * Extension capability list.
@@ -668,6 +669,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_IRQ_XICS 92
 #define KVM_CAP_ARM_EL1_32BIT 93
 #define KVM_CAP_SPAPR_MULTITCE 94
+#define KVM_CAP_EXT_EMUL_CPUID 95
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -843,6 +845,10 @@ struct kvm_device_attr {
 #define KVM_DEV_TYPE_FSL_MPIC_20	1
 #define KVM_DEV_TYPE_FSL_MPIC_42	2
 #define KVM_DEV_TYPE_XICS		3
+#define KVM_DEV_TYPE_VFIO		4
+#define  KVM_DEV_VFIO_GROUP			1
+#define   KVM_DEV_VFIO_GROUP_ADD			1
+#define   KVM_DEV_VFIO_GROUP_DEL			2
 
 /*
  * ioctls for VM fds
@@ -1012,6 +1018,7 @@ struct kvm_s390_ucas_mapping {
 /* VM is being stopped by host */
 #define KVM_KVMCLOCK_CTRL	  _IO(KVMIO,   0xad)
 #define KVM_ARM_VCPU_INIT	  _IOW(KVMIO,  0xae, struct kvm_vcpu_init)
+#define KVM_ARM_PREFERRED_TARGET  _IOR(KVMIO,  0xaf, struct kvm_vcpu_init)
 #define KVM_GET_REG_LIST	  _IOWR(KVMIO, 0xb0, struct kvm_reg_list)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)

[Qemu-devel] [PATCH 2/2] vfio-pci: Make use of new KVM-VFIO device

[Alex Williamson]

Add and remove groups from the KVM virtual VFIO device as we make
use of them.  This allows KVM to optimize for performance and
correctness based on properties of the group.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---
 hw/misc/vfio.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index fe95e03..5a8c305 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -208,6 +208,17 @@ static QLIST_HEAD(, VFIOContainer)
 static QLIST_HEAD(, VFIOGroup)
     group_list = QLIST_HEAD_INITIALIZER(group_list);
 
+#ifdef CONFIG_KVM
+/*
+ * We have a single VFIO pseudo device per KVM VM.  Once created it lives
+ * for the life of the VM.  Closing the file descriptor only drops our
+ * reference to it and the device's reference to kvm.  Therefore once
+ * initialized, this file descriptor is only released on QEMU exit and
+ * we'll re-use it should another vfio device be attached before then.
+ */
+static int vfio_kvm_device_fd = -1;
+#endif
+
 static void vfio_disable_interrupts(VFIODevice *vdev);
 static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
@@ -3041,6 +3052,59 @@ static void vfio_pci_reset_handler(void *opaque)
     }
 }
 
+static void vfio_kvm_device_add_group(VFIOGroup *group)
+{
+#ifdef CONFIG_KVM
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_GROUP,
+        .attr = KVM_DEV_VFIO_GROUP_ADD,
+        .addr = (uint64_t)(unsigned long)&group->fd,
+    };
+
+    if (!kvm_enabled()) {
+        return;
+    }
+
+    if (vfio_kvm_device_fd < 0) {
+        struct kvm_create_device cd = {
+            .type = KVM_DEV_TYPE_VFIO,
+        };
+
+        if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
+            DPRINTF("KVM_CREATE_DEVICE: %m\n");
+            return;
+        }
+
+        vfio_kvm_device_fd = cd.fd;
+    }
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to add group %d to KVM VFIO device: %m",
+                     group->groupid);
+    }
+#endif
+}
+
+static void vfio_kvm_device_del_group(VFIOGroup *group)
+{
+#ifdef CONFIG_KVM
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_GROUP,
+        .attr = KVM_DEV_VFIO_GROUP_DEL,
+        .addr = (uint64_t)(unsigned long)&group->fd,
+    };
+
+    if (vfio_kvm_device_fd < 0) {
+        return;
+    }
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to remove group %d to KVM VFIO device: %m",
+                     group->groupid);
+    }
+#endif
+}
+
 static int vfio_connect_container(VFIOGroup *group)
 {
     VFIOContainer *container;
@@ -3189,6 +3253,8 @@ static VFIOGroup *vfio_get_group(int groupid)
 
     QLIST_INSERT_HEAD(&group_list, group, next);
 
+    vfio_kvm_device_add_group(group);
+
     return group;
 }
 
@@ -3198,6 +3264,7 @@ static void vfio_put_group(VFIOGroup *group)
         return;
     }
 
+    vfio_kvm_device_del_group(group);
     vfio_disconnect_container(group);
     QLIST_REMOVE(group, next);
     DPRINTF("vfio_put_group: close group->fd\n");