[Qemu-devel] [RFC PATCH] vfio-pci: Make use of new KVM-VFIO device

[Alex Williamson <alex.williamson@redhat.com>]

Add and remove groups from the KVM virtual VFIO device as we make
use of them.  This allows KVM to optimize for performance and
correctness based on properties of the group.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---

This patch is enabled by:

[RFC PATCH 0/3] kvm/vfio: Manage KVM IOMMU coherency with virtual VFIO device

(kvm list for those watching from qemu-devel)

 hw/misc/vfio.c |   61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index 98d372f..02b0f5a 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -53,6 +53,8 @@
 #define VFIO_ALLOW_MMAP 1
 #define VFIO_ALLOW_KVM_INTX 1
 
+int vfio_kvm_device_fd = -1;
+
 struct VFIODevice;
 
 typedef struct VFIOQuirk {
@@ -3032,6 +3034,61 @@ static void vfio_pci_reset_handler(void *opaque)
     }
 }
 
+static void vfio_kvm_device_add_group(VFIOGroup *group)
+{
+#ifdef CONFIG_KVM
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_ADD_GROUP,
+        .addr = group->fd,
+    };
+
+    if (vfio_kvm_device_fd < 0) {
+        struct kvm_create_device cd = {
+            .type = KVM_DEV_TYPE_VFIO,
+        };
+
+        if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
+            DPRINTF("KVM_CREATE_DEVICE: %m\n");
+            return;
+        }
+
+        vfio_kvm_device_fd = cd.fd;
+    }
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to add group %d to KVM VFIO device: %m",
+                     group->groupid);
+    }
+#endif
+}
+
+static void vfio_kvm_device_del_group(VFIOGroup *group)
+{
+#ifdef CONFIG_KVM
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_DEL_GROUP,
+        .addr = group->fd,
+    };
+
+    if (vfio_kvm_device_fd < 0) {
+        return;
+    }
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to remove group %d to KVM VFIO device: %m",
+                     group->groupid);
+    }
+#endif
+}
+
+static void vfio_kvm_device_cleanup(void)
+{
+	if (vfio_kvm_device_fd >= 0) {
+		close(vfio_kvm_device_fd);
+		vfio_kvm_device_fd = -1;
+	}
+}
+
 static int vfio_connect_container(VFIOGroup *group)
 {
     VFIOContainer *container;
@@ -3180,6 +3237,8 @@ static VFIOGroup *vfio_get_group(int groupid)
 
     QLIST_INSERT_HEAD(&group_list, group, next);
 
+    vfio_kvm_device_add_group(group);
+
     return group;
 }
 
@@ -3189,6 +3248,7 @@ static void vfio_put_group(VFIOGroup *group)
         return;
     }
 
+    vfio_kvm_device_del_group(group);
     vfio_disconnect_container(group);
     QLIST_REMOVE(group, next);
     DPRINTF("vfio_put_group: close group->fd\n");
@@ -3197,6 +3257,7 @@ static void vfio_put_group(VFIOGroup *group)
 
     if (QLIST_EMPTY(&group_list)) {
         qemu_unregister_reset(vfio_pci_reset_handler, NULL);
+        vfio_kvm_device_cleanup();
     }
 }