[Qemu-devel] [PATCH] virtio: Implement userspace forwarding for host notifiers

[Pavel Fedin <p.fedin@samsung.com>]

If you happen to have a kernel with ioeventfd support enabled, but
missing support for them in KVM, and you attempt to enable vhost by
setting vhost=on, qemu aborts with error:

kvm_mem_ioeventfd_add: error adding ioeventfd: Function not implemented

This patch adds a mechanism which allows to emulate KVM binding by
triggering the related notifiers via the userspace. The first time the
emulation is used, a warning is displayed, so that the user knows about
potential performance impact:

2015-11-19T09:35:16.618380Z qemu-system-aarch64: KVM does not support eventfd binding, using userspace event forwarding (slow)

This problem can be observed with libvirt, which checks for /dev/vhost-net
availability and just inserts "vhost=on" automatically in this case; on an
ARM64 system using stock kernel 3.18.0 with CONFIG_IOEVENTFD enabled in
expert settings.

Signed-off-by: Pavel Fedin <p.fedin@samsung.com>
---
 hw/virtio/virtio-mmio.c    | 15 +++++++++---
 hw/virtio/virtio-pci.c     | 61 ++++++++++++++++++++++++++--------------------
 hw/virtio/virtio.c         | 24 +++++++++++++++++-
 include/hw/virtio/virtio.h |  1 +
 4 files changed, 69 insertions(+), 32 deletions(-)

diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index 16621fa..69d4cbc 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -110,11 +110,18 @@ static int virtio_mmio_set_host_notifier_internal(VirtIOMMIOProxy *proxy,
             return r;
         }
         virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
-        memory_region_add_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
-                                  true, n, notifier);
+
+        if (kvm_eventfds_enabled()) {
+            memory_region_add_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
+                                      true, n, notifier);
+        } else if (!set_handler) {
+            virtio_queue_set_host_notifier_forwarding(vq);
+        }
     } else {
-        memory_region_del_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
-                                  true, n, notifier);
+        if (kvm_eventfds_enabled()) {
+            memory_region_del_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
+                                      true, n, notifier);
+        }
         virtio_queue_set_host_notifier_fd_handler(vq, false, false);
         event_notifier_cleanup(notifier);
     }
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index 96be4fd..b27a630 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -293,41 +293,48 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
             return r;
         }
         virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
-        if (modern) {
-            if (fast_mmio) {
-                memory_region_add_eventfd(modern_mr, modern_addr, 0,
-                                          false, n, notifier);
-            } else {
-                memory_region_add_eventfd(modern_mr, modern_addr, 2,
-                                          false, n, notifier);
+
+        if (kvm_eventfds_enabled()) {
+            if (modern) {
+                if (fast_mmio) {
+                    memory_region_add_eventfd(modern_mr, modern_addr, 0,
+                                              false, n, notifier);
+                } else {
+                    memory_region_add_eventfd(modern_mr, modern_addr, 2,
+                                              false, n, notifier);
+                }
+                if (modern_pio) {
+                    memory_region_add_eventfd(modern_notify_mr, 0, 2,
+                                                  true, n, notifier);
+                }
             }
-            if (modern_pio) {
-                memory_region_add_eventfd(modern_notify_mr, 0, 2,
-                                              true, n, notifier);
+            if (legacy) {
+                memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
+                                          true, n, notifier);
             }
-        }
-        if (legacy) {
-            memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
-                                      true, n, notifier);
+        } else if (!set_handler) {
+            virtio_queue_set_host_notifier_forwarding(vq);
         }
     } else {
-        if (modern) {
-            if (fast_mmio) {
-                memory_region_del_eventfd(modern_mr, modern_addr, 0,
-                                          false, n, notifier);
-            } else {
-                memory_region_del_eventfd(modern_mr, modern_addr, 2,
-                                          false, n, notifier);
+        if (kvm_eventfds_enabled()) {
+            if (modern) {
+                if (fast_mmio) {
+                    memory_region_del_eventfd(modern_mr, modern_addr, 0,
+                                              false, n, notifier);
+                } else {
+                    memory_region_del_eventfd(modern_mr, modern_addr, 2,
+                                              false, n, notifier);
+                }
+                if (modern_pio) {
+                    memory_region_del_eventfd(modern_notify_mr, 0, 2,
+                                              true, n, notifier);
+                }
             }
-            if (modern_pio) {
-                memory_region_del_eventfd(modern_notify_mr, 0, 2,
+            if (legacy) {
+                memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
                                           true, n, notifier);
             }
         }
-        if (legacy) {
-            memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
-                                      true, n, notifier);
-        }
         virtio_queue_set_host_notifier_fd_handler(vq, false, false);
         event_notifier_cleanup(notifier);
     }
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index 1edef59..6fe268f 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -89,6 +89,7 @@ struct VirtQueue
     VirtIODevice *vdev;
     EventNotifier guest_notifier;
     EventNotifier host_notifier;
+    bool forward_host_notifier;
     QLIST_ENTRY(VirtQueue) node;
 };
 
@@ -969,7 +970,13 @@ void virtio_queue_notify_vq(VirtQueue *vq)
 
 void virtio_queue_notify(VirtIODevice *vdev, int n)
 {
-    virtio_queue_notify_vq(&vdev->vq[n]);
+    VirtQueue *vq = &vdev->vq[n];
+
+    if (vq->forward_host_notifier) {
+        event_notifier_set(&vq->host_notifier);
+    } else {
+        virtio_queue_notify_vq(&vdev->vq[n]);
+    }
 }
 
 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
@@ -1715,6 +1722,21 @@ void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
         /* Test and clear notifier before after disabling event,
          * in case poll callback didn't have time to run. */
         virtio_queue_host_notifier_read(&vq->host_notifier);
+
+        vq->forward_host_notifier = false;
+    }
+}
+
+static bool forwarding_warning;
+
+void virtio_queue_set_host_notifier_forwarding(VirtQueue *vq)
+{
+    vq->forward_host_notifier = true;
+
+    if (!forwarding_warning) {
+        forwarding_warning = true;
+        error_report("KVM does not support eventfd binding, "
+                     "using userspace event forwarding (slow)");
     }
 }
 
diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
index 205fadf..f288ccb 100644
--- a/include/hw/virtio/virtio.h
+++ b/include/hw/virtio/virtio.h
@@ -245,6 +245,7 @@ void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                bool set_handler);
+void virtio_queue_set_host_notifier_forwarding(VirtQueue *vq);
 void virtio_queue_notify_vq(VirtQueue *vq);
 void virtio_irq(VirtQueue *vq);
 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector);
-- 
1.9.5.msysgit.0