[Qemu-devel] [PATCH v3 0/3] virtio-mmio: introduce eventfd support

[Qemu-devel] [PATCH v3 0/3] virtio-mmio: introduce eventfd support

[Pavel Fedin]

 This patch set introduces eventfd support for virio-mmio. It was originally
published by Ying-Shiuan Pan but never got it to upstream:
https://lists.gnu.org/archive/html/qemu-devel/2014-02/msg00715.html
 I have updated and successfully tested it with vhost-net. I confirm that this
solution significantly improves the network performance even without irqfd.
 I would like to upstream it, since virtio-mmio is still there. I know that
some of you consider it deprecated, however i believe this is not entirely
true. Because you can add it to machine models which are not supposed to
have PCI (like vexpress).
 An old patch set relied on additional "eventfd" option in order to disable
the support if not implemented in kernel. My version simply checks
kvm_eventfds_enabled() for this purpose, so backwards compatibility is much
better.
 I decided to leave this set in three parts because ioeventfd support should
be enabled only when both host and guest notifiers are in place. I believe
it will not work with partial implementation, at least because vhost-net
requires both sets of eventfds. In this version i added correct reset handling.



Ying-Shiuan Pan (3):
  virtio-mmio: introduce set_host_notifier()
  virtio-mmio: introduce set_guest_notifiers
  virtio-mmio: start ioeventfd when status gets DRIVER_OK

 hw/virtio/virtio-mmio.c | 181 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 181 insertions(+)

-- 
1.9.5.msysgit.0

[Qemu-devel] [PATCH v3 1/3] virtio-mmio: introduce set_host_notifier()

[Pavel Fedin]

From: Ying-Shiuan Pan <yingshiuan.pan@gmail.com>

set_host_notifier() is introduced into virtio-mmio now. Most of codes came
from virtio-pci.

Signed-off-by: Ying-Shiuan Pan <yingshiuan.pan@gmail.com>
Signed-off-by: Pavel Fedin <p.fedin@samsung.com>
---
 hw/virtio/virtio-mmio.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index 10123f3..da56b9c 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -23,6 +23,7 @@
 #include "hw/virtio/virtio.h"
 #include "qemu/host-utils.h"
 #include "hw/virtio/virtio-bus.h"
+#include "qemu/error-report.h"
 
 /* #define DEBUG_VIRTIO_MMIO */
 
@@ -87,8 +88,59 @@ typedef struct {
     uint32_t guest_page_shift;
     /* virtio-bus */
     VirtioBusState bus;
+    bool ioeventfd_disabled;
+    bool ioeventfd_started;
 } VirtIOMMIOProxy;
 
+static int virtio_mmio_set_host_notifier_internal(VirtIOMMIOProxy *proxy,
+                                                  int n, bool assign,
+                                                  bool set_handler)
+{
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    VirtQueue *vq = virtio_get_queue(vdev, n);
+    EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
+    int r = 0;
+
+    if (assign) {
+        r = event_notifier_init(notifier, 1);
+        if (r < 0) {
+            error_report("%s: unable to init event notifier: %d",
+                         __func__, r);
+            return r;
+        }
+        virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
+        memory_region_add_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
+                                  true, n, notifier);
+    } else {
+        memory_region_del_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
+                                  true, n, notifier);
+        virtio_queue_set_host_notifier_fd_handler(vq, false, false);
+        event_notifier_cleanup(notifier);
+    }
+    return r;
+}
+
+static void virtio_mmio_stop_ioeventfd(VirtIOMMIOProxy *proxy)
+{
+    int r;
+    int n;
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    if (!proxy->ioeventfd_started) {
+        return;
+    }
+
+    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            continue;
+        }
+
+        r = virtio_mmio_set_host_notifier_internal(proxy, n, false, false);
+        assert(r >= 0);
+    }
+    proxy->ioeventfd_started = false;
+}
+
 static uint64_t virtio_mmio_read(void *opaque, hwaddr offset, unsigned size)
 {
     VirtIOMMIOProxy *proxy = (VirtIOMMIOProxy *)opaque;
@@ -336,12 +388,32 @@ static void virtio_mmio_reset(DeviceState *d)
 {
     VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
 
+    virtio_mmio_stop_ioeventfd(proxy);
     virtio_bus_reset(&proxy->bus);
     proxy->host_features_sel = 0;
     proxy->guest_features_sel = 0;
     proxy->guest_page_shift = 0;
 }
 
+static int virtio_mmio_set_host_notifier(DeviceState *opaque, int n,
+                                         bool assign)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(opaque);
+
+    /* Stop using ioeventfd for virtqueue kick if the device starts using host
+     * notifiers.  This makes it easy to avoid stepping on each others' toes.
+     */
+    proxy->ioeventfd_disabled = assign;
+    if (assign) {
+        virtio_mmio_stop_ioeventfd(proxy);
+    }
+    /* We don't need to start here: it's not needed because backend
+     * currently only stops on status change away from ok,
+     * reset, vmstop and such. If we do add code to start here,
+     * need to check vmstate, device state etc. */
+    return virtio_mmio_set_host_notifier_internal(proxy, n, assign, false);
+}
+
 /* virtio-mmio device */
 
 /* This is called by virtio-bus just after the device is plugged. */
@@ -399,6 +471,7 @@ static void virtio_mmio_bus_class_init(ObjectClass *klass, void *data)
     k->notify = virtio_mmio_update_irq;
     k->save_config = virtio_mmio_save_config;
     k->load_config = virtio_mmio_load_config;
+    k->set_host_notifier = virtio_mmio_set_host_notifier;
     k->get_features = virtio_mmio_get_features;
     k->device_plugged = virtio_mmio_device_plugged;
     k->has_variable_vring_alignment = true;
-- 
1.9.5.msysgit.0

[Qemu-devel] [PATCH v3 2/3] virtio-mmio: introduce set_guest_notifiers

[Pavel Fedin]

From: Ying-Shiuan Pan <yingshiuan.pan@gmail.com>

Same as host notifier of virtio-mmio, most of codes came from virtio-pci.
The kvm-arm does not yet support irqfd, need to fix the hard-coded part after
kvm-arm gets irqfd support.

Signed-off-by: Ying-Shiuan Pan <yingshiuan.pan@gmail.com>
Signed-off-by: Pavel Fedin <p.fedin@samsung.com>
---
 hw/virtio/virtio-mmio.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index da56b9c..a4c4d89 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -395,6 +395,66 @@ static void virtio_mmio_reset(DeviceState *d)
     proxy->guest_page_shift = 0;
 }
 
+static int virtio_mmio_set_guest_notifier(DeviceState *d, int n, bool assign,
+                                          bool with_irqfd)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
+    VirtQueue *vq = virtio_get_queue(vdev, n);
+    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
+
+    if (assign) {
+        int r = event_notifier_init(notifier, 0);
+        if (r < 0) {
+            return r;
+        }
+        virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
+    } else {
+        virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
+        event_notifier_cleanup(notifier);
+    }
+
+    if (vdc->guest_notifier_mask) {
+        vdc->guest_notifier_mask(vdev, n, !assign);
+    }
+
+    return 0;
+}
+
+static int virtio_mmio_set_guest_notifiers(DeviceState *d, int nvqs,
+                                           bool assign)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    /* TODO: need to check if kvm-arm supports irqfd */
+    bool with_irqfd = false;
+    int r, n;
+
+    nvqs = MIN(nvqs, VIRTIO_PCI_QUEUE_MAX);
+
+    for (n = 0; n < nvqs; n++) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            break;
+        }
+
+        r = virtio_mmio_set_guest_notifier(d, n, assign, with_irqfd);
+        if (r < 0) {
+            goto assign_error;
+        }
+    }
+
+    return 0;
+
+assign_error:
+    /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
+    assert(assign);
+    while (--n >= 0) {
+        virtio_mmio_set_guest_notifier(d, n, !assign, false);
+    }
+    return r;
+}
+
 static int virtio_mmio_set_host_notifier(DeviceState *opaque, int n,
                                          bool assign)
 {
@@ -472,6 +532,7 @@ static void virtio_mmio_bus_class_init(ObjectClass *klass, void *data)
     k->save_config = virtio_mmio_save_config;
     k->load_config = virtio_mmio_load_config;
     k->set_host_notifier = virtio_mmio_set_host_notifier;
+    k->set_guest_notifiers = virtio_mmio_set_guest_notifiers;
     k->get_features = virtio_mmio_get_features;
     k->device_plugged = virtio_mmio_device_plugged;
     k->has_variable_vring_alignment = true;
-- 
1.9.5.msysgit.0

[Qemu-devel] [PATCH v3 3/3] virtio-mmio: start ioeventfd when status gets DRIVER_OK

[Pavel Fedin]

From: Ying-Shiuan Pan <yingshiuan.pan@gmail.com>

Signed-off-by: Ying-Shiuan Pan <yingshiuan.pan@gmail.com>
Signed-off-by: Pavel Fedin <p.fedin@samsung.com>
---
 hw/virtio/virtio-mmio.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index a4c4d89..c12d3fc 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -22,6 +22,7 @@
 #include "hw/sysbus.h"
 #include "hw/virtio/virtio.h"
 #include "qemu/host-utils.h"
+#include "sysemu/kvm.h"
 #include "hw/virtio/virtio-bus.h"
 #include "qemu/error-report.h"
 
@@ -120,6 +121,43 @@ static int virtio_mmio_set_host_notifier_internal(VirtIOMMIOProxy *proxy,
     return r;
 }
 
+static void virtio_mmio_start_ioeventfd(VirtIOMMIOProxy *proxy)
+{
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    int n, r;
+
+    if (!kvm_eventfds_enabled() ||
+        proxy->ioeventfd_disabled ||
+        proxy->ioeventfd_started) {
+        return;
+    }
+
+    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            continue;
+        }
+
+        r = virtio_mmio_set_host_notifier_internal(proxy, n, true, true);
+        if (r < 0) {
+            goto assign_error;
+        }
+    }
+    proxy->ioeventfd_started = true;
+    return;
+
+assign_error:
+    while (--n >= 0) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            continue;
+        }
+
+        r = virtio_mmio_set_host_notifier_internal(proxy, n, false, false);
+        assert(r >= 0);
+    }
+    proxy->ioeventfd_started = false;
+    error_report("%s: failed. Fallback to a userspace (slower).", __func__);
+}
+
 static void virtio_mmio_stop_ioeventfd(VirtIOMMIOProxy *proxy)
 {
     int r;
@@ -318,7 +356,16 @@ static void virtio_mmio_write(void *opaque, hwaddr offset, uint64_t value,
         virtio_update_irq(vdev);
         break;
     case VIRTIO_MMIO_STATUS:
+        if (!(value & VIRTIO_CONFIG_S_DRIVER_OK)) {
+            virtio_mmio_stop_ioeventfd(proxy);
+        }
+
         virtio_set_status(vdev, value & 0xff);
+
+        if (value & VIRTIO_CONFIG_S_DRIVER_OK) {
+            virtio_mmio_start_ioeventfd(proxy);
+        }
+
         if (vdev->status == 0) {
             virtio_reset(vdev);
         }
-- 
1.9.5.msysgit.0

Re: [Qemu-devel] [PATCH v3 0/3] virtio-mmio: introduce eventfd support

[Peter Maydell]

On 12 May 2015 at 09:10, Pavel Fedin <p.fedin@samsung.com> wrote:
>  This patch set introduces eventfd support for virio-mmio. It was originally
> published by Ying-Shiuan Pan but never got it to upstream:
> https://lists.gnu.org/archive/html/qemu-devel/2014-02/msg00715.html
>  I have updated and successfully tested it with vhost-net. I confirm that this
> solution significantly improves the network performance even without irqfd.
>  I would like to upstream it, since virtio-mmio is still there. I know that
> some of you consider it deprecated, however i believe this is not entirely
> true. Because you can add it to machine models which are not supposed to
> have PCI (like vexpress).
>  An old patch set relied on additional "eventfd" option in order to disable
> the support if not implemented in kernel. My version simply checks
> kvm_eventfds_enabled() for this purpose, so backwards compatibility is much
> better.
>  I decided to leave this set in three parts because ioeventfd support should
> be enabled only when both host and guest notifiers are in place. I believe
> it will not work with partial implementation, at least because vhost-net
> requires both sets of eventfds. In this version i added correct reset handling.

Hi Pavel. I've been going through my to-review folder and found this
series from back in May lurking in it. Sorry I never got to reviewing it.
Do you think it's still a change worth having? If so, please can you
rebase and resend and I'll look at it this time around.

thanks
-- PMM

Re: [Qemu-devel] [PATCH v3 0/3] virtio-mmio: introduce eventfd support

[Yingshiuan Pan]

[-- Attachment #1: Type: text/plain, Size: 1860 bytes --]

Hi Peter,

I think this patch set was already merged by Michael in Jun. :).
http://git.qemu.org/?p=qemu.git;a=commit;h=434027badb421863b85ffdb4769966533c001cfa

--
Best Regards,
Yingshiuan Pan

2015-09-19 0:24 GMT+08:00 Peter Maydell <peter.maydell@linaro.org>:

> On 12 May 2015 at 09:10, Pavel Fedin <p.fedin@samsung.com> wrote:
> >  This patch set introduces eventfd support for virio-mmio. It was
> originally
> > published by Ying-Shiuan Pan but never got it to upstream:
> > https://lists.gnu.org/archive/html/qemu-devel/2014-02/msg00715.html
> >  I have updated and successfully tested it with vhost-net. I confirm
> that this
> > solution significantly improves the network performance even without
> irqfd.
> >  I would like to upstream it, since virtio-mmio is still there. I know
> that
> > some of you consider it deprecated, however i believe this is not
> entirely
> > true. Because you can add it to machine models which are not supposed to
> > have PCI (like vexpress).
> >  An old patch set relied on additional "eventfd" option in order to
> disable
> > the support if not implemented in kernel. My version simply checks
> > kvm_eventfds_enabled() for this purpose, so backwards compatibility is
> much
> > better.
> >  I decided to leave this set in three parts because ioeventfd support
> should
> > be enabled only when both host and guest notifiers are in place. I
> believe
> > it will not work with partial implementation, at least because vhost-net
> > requires both sets of eventfds. In this version i added correct reset
> handling.
>
> Hi Pavel. I've been going through my to-review folder and found this
> series from back in May lurking in it. Sorry I never got to reviewing it.
> Do you think it's still a change worth having? If so, please can you
> rebase and resend and I'll look at it this time around.
>
> thanks
> -- PMM
>

[-- Attachment #2: Type: text/html, Size: 2771 bytes --]

Re: [Qemu-devel] [PATCH v3 0/3] virtio-mmio: introduce eventfd support

[Peter Maydell]

On 18 September 2015 at 17:56, Yingshiuan Pan <yingshiuan.pan@gmail.com> wrote:
> Hi Peter,
>
> I think this patch set was already merged by Michael in Jun. :).
> http://git.qemu.org/?p=qemu.git;a=commit;h=434027badb421863b85ffdb4769966533c001cfa

That answers that question :-). Thanks for digging up the git
commit for me.

-- PMM