[Qemu-devel] [PATCH v2 1/8] virtio-blk: use batch notify in non-dataplane case

[Stefan Hajnoczi <stefanha@redhat.com>]

Commit 5b2ffbe4d99843fd8305c573a100047a8c962327 ("virtio-blk: dataplane:
notify guest as a batch") deferred guest notification to a BH in order
batch notifications.  This optimization is not specific to dataplane so
move it to the generic virtio-blk code that is shared by both dataplane
and non-dataplane code.

Use the AioContext notifier to detect when dataplane moves the
BlockBackend to the IOThread's AioContext.  This is necessary so the
notification BH is always created in the current AioContext.

Note that this patch adds a flush function to force pending
notifications.  This way we can ensure notifications do not cross device
reset or vmstate saving.

Cc: Ming Lei <tom.leiming@gmail.com>
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/block/dataplane/virtio-blk.c | 10 -------
 hw/block/virtio-blk.c           | 60 ++++++++++++++++++++++++++++++++++++-----
 include/hw/virtio/virtio-blk.h  |  1 +
 3 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index 3cb97c9..e0ac4f4 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -35,7 +35,6 @@ struct VirtIOBlockDataPlane {
     VirtIODevice *vdev;
     VirtQueue *vq;                  /* virtqueue vring */
     EventNotifier *guest_notifier;  /* irq */
-    QEMUBH *bh;                     /* bh for guest notification */
 
     Notifier insert_notifier, remove_notifier;
 
@@ -54,13 +53,6 @@ struct VirtIOBlockDataPlane {
 /* Raise an interrupt to signal guest, if necessary */
 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s)
 {
-    qemu_bh_schedule(s->bh);
-}
-
-static void notify_guest_bh(void *opaque)
-{
-    VirtIOBlockDataPlane *s = opaque;
-
     if (!virtio_should_notify(s->vdev, s->vq)) {
         return;
     }
@@ -156,7 +148,6 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
         object_ref(OBJECT(s->iothread));
     }
     s->ctx = iothread_get_aio_context(s->iothread);
-    s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
 
     s->insert_notifier.notify = data_plane_blk_insert_notifier;
     s->remove_notifier.notify = data_plane_blk_remove_notifier;
@@ -179,7 +170,6 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
     data_plane_remove_op_blockers(s);
     notifier_remove(&s->insert_notifier);
     notifier_remove(&s->remove_notifier);
-    qemu_bh_delete(s->bh);
     object_unref(OBJECT(s->iothread));
     g_free(s);
 }
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index 284e646..c108414 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -45,20 +45,57 @@ void virtio_blk_free_request(VirtIOBlockReq *req)
     }
 }
 
+/* Many requests can complete in an event loop iteration, we only notify the
+ * guest once.
+ */
+static void virtio_blk_batch_notify_bh(void *opaque)
+{
+    VirtIOBlock *s = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(s);
+
+    if (s->dataplane_started && !s->dataplane_disabled) {
+        virtio_blk_data_plane_notify(s->dataplane);
+    } else {
+        virtio_notify(vdev, s->vq);
+    }
+}
+
+/* Force batch notifications to run */
+static void virtio_blk_batch_notify_flush(VirtIOBlock *s)
+{
+    qemu_bh_cancel(s->batch_notify_bh);
+    virtio_blk_batch_notify_bh(s);
+}
+
+static void virtio_blk_attached_aio_context(AioContext *new_context,
+                                            void *opaque)
+{
+    VirtIOBlock *s = opaque;
+
+    assert(!s->batch_notify_bh);
+    s->batch_notify_bh = aio_bh_new(new_context, virtio_blk_batch_notify_bh,
+                                    s);
+    qemu_bh_schedule(s->batch_notify_bh); /* in case notify was pending */
+}
+
+static void virtio_blk_detach_aio_context(void *opaque)
+{
+    VirtIOBlock *s = opaque;
+
+    assert(s->batch_notify_bh);
+    qemu_bh_delete(s->batch_notify_bh);
+    s->batch_notify_bh = NULL;
+}
+
 static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
 {
     VirtIOBlock *s = req->dev;
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
     trace_virtio_blk_req_complete(req, status);
 
     stb_p(&req->in->status, status);
     virtqueue_push(s->vq, &req->elem, req->in_len);
-    if (s->dataplane_started && !s->dataplane_disabled) {
-        virtio_blk_data_plane_notify(s->dataplane);
-    } else {
-        virtio_notify(vdev, s->vq);
-    }
+    qemu_bh_schedule(s->batch_notify_bh);
 }
 
 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
@@ -664,6 +701,8 @@ static void virtio_blk_reset(VirtIODevice *vdev)
     if (s->dataplane) {
         virtio_blk_data_plane_stop(s->dataplane);
     }
+
+    virtio_blk_batch_notify_flush(s);
     aio_context_release(ctx);
 
     blk_set_enable_write_cache(s->blk, s->original_wce);
@@ -801,6 +840,8 @@ static void virtio_blk_save(QEMUFile *f, void *opaque)
         virtio_blk_data_plane_stop(s->dataplane);
     }
 
+    virtio_blk_batch_notify_flush(s);
+
     virtio_save(vdev, f);
 }
     
@@ -896,6 +937,10 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    blk_add_aio_context_notifier(s->blk, virtio_blk_attached_aio_context,
+                                 virtio_blk_detach_aio_context, s);
+    virtio_blk_attached_aio_context(blk_get_aio_context(s->blk), s);
+
     s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
     register_savevm(dev, "virtio-blk", virtio_blk_id++, 2,
                     virtio_blk_save, virtio_blk_load, s);
@@ -910,6 +955,9 @@ static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
 
+    blk_remove_aio_context_notifier(s->blk, virtio_blk_attached_aio_context,
+                                    virtio_blk_detach_aio_context, s);
+    virtio_blk_detach_aio_context(s);
     virtio_blk_data_plane_destroy(s->dataplane);
     s->dataplane = NULL;
     qemu_del_vm_change_state_handler(s->change);
diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
index 8f2b056..b3834bc 100644
--- a/include/hw/virtio/virtio-blk.h
+++ b/include/hw/virtio/virtio-blk.h
@@ -49,6 +49,7 @@ typedef struct VirtIOBlock {
     VirtQueue *vq;
     void *rq;
     QEMUBH *bh;
+    QEMUBH *batch_notify_bh;
     VirtIOBlkConf conf;
     unsigned short sector_mask;
     bool original_wce;
-- 
2.5.5