[Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O as a batch

[Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O as a batch

[Ming Lei]

Hi,

The commit 580b6b2aa2(dataplane: use the QEMU block layer for I/O)
introduces ~40% throughput regression on virtio-blk dataplane, and
one of causes is that submitting I/O as a batch is removed.

This patchset trys to introduce this mechanism on block, at least,
linux-aio can benefit from that.

With these patches, it is observed that thoughout on virtio-blk
dataplane can be improved a lot, see data in commit log of patch
3/3.

It should be possible to apply the batch mechanism to other devices
(such as virtio-scsi) too.

TODO:
	- support queuing I/O to multi files for scsi devies, which
    need some changes to linux-aio

V6:
	- fix requests leak if part of them arn't submitted successfully,
	pointed by Stefan
	- linux-aio.c coding style fix

V5:
	- rebase on v2.1.0-rc0 of qemu.git/master
	- block/linux-aio.c code style fix
	- don't flush io queue before flush, pointed by Paolo

V4:
	- support other non-raw formats with under-optimized performance
	- use reference counter for plug & unplug
	- flush io queue before sending flush command

V3:
	- only support submitting I/O as a batch for raw format, pointed by
    Kevin

V2:
	- define return value of bdrv_io_unplug as void, suggested by Paolo
	- avoid busy-wait for handling io_submit
V1:
	- move queuing io stuff into linux-aio.c as suggested by Paolo


Thanks,
--
Ming Lei

[Qemu-devel] [PATCH v6 1/3] block: block: introduce APIs for submitting IO as a batch

[Ming Lei]

This patch introduces three APIs so that following
patches can support queuing I/O requests and submitting them
as a batch for improving I/O performance.

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 block.c                   |   31 +++++++++++++++++++++++++++++++
 include/block/block.h     |    4 ++++
 include/block/block_int.h |    5 +++++
 3 files changed, 40 insertions(+)

diff --git a/block.c b/block.c
index c8379ca..1aa452f 100644
--- a/block.c
+++ b/block.c
@@ -1905,6 +1905,7 @@ void bdrv_drain_all(void)
             bool bs_busy;
 
             aio_context_acquire(aio_context);
+            bdrv_flush_io_queue(bs);
             bdrv_start_throttled_reqs(bs);
             bs_busy = bdrv_requests_pending(bs);
             bs_busy |= aio_poll(aio_context, bs_busy);
@@ -5786,3 +5787,33 @@ BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
 
     return to_replace_bs;
 }
+
+void bdrv_io_plug(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (drv && drv->bdrv_io_plug) {
+        drv->bdrv_io_plug(bs);
+    } else if (bs->file) {
+        bdrv_io_plug(bs->file);
+    }
+}
+
+void bdrv_io_unplug(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (drv && drv->bdrv_io_unplug) {
+        drv->bdrv_io_unplug(bs);
+    } else if (bs->file) {
+        bdrv_io_unplug(bs->file);
+    }
+}
+
+void bdrv_flush_io_queue(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (drv && drv->bdrv_flush_io_queue) {
+        drv->bdrv_flush_io_queue(bs);
+    } else if (bs->file) {
+        bdrv_flush_io_queue(bs->file);
+    }
+}
diff --git a/include/block/block.h b/include/block/block.h
index baecc26..32d3676 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -584,4 +584,8 @@ AioContext *bdrv_get_aio_context(BlockDriverState *bs);
  */
 void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context);
 
+void bdrv_io_plug(BlockDriverState *bs);
+void bdrv_io_unplug(BlockDriverState *bs);
+void bdrv_flush_io_queue(BlockDriverState *bs);
+
 #endif
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 8f8e65e..f6c3bef 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -261,6 +261,11 @@ struct BlockDriver {
     void (*bdrv_attach_aio_context)(BlockDriverState *bs,
                                     AioContext *new_context);
 
+    /* io queue for linux-aio */
+    void (*bdrv_io_plug)(BlockDriverState *bs);
+    void (*bdrv_io_unplug)(BlockDriverState *bs);
+    void (*bdrv_flush_io_queue)(BlockDriverState *bs);
+
     QLIST_ENTRY(BlockDriver) list;
 };
 
-- 
1.7.9.5

[Qemu-devel] [PATCH v6 2/3] linux-aio: implement io plug, unplug and flush io queue

[Ming Lei]

This patch implements .bdrv_io_plug, .bdrv_io_unplug and
.bdrv_flush_io_queue callbacks for linux-aio Block Drivers,
so that submitting I/O as a batch can be supported on linux-aio.

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 block/linux-aio.c |   96 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 block/raw-aio.h   |    2 ++
 block/raw-posix.c |   45 +++++++++++++++++++++++++
 3 files changed, 141 insertions(+), 2 deletions(-)

diff --git a/block/linux-aio.c b/block/linux-aio.c
index f0a2c08..f8a5230 100644
--- a/block/linux-aio.c
+++ b/block/linux-aio.c
@@ -25,6 +25,8 @@
  */
 #define MAX_EVENTS 128
 
+#define MAX_QUEUED_IO  128
+
 struct qemu_laiocb {
     BlockDriverAIOCB common;
     struct qemu_laio_state *ctx;
@@ -36,9 +38,19 @@ struct qemu_laiocb {
     QLIST_ENTRY(qemu_laiocb) node;
 };
 
+typedef struct {
+    struct iocb *iocbs[MAX_QUEUED_IO];
+    int plugged;
+    unsigned int size;
+    unsigned int idx;
+} LaioQueue;
+
 struct qemu_laio_state {
     io_context_t ctx;
     EventNotifier e;
+
+    /* io queue for submit at batch */
+    LaioQueue io_q;
 };
 
 static inline ssize_t io_event_ret(struct io_event *ev)
@@ -135,6 +147,79 @@ static const AIOCBInfo laio_aiocb_info = {
     .cancel             = laio_cancel,
 };
 
+static void ioq_init(LaioQueue *io_q)
+{
+    io_q->size = MAX_QUEUED_IO;
+    io_q->idx = 0;
+    io_q->plugged = 0;
+}
+
+static int ioq_submit(struct qemu_laio_state *s)
+{
+    int ret, i = 0;
+    int len = s->io_q.idx;
+
+    do {
+        ret = io_submit(s->ctx, len, s->io_q.iocbs);
+    } while (i++ < 3 && ret == -EAGAIN);
+
+    /* empty io queue */
+    s->io_q.idx = 0;
+
+    if (ret < 0) {
+        i = 0;
+    } else {
+        i = ret;
+    }
+
+    for (; i < len; i++) {
+        struct qemu_laiocb *laiocb =
+            container_of(s->io_q.iocbs[i], struct qemu_laiocb, iocb);
+
+        laiocb->ret = ret;
+        qemu_laio_process_completion(s, laiocb);
+    }
+    return ret;
+}
+
+static void ioq_enqueue(struct qemu_laio_state *s, struct iocb *iocb)
+{
+    unsigned int idx = s->io_q.idx;
+
+    s->io_q.iocbs[idx++] = iocb;
+    s->io_q.idx = idx;
+
+    /* submit immediately if queue is full */
+    if (idx == s->io_q.size) {
+        ioq_submit(s);
+    }
+}
+
+void laio_io_plug(BlockDriverState *bs, void *aio_ctx)
+{
+    struct qemu_laio_state *s = aio_ctx;
+
+    s->io_q.plugged++;
+}
+
+int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug)
+{
+    struct qemu_laio_state *s = aio_ctx;
+    int ret = 0;
+
+    assert(s->io_q.plugged > 0 || !unplug);
+
+    if (unplug && --s->io_q.plugged > 0) {
+        return 0;
+    }
+
+    if (s->io_q.idx > 0) {
+        ret = ioq_submit(s);
+    }
+
+    return ret;
+}
+
 BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque, int type)
@@ -168,8 +253,13 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
     }
     io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
 
-    if (io_submit(s->ctx, 1, &iocbs) < 0)
-        goto out_free_aiocb;
+    if (!s->io_q.plugged) {
+        if (io_submit(s->ctx, 1, &iocbs) < 0) {
+            goto out_free_aiocb;
+        }
+    } else {
+        ioq_enqueue(s, iocbs);
+    }
     return &laiocb->common;
 
 out_free_aiocb:
@@ -204,6 +294,8 @@ void *laio_init(void)
         goto out_close_efd;
     }
 
+    ioq_init(&s->io_q);
+
     return s;
 
 out_close_efd:
diff --git a/block/raw-aio.h b/block/raw-aio.h
index 8cf084e..e18c975 100644
--- a/block/raw-aio.h
+++ b/block/raw-aio.h
@@ -40,6 +40,8 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
         BlockDriverCompletionFunc *cb, void *opaque, int type);
 void laio_detach_aio_context(void *s, AioContext *old_context);
 void laio_attach_aio_context(void *s, AioContext *new_context);
+void laio_io_plug(BlockDriverState *bs, void *aio_ctx);
+int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug);
 #endif
 
 #ifdef _WIN32
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 825a0c8..808b2d8 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -1057,6 +1057,36 @@ static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
                        cb, opaque, type);
 }
 
+static void raw_aio_plug(BlockDriverState *bs)
+{
+#ifdef CONFIG_LINUX_AIO
+    BDRVRawState *s = bs->opaque;
+    if (s->use_aio) {
+        laio_io_plug(bs, s->aio_ctx);
+    }
+#endif
+}
+
+static void raw_aio_unplug(BlockDriverState *bs)
+{
+#ifdef CONFIG_LINUX_AIO
+    BDRVRawState *s = bs->opaque;
+    if (s->use_aio) {
+        laio_io_unplug(bs, s->aio_ctx, true);
+    }
+#endif
+}
+
+static void raw_aio_flush_io_queue(BlockDriverState *bs)
+{
+#ifdef CONFIG_LINUX_AIO
+    BDRVRawState *s = bs->opaque;
+    if (s->use_aio) {
+        laio_io_unplug(bs, s->aio_ctx, false);
+    }
+#endif
+}
+
 static BlockDriverAIOCB *raw_aio_readv(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque)
@@ -1528,6 +1558,9 @@ static BlockDriver bdrv_file = {
     .bdrv_aio_flush = raw_aio_flush,
     .bdrv_aio_discard = raw_aio_discard,
     .bdrv_refresh_limits = raw_refresh_limits,
+    .bdrv_io_plug = raw_aio_plug,
+    .bdrv_io_unplug = raw_aio_unplug,
+    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate = raw_truncate,
     .bdrv_getlength = raw_getlength,
@@ -1927,6 +1960,9 @@ static BlockDriver bdrv_host_device = {
     .bdrv_aio_flush	= raw_aio_flush,
     .bdrv_aio_discard   = hdev_aio_discard,
     .bdrv_refresh_limits = raw_refresh_limits,
+    .bdrv_io_plug = raw_aio_plug,
+    .bdrv_io_unplug = raw_aio_unplug,
+    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength	= raw_getlength,
@@ -2072,6 +2108,9 @@ static BlockDriver bdrv_host_floppy = {
     .bdrv_aio_writev    = raw_aio_writev,
     .bdrv_aio_flush	= raw_aio_flush,
     .bdrv_refresh_limits = raw_refresh_limits,
+    .bdrv_io_plug = raw_aio_plug,
+    .bdrv_io_unplug = raw_aio_unplug,
+    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength      = raw_getlength,
@@ -2200,6 +2239,9 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_aio_writev    = raw_aio_writev,
     .bdrv_aio_flush	= raw_aio_flush,
     .bdrv_refresh_limits = raw_refresh_limits,
+    .bdrv_io_plug = raw_aio_plug,
+    .bdrv_io_unplug = raw_aio_unplug,
+    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength      = raw_getlength,
@@ -2334,6 +2376,9 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_aio_writev    = raw_aio_writev,
     .bdrv_aio_flush	= raw_aio_flush,
     .bdrv_refresh_limits = raw_refresh_limits,
+    .bdrv_io_plug = raw_aio_plug,
+    .bdrv_io_unplug = raw_aio_unplug,
+    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength      = raw_getlength,
-- 
1.7.9.5

[Qemu-devel] [PATCH v6 3/3] dataplane: submit I/O as a batch

[Ming Lei]

Before commit 580b6b2aa2(dataplane: use the QEMU block
layer for I/O), dataplane for virtio-blk submits block
I/O as a batch.

This commit 580b6b2aa2 replaces the custom linux AIO
implementation(including submit I/O as a batch) with QEMU
block layer, but this commit causes ~40% throughput regression
on virtio-blk performance, and removing submitting I/O
as a batch is one of the causes.

This patch applies the newly introduced bdrv_io_plug() and
bdrv_io_unplug() interfaces to support submitting I/O
at batch for Qemu block layer, and in my test, the change
can improve throughput by ~30% with 'aio=native'.

Following my fio test script:

	[global]
	direct=1
	size=4G
	bsrange=4k-4k
	timeout=40
	numjobs=4
	ioengine=libaio
	iodepth=64
	filename=/dev/vdc
	group_reporting=1

	[f]
	rw=randread

Result on one of my small machine(host: x86_64, 2cores, 4thread, guest: 4cores):
	- qemu master: 65K IOPS
	- qemu master with these patches: 92K IOPS
	- 2.0.0 release(dataplane using custom linux aio): 104K IOPS

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 hw/block/dataplane/virtio-blk.c |    2 ++
 1 file changed, 2 insertions(+)

diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index 20f12c7..227bb15 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -77,6 +77,7 @@ static void handle_notify(EventNotifier *e)
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
 
     event_notifier_test_and_clear(&s->host_notifier);
+    bdrv_io_plug(s->blk->conf.bs);
     for (;;) {
         MultiReqBuffer mrb = {
             .num_writes = 0,
@@ -115,6 +116,7 @@ static void handle_notify(EventNotifier *e)
             break;
         }
     }
+    bdrv_io_unplug(s->blk->conf.bs);
 }
 
 /* Context: QEMU global mutex held */
-- 
1.7.9.5

Re: [Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O as a batch

[Stefan Hajnoczi]

[-- Attachment #1: Type: text/plain, Size: 1793 bytes --]

On Fri, Jul 04, 2014 at 06:04:32PM +0800, Ming Lei wrote:
> Hi,
> 
> The commit 580b6b2aa2(dataplane: use the QEMU block layer for I/O)
> introduces ~40% throughput regression on virtio-blk dataplane, and
> one of causes is that submitting I/O as a batch is removed.
> 
> This patchset trys to introduce this mechanism on block, at least,
> linux-aio can benefit from that.
> 
> With these patches, it is observed that thoughout on virtio-blk
> dataplane can be improved a lot, see data in commit log of patch
> 3/3.
> 
> It should be possible to apply the batch mechanism to other devices
> (such as virtio-scsi) too.
> 
> TODO:
> 	- support queuing I/O to multi files for scsi devies, which
>     need some changes to linux-aio
> 
> V6:
> 	- fix requests leak if part of them arn't submitted successfully,
> 	pointed by Stefan
> 	- linux-aio.c coding style fix
> 
> V5:
> 	- rebase on v2.1.0-rc0 of qemu.git/master
> 	- block/linux-aio.c code style fix
> 	- don't flush io queue before flush, pointed by Paolo
> 
> V4:
> 	- support other non-raw formats with under-optimized performance
> 	- use reference counter for plug & unplug
> 	- flush io queue before sending flush command
> 
> V3:
> 	- only support submitting I/O as a batch for raw format, pointed by
>     Kevin
> 
> V2:
> 	- define return value of bdrv_io_unplug as void, suggested by Paolo
> 	- avoid busy-wait for handling io_submit
> V1:
> 	- move queuing io stuff into linux-aio.c as suggested by Paolo

Thanks, applied to my block tree:
https://github.com/stefanha/qemu/commits/block

In Patch 2 we should complete requests with -EIO if io_submit() returned
0 <= ret < len.  I fixed this up when applying because the patch was
completing with a bogus ret value.

Stefan

[-- Attachment #2: Type: application/pgp-signature, Size: 473 bytes --]

Re: [Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O asa batch

[Zhang Haoyu]

>> Hi,
>> 
>> The commit 580b6b2aa2(dataplane: use the QEMU block layer for I/O)
>> introduces ~40% throughput regression on virtio-blk dataplane, and
>> one of causes is that submitting I/O as a batch is removed.
>> 
>> This patchset trys to introduce this mechanism on block, at least,
>> linux-aio can benefit from that.
>> 
>> With these patches, it is observed that thoughout on virtio-blk
>> dataplane can be improved a lot, see data in commit log of patch
>> 3/3.
>> 
>> It should be possible to apply the batch mechanism to other devices
>> (such as virtio-scsi) too.
>> 
>> TODO:
>> 	- support queuing I/O to multi files for scsi devies, which
>>     need some changes to linux-aio
>> 
>> V6:
>> 	- fix requests leak if part of them arn't submitted successfully,
>> 	pointed by Stefan
>> 	- linux-aio.c coding style fix
>> 
>> V5:
>> 	- rebase on v2.1.0-rc0 of qemu.git/master
>> 	- block/linux-aio.c code style fix
>> 	- don't flush io queue before flush, pointed by Paolo
>> 
>> V4:
>> 	- support other non-raw formats with under-optimized performance
>> 	- use reference counter for plug & unplug
>> 	- flush io queue before sending flush command
>> 
>> V3:
>> 	- only support submitting I/O as a batch for raw format, pointed by
>>     Kevin
>> 
>> V2:
>> 	- define return value of bdrv_io_unplug as void, suggested by Paolo
>> 	- avoid busy-wait for handling io_submit
>> V1:
>> 	- move queuing io stuff into linux-aio.c as suggested by Paolo
>
>Thanks, applied to my block tree:
>https://github.com/stefanha/qemu/commits/block
>
Can we use the queued io data as caches, 
io write will directly return and tell the guest the io is completed after the io is enqueued, 
better user experience for burst io,
and io-read will firstly search the io queue, if matched data found, directly get the data from the queue, 
if not, then read the data from the disk or host page cache.
Any ideas?

Thanks,
Zhang Haoyu

>In Patch 2 we should complete requests with -EIO if io_submit() returned
>0 <= ret < len.  I fixed this up when applying because the patch was
>completing with a bogus ret value.
>
>Stefan

Re: [Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O asa batch

[Fam Zheng]

On Tue, 08/26 15:31, Zhang Haoyu wrote:
> Can we use the queued io data as caches, 
> io write will directly return and tell the guest the io is completed after the io is enqueued, 
> better user experience for burst io,
> and io-read will firstly search the io queue, if matched data found, directly get the data from the queue, 
> if not, then read the data from the disk or host page cache.
> Any ideas?

Guest kernel already has a page cache that exactly does this, also keeping a
copy of guest request data in qemu may hurt the bandwidth in some cases.

Fam

Re: [Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O asabatch

[Zhang Haoyu]

>> Can we use the queued io data as caches, 
>> io write will directly return and tell the guest the io is completed after the io is enqueued, 
>> better user experience for burst io,
>> and io-read will firstly search the io queue, if matched data found, directly get the data from the queue, 
>> if not, then read the data from the disk or host page cache.
>> Any ideas?
>
>Guest kernel already has a page cache that exactly does this, also keeping a
>copy of guest request data in qemu may hurt the bandwidth in some cases.
>
You are right.
Does the io merging in queue is worthy to be performed ?

Thanks,
Zhang Haoyu
>Fam

Re: [Qemu-devel] [PATCH v6 0/3] linux-aio: introduce submit I/O as a batch

[Zhang Haoyu]

>>> Hi,
>>> 
>>> The commit 580b6b2aa2(dataplane: use the QEMU block layer for I/O)
>>> introduces ~40% throughput regression on virtio-blk dataplane, and
>>> one of causes is that submitting I/O as a batch is removed.
>>> 
>>> This patchset trys to introduce this mechanism on block, at least,
>>> linux-aio can benefit from that.
>>> 
>>> With these patches, it is observed that thoughout on virtio-blk
>>> dataplane can be improved a lot, see data in commit log of patch
>>> 3/3.
>>> 
>>> It should be possible to apply the batch mechanism to other devices
>>> (such as virtio-scsi) too.
>>> 
>>> TODO:
>>> 	- support queuing I/O to multi files for scsi devies, which
>>>     need some changes to linux-aio
>>> 
>>> V6:
>>> 	- fix requests leak if part of them arn't submitted successfully,
>>> 	pointed by Stefan
>>> 	- linux-aio.c coding style fix
>>> 
>>> V5:
>>> 	- rebase on v2.1.0-rc0 of qemu.git/master
>>> 	- block/linux-aio.c code style fix
>>> 	- don't flush io queue before flush, pointed by Paolo
>>> 
>>> V4:
>>> 	- support other non-raw formats with under-optimized performance
>>> 	- use reference counter for plug & unplug
>>> 	- flush io queue before sending flush command
>>> 
>>> V3:
>>> 	- only support submitting I/O as a batch for raw format, pointed by
>>>     Kevin
>>> 
>>> V2:
>>> 	- define return value of bdrv_io_unplug as void, suggested by Paolo
>>> 	- avoid busy-wait for handling io_submit
>>> V1:
>>> 	- move queuing io stuff into linux-aio.c as suggested by Paolo
>>
>>Thanks, applied to my block tree:
>>https://github.com/stefanha/qemu/commits/block
>>
>Can we use the queued io data as caches, 
>io write will directly return and tell the guest the io is completed after the io is enqueued, 
>better user experience for burst io,
>and io-read will firstly search the io queue, if matched data found, directly get the data from the queue, 
>if not, then read the data from the disk or host page cache.

Because host page cache cannot assure the order of io-write, leading to VM image data corruption if suddenly poweroff happened,
but we can assure the order of io-write in queue, and use cache=none/writethrough/directsync to submit the batched IO to disk directly, 
bypass host page cache.
IO merging also can be performed in the queue.

>Any ideas?
>
>Thanks,
>Zhang Haoyu
>
>>In Patch 2 we should complete requests with -EIO if io_submit() returned
>>0 <= ret < len.  I fixed this up when applying because the patch was
>>completing with a bogus ret value.
>>
>>Stefan