[patch 0/2] virtio-blk async IO

[patch 0/2] virtio-blk async IO

[Marcelo Tosatti]

Use the asynchronous version of block IO functions, otherwise guests can block
for long periods of time waiting for the operations to complete.

-- 


-------------------------------------------------------------------------
This SF.net email is sponsored by the 2008 JavaOne(SM) Conference 
Don't miss this year's exciting event. There's still time to save $100. 
Use priority code J8TL2D2. 
http://ad.doubleclick.net/clk;198757673;13503038;p?http://java.sun.com/javaone

[patch 1/2] QEMU/KVM: provide a reset method for virtio

[Marcelo Tosatti]

[-- Attachment #1: virtio-reset --]
[-- Type: text/plain, Size: 1346 bytes --]

So drivers can do whatever necessary on reset.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: kvm-userspace.aio/qemu/hw/virtio.c
===================================================================
--- kvm-userspace.aio.orig/qemu/hw/virtio.c
+++ kvm-userspace.aio/qemu/hw/virtio.c
@@ -166,6 +166,9 @@ void virtio_reset(void *opaque)
     VirtIODevice *vdev = opaque;
     int i;
 
+    if (vdev->reset)
+        vdev->reset(vdev);
+
     vdev->features = 0;
     vdev->queue_sel = 0;
     vdev->status = 0;
Index: kvm-userspace.aio/qemu/hw/virtio.h
===================================================================
--- kvm-userspace.aio.orig/qemu/hw/virtio.h
+++ kvm-userspace.aio/qemu/hw/virtio.h
@@ -119,6 +119,7 @@ struct VirtIODevice
     uint32_t (*get_features)(VirtIODevice *vdev);
     void (*set_features)(VirtIODevice *vdev, uint32_t val);
     void (*update_config)(VirtIODevice *vdev, uint8_t *config);
+    void (*reset)(VirtIODevice *vdev);
     VirtQueue vq[VIRTIO_PCI_QUEUE_MAX];
 };
 

-- 


-------------------------------------------------------------------------
This SF.net email is sponsored by the 2008 JavaOne(SM) Conference 
Don't miss this year's exciting event. There's still time to save $100. 
Use priority code J8TL2D2. 
http://ad.doubleclick.net/clk;198757673;13503038;p?http://java.sun.com/javaone

[patch 2/2] QEMU/KVM: virtio-blk async IO

[Marcelo Tosatti]

[-- Attachment #1: virtio-blk-async --]
[-- Type: text/plain, Size: 4632 bytes --]

virtio-blk should not use synchronous requests, as that can blocks vcpus 
outside of guest mode for large periods of time for no reason.

The generic block layer could complete AIO's before re-entering guest mode,
so that cached reads and writes can be reported ASAP, a job for the block layer.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: kvm-userspace.aio/qemu/hw/virtio-blk.c
===================================================================
--- kvm-userspace.aio.orig/qemu/hw/virtio-blk.c
+++ kvm-userspace.aio/qemu/hw/virtio-blk.c
@@ -77,54 +77,117 @@ static VirtIOBlock *to_virtio_blk(VirtIO
     return (VirtIOBlock *)vdev;
 }
 
+typedef struct VirtIOBlockReq
+{
+    VirtIODevice *vdev;
+    VirtQueue *vq;
+    struct iovec in_sg_status;
+    unsigned int pending;
+    unsigned int len;
+    unsigned int elem_idx;
+    int status;
+} VirtIOBlockReq;
+
+static void virtio_blk_rw_complete(void *opaque, int ret)
+{
+    VirtIOBlockReq *req = opaque;
+    struct virtio_blk_inhdr *in;
+    VirtQueueElement elem;
+
+    req->status |= ret;
+    if (--req->pending > 0)
+        return;
+
+    elem.index = req->elem_idx;
+    in = (void *)req->in_sg_status.iov_base;
+
+    in->status = req->status ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
+    virtqueue_push(req->vq, &elem, req->len);
+    virtio_notify(req->vdev, req->vq);
+    qemu_free(req);
+}
+
 static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBlock *s = to_virtio_blk(vdev);
     VirtQueueElement elem;
+    VirtIOBlockReq *req;
     unsigned int count;
 
     while ((count = virtqueue_pop(vq, &elem)) != 0) {
 	struct virtio_blk_inhdr *in;
 	struct virtio_blk_outhdr *out;
-	unsigned int wlen;
 	off_t off;
 	int i;
 
+	/*
+	 * FIXME: limit the number of in-flight requests
+	 */
+	req = qemu_malloc(sizeof(VirtIOBlockReq));
+	if (!req)
+	    return;
+	memset(req, 0, sizeof(*req));
+	memcpy(&req->in_sg_status, &elem.in_sg[elem.in_num - 1],
+	       sizeof(req->in_sg_status));
+	req->vdev = vdev;
+	req->vq = vq;
+	req->elem_idx = elem.index;
+
 	out = (void *)elem.out_sg[0].iov_base;
 	in = (void *)elem.in_sg[elem.in_num - 1].iov_base;
 	off = out->sector;
 
 	if (out->type & VIRTIO_BLK_T_SCSI_CMD) {
-	    wlen = sizeof(*in);
+	    unsigned int len = sizeof(*in);
+
 	    in->status = VIRTIO_BLK_S_UNSUPP;
+	    virtqueue_push(vq, &elem, len);
+	    virtio_notify(vdev, vq);
+	    qemu_free(req);
+
 	} else if (out->type & VIRTIO_BLK_T_OUT) {
-	    wlen = sizeof(*in);
+	    req->pending = elem.out_num - 1;
 
 	    for (i = 1; i < elem.out_num; i++) {
-		bdrv_write(s->bs, off,
+		bdrv_aio_write(s->bs, off,
 			   elem.out_sg[i].iov_base,
-			   elem.out_sg[i].iov_len / 512);
+			   elem.out_sg[i].iov_len / 512,
+			   virtio_blk_rw_complete,
+			   req);
 		off += elem.out_sg[i].iov_len / 512;
+		req->len += elem.out_sg[i].iov_len;
 	    }
 
-	    in->status = VIRTIO_BLK_S_OK;
 	} else {
-	    wlen = sizeof(*in);
+	    req->pending = elem.in_num - 1;
 
 	    for (i = 0; i < elem.in_num - 1; i++) {
-		bdrv_read(s->bs, off,
+		bdrv_aio_read(s->bs, off,
 			  elem.in_sg[i].iov_base,
-			  elem.in_sg[i].iov_len / 512);
+			  elem.in_sg[i].iov_len / 512,
+			  virtio_blk_rw_complete,
+			  req);
 		off += elem.in_sg[i].iov_len / 512;
-		wlen += elem.in_sg[i].iov_len;
+		req->len += elem.in_sg[i].iov_len;
 	    }
-
-	    in->status = VIRTIO_BLK_S_OK;
 	}
-
-	virtqueue_push(vq, &elem, wlen);
-	virtio_notify(vdev, vq);
     }
+    /*
+     * FIXME: Want to check for completions before returning to guest mode,
+     * so cached reads and writes are reported as quickly as possible. But
+     * that should be done in the generic block layer.
+     */
+}
+
+static void virtio_blk_reset(VirtIODevice *vdev)
+{
+    VirtIOBlock *s = to_virtio_blk(vdev);
+
+    /*
+     * This should cancel pending requests, but can't do nicely until there
+     * are per-device request lists.
+     */
+    qemu_aio_flush();
 }
 
 static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config)
@@ -156,6 +219,7 @@ void *virtio_blk_init(PCIBus *bus, uint1
 
     s->vdev.update_config = virtio_blk_update_config;
     s->vdev.get_features = virtio_blk_get_features;
+    s->vdev.reset = virtio_blk_reset;
     s->bs = bs;
     bs->devfn = s->vdev.pci_dev.devfn;
 

-- 


-------------------------------------------------------------------------
This SF.net email is sponsored by the 2008 JavaOne(SM) Conference 
Don't miss this year's exciting event. There's still time to save $100. 
Use priority code J8TL2D2. 
http://ad.doubleclick.net/clk;198757673;13503038;p?http://java.sun.com/javaone

Re: [patch 0/2] virtio-blk async IO

[Gerd von Egidy]

Hi Marcelo,

> Use the asynchronous version of block IO functions, otherwise guests can
> block for long periods of time waiting for the operations to complete.

just tried these patches. Results are similar to the last ones: the guest 
comes up fine but after running 2 or 3 minutes of bonnie++ the guest-vm 
hangs. This time I used screen on the guest console to try switching to 
another process - hanging too.

Here is the kvm_stat --once output:

efer_reload                    0         0
exits                    3325114       196
fpu_reload                185671         0
halt_exits                 18692        29
halt_wakeup                24807         0
host_state_reload        1387308        59
insn_emulation           1924291       130
insn_emulation_fail            0         0
invlpg                         0         0
io_exits                  350020        30
irq_exits                 225446         3
irq_window                     0         0
mmio_exits                917561         0
mmu_cache_miss             55436         0
mmu_flooded                64416         0
mmu_pde_zapped             46914         0
mmu_pte_updated           565547         0
mmu_pte_write             650181         0
mmu_recycled                   0         0
mmu_shadow_zapped          64416         0
pf_fixed                 1229672         0
pf_guest                   94338         0
remote_tlb_flush               0         0
request_irq                    0         0
signal_exits                   1         0
tlb_flush                 602678         4

Kind regards,

Gerd

-- 
Address (better: trap) for people I really don't want to get mail from:
james(at)cactusamerica.com

-------------------------------------------------------------------------
This SF.net email is sponsored by the 2008 JavaOne(SM) Conference 
Don't miss this year's exciting event. There's still time to save $100. 
Use priority code J8TL2D2. 
http://ad.doubleclick.net/clk;198757673;13503038;p?http://java.sun.com/javaone

[patch 0/2] virtio-blk async IO (v3)

[Marcelo Tosatti]

Resending the virtio-blk async patches, now that the reason for Gerd's
hangs are known.

The above results are on host hot-cached data, cold-cache data workloads
are the real winners, where the current code waits until each read
request is finished before submitting the next one.

ide:
Version  1.03       ------Sequential Output------ --Sequential Input- --Random-
                    -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
localhost.locald 2G 36618  88 107274  31 80361  49 44311  98 292436  94 +++++ +++

virtio-blk:

Version  1.03       ------Sequential Output------ --Sequential Input- --Random-
                    -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
localhost.locald 2G 39738  89 108389  31 82313  48 43943  98 290500  94 +++++ +++

virtio-blk-async:

Version  1.03       ------Sequential Output------ --Sequential Input- --Random-
                    -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
localhost.locald 2G 40781  92 102806  34 86887  36 44339  97 347461  78 +++++ +++


-- 

[patch 1/2] QEMU/KVM: provide a reset method for virtio

[Marcelo Tosatti]

[-- Attachment #1: virtio-reset --]
[-- Type: text/plain, Size: 1030 bytes --]

So drivers can do whatever necessary on reset.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: kvm-userspace.vblk/qemu/hw/virtio.c
===================================================================
--- kvm-userspace.vblk.orig/qemu/hw/virtio.c
+++ kvm-userspace.vblk/qemu/hw/virtio.c
@@ -207,6 +207,9 @@ void virtio_reset(void *opaque)
     VirtIODevice *vdev = opaque;
     int i;
 
+    if (vdev->reset)
+        vdev->reset(vdev);
+
     vdev->features = 0;
     vdev->queue_sel = 0;
     vdev->status = 0;
Index: kvm-userspace.vblk/qemu/hw/virtio.h
===================================================================
--- kvm-userspace.vblk.orig/qemu/hw/virtio.h
+++ kvm-userspace.vblk/qemu/hw/virtio.h
@@ -116,6 +116,7 @@ struct VirtIODevice
     uint32_t (*get_features)(VirtIODevice *vdev);
     void (*set_features)(VirtIODevice *vdev, uint32_t val);
     void (*update_config)(VirtIODevice *vdev, uint8_t *config);
+    void (*reset)(VirtIODevice *vdev);
     VirtQueue vq[VIRTIO_PCI_QUEUE_MAX];
 };
 

--