[Qemu-devel] [PATCH v5 09/15] qmp: add query-block-jobs

[Qemu-devel] [PATCH v5 09/15] qmp: add query-block-jobs

[Stefan Hajnoczi]

Add query-block-jobs, which shows the progress of ongoing block device
operations.

Signed-off-by: Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
---
 blockdev.c       |   33 +++++++++++++++++++++++++++++++++
 hmp.c            |   36 ++++++++++++++++++++++++++++++++++++
 hmp.h            |    1 +
 monitor.c        |    7 +++++++
 qapi-schema.json |   32 ++++++++++++++++++++++++++++++++
 qmp-commands.hx  |    6 ++++++
 6 files changed, 115 insertions(+), 0 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index 35de3bc..4549c9e 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -989,3 +989,36 @@ void qmp_block_job_cancel(const char *device, Error **errp)
     trace_qmp_block_job_cancel(job);
     block_job_cancel(job);
 }
+
+static void do_qmp_query_block_jobs_one(void *opaque, BlockDriverState *bs)
+{
+    BlockJobInfoList **prev = opaque;
+    BlockJob *job = bs->job;
+
+    if (job) {
+        BlockJobInfoList *elem;
+        BlockJobInfo *info = g_new(BlockJobInfo, 1);
+        *info = (BlockJobInfo){
+            .type   = g_strdup(job->job_type->job_type),
+            .device = g_strdup(bdrv_get_device_name(bs)),
+            .len    = job->len,
+            .offset = job->offset,
+            .speed  = job->speed,
+        };
+
+        elem = g_new0(BlockJobInfoList, 1);
+        elem->value = info;
+
+        (*prev)->next = elem;
+        *prev = elem;
+    }
+}
+
+BlockJobInfoList *qmp_query_block_jobs(Error **errp)
+{
+    /* Dummy is a fake list element for holding the head pointer */
+    BlockJobInfoList dummy = {};
+    BlockJobInfoList *prev = &dummy;
+    bdrv_iterate(do_qmp_query_block_jobs_one, &prev);
+    return dummy.next;
+}
diff --git a/hmp.c b/hmp.c
index 851885b..76e89f8 100644
--- a/hmp.c
+++ b/hmp.c
@@ -507,6 +507,42 @@ void hmp_info_pci(Monitor *mon)
     qapi_free_PciInfoList(info);
 }
 
+void hmp_info_block_jobs(Monitor *mon)
+{
+    BlockJobInfoList *list;
+    Error *err = NULL;
+
+    list = qmp_query_block_jobs(&err);
+    assert(!err);
+
+    if (!list) {
+        monitor_printf(mon, "No active jobs\n");
+        return;
+    }
+
+    while (list) {
+        if (strcmp(list->value->type, "stream") == 0) {
+            monitor_printf(mon, "Streaming device %s: Completed %" PRId64
+                           " of %" PRId64 " bytes, speed limit %" PRId64
+                           " bytes/s\n",
+                           list->value->device,
+                           list->value->offset,
+                           list->value->len,
+                           list->value->speed);
+        } else {
+            monitor_printf(mon, "Type %s, device %s: Completed %" PRId64
+                           " of %" PRId64 " bytes, speed limit %" PRId64
+                           " bytes/s\n",
+                           list->value->type,
+                           list->value->device,
+                           list->value->offset,
+                           list->value->len,
+                           list->value->speed);
+        }
+        list = list->next;
+    }
+}
+
 void hmp_quit(Monitor *mon, const QDict *qdict)
 {
     monitor_suspend(mon);
diff --git a/hmp.h b/hmp.h
index 0ad2004..23bfca2 100644
--- a/hmp.h
+++ b/hmp.h
@@ -32,6 +32,7 @@ void hmp_info_vnc(Monitor *mon);
 void hmp_info_spice(Monitor *mon);
 void hmp_info_balloon(Monitor *mon);
 void hmp_info_pci(Monitor *mon);
+void hmp_info_block_jobs(Monitor *mon);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
diff --git a/monitor.c b/monitor.c
index 01850ca..f96a296 100644
--- a/monitor.c
+++ b/monitor.c
@@ -2483,6 +2483,13 @@ static mon_cmd_t info_cmds[] = {
         .mhandler.info = hmp_info_blockstats,
     },
     {
+        .name       = "block-jobs",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show progress of ongoing block device operations",
+        .mhandler.info = hmp_info_block_jobs,
+    },
+    {
         .name       = "registers",
         .args_type  = "",
         .params     = "",
diff --git a/qapi-schema.json b/qapi-schema.json
index 3d23ce2..b4f6b15 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -845,6 +845,38 @@
 { 'command': 'query-pci', 'returns': ['PciInfo'] }
 
 ##
+# @BlockJobInfo:
+#
+# Information about a long-running block device operation.
+#
+# @type: the job type ('stream' for image streaming)
+#
+# @device: the block device name
+#
+# @len: the maximum progress value
+#
+# @offset: the current progress value
+#
+# @speed: the rate limit, bytes per second
+#
+# Since: 1.1
+##
+{ 'type': 'BlockJobInfo',
+  'data': {'type': 'str', 'device': 'str', 'len': 'int',
+           'offset': 'int', 'speed': 'int'} }
+
+##
+# @query-block-jobs:
+#
+# Return information about long-running block device operations.
+#
+# Returns: a list of @BlockJobInfo for each active block job
+#
+# Since: 1.1
+##
+{ 'command': 'query-block-jobs', 'returns': ['BlockJobInfo'] }
+
+##
 # @quit:
 #
 # This command will cause the QEMU process to exit gracefully.  While every
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 0a0335f..4be6632 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -2029,6 +2029,12 @@ EQMP
     },
 
     {
+        .name       = "query-block-jobs",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_input_query_block_jobs,
+    },
+
+    {
         .name       = "qom-list",
         .args_type  = "path:s",
         .mhandler.cmd_new = qmp_marshal_input_qom_list,
-- 
1.7.7.3

[Qemu-devel] [PATCH v5 15/15] test: add image streaming test cases

[Stefan Hajnoczi]

python test-stream.py

Signed-off-by: Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
---
 test-stream.py |  208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 208 insertions(+), 0 deletions(-)
 create mode 100644 test-stream.py

diff --git a/test-stream.py b/test-stream.py
new file mode 100644
index 0000000..16cbe5d
--- /dev/null
+++ b/test-stream.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+import unittest
+import subprocess
+import re
+import os
+import sys; sys.path.append('QMP/')
+import qmp
+
+def qemu_img(*args):
+    devnull = open('/dev/null', 'r+')
+    return subprocess.call(['./qemu-img'] + list(args), stdin=devnull, stdout=devnull)
+
+def qemu_io(*args):
+    args = ['./qemu-io'] + list(args)
+    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
+
+class VM(object):
+    def __init__(self):
+        self._monitor_path = '/tmp/qemu-mon.%d' % os.getpid()
+        self._qemu_log_path = '/tmp/qemu-log.%d' % os.getpid()
+        self._args = ['x86_64-softmmu/qemu-system-x86_64',
+                      '-chardev', 'socket,id=mon,path=' + self._monitor_path,
+                      '-mon', 'chardev=mon,mode=control', '-nographic']
+        self._num_drives = 0
+
+    def add_drive(self, path, opts=''):
+        options = ['if=virtio',
+                   'cache=none',
+                   'file=%s' % path,
+                   'id=drive%d' % self._num_drives]
+        if opts:
+            options.append(opts)
+
+        self._args.append('-drive')
+        self._args.append(','.join(options))
+        self._num_drives += 1
+        return self
+
+    def launch(self):
+        devnull = open('/dev/null', 'rb')
+        qemulog = open(self._qemu_log_path, 'wb')
+        self._qmp = qmp.QEMUMonitorProtocol(self._monitor_path, server=True)
+        self._popen = subprocess.Popen(self._args, stdin=devnull, stdout=qemulog,
+                                       stderr=subprocess.STDOUT)
+        self._qmp.accept()
+
+    def shutdown(self):
+        self._qmp.cmd('quit')
+        self._popen.wait()
+        os.remove(self._monitor_path)
+        os.remove(self._qemu_log_path)
+
+    def qmp(self, cmd, **args):
+        return self._qmp.cmd(cmd, args=args)
+
+    def get_qmp_events(self, wait=False):
+        events = self._qmp.get_events(wait=wait)
+        self._qmp.clear_events()
+        return events
+
+index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
+
+class QMPTestCase(unittest.TestCase):
+    def dictpath(self, d, path):
+        """Traverse a path in a nested dict"""
+        for component in path.split('/'):
+            m = index_re.match(component)
+            if m:
+                component, idx = m.groups()
+                idx = int(idx)
+
+            if not isinstance(d, dict) or component not in d:
+                self.fail('failed path traversal for "%s" in "%s"' % (path, str(d)))
+            d = d[component]
+
+            if m:
+                if not isinstance(d, list):
+                    self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d)))
+                try:
+                    d = d[idx]
+                except IndexError:
+                    self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
+        return d
+
+    def assert_qmp(self, d, path, value):
+        result = self.dictpath(d, path)
+        self.assertEqual(result, value, 'values not equal "%s" and "%s"' % (str(result), str(value)))
+
+    def assert_no_active_streams(self):
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return', [])
+
+class TestSingleDrive(QMPTestCase):
+    image_len = 1 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', 'backing.img', str(TestSingleDrive.image_len))
+        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
+        self.vm = VM().add_drive('test.qed')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove('test.qed')
+        os.remove('backing.img')
+
+    def test_stream(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block_stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+
+        self.assertFalse('sectors not allocated' in qemu_io('-c', 'map', 'test.qed'),
+                         'image file not fully populated after streaming')
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('block_stream', device='nonexistent')
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+class TestStreamStop(QMPTestCase):
+    image_len = 8 * 1024 * 1024 * 1024 # GB
+
+    def setUp(self):
+        qemu_img('create', 'backing.img', str(TestStreamStop.image_len))
+        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
+        self.vm = VM().add_drive('test.qed')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove('test.qed')
+        os.remove('backing.img')
+
+    def test_stream_stop(self):
+        import time
+
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block_stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        time.sleep(1)
+        events = self.vm.get_qmp_events(wait=False)
+        self.assertEqual(events, [], 'unexpected QMP event: %s' % events)
+
+        self.vm.qmp('block_job_cancel', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        cancelled = False
+        while not cancelled:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_CANCELLED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    cancelled = True
+
+        self.assert_no_active_streams()
+
+class TestSetSpeed(QMPTestCase):
+    image_len = 80 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', 'backing.img', str(TestSetSpeed.image_len))
+        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
+        self.vm = VM().add_drive('test.qed')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove('test.qed')
+        os.remove('backing.img')
+
+    # This doesn't print or verify anything, only use it via "test-stream.py TestSetSpeed"
+    def test_stream_set_speed(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block_stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block_job_set_speed', device='drive0', value=8 * 1024 * 1024)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+
+if __name__ == '__main__':
+    unittest.main()
-- 
1.7.7.3

[Qemu-devel] [PATCH v5 00/15] block: generic image streaming

[Stefan Hajnoczi]

Note: This is a resend of v5 because the emails I sent earlier today
disappeared.

This series adds the 'block_stream' command which copies the contents of a
backing file into the image file while the VM is running.  The series builds on
the zero detection features which I sent out before Christmas. I suggest
grabbing my git tree to try it out without merging this dependency:

https://github.com/stefanha/qemu/tree/image-streaming-api

The image streaming HMP/QMP commands are documented in the patch and also
described here:

http://wiki.qemu.org/Features/LiveBlockMigration/ImageStreamingAPI

The basic idea is to execute 'block_stream virtio0' while the guest is running.
Progress can be monitored using 'info block-jobs'.  When the streaming
operation completes it raises a QMP event.

Note: The last patch includes includes a Python test script called
test-stream.py, I do not propose to merge it.  When run in a QEMU source tree
it performs basic image streaming QMP tests.

v5:
 * Handle block_job_create() failure [Luiz]
 * Mark BLOCK_JOB_COMPLETED error field optional [Luiz]
 * Mark block_stream base parameter optional [Luiz]
 * Check bdrv_getlength() failure and don't call twice [Kevin]
 * Rename "id" to "backing_file" in bdrv_find_backing_image() [Kevin]
 * Rename BaseIdNotFound qerror to BaseNotFound [Kevin]
 * Document BaseNotFound qerror from block_stream
 * Document that qemu_co_sleep_ns() needs main loop [Kevin]
 * Make bdrv_co_is_allocated_base() private to block/stream.c [Kevin]
 * Clean up commit messages

v4:
 * Drop SQMP/EQMP docs from qmp-commands.hx [Luiz]
 * Follow QAPI doc conventions [Luiz]
 * Document QMP events in QMP/qmp-events.txt [Luiz]
 * Protect against hotplug, resize, eject, etc [Kevin]
 * Move block job functions from header to block.c [Kevin]
 * Return error from bdrg_change_backing_file() [Kevin]
 * Merge Marcelo's block_stream base partial streaming series [Marcelo]

Marcelo Tosatti (4):
  block: add bdrv_find_backing_image
  add QERR_BASE_NOT_FOUND
  block: add support for partial streaming
  docs: describe live block operations

Stefan Hajnoczi (11):
  coroutine: add co_sleep_ns() coroutine sleep function
  block: check bdrv_in_use() before blockdev operations
  block: add BlockJob interface for long-running operations
  block: add image streaming block job
  block: rate-limit streaming operations
  qmp: add block_stream command
  qmp: add block_job_set_speed command
  qmp: add block_job_cancel command
  qmp: add query-block-jobs
  blockdev: make image streaming safe across hotplug
  test: add image streaming test cases

 Makefile.objs           |    2 +
 QMP/qmp-events.txt      |   53 ++++++++++
 block.c                 |   70 +++++++++++++
 block.h                 |    2 +
 block/stream.c          |  260 +++++++++++++++++++++++++++++++++++++++++++++++
 block_int.h             |   44 ++++++++
 blockdev.c              |  199 +++++++++++++++++++++++++++++++++++-
 docs/live-block-ops.txt |   58 +++++++++++
 hmp-commands.hx         |   41 ++++++++
 hmp.c                   |   68 ++++++++++++
 hmp.h                   |    4 +
 monitor.c               |   13 +++
 monitor.h               |    2 +
 qapi-schema.json        |  116 +++++++++++++++++++++
 qemu-coroutine-sleep.c  |   38 +++++++
 qemu-coroutine.h        |    9 ++
 qerror.c                |    8 ++
 qerror.h                |    6 +
 qmp-commands.hx         |   24 +++++
 test-stream.py          |  208 +++++++++++++++++++++++++++++++++++++
 trace-events            |    9 ++
 21 files changed, 1233 insertions(+), 1 deletions(-)
 create mode 100644 block/stream.c
 create mode 100644 docs/live-block-ops.txt
 create mode 100644 qemu-coroutine-sleep.c
 create mode 100644 test-stream.py

-- 
1.7.7.3

[Qemu-devel] [PATCH v5 15/15] test: add image streaming test cases

[Stefan Hajnoczi]

python test-stream.py

Signed-off-by: Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
---
 test-stream.py |  208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 208 insertions(+), 0 deletions(-)
 create mode 100644 test-stream.py

diff --git a/test-stream.py b/test-stream.py
new file mode 100644
index 0000000..16cbe5d
--- /dev/null
+++ b/test-stream.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+import unittest
+import subprocess
+import re
+import os
+import sys; sys.path.append('QMP/')
+import qmp
+
+def qemu_img(*args):
+    devnull = open('/dev/null', 'r+')
+    return subprocess.call(['./qemu-img'] + list(args), stdin=devnull, stdout=devnull)
+
+def qemu_io(*args):
+    args = ['./qemu-io'] + list(args)
+    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
+
+class VM(object):
+    def __init__(self):
+        self._monitor_path = '/tmp/qemu-mon.%d' % os.getpid()
+        self._qemu_log_path = '/tmp/qemu-log.%d' % os.getpid()
+        self._args = ['x86_64-softmmu/qemu-system-x86_64',
+                      '-chardev', 'socket,id=mon,path=' + self._monitor_path,
+                      '-mon', 'chardev=mon,mode=control', '-nographic']
+        self._num_drives = 0
+
+    def add_drive(self, path, opts=''):
+        options = ['if=virtio',
+                   'cache=none',
+                   'file=%s' % path,
+                   'id=drive%d' % self._num_drives]
+        if opts:
+            options.append(opts)
+
+        self._args.append('-drive')
+        self._args.append(','.join(options))
+        self._num_drives += 1
+        return self
+
+    def launch(self):
+        devnull = open('/dev/null', 'rb')
+        qemulog = open(self._qemu_log_path, 'wb')
+        self._qmp = qmp.QEMUMonitorProtocol(self._monitor_path, server=True)
+        self._popen = subprocess.Popen(self._args, stdin=devnull, stdout=qemulog,
+                                       stderr=subprocess.STDOUT)
+        self._qmp.accept()
+
+    def shutdown(self):
+        self._qmp.cmd('quit')
+        self._popen.wait()
+        os.remove(self._monitor_path)
+        os.remove(self._qemu_log_path)
+
+    def qmp(self, cmd, **args):
+        return self._qmp.cmd(cmd, args=args)
+
+    def get_qmp_events(self, wait=False):
+        events = self._qmp.get_events(wait=wait)
+        self._qmp.clear_events()
+        return events
+
+index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
+
+class QMPTestCase(unittest.TestCase):
+    def dictpath(self, d, path):
+        """Traverse a path in a nested dict"""
+        for component in path.split('/'):
+            m = index_re.match(component)
+            if m:
+                component, idx = m.groups()
+                idx = int(idx)
+
+            if not isinstance(d, dict) or component not in d:
+                self.fail('failed path traversal for "%s" in "%s"' % (path, str(d)))
+            d = d[component]
+
+            if m:
+                if not isinstance(d, list):
+                    self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d)))
+                try:
+                    d = d[idx]
+                except IndexError:
+                    self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
+        return d
+
+    def assert_qmp(self, d, path, value):
+        result = self.dictpath(d, path)
+        self.assertEqual(result, value, 'values not equal "%s" and "%s"' % (str(result), str(value)))
+
+    def assert_no_active_streams(self):
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return', [])
+
+class TestSingleDrive(QMPTestCase):
+    image_len = 1 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', 'backing.img', str(TestSingleDrive.image_len))
+        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
+        self.vm = VM().add_drive('test.qed')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove('test.qed')
+        os.remove('backing.img')
+
+    def test_stream(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block_stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+
+        self.assertFalse('sectors not allocated' in qemu_io('-c', 'map', 'test.qed'),
+                         'image file not fully populated after streaming')
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('block_stream', device='nonexistent')
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+class TestStreamStop(QMPTestCase):
+    image_len = 8 * 1024 * 1024 * 1024 # GB
+
+    def setUp(self):
+        qemu_img('create', 'backing.img', str(TestStreamStop.image_len))
+        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
+        self.vm = VM().add_drive('test.qed')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove('test.qed')
+        os.remove('backing.img')
+
+    def test_stream_stop(self):
+        import time
+
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block_stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        time.sleep(1)
+        events = self.vm.get_qmp_events(wait=False)
+        self.assertEqual(events, [], 'unexpected QMP event: %s' % events)
+
+        self.vm.qmp('block_job_cancel', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        cancelled = False
+        while not cancelled:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_CANCELLED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    cancelled = True
+
+        self.assert_no_active_streams()
+
+class TestSetSpeed(QMPTestCase):
+    image_len = 80 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', 'backing.img', str(TestSetSpeed.image_len))
+        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
+        self.vm = VM().add_drive('test.qed')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove('test.qed')
+        os.remove('backing.img')
+
+    # This doesn't print or verify anything, only use it via "test-stream.py TestSetSpeed"
+    def test_stream_set_speed(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block_stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block_job_set_speed', device='drive0', value=8 * 1024 * 1024)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+
+if __name__ == '__main__':
+    unittest.main()
-- 
1.7.7.3

Re: [Qemu-devel] [PATCH v5 15/15] test: add image streaming test cases

[Stefan Hajnoczi]

Hi Lucas,
The Python script below verifies the image streaming feature.  It's
built on the standard library "unittest" module, as well as QEMU's
qmp.py module.  It spawns a qemu process and creates necessary disk
image files.  The tests themselves issue QMP commands and check their
result or wait for QMP events to be raised.

I think this sort of test could be done with kvm-autotest but I don't
see much usage of cmd_qmp() in client/tests/kvm/tests/.  I'm curious
how you would approach this.  The high-level steps are:

1. Create a backing file.
2. Create a test QED image file using the backing file.
3. Issue "block_stream device=drive0" to the running VM.  This should
return no value.
4. Wait for the BLOCK_JOB_COMPLETED QMP event and check its fields -
they must contain expected values.
5. Ensure "query-block-job" does not show any active jobs anymore.
6. Use qemu-io's map command to verify that the image stays compact
and isn't bloated with actual zero bytes (this is kind of unrelated to
the rest of the test).

The other test cases share much of the same building blocks as
TestSingleDrive, so they are less interesting.

Would it be possible to look at TestSingleDrive below and give a
kvm-autotest equivalent?

Thanks,
Stefan

On Fri, Jan 13, 2012 at 1:14 PM, Stefan Hajnoczi
<stefanha@linux.vnet.ibm.com> wrote:
> python test-stream.py
>
> Signed-off-by: Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
> ---
>  test-stream.py |  208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 208 insertions(+), 0 deletions(-)
>  create mode 100644 test-stream.py
>
> diff --git a/test-stream.py b/test-stream.py
> new file mode 100644
> index 0000000..16cbe5d
> --- /dev/null
> +++ b/test-stream.py
> @@ -0,0 +1,208 @@
> +#!/usr/bin/env python
> +import unittest
> +import subprocess
> +import re
> +import os
> +import sys; sys.path.append('QMP/')
> +import qmp
> +
> +def qemu_img(*args):
> +    devnull = open('/dev/null', 'r+')
> +    return subprocess.call(['./qemu-img'] + list(args), stdin=devnull, stdout=devnull)
> +
> +def qemu_io(*args):
> +    args = ['./qemu-io'] + list(args)
> +    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
> +
> +class VM(object):
> +    def __init__(self):
> +        self._monitor_path = '/tmp/qemu-mon.%d' % os.getpid()
> +        self._qemu_log_path = '/tmp/qemu-log.%d' % os.getpid()
> +        self._args = ['x86_64-softmmu/qemu-system-x86_64',
> +                      '-chardev', 'socket,id=mon,path=' + self._monitor_path,
> +                      '-mon', 'chardev=mon,mode=control', '-nographic']
> +        self._num_drives = 0
> +
> +    def add_drive(self, path, opts=''):
> +        options = ['if=virtio',
> +                   'cache=none',
> +                   'file=%s' % path,
> +                   'id=drive%d' % self._num_drives]
> +        if opts:
> +            options.append(opts)
> +
> +        self._args.append('-drive')
> +        self._args.append(','.join(options))
> +        self._num_drives += 1
> +        return self
> +
> +    def launch(self):
> +        devnull = open('/dev/null', 'rb')
> +        qemulog = open(self._qemu_log_path, 'wb')
> +        self._qmp = qmp.QEMUMonitorProtocol(self._monitor_path, server=True)
> +        self._popen = subprocess.Popen(self._args, stdin=devnull, stdout=qemulog,
> +                                       stderr=subprocess.STDOUT)
> +        self._qmp.accept()
> +
> +    def shutdown(self):
> +        self._qmp.cmd('quit')
> +        self._popen.wait()
> +        os.remove(self._monitor_path)
> +        os.remove(self._qemu_log_path)
> +
> +    def qmp(self, cmd, **args):
> +        return self._qmp.cmd(cmd, args=args)
> +
> +    def get_qmp_events(self, wait=False):
> +        events = self._qmp.get_events(wait=wait)
> +        self._qmp.clear_events()
> +        return events
> +
> +index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
> +
> +class QMPTestCase(unittest.TestCase):
> +    def dictpath(self, d, path):
> +        """Traverse a path in a nested dict"""
> +        for component in path.split('/'):
> +            m = index_re.match(component)
> +            if m:
> +                component, idx = m.groups()
> +                idx = int(idx)
> +
> +            if not isinstance(d, dict) or component not in d:
> +                self.fail('failed path traversal for "%s" in "%s"' % (path, str(d)))
> +            d = d[component]
> +
> +            if m:
> +                if not isinstance(d, list):
> +                    self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d)))
> +                try:
> +                    d = d[idx]
> +                except IndexError:
> +                    self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
> +        return d
> +
> +    def assert_qmp(self, d, path, value):
> +        result = self.dictpath(d, path)
> +        self.assertEqual(result, value, 'values not equal "%s" and "%s"' % (str(result), str(value)))
> +
> +    def assert_no_active_streams(self):
> +        result = self.vm.qmp('query-block-jobs')
> +        self.assert_qmp(result, 'return', [])
> +
> +class TestSingleDrive(QMPTestCase):
> +    image_len = 1 * 1024 * 1024 # MB
> +
> +    def setUp(self):
> +        qemu_img('create', 'backing.img', str(TestSingleDrive.image_len))
> +        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
> +        self.vm = VM().add_drive('test.qed')
> +        self.vm.launch()
> +
> +    def tearDown(self):
> +        self.vm.shutdown()
> +        os.remove('test.qed')
> +        os.remove('backing.img')
> +
> +    def test_stream(self):
> +        self.assert_no_active_streams()
> +
> +        result = self.vm.qmp('block_stream', device='drive0')
> +        self.assert_qmp(result, 'return', {})
> +
> +        completed = False
> +        while not completed:
> +            for event in self.vm.get_qmp_events(wait=True):
> +                if event['event'] == 'BLOCK_JOB_COMPLETED':
> +                    self.assert_qmp(event, 'data/type', 'stream')
> +                    self.assert_qmp(event, 'data/device', 'drive0')
> +                    self.assert_qmp(event, 'data/offset', self.image_len)
> +                    self.assert_qmp(event, 'data/len', self.image_len)
> +                    completed = True
> +
> +        self.assert_no_active_streams()
> +
> +        self.assertFalse('sectors not allocated' in qemu_io('-c', 'map', 'test.qed'),
> +                         'image file not fully populated after streaming')
> +
> +    def test_device_not_found(self):
> +        result = self.vm.qmp('block_stream', device='nonexistent')
> +        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
> +
> +class TestStreamStop(QMPTestCase):
> +    image_len = 8 * 1024 * 1024 * 1024 # GB
> +
> +    def setUp(self):
> +        qemu_img('create', 'backing.img', str(TestStreamStop.image_len))
> +        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
> +        self.vm = VM().add_drive('test.qed')
> +        self.vm.launch()
> +
> +    def tearDown(self):
> +        self.vm.shutdown()
> +        os.remove('test.qed')
> +        os.remove('backing.img')
> +
> +    def test_stream_stop(self):
> +        import time
> +
> +        self.assert_no_active_streams()
> +
> +        result = self.vm.qmp('block_stream', device='drive0')
> +        self.assert_qmp(result, 'return', {})
> +
> +        time.sleep(1)
> +        events = self.vm.get_qmp_events(wait=False)
> +        self.assertEqual(events, [], 'unexpected QMP event: %s' % events)
> +
> +        self.vm.qmp('block_job_cancel', device='drive0')
> +        self.assert_qmp(result, 'return', {})
> +
> +        cancelled = False
> +        while not cancelled:
> +            for event in self.vm.get_qmp_events(wait=True):
> +                if event['event'] == 'BLOCK_JOB_CANCELLED':
> +                    self.assert_qmp(event, 'data/type', 'stream')
> +                    self.assert_qmp(event, 'data/device', 'drive0')
> +                    cancelled = True
> +
> +        self.assert_no_active_streams()
> +
> +class TestSetSpeed(QMPTestCase):
> +    image_len = 80 * 1024 * 1024 # MB
> +
> +    def setUp(self):
> +        qemu_img('create', 'backing.img', str(TestSetSpeed.image_len))
> +        qemu_img('create', '-f', 'qed', '-o', 'backing_file=backing.img', 'test.qed')
> +        self.vm = VM().add_drive('test.qed')
> +        self.vm.launch()
> +
> +    def tearDown(self):
> +        self.vm.shutdown()
> +        os.remove('test.qed')
> +        os.remove('backing.img')
> +
> +    # This doesn't print or verify anything, only use it via "test-stream.py TestSetSpeed"
> +    def test_stream_set_speed(self):
> +        self.assert_no_active_streams()
> +
> +        result = self.vm.qmp('block_stream', device='drive0')
> +        self.assert_qmp(result, 'return', {})
> +
> +        result = self.vm.qmp('block_job_set_speed', device='drive0', value=8 * 1024 * 1024)
> +        self.assert_qmp(result, 'return', {})
> +
> +        completed = False
> +        while not completed:
> +            for event in self.vm.get_qmp_events(wait=True):
> +                if event['event'] == 'BLOCK_JOB_COMPLETED':
> +                    self.assert_qmp(event, 'data/type', 'stream')
> +                    self.assert_qmp(event, 'data/device', 'drive0')
> +                    self.assert_qmp(event, 'data/offset', self.image_len)
> +                    self.assert_qmp(event, 'data/len', self.image_len)
> +                    completed = True
> +
> +        self.assert_no_active_streams()
> +
> +if __name__ == '__main__':
> +    unittest.main()
> --
> 1.7.7.3
>
>

Re: [Qemu-devel] [PATCH v5 15/15] test: add image streaming test cases

[Lucas Meneghel Rodrigues]

On 01/13/2012 02:49 PM, Stefan Hajnoczi wrote:
> Hi Lucas,
> The Python script below verifies the image streaming feature.  It's
> built on the standard library "unittest" module, as well as QEMU's
> qmp.py module.  It spawns a qemu process and creates necessary disk
> image files.  The tests themselves issue QMP commands and check their
> result or wait for QMP events to be raised.
>
> I think this sort of test could be done with kvm-autotest but I don't
> see much usage of cmd_qmp() in client/tests/kvm/tests/.  I'm curious
> how you would approach this.  The high-level steps are:
>
> 1. Create a backing file.
> 2. Create a test QED image file using the backing file.
> 3. Issue "block_stream device=drive0" to the running VM.  This should
> return no value.
> 4. Wait for the BLOCK_JOB_COMPLETED QMP event and check its fields -
> they must contain expected values.
> 5. Ensure "query-block-job" does not show any active jobs anymore.
> 6. Use qemu-io's map command to verify that the image stays compact
> and isn't bloated with actual zero bytes (this is kind of unrelated to
> the rest of the test).
>
> The other test cases share much of the same building blocks as
> TestSingleDrive, so they are less interesting.
>
> Would it be possible to look at TestSingleDrive below and give a
> kvm-autotest equivalent?

Yes Stefan, sorry for the late reply. I was in FUDCon, therefore taking 
care of some Fedora related autotest stuff, but I'm putting on my todo 
list to create a KVM autotest equivalent of it.

Cheers,

Lucas

Re: [Qemu-devel] [PATCH v5 15/15] test: add image streaming test cases

[Stefan Hajnoczi]

On Tue, Jan 17, 2012 at 7:07 PM, Lucas Meneghel Rodrigues
<lmr@redhat.com> wrote:
> On 01/13/2012 02:49 PM, Stefan Hajnoczi wrote:
>>
>> Hi Lucas,
>> The Python script below verifies the image streaming feature.  It's
>> built on the standard library "unittest" module, as well as QEMU's
>> qmp.py module.  It spawns a qemu process and creates necessary disk
>> image files.  The tests themselves issue QMP commands and check their
>> result or wait for QMP events to be raised.
>>
>> I think this sort of test could be done with kvm-autotest but I don't
>> see much usage of cmd_qmp() in client/tests/kvm/tests/.  I'm curious
>> how you would approach this.  The high-level steps are:
>>
>> 1. Create a backing file.
>> 2. Create a test QED image file using the backing file.
>> 3. Issue "block_stream device=drive0" to the running VM.  This should
>> return no value.
>> 4. Wait for the BLOCK_JOB_COMPLETED QMP event and check its fields -
>> they must contain expected values.
>> 5. Ensure "query-block-job" does not show any active jobs anymore.
>> 6. Use qemu-io's map command to verify that the image stays compact
>> and isn't bloated with actual zero bytes (this is kind of unrelated to
>> the rest of the test).
>>
>> The other test cases share much of the same building blocks as
>> TestSingleDrive, so they are less interesting.
>>
>> Would it be possible to look at TestSingleDrive below and give a
>> kvm-autotest equivalent?
>
>
> Yes Stefan, sorry for the late reply. I was in FUDCon, therefore taking care
> of some Fedora related autotest stuff, but I'm putting on my todo list to
> create a KVM autotest equivalent of it.

Great, thank you!

Stefan