[Qemu-devel] [PULL 0/7] Block layer patches for 2.12.0-rc3

[Qemu-devel] [PULL 0/7] Block layer patches for 2.12.0-rc3

[Kevin Wolf]

The following changes since commit df6378eb0e6cfd58a22a1c3ff8fa4a9039f1eaa8:

  Merge remote-tracking branch 'remotes/kraxel/tags/ui-20180410-pull-request' into staging (2018-04-10 14:04:27 +0100)

are available in the git repository at:

  git://repo.or.cz/qemu/kevin.git tags/for-upstream

for you to fetch changes up to c1de5696d6a25b426432c147dfd7fb8a9eb86b89:

  qemu-iotests: update 185 output (2018-04-10 16:34:38 +0200)

----------------------------------------------------------------
Block layer patches

----------------------------------------------------------------
Kevin Wolf (2):
      qemu-iotests: Remove _supported_fmt dmg
      commit/stream: Reset delay_ns

Philippe Mathieu-Daudé (1):
      hw/block/pflash_cfi: fix off-by-one error

Stefan Hajnoczi (1):
      qemu-iotests: update 185 output

Vladimir Sementsov-Ogievskiy (3):
      iotests.py: support unsupported_fmts in main()
      iotests.py: improve verify_image_format helper
      iotests: blacklist bochs and cloop for 205 and 208

 block/commit.c                |  2 ++
 block/stream.c                |  2 ++
 hw/block/pflash_cfi01.c       | 10 ++++------
 hw/block/pflash_cfi02.c       |  9 ++++-----
 tests/qemu-iotests/183        |  2 +-
 tests/qemu-iotests/185        | 10 ++++++----
 tests/qemu-iotests/185.out    | 12 +++++++-----
 tests/qemu-iotests/194        |  2 +-
 tests/qemu-iotests/205        |  2 +-
 tests/qemu-iotests/208        |  2 ++
 tests/qemu-iotests/iotests.py | 19 ++++++++++++++-----
 11 files changed, 44 insertions(+), 28 deletions(-)

[Qemu-devel] [PULL 1/7] iotests.py: support unsupported_fmts in main()

[Kevin Wolf]

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/iotests.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index b5d7945af8..1623257e32 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -553,7 +553,8 @@ def verify_quorum():
     if not supports_quorum():
         notrun('quorum support missing')
 
-def main(supported_fmts=[], supported_oses=['linux'], supported_cache_modes=[]):
+def main(supported_fmts=[], supported_oses=['linux'], supported_cache_modes=[],
+         unsupported_fmts=[]):
     '''Run tests'''
 
     global debug
@@ -568,7 +569,7 @@ def main(supported_fmts=[], supported_oses=['linux'], supported_cache_modes=[]):
 
     debug = '-d' in sys.argv
     verbosity = 1
-    verify_image_format(supported_fmts)
+    verify_image_format(supported_fmts, unsupported_fmts)
     verify_platform(supported_oses)
     verify_cache_mode(supported_cache_modes)
 
-- 
2.13.6

[Qemu-devel] [PULL 2/7] hw/block/pflash_cfi: fix off-by-one error

[Kevin Wolf]

From: Philippe Mathieu-Daudé <f4bug@amsat.org>

ASAN reported:

    hw/block/pflash_cfi02.c:245:33: runtime error: index 82 out of bounds for type 'uint8_t [82]'

Since the 'cfi_len' member is not used, remove it to keep the code safer.

Cc: qemu-stable@nongnu.org
Reported-by: AddressSanitizer
Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 hw/block/pflash_cfi01.c | 10 ++++------
 hw/block/pflash_cfi02.c |  9 ++++-----
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
index 1113ab1ccf..2e8284001d 100644
--- a/hw/block/pflash_cfi01.c
+++ b/hw/block/pflash_cfi01.c
@@ -90,7 +90,6 @@ struct pflash_t {
     uint16_t ident1;
     uint16_t ident2;
     uint16_t ident3;
-    uint8_t cfi_len;
     uint8_t cfi_table[0x52];
     uint64_t counter;
     unsigned int writeblock_size;
@@ -153,7 +152,7 @@ static uint32_t pflash_cfi_query(pflash_t *pfl, hwaddr offset)
     boff = offset >> (ctz32(pfl->bank_width) +
                       ctz32(pfl->max_device_width) - ctz32(pfl->device_width));
 
-    if (boff > pfl->cfi_len) {
+    if (boff >= sizeof(pfl->cfi_table)) {
         return 0;
     }
     /* Now we will construct the CFI response generated by a single
@@ -385,10 +384,10 @@ static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
                 boff = boff >> 2;
             }
 
-            if (boff > pfl->cfi_len) {
-                ret = 0;
-            } else {
+            if (boff < sizeof(pfl->cfi_table)) {
                 ret = pfl->cfi_table[boff];
+            } else {
+                ret = 0;
             }
         } else {
             /* If we have a read larger than the bank_width, combine multiple
@@ -791,7 +790,6 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
     pfl->cmd = 0;
     pfl->status = 0;
     /* Hardcoded CFI table */
-    pfl->cfi_len = 0x52;
     /* Standard "QRY" string */
     pfl->cfi_table[0x10] = 'Q';
     pfl->cfi_table[0x11] = 'R';
diff --git a/hw/block/pflash_cfi02.c b/hw/block/pflash_cfi02.c
index c81ddd3a99..75d1ae1026 100644
--- a/hw/block/pflash_cfi02.c
+++ b/hw/block/pflash_cfi02.c
@@ -83,7 +83,6 @@ struct pflash_t {
     uint16_t ident3;
     uint16_t unlock_addr0;
     uint16_t unlock_addr1;
-    uint8_t cfi_len;
     uint8_t cfi_table[0x52];
     QEMUTimer *timer;
     /* The device replicates the flash memory across its memory space.  Emulate
@@ -235,10 +234,11 @@ static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
         break;
     case 0x98:
         /* CFI query mode */
-        if (boff > pfl->cfi_len)
-            ret = 0;
-        else
+        if (boff < sizeof(pfl->cfi_table)) {
             ret = pfl->cfi_table[boff];
+        } else {
+            ret = 0;
+        }
         break;
     }
 
@@ -663,7 +663,6 @@ static void pflash_cfi02_realize(DeviceState *dev, Error **errp)
     pfl->cmd = 0;
     pfl->status = 0;
     /* Hardcoded CFI table (mostly from SG29 Spansion flash) */
-    pfl->cfi_len = 0x52;
     /* Standard "QRY" string */
     pfl->cfi_table[0x10] = 'Q';
     pfl->cfi_table[0x11] = 'R';
-- 
2.13.6

[Qemu-devel] [PULL 3/7] iotests.py: improve verify_image_format helper

[Kevin Wolf]

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Support "generic" formats like in bash tests with their
   _supported_fmt generic
The test, supporting "generic" formats will run if IMGFMT_GENERIC =
true, which is default, except for bochs and cloop. However, you can
use verify_image_format(['generic', 'bochs']), which will run for all
except cloop (for this moment).

Also, add an assert (we don't want set both arguments) and remove
duplication.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/iotests.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 1623257e32..97131b1f86 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -532,9 +532,17 @@ def notrun(reason):
     sys.exit(0)
 
 def verify_image_format(supported_fmts=[], unsupported_fmts=[]):
-    if supported_fmts and (imgfmt not in supported_fmts):
-        notrun('not suitable for this image format: %s' % imgfmt)
-    if unsupported_fmts and (imgfmt in unsupported_fmts):
+    assert not (supported_fmts and unsupported_fmts)
+
+    if 'generic' in supported_fmts and \
+            os.environ.get('IMGFMT_GENERIC', 'true') == 'true':
+        # similar to
+        #   _supported_fmt generic
+        # for bash tests
+        return
+
+    not_sup = supported_fmts and (imgfmt not in supported_fmts)
+    if not_sup or (imgfmt in unsupported_fmts):
         notrun('not suitable for this image format: %s' % imgfmt)
 
 def verify_platform(supported_oses=['linux']):
-- 
2.13.6

[Qemu-devel] [PULL 4/7] iotests: blacklist bochs and cloop for 205 and 208

[Kevin Wolf]

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Blacklist these formats, as they don't support image creation, as they
say:
    > ./qemu-img create -f bochs x 1m
    qemu-img: x: Format driver 'bochs' does not support image creation

    > ./qemu-img create -f cloop x 1m
    qemu-img: x: Format driver 'cloop' does not support image creation

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/205 | 2 +-
 tests/qemu-iotests/208 | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/qemu-iotests/205 b/tests/qemu-iotests/205
index e7b2eae51d..31b2f5707a 100755
--- a/tests/qemu-iotests/205
+++ b/tests/qemu-iotests/205
@@ -153,4 +153,4 @@ class TestNbdServerRemove(iotests.QMPTestCase):
 
 
 if __name__ == '__main__':
-    iotests.main()
+    iotests.main(supported_fmts=['generic'])
diff --git a/tests/qemu-iotests/208 b/tests/qemu-iotests/208
index 18f59ada94..1e202388dc 100755
--- a/tests/qemu-iotests/208
+++ b/tests/qemu-iotests/208
@@ -22,6 +22,8 @@
 
 import iotests
 
+iotests.verify_image_format(supported_fmts=['generic'])
+
 with iotests.FilePath('disk.img') as disk_img_path, \
      iotests.FilePath('disk-snapshot.img') as disk_snapshot_img_path, \
      iotests.FilePath('nbd.sock') as nbd_sock_path, \
-- 
2.13.6

[Qemu-devel] [PULL 5/7] qemu-iotests: Remove _supported_fmt dmg

[Kevin Wolf]

qemu-iotests doesn't support dmg, and the dmg block driver doesn't
support image creation. Two test cases declare dmg as supported, but
that's obviously wrong for both reasons. Remove the declaration.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
---
 tests/qemu-iotests/183 | 2 +-
 tests/qemu-iotests/194 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/qemu-iotests/183 b/tests/qemu-iotests/183
index 20268ff7a1..c49e1ad6ef 100755
--- a/tests/qemu-iotests/183
+++ b/tests/qemu-iotests/183
@@ -43,7 +43,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 . ./common.qemu
 
-_supported_fmt qcow2 raw qed dmg quorum
+_supported_fmt qcow2 raw qed quorum
 _supported_proto file
 _supported_os Linux
 
diff --git a/tests/qemu-iotests/194 b/tests/qemu-iotests/194
index 1d4214aca3..d746ab1e21 100755
--- a/tests/qemu-iotests/194
+++ b/tests/qemu-iotests/194
@@ -21,7 +21,7 @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['qcow2', 'qed', 'raw', 'dmg'])
+iotests.verify_image_format(supported_fmts=['qcow2', 'qed', 'raw'])
 iotests.verify_platform(['linux'])
 
 with iotests.FilePath('source.img') as source_img_path, \
-- 
2.13.6

[Qemu-devel] [PULL 6/7] commit/stream: Reset delay_ns

[Kevin Wolf]

Streaming and the commit block job only want to apply throttling when
they actually copied data instead of skipping it, so they made the
calculation of delay_ns conditional. However, delay_ns isn't reset when
skipping some sectors, so instead of not waiting, the old delay is
applied again.

Properly reset delay_ns where needed.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
---
 block/commit.c | 2 ++
 block/stream.c | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/block/commit.c b/block/commit.c
index ab4fa3c3cf..1432baeef4 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -202,6 +202,8 @@ static void coroutine_fn commit_run(void *opaque)
 
         if (copy && s->common.speed) {
             delay_ns = ratelimit_calculate_delay(&s->limit, n);
+        } else {
+            delay_ns = 0;
         }
     }
 
diff --git a/block/stream.c b/block/stream.c
index f3b53f49e2..1a85708fcf 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -188,6 +188,8 @@ static void coroutine_fn stream_run(void *opaque)
         s->common.offset += n;
         if (copy && s->common.speed) {
             delay_ns = ratelimit_calculate_delay(&s->limit, n);
+        } else {
+            delay_ns = 0;
         }
     }
 
-- 
2.13.6

[Qemu-devel] [PULL 7/7] qemu-iotests: update 185 output

[Kevin Wolf]

From: Stefan Hajnoczi <stefanha@redhat.com>

Commit 4486e89c219c0d1b9bd8dfa0b1dd5b0d51ff2268 ("vl: introduce
vm_shutdown()") added a bdrv_drain_all() call.  As a side-effect of the
drain operation the block job iterates one more time than before.  The
185 output no longer matches and the test is failing now.

It may be possible to avoid the superfluous block job iteration, but
that type of patch is not suitable late in the QEMU 2.12 release cycle.

This patch simply updates the 185 output file.  The new behavior is
correct, just not optimal, so make the test pass again.

Fixes: 4486e89c219c0d1b9bd8dfa0b1dd5b0d51ff2268 ("vl: introduce vm_shutdown()")
Cc: Kevin Wolf <kwolf@redhat.com>
Cc: QingFeng Hao <haoqf@linux.vnet.ibm.com>
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: QingFeng Hao <haoqf@linux.vnet.ibm.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/185     | 10 ++++++----
 tests/qemu-iotests/185.out | 12 +++++++-----
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/tests/qemu-iotests/185 b/tests/qemu-iotests/185
index f5b47e4c1a..298d88d04e 100755
--- a/tests/qemu-iotests/185
+++ b/tests/qemu-iotests/185
@@ -92,9 +92,8 @@ echo === Start commit job and exit qemu ===
 echo
 
 # Note that the reference output intentionally includes the 'offset' field in
-# BLOCK_JOB_CANCELLED events for all of the following block jobs. They are
-# predictable and any change in the offsets would hint at a bug in the job
-# throttling code.
+# BLOCK_JOB_* events for all of the following block jobs. They are predictable
+# and any change in the offsets would hint at a bug in the job throttling code.
 #
 # In order to achieve these predictable offsets, all of the following tests
 # use speed=65536. Each job will perform exactly one iteration before it has
@@ -102,11 +101,14 @@ echo
 # command to be received (after receiving the command, the rest runs
 # synchronously, so jobs can arbitrarily continue or complete).
 #
+# Jobs present while QEMU is terminating iterate once more due to
+# bdrv_drain_all().
+#
 # The buffer size for commit and streaming is 512k (waiting for 8 seconds after
 # the first request), for active commit and mirror it's large enough to cover
 # the full 4M, and for backup it's the qcow2 cluster size, which we know is
 # 64k. As all of these are at least as large as the speed, we are sure that the
-# offset doesn't advance after the first iteration before qemu exits.
+# offset advances exactly twice before qemu exits.
 
 _send_qemu_cmd $h \
     "{ 'execute': 'block-commit',
diff --git a/tests/qemu-iotests/185.out b/tests/qemu-iotests/185.out
index 57eaf8d699..2c4b04de73 100644
--- a/tests/qemu-iotests/185.out
+++ b/tests/qemu-iotests/185.out
@@ -20,7 +20,7 @@ Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 backing_file=TEST_DIR/t.q
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 67108864, "offset": 524288, "speed": 65536, "type": "commit"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 67108864, "offset": 1048576, "speed": 65536, "type": "commit"}}
 
 === Start active commit job and exit qemu ===
 
@@ -28,16 +28,18 @@ Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 backing_file=TEST_DIR/t.q
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 4194304, "offset": 4194304, "speed": 65536, "type": "commit"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "disk", "len": 4194304, "offset": 4194304, "speed": 65536, "type": "commit"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "disk", "len": 4194304, "offset": 4194304, "speed": 65536, "type": "commit"}}
 
 === Start mirror job and exit qemu ===
 
 {"return": {}}
 Formatting 'TEST_DIR/t.qcow2.copy', fmt=qcow2 size=67108864 cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "disk", "len": 4194304, "offset": 4194304, "speed": 65536, "type": "mirror"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 4194304, "offset": 4194304, "speed": 65536, "type": "mirror"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "disk", "len": 4194304, "offset": 4194304, "speed": 65536, "type": "mirror"}}
 
 === Start backup job and exit qemu ===
 
@@ -46,7 +48,7 @@ Formatting 'TEST_DIR/t.qcow2.copy', fmt=qcow2 size=67108864 cluster_size=65536 l
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 67108864, "offset": 65536, "speed": 65536, "type": "backup"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 67108864, "offset": 131072, "speed": 65536, "type": "backup"}}
 
 === Start streaming job and exit qemu ===
 
@@ -54,6 +56,6 @@ Formatting 'TEST_DIR/t.qcow2.copy', fmt=qcow2 size=67108864 cluster_size=65536 l
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 67108864, "offset": 524288, "speed": 65536, "type": "stream"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "disk", "len": 67108864, "offset": 1048576, "speed": 65536, "type": "stream"}}
 No errors were found on the image.
 *** done
-- 
2.13.6

Re: [Qemu-devel] [PULL 0/7] Block layer patches for 2.12.0-rc3

[Peter Maydell]

On 10 April 2018 at 16:37, Kevin Wolf <kwolf@redhat.com> wrote:
> The following changes since commit df6378eb0e6cfd58a22a1c3ff8fa4a9039f1eaa8:
>
>   Merge remote-tracking branch 'remotes/kraxel/tags/ui-20180410-pull-request' into staging (2018-04-10 14:04:27 +0100)
>
> are available in the git repository at:
>
>   git://repo.or.cz/qemu/kevin.git tags/for-upstream
>
> for you to fetch changes up to c1de5696d6a25b426432c147dfd7fb8a9eb86b89:
>
>   qemu-iotests: update 185 output (2018-04-10 16:34:38 +0200)
>
> ----------------------------------------------------------------
> Block layer patches
>

Applied, thanks.

-- PMM