[Qemu-devel] [PATCH 0/2] qemu-iotests: Test qcow2 header rewrites

[Qemu-devel] [PATCH 0/2] qemu-iotests: Test qcow2 header rewrites

[Kevin Wolf]

Some bugs related to rewriting the qcow2 header (as it happens e.g. during
qemu-img rebase) have been fixed recently. Here is the test case for it.

Kevin Wolf (2):
  qemu-iotests: qcow2.py
  qemu-iotests: Test unknown qcow2 header extensions

 tests/qemu-iotests/031      |   72 +++++++++++++++
 tests/qemu-iotests/031.out  |   76 ++++++++++++++++
 tests/qemu-iotests/group    |    1 +
 tests/qemu-iotests/qcow2.py |  207 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 356 insertions(+), 0 deletions(-)
 create mode 100755 tests/qemu-iotests/031
 create mode 100644 tests/qemu-iotests/031.out
 create mode 100755 tests/qemu-iotests/qcow2.py

-- 
1.7.6.5

[Qemu-devel] [PATCH 1/2] qemu-iotests: qcow2.py

[Kevin Wolf]

This adds a tool that is meant to inspect and edit qcow2 files in a
low-level way, that wouldn't be possible with qemu-img/io, for example
by adding yet unknown extensions or flags. This way we can test whether
qemu deals properly with future backwards compatible extensions.

For now, let's start with the image header and header extensions.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/qcow2.py |  207 +++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 207 insertions(+), 0 deletions(-)
 create mode 100755 tests/qemu-iotests/qcow2.py

diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
new file mode 100755
index 0000000..bfb47e8
--- /dev/null
+++ b/tests/qemu-iotests/qcow2.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+
+import sys
+import struct
+import string
+
+class QcowHeaderExtension:
+
+    def __init__(self, magic, length, data):
+        self.magic  = magic
+        self.length = length
+        self.data   = data
+
+    @classmethod
+    def create(cls, magic, data):
+        return QcowHeaderExtension(magic, len(data), data)
+
+class QcowHeader:
+
+    uint32_t = 'I'
+    uint64_t = 'Q'
+
+    fields = [
+        # Version 2 header fields
+        [ uint32_t, '%#x',  'magic' ],
+        [ uint32_t, '%d',   'version' ],
+        [ uint64_t, '%#x',  'backing_file_offset' ],
+        [ uint32_t, '%#x',  'backing_file_size' ],
+        [ uint32_t, '%d',   'cluster_bits' ],
+        [ uint64_t, '%d',   'size' ],
+        [ uint32_t, '%d',   'crypt_method' ],
+        [ uint32_t, '%d',   'l1_size' ],
+        [ uint64_t, '%#x',  'l1_table_offset' ],
+        [ uint64_t, '%#x',  'refcount_table_offset' ],
+        [ uint32_t, '%d',   'refcount_table_clusters' ],
+        [ uint32_t, '%d',   'nb_snapshots' ],
+        [ uint64_t, '%#x',  'snapshot_offset' ],
+    ];
+
+    fmt = '>' + ''.join(field[0] for field in fields)
+
+    def __init__(self, fd):
+
+        buf_size = struct.calcsize(QcowHeader.fmt)
+
+        fd.seek(0)
+        buf = fd.read(buf_size)
+
+        header = struct.unpack(QcowHeader.fmt, buf)
+        self.__dict__ = dict((field[2], header[i])
+            for i, field in enumerate(QcowHeader.fields))
+
+        self.cluster_size = 1 << self.cluster_bits
+
+        fd.seek(self.get_header_length())
+        self.load_extensions(fd)
+
+        if self.backing_file_offset:
+            fd.seek(self.backing_file_offset)
+            self.backing_file = fd.read(self.backing_file_size)
+        else:
+            self.backing_file = None
+
+    def get_header_length(self):
+        if self.version == 2:
+            return 72
+        else:
+            raise Exception("version != 2 not supported")
+
+    def load_extensions(self, fd):
+        self.extensions = []
+
+        if self.backing_file_offset != 0:
+            end = min(self.cluster_size, self.backing_file_offset)
+        else:
+            end = self.cluster_size
+
+        while fd.tell() < end:
+            (magic, length) = struct.unpack('>II', fd.read(8))
+            if magic == 0:
+                break
+            else:
+                padded = (length + 7) & ~7
+                data = fd.read(padded)
+                self.extensions.append(QcowHeaderExtension(magic, length, data))
+
+    def update_extensions(self, fd):
+
+        fd.seek(self.get_header_length())
+        extensions = self.extensions
+        extensions.append(QcowHeaderExtension(0, 0, ""))
+        for ex in extensions:
+            buf = struct.pack('>II', ex.magic, ex.length)
+            fd.write(buf)
+            fd.write(ex.data)
+
+        if self.backing_file != None:
+            self.backing_file_offset = fd.tell()
+            fd.write(self.backing_file)
+
+        if fd.tell() > self.cluster_size:
+            raise Exception("I think I just broke the image...")
+
+
+    def update(self, fd):
+        header_bytes = self.get_header_length()
+
+        self.update_extensions(fd)
+
+        fd.seek(0)
+        header = tuple(self.__dict__[f] for t, p, f in QcowHeader.fields)
+        buf = struct.pack(QcowHeader.fmt, *header)
+        buf = buf[0:header_bytes-1]
+        fd.write(buf)
+
+    def dump(self):
+        for f in QcowHeader.fields:
+            print "%-25s" % f[2], f[1] % self.__dict__[f[2]]
+        print ""
+
+    def dump_extensions(self):
+        for ex in self.extensions:
+
+            data = ex.data[:ex.length]
+            if all(c in string.printable for c in data):
+                data = "'%s'" % data
+            else:
+                data = "<binary>"
+
+            print "Header extension:"
+            print "%-25s %#x" % ("magic", ex.magic)
+            print "%-25s %d" % ("length", ex.length)
+            print "%-25s %s" % ("data", data)
+            print ""
+
+
+def cmd_dump_header(fd):
+    h = QcowHeader(fd)
+    h.dump()
+    h.dump_extensions()
+
+def cmd_add_header_ext(fd, magic, data):
+    try:
+        magic = int(magic, 0)
+    except:
+        print "'%s' is not a valid magic number" % magic
+        sys.exit(1)
+
+    h = QcowHeader(fd)
+    h.extensions.append(QcowHeaderExtension.create(magic, data))
+    h.update(fd)
+
+def cmd_del_header_ext(fd, magic):
+    try:
+        magic = int(magic, 0)
+    except:
+        print "'%s' is not a valid magic number" % magic
+        sys.exit(1)
+
+    h = QcowHeader(fd)
+    found = False
+
+    for ex in h.extensions:
+        if ex.magic == magic:
+            found = True
+            h.extensions.remove(ex)
+
+    if not found:
+        print "No such header extension"
+        return
+
+    h.update(fd)
+
+cmds = [
+    [ 'dump-header',    cmd_dump_header,    0, 'Dump image header and header extensions' ],
+    [ 'add-header-ext', cmd_add_header_ext, 2, 'Add a header extension' ],
+    [ 'del-header-ext', cmd_del_header_ext, 1, 'Delete a header extension' ],
+]
+
+def main(filename, cmd, args):
+    fd = open(filename, "r+b")
+    try:
+        for name, handler, num_args, desc in cmds:
+            if name != cmd:
+                continue
+            elif len(args) != num_args:
+                usage()
+                return
+            else:
+                handler(fd, *args)
+                return
+        print "Unknown command '%s'" % cmd
+    finally:
+        fd.close()
+
+def usage():
+    print "Usage: %s <file> <cmd> [<arg>, ...]" % sys.argv[0]
+    print ""
+    print "Supported commands:"
+    for name, handler, num_args, desc in cmds:
+        print "    %-20s - %s" % (name, desc)
+
+if len(sys.argv) < 3:
+    usage()
+    sys.exit(1)
+
+main(sys.argv[1], sys.argv[2], sys.argv[3:])
-- 
1.7.6.5

[Qemu-devel] [PATCH 2/2] qemu-iotests: Test unknown qcow2 header extensions

[Kevin Wolf]

The immportant thing here is that header extensions don't get silently
dropped when the header is rewritten, e.g. during a rebase.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/031     |   72 +++++++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/031.out |   76 ++++++++++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/group   |    1 +
 3 files changed, 149 insertions(+), 0 deletions(-)
 create mode 100755 tests/qemu-iotests/031
 create mode 100644 tests/qemu-iotests/031.out

diff --git a/tests/qemu-iotests/031 b/tests/qemu-iotests/031
new file mode 100755
index 0000000..6365f28
--- /dev/null
+++ b/tests/qemu-iotests/031
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Test that all qcow2 header extensions survive a header rewrite
+#
+# Copyright (C) 2011 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.pattern
+
+# This tests qcow2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+CLUSTER_SIZE=65536
+echo
+echo === Create image with unknown header extension ===
+echo
+_make_test_img 64M
+./qcow2.py $TEST_IMG add-header-ext 0x12345678 "This is a test header extension"
+./qcow2.py $TEST_IMG dump-header
+_check_test_img
+
+echo
+echo === Rewrite header with no backing file ===
+echo
+$QEMU_IMG rebase -u -b "" $TEST_IMG
+./qcow2.py $TEST_IMG dump-header
+_check_test_img
+
+echo
+echo === Add a backing file and format ===
+echo
+$QEMU_IMG rebase -u -b "/some/backing/file/path" -F host_device $TEST_IMG
+./qcow2.py $TEST_IMG dump-header
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out
new file mode 100644
index 0000000..0f1bf68
--- /dev/null
+++ b/tests/qemu-iotests/031.out
@@ -0,0 +1,76 @@
+QA output created by 031
+
+=== Create image with unknown header extension ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 cluster_size=65536 
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+
+Header extension:
+magic                     0x12345678
+length                    31
+data                      'This is a test header extension'
+
+No errors were found on the image.
+
+=== Rewrite header with no backing file ===
+
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+
+Header extension:
+magic                     0x12345678
+length                    31
+data                      'This is a test header extension'
+
+No errors were found on the image.
+
+=== Add a backing file and format ===
+
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x90
+backing_file_size         0x17
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+
+Header extension:
+magic                     0xe2792aca
+length                    11
+data                      'host_device'
+
+Header extension:
+magic                     0x12345678
+length                    31
+data                      'This is a test header extension'
+
+*** done
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index b549f10..1742ede 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -37,3 +37,4 @@
 028 rw backing auto
 029 rw auto quick
 030 rw auto
+031 rw auto quick
-- 
1.7.6.5