[Qemu-devel] [PATCH v4 0/3] Change the backing file of images

[Qemu-devel] [PATCH v4 0/3] Change the backing file of images

[Kevin Wolf]

This patch series adds the functionality to change the backing file of existing
images. This includes simple renaming of a backing file without converting the
COW image as well as safe rebasing onto a different backing file that has some
changes compared to the current backing file.

v2:
- Added missing braces in copied code

v3:
- Added some missing error checks
- Fixed storing the backing file format in qcow2
- Open backing files with right format

v4:
- Handle bdrv_pwrite failure

Kevin Wolf (3):
  block: Introduce BDRV_O_NO_BACKING
  block: Add bdrv_change_backing_file
  qemu-img rebase

 block.c          |   24 +++++-
 block.h          |    3 +
 block/qcow2.c    |  101 ++++++++++++++++++++++++
 block_int.h      |    3 +
 qemu-img-cmds.hx |    6 ++
 qemu-img.c       |  227 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 6 files changed, 361 insertions(+), 3 deletions(-)

[Qemu-devel] [PATCH v4 1/3] block: Introduce BDRV_O_NO_BACKING

[Kevin Wolf]

If an image references a backing file that doesn't exist, qemu-img info fails
to open this image. Exactly in this case the info would be valuable, though:
the user might want to find out which file is missing.

This patch introduces a BDRV_O_NO_BACKING flag to ignore the backing file when
opening the image. qemu-img info is the first user and provides info now even
if the backing file is invalid.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block.c    |    4 ++--
 block.h    |    1 +
 qemu-img.c |    2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/block.c b/block.c
index 30ae2b1..994109e 100644
--- a/block.c
+++ b/block.c
@@ -477,7 +477,7 @@ int bdrv_open2(BlockDriverState *bs, const char *filename, int flags,
         unlink(filename);
     }
 #endif
-    if (bs->backing_file[0] != '\0') {
+    if ((flags & BDRV_O_NO_BACKING) == 0 && bs->backing_file[0] != '\0') {
         /* if there is a backing file, use it */
         BlockDriver *back_drv = NULL;
         bs->backing_hd = bdrv_new("");
@@ -1352,7 +1352,7 @@ const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
 void bdrv_get_backing_filename(BlockDriverState *bs,
                                char *filename, int filename_size)
 {
-    if (!bs->backing_hd) {
+    if (!bs->backing_file) {
         pstrcpy(filename, filename_size, "");
     } else {
         pstrcpy(filename, filename_size, bs->backing_file);
diff --git a/block.h b/block.h
index fa51ddf..f660d5f 100644
--- a/block.h
+++ b/block.h
@@ -39,6 +39,7 @@ typedef struct QEMUSnapshotInfo {
 #define BDRV_O_NOCACHE     0x0020 /* do not use the host page cache */
 #define BDRV_O_CACHE_WB    0x0040 /* use write-back caching */
 #define BDRV_O_NATIVE_AIO  0x0080 /* use native AIO instead of the thread pool */
+#define BDRV_O_NO_BACKING  0x0100 /* don't open the backing file */
 
 #define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB)
 
diff --git a/qemu-img.c b/qemu-img.c
index 1d97f2e..5ad88bf 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -884,7 +884,7 @@ static int img_info(int argc, char **argv)
     } else {
         drv = NULL;
     }
-    if (bdrv_open2(bs, filename, BRDV_O_FLAGS, drv) < 0) {
+    if (bdrv_open2(bs, filename, BRDV_O_FLAGS | BDRV_O_NO_BACKING, drv) < 0) {
         error("Could not open '%s'", filename);
     }
     bdrv_get_format(bs, fmt_name, sizeof(fmt_name));
-- 
1.6.2.5

[Qemu-devel] [PATCH v4 2/3] block: Add bdrv_change_backing_file

[Kevin Wolf]

Introduce the functions needed to change the backing file of an image. The
function is implemented for qcow2.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block.c       |   20 +++++++++++
 block.h       |    2 +
 block/qcow2.c |  101 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 block_int.h   |    3 ++
 4 files changed, 126 insertions(+), 0 deletions(-)

diff --git a/block.c b/block.c
index 994109e..c7bc837 100644
--- a/block.c
+++ b/block.c
@@ -597,6 +597,26 @@ int bdrv_commit(BlockDriverState *bs)
     return 0;
 }
 
+/*
+ * Return values:
+ * 0        - success
+ * -EINVAL  - backing format specified, but no file
+ * -ENOSPC  - can't update the backing file because no space is left in the
+ *            image file header
+ * -ENOTSUP - format driver doesn't support changing the backing file
+ */
+int bdrv_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (drv->bdrv_change_backing_file != NULL) {
+        return drv->bdrv_change_backing_file(bs, backing_file, backing_fmt);
+    } else {
+        return -ENOTSUP;
+    }
+}
+
 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
                                    size_t size)
 {
diff --git a/block.h b/block.h
index f660d5f..bee9ec5 100644
--- a/block.h
+++ b/block.h
@@ -83,6 +83,8 @@ int64_t bdrv_getlength(BlockDriverState *bs);
 void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr);
 void bdrv_guess_geometry(BlockDriverState *bs, int *pcyls, int *pheads, int *psecs);
 int bdrv_commit(BlockDriverState *bs);
+int bdrv_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt);
 void bdrv_register(BlockDriver *bdrv);
 
 /* async block I/O */
diff --git a/block/qcow2.c b/block/qcow2.c
index 984264b..4f9ef91 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -658,6 +658,105 @@ static void qcow_close(BlockDriverState *bs)
     bdrv_delete(s->hd);
 }
 
+/*
+ * Updates the variable length parts of the qcow2 header, i.e. the backing file
+ * name and all extensions. qcow2 was not designed to allow such changes, so if
+ * we run out of space (we can only use the first cluster) this function may
+ * fail.
+ *
+ * Returns 0 on success, -errno in error cases.
+ */
+static int qcow2_update_ext_header(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt)
+{
+    size_t backing_file_len = 0;
+    size_t backing_fmt_len = 0;
+    BDRVQcowState *s = bs->opaque;
+    QCowExtension ext_backing_fmt = {0, 0};
+    int ret;
+
+    /* Backing file format doesn't make sense without a backing file */
+    if (backing_fmt && !backing_file) {
+        return -EINVAL;
+    }
+
+    /* Prepare the backing file format extension if needed */
+    if (backing_fmt) {
+        ext_backing_fmt.len = cpu_to_be32(strlen(backing_fmt));
+        ext_backing_fmt.magic = cpu_to_be32(QCOW_EXT_MAGIC_BACKING_FORMAT);
+        backing_fmt_len = ((sizeof(ext_backing_fmt)
+            + strlen(backing_fmt) + 7) & ~7);
+    }
+
+    /* Check if we can fit the new header into the first cluster */
+    if (backing_file) {
+        backing_file_len = strlen(backing_file);
+    }
+
+    size_t header_size = sizeof(QCowHeader) + backing_file_len
+        + backing_fmt_len;
+
+    if (header_size > s->cluster_size) {
+        return -ENOSPC;
+    }
+
+    /* Rewrite backing file name and qcow2 extensions */
+    size_t ext_size = header_size - sizeof(QCowHeader);
+    uint8_t buf[ext_size];
+    size_t offset = 0;
+    size_t backing_file_offset = 0;
+
+    if (backing_file) {
+        if (backing_fmt) {
+            int padding = backing_fmt_len -
+                (sizeof(ext_backing_fmt) + strlen(backing_fmt));
+
+            memcpy(buf + offset, &ext_backing_fmt, sizeof(ext_backing_fmt));
+            offset += sizeof(ext_backing_fmt);
+
+            memcpy(buf + offset, backing_fmt, strlen(backing_fmt));
+            offset += strlen(backing_fmt);
+
+            memset(buf + offset, 0, padding);
+            offset += padding;
+        }
+
+        memcpy(buf + offset, backing_file, backing_file_len);
+        backing_file_offset = sizeof(QCowHeader) + offset;
+    }
+
+    ret = bdrv_pwrite(s->hd, sizeof(QCowHeader), buf, ext_size);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Update header fields */
+    uint64_t be_backing_file_offset = cpu_to_be64(backing_file_offset);
+    uint32_t be_backing_file_size = cpu_to_be32(backing_file_len);
+
+    ret = bdrv_pwrite(s->hd, offsetof(QCowHeader, backing_file_offset),
+        &be_backing_file_offset, sizeof(uint64_t));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = bdrv_pwrite(s->hd, offsetof(QCowHeader, backing_file_size),
+        &be_backing_file_size, sizeof(uint32_t));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = 0;
+fail:
+    return ret;
+}
+
+static int qcow2_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt)
+{
+    return qcow2_update_ext_header(bs, backing_file, backing_fmt);
+}
+
 static int get_bits_from_size(size_t size)
 {
     int res = 0;
@@ -1137,6 +1236,8 @@ static BlockDriver bdrv_qcow2 = {
     .bdrv_save_vmstate    = qcow_save_vmstate,
     .bdrv_load_vmstate    = qcow_load_vmstate,
 
+    .bdrv_change_backing_file   = qcow2_change_backing_file,
+
     .create_options = qcow_create_options,
     .bdrv_check = qcow_check,
 };
diff --git a/block_int.h b/block_int.h
index 9a3b2e0..a0ebd90 100644
--- a/block_int.h
+++ b/block_int.h
@@ -98,6 +98,9 @@ struct BlockDriver {
     int (*bdrv_load_vmstate)(BlockDriverState *bs, uint8_t *buf,
                              int64_t pos, int size);
 
+    int (*bdrv_change_backing_file)(BlockDriverState *bs,
+        const char *backing_file, const char *backing_fmt);
+
     /* removable device specific */
     int (*bdrv_is_inserted)(BlockDriverState *bs);
     int (*bdrv_media_changed)(BlockDriverState *bs);
-- 
1.6.2.5

[Qemu-devel] [PATCH v4 3/3] qemu-img rebase

[Kevin Wolf]

This adds a rebase subcommand to qemu-img which allows to change the backing
file of an image.

In default mode, both the current and the new backing file need to exist, and
after the rebase, the COW image is guaranteed to have the same guest visible
content as before. To achieve this, old and new backing file are compared and,
if necessary, data is copied from the old backing file into the COW image.

With -u an unsafe mode is enabled that doesn't require the backing files to
exist. It merely changes the backing file reference in the COW image. This is
useful for renaming or moving the backing file. The user is responsible to make
sure that the new backing file has no changes compared to the old one, or
corruption may occur.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 qemu-img-cmds.hx |    6 ++
 qemu-img.c       |  225 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 231 insertions(+), 0 deletions(-)

diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index 641bd87..f28ef36 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -43,5 +43,11 @@ DEF("snapshot", img_snapshot,
     "snapshot [-l | -a snapshot | -c snapshot | -d snapshot] filename")
 STEXI
 @item snapshot [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot}] @var{filename}
+ETEXI
+
+DEF("rebase", img_rebase,
+    "rebase [-u] -b backing_file [-F backing_fmt] filename")
+STEXI
+@item rebase [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
 @end table
 ETEXI
diff --git a/qemu-img.c b/qemu-img.c
index 5ad88bf..48b9315 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -81,6 +81,9 @@ static void help(void)
            "    name=value format. Use -o ? for an overview of the options supported by the\n"
            "    used format\n"
            "  '-c' indicates that target image must be compressed (qcow format only)\n"
+           "  '-u' enables unsafe rebasing. It is assumed that old and new backing file\n"
+           "       match exactly. The image doesn't need a working backing file before\n"
+           "       rebasing in this case (useful for renaming the backing file)\n"
            "  '-h' with or without a command shows this help and lists the supported formats\n"
            "\n"
            "Parameters to snapshot subcommand:\n"
@@ -527,6 +530,37 @@ static int is_allocated_sectors(const uint8_t *buf, int n, int *pnum)
     return v;
 }
 
+/*
+ * Compares two buffers sector by sector. Returns 0 if the first sector of both
+ * buffers matches, non-zero otherwise.
+ *
+ * pnum is set to the number of sectors (including and immediately following
+ * the first one) that are known to have the same comparison result
+ */
+static int compare_sectors(const uint8_t *buf1, const uint8_t *buf2, int n,
+    int *pnum)
+{
+    int res, i;
+
+    if (n <= 0) {
+        *pnum = 0;
+        return 0;
+    }
+
+    res = !!memcmp(buf1, buf2, 512);
+    for(i = 1; i < n; i++) {
+        buf1 += 512;
+        buf2 += 512;
+
+        if (!!memcmp(buf1, buf2, 512) != res) {
+            break;
+        }
+    }
+
+    *pnum = i;
+    return res;
+}
+
 #define IO_BUF_SIZE (2 * 1024 * 1024)
 
 static int img_convert(int argc, char **argv)
@@ -1033,6 +1067,197 @@ static int img_snapshot(int argc, char **argv)
     return 0;
 }
 
+static int img_rebase(int argc, char **argv)
+{
+    BlockDriverState *bs, *bs_old_backing, *bs_new_backing;
+    BlockDriver *old_backing_drv, *new_backing_drv;
+    char *filename;
+    const char *out_basefmt, *out_baseimg;
+    int c, flags, ret;
+    int unsafe = 0;
+
+    /* Parse commandline parameters */
+    out_baseimg = NULL;
+    out_basefmt = NULL;
+
+    for(;;) {
+        c = getopt(argc, argv, "uhF:b:");
+        if (c == -1)
+            break;
+        switch(c) {
+        case 'h':
+            help();
+            return 0;
+        case 'F':
+            out_basefmt = optarg;
+            break;
+        case 'b':
+            out_baseimg = optarg;
+            break;
+        case 'u':
+            unsafe = 1;
+            break;
+        }
+    }
+
+    if ((optind >= argc) || !out_baseimg)
+        help();
+    filename = argv[optind++];
+
+    /*
+     * Open the images.
+     *
+     * Ignore the old backing file for unsafe rebase in case we want to correct
+     * the reference to a renamed or moved backing file.
+     */
+    bs = bdrv_new("");
+    if (!bs)
+        error("Not enough memory");
+
+    flags = BRDV_O_FLAGS | (unsafe ? BDRV_O_NO_BACKING : 0);
+    if (bdrv_open2(bs, filename, flags, NULL) < 0) {
+        error("Could not open '%s'", filename);
+    }
+
+    /* Find the right drivers for the backing files */
+    old_backing_drv = NULL;
+    new_backing_drv = NULL;
+
+    if (!unsafe && bs->backing_format[0] != '\0') {
+        old_backing_drv = bdrv_find_format(bs->backing_format);
+        if (old_backing_drv == NULL) {
+            error("Invalid format name: '%s'", bs->backing_format);
+        }
+    }
+
+    if (out_basefmt != NULL) {
+        new_backing_drv = bdrv_find_format(out_basefmt);
+        if (new_backing_drv == NULL) {
+            error("Invalid format name: '%s'", out_basefmt);
+        }
+    }
+
+    /* For safe rebasing we need to compare old and new backing file */
+    if (unsafe) {
+        /* Make the compiler happy */
+        bs_old_backing = NULL;
+        bs_new_backing = NULL;
+    } else {
+        char backing_name[1024];
+
+        bs_old_backing = bdrv_new("old_backing");
+        bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
+        if (bdrv_open2(bs_old_backing, backing_name, BRDV_O_FLAGS,
+            old_backing_drv))
+        {
+            error("Could not open old backing file '%s'", backing_name);
+            return -1;
+        }
+
+        bs_new_backing = bdrv_new("new_backing");
+        if (bdrv_open2(bs_new_backing, out_baseimg, BRDV_O_FLAGS,
+            new_backing_drv))
+        {
+            error("Could not open new backing file '%s'", backing_name);
+            return -1;
+        }
+    }
+
+    /*
+     * Check each unallocated cluster in the COW file. If it is unallocated,
+     * accesses go to the backing file. We must therefore compare this cluster
+     * in the old and new backing file, and if they differ we need to copy it
+     * from the old backing file into the COW file.
+     *
+     * If qemu-img crashes during this step, no harm is done. The content of
+     * the image is the same as the original one at any time.
+     */
+    if (!unsafe) {
+        uint64_t num_sectors;
+        uint64_t sector;
+        int n, n1;
+        uint8_t buf_old[IO_BUF_SIZE];
+        uint8_t buf_new[IO_BUF_SIZE];
+
+        bdrv_get_geometry(bs, &num_sectors);
+
+        for (sector = 0; sector < num_sectors; sector += n) {
+
+            /* How many sectors can we handle with the next read? */
+            if (sector + (IO_BUF_SIZE / 512) <= num_sectors) {
+                n = (IO_BUF_SIZE / 512);
+            } else {
+                n = num_sectors - sector;
+            }
+
+            /* If the cluster is allocated, we don't need to take action */
+            if (bdrv_is_allocated(bs, sector, n, &n1)) {
+                n = n1;
+                continue;
+            }
+
+            /* Read old and new backing file */
+            if (bdrv_read(bs_old_backing, sector, buf_old, n) < 0) {
+                error("error while reading from old backing file");
+            }
+            if (bdrv_read(bs_new_backing, sector, buf_new, n) < 0) {
+                error("error while reading from new backing file");
+            }
+
+            /* If they differ, we need to write to the COW file */
+            uint64_t written = 0;
+
+            while (written < n) {
+                int pnum;
+
+                if (compare_sectors(buf_old + written * 512,
+                    buf_new + written * 512, n, &pnum))
+                {
+                    ret = bdrv_write(bs, sector + written,
+                        buf_old + written * 512, pnum);
+                    if (ret < 0) {
+                        error("Error while writing to COW image: %s",
+                            strerror(-ret));
+                    }
+                }
+
+                written += pnum;
+            }
+        }
+    }
+
+    /*
+     * Change the backing file. All clusters that are different from the old
+     * backing file are overwritten in the COW file now, so the visible content
+     * doesn't change when we switch the backing file.
+     */
+    ret = bdrv_change_backing_file(bs, out_baseimg, out_basefmt);
+    if (ret == -ENOSPC) {
+        error("Could not change the backing file to '%s': No space left in "
+            "the file header", out_baseimg);
+    } else if (ret < 0) {
+        error("Could not change the backing file to '%s': %s",
+            out_baseimg, strerror(-ret));
+    }
+
+    /*
+     * TODO At this point it is possible to check if any clusters that are
+     * allocated in the COW file are the same in the backing file. If so, they
+     * could be dropped from the COW file. Don't do this before switching the
+     * backing file, in case of a crash this would lead to corruption.
+     */
+
+    /* Cleanup */
+    if (!unsafe) {
+        bdrv_delete(bs_old_backing);
+        bdrv_delete(bs_new_backing);
+    }
+
+    bdrv_delete(bs);
+
+    return 0;
+}
+
 static const img_cmd_t img_cmds[] = {
 #define DEF(option, callback, arg_string)        \
     { option, callback },
-- 
1.6.2.5

Re: [Qemu-devel] [PATCH v4 1/3] block: Introduce BDRV_O_NO_BACKING

[Anthony Liguori]

On 01/12/2010 05:55 AM, Kevin Wolf wrote:
> If an image references a backing file that doesn't exist, qemu-img info fails
> to open this image. Exactly in this case the info would be valuable, though:
> the user might want to find out which file is missing.
>
> This patch introduces a BDRV_O_NO_BACKING flag to ignore the backing file when
> opening the image. qemu-img info is the first user and provides info now even
> if the backing file is invalid.
>
> Signed-off-by: Kevin Wolf<kwolf@redhat.com>
>    

Applied all.  Thanks.

Regards,

Anthony Liguori
> ---
>   block.c    |    4 ++--
>   block.h    |    1 +
>   qemu-img.c |    2 +-
>   3 files changed, 4 insertions(+), 3 deletions(-)
>
> diff --git a/block.c b/block.c
> index 30ae2b1..994109e 100644
> --- a/block.c
> +++ b/block.c
> @@ -477,7 +477,7 @@ int bdrv_open2(BlockDriverState *bs, const char *filename, int flags,
>           unlink(filename);
>       }
>   #endif
> -    if (bs->backing_file[0] != '\0') {
> +    if ((flags&  BDRV_O_NO_BACKING) == 0&&  bs->backing_file[0] != '\0') {
>           /* if there is a backing file, use it */
>           BlockDriver *back_drv = NULL;
>           bs->backing_hd = bdrv_new("");
> @@ -1352,7 +1352,7 @@ const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
>   void bdrv_get_backing_filename(BlockDriverState *bs,
>                                  char *filename, int filename_size)
>   {
> -    if (!bs->backing_hd) {
> +    if (!bs->backing_file) {
>           pstrcpy(filename, filename_size, "");
>       } else {
>           pstrcpy(filename, filename_size, bs->backing_file);
> diff --git a/block.h b/block.h
> index fa51ddf..f660d5f 100644
> --- a/block.h
> +++ b/block.h
> @@ -39,6 +39,7 @@ typedef struct QEMUSnapshotInfo {
>   #define BDRV_O_NOCACHE     0x0020 /* do not use the host page cache */
>   #define BDRV_O_CACHE_WB    0x0040 /* use write-back caching */
>   #define BDRV_O_NATIVE_AIO  0x0080 /* use native AIO instead of the thread pool */
> +#define BDRV_O_NO_BACKING  0x0100 /* don't open the backing file */
>
>   #define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB)
>
> diff --git a/qemu-img.c b/qemu-img.c
> index 1d97f2e..5ad88bf 100644
> --- a/qemu-img.c
> +++ b/qemu-img.c
> @@ -884,7 +884,7 @@ static int img_info(int argc, char **argv)
>       } else {
>           drv = NULL;
>       }
> -    if (bdrv_open2(bs, filename, BRDV_O_FLAGS, drv)<  0) {
> +    if (bdrv_open2(bs, filename, BRDV_O_FLAGS | BDRV_O_NO_BACKING, drv)<  0) {
>           error("Could not open '%s'", filename);
>       }
>       bdrv_get_format(bs, fmt_name, sizeof(fmt_name));
>