[Qemu-devel] [PATCH 0/5 v2] qcow2 info: show highest-allocation and num-free-bytes

[Qemu-devel] [PATCH 0/5 v2] qcow2 info: show highest-allocation and num-free-bytes

[Uri Lublin]

Hello,

The following 5 enhance "info blockstats" for qcow2 images (and
other formats that implement bdrv_get_info), to show 
higest-allocated-offset and number-of-free-bytes (below
the higest-allocated-offset) of the image.

One of the main usage for this information is to know and act
upon a case of end-of-storage-space. Instead of waiting for
-ENOSPACE (which now may cause the VM to stop), one can
define a threshold and extend the diskspace allocated for
the image.
This info is also good for management systems, so they
can report the user disk space status.

Also with highest_alloc and num_free_bytes, we can have a pretty 
good idea about how fragmented a qcow2 images is, and try to
defragment it later.

Patches 1,2,3, Provides information about the highest allocated byte (offset)
Patch 4, Provides information about the number of free bytes 
         below highest_alloc.
Patch 5, Provides info about highest_alloc and num_free_bytes through 
         qemu-img info


Changes from v1: better description of use cases.

Regards,
    Uri.

[Qemu-devel] [PATCH 1/5 v2] block-qcow2: keep highest allocated byte

[Uri Lublin]

We want to know the highest written offset for qcow2 images.
This gives a pretty good (and easy to calculate) estimation to how
much more allocation can be done for the block device.

It can be usefull for allocating more diskspace for that image
(if possible, e.g. lvm) before we run out-of-disk-space

Signed-off-by: Uri Lublin <uril@redhat.com>
---
 block-qcow2.c |   34 ++++++++++++++++++++++++++++++++++
 1 files changed, 34 insertions(+), 0 deletions(-)

diff --git a/block-qcow2.c b/block-qcow2.c
index d4556ef..e016e8f 100644
--- a/block-qcow2.c
+++ b/block-qcow2.c
@@ -143,6 +143,9 @@ typedef struct BDRVQcowState {
     uint32_t crypt_method_header;
     AES_KEY aes_encrypt_key;
     AES_KEY aes_decrypt_key;
+
+    int64_t highest_alloc; /* highest cluester allocated (in clusters) */
+
     uint64_t snapshots_offset;
     int snapshots_size;
     int nb_snapshots;
@@ -170,6 +173,8 @@ static void free_clusters(BlockDriverState *bs,
 #ifdef DEBUG_ALLOC
 static void check_refcounts(BlockDriverState *bs);
 #endif
+static void scan_refcount(BlockDriverState *bs, int64_t *high);
+
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
 {
@@ -278,6 +283,8 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)
     if (refcount_init(bs) < 0)
         goto fail;
 
+    scan_refcount(bs, &s->highest_alloc);
+
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
         len = header.backing_file_size;
@@ -2206,6 +2213,29 @@ static int load_refcount_block(BlockDriverState *bs,
     return 0;
 }
 
+static void scan_refcount(BlockDriverState *bs, int64_t *high)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t refcnt_index, cluster_index, cluster_end, h = 0;
+
+    for (refcnt_index=0; refcnt_index < s->refcount_table_size; refcnt_index++){
+        if (s->refcount_table[refcnt_index] == 0) {
+            continue;
+        }
+        cluster_index = refcnt_index << (s->cluster_bits - REFCOUNT_SHIFT);
+        cluster_end = (refcnt_index + 1) << (s->cluster_bits - REFCOUNT_SHIFT);
+        for ( ; cluster_index < cluster_end; cluster_index++) {
+            if (get_refcount(bs, cluster_index) == 0)
+                /* do nothing -- reserved for free counting */;
+            else
+                h = cluster_index;
+        }
+    }
+
+    if (high)
+        *high = (h+1);
+}
+
 static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
 {
     BDRVQcowState *s = bs->opaque;
@@ -2246,6 +2276,10 @@ retry:
             size,
             (s->free_cluster_index - nb_clusters) << s->cluster_bits);
 #endif
+
+    if (s->highest_alloc < s->free_cluster_index)
+        s->highest_alloc = s->free_cluster_index;
+
     return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
 }
 
-- 
1.6.0.6

[Qemu-devel] [PATCH 2/5 v2] block-qcow2: export highest_allocated through BlockDriverInfo and get_info()

[Uri Lublin]

Signed-off-by: Uri Lublin <uril@redhat.com>
---
 block-qcow2.c |    1 +
 block.h       |    1 +
 2 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/block-qcow2.c b/block-qcow2.c
index e016e8f..7c00ee8 100644
--- a/block-qcow2.c
+++ b/block-qcow2.c
@@ -1671,6 +1671,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->cluster_size = s->cluster_size;
     bdi->vm_state_offset = (int64_t)s->l1_vm_state_index <<
         (s->cluster_bits + s->l2_bits);
+    bdi->highest_alloc = s->highest_alloc << s->cluster_bits;
     return 0;
 }
 
diff --git a/block.h b/block.h
index c3314a1..9c64af3 100644
--- a/block.h
+++ b/block.h
@@ -25,6 +25,7 @@ typedef struct BlockDriverInfo {
     int cluster_size;
     /* offset at which the VM state can be saved (0 if not possible) */
     int64_t vm_state_offset;
+    int64_t highest_alloc; /* highest allocated block offset (in bytes) */
 } BlockDriverInfo;
 
 typedef struct QEMUSnapshotInfo {
-- 
1.6.0.6

[Qemu-devel] [PATCH 3/5 v2] info blockstats: show highest_allocated if exists

[Uri Lublin]

Signed-off-by: Uri Lublin <uril@redhat.com>
---
 block.c |    7 ++++++-
 1 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/block.c b/block.c
index 3250327..0058531 100644
--- a/block.c
+++ b/block.c
@@ -1077,6 +1077,7 @@ void bdrv_info(void)
 void bdrv_info_stats (void)
 {
     BlockDriverState *bs;
+    BlockDriverInfo bdi;
 
     for (bs = bdrv_first; bs != NULL; bs = bs->next) {
 	term_printf ("%s:"
@@ -1084,10 +1085,14 @@ void bdrv_info_stats (void)
 		     " wr_bytes=%" PRIu64
 		     " rd_operations=%" PRIu64
 		     " wr_operations=%" PRIu64
-		     "\n",
+                     ,
 		     bs->device_name,
 		     bs->rd_bytes, bs->wr_bytes,
 		     bs->rd_ops, bs->wr_ops);
+        if (bdrv_get_info(bs, &bdi) == 0)
+            term_printf(" high=%" PRIu64,
+                        bdi.highest_alloc);
+        term_printf("\n");
     }
 }
 
-- 
1.6.0.6

[Qemu-devel] [PATCH 4/5 v2] qcow2 format: keep 'num_free_bytes', and show it upon 'info blockstats'

[Uri Lublin]

'num_free_bytes' is the number of non-allocated bytes below highest-allocation.
It's useful, together with the highest-allocation, to figure out how
fragmented the image is, and how likely it will run out-of-space soon.

For example when the highest allocation is high (almost end-of-disk), but 
many bytes (clusters) are free, and can be re-allocated when neeeded, than
we know it's probably not going to reach end-of-disk-space soon.

Added bookkeeping to block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists

Signed-off-by: Uri Lublin <uril@redhat.com>
---
 block-qcow2.c |   36 ++++++++++++++++++++++++++++--------
 block.c       |    5 +++--
 block.h       |    1 +
 3 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/block-qcow2.c b/block-qcow2.c
index 7c00ee8..8a5b621 100644
--- a/block-qcow2.c
+++ b/block-qcow2.c
@@ -145,6 +145,7 @@ typedef struct BDRVQcowState {
     AES_KEY aes_decrypt_key;
 
     int64_t highest_alloc; /* highest cluester allocated (in clusters) */
+    int64_t nc_free;       /* num of free clusters below highest_alloc */
 
     uint64_t snapshots_offset;
     int snapshots_size;
@@ -173,7 +174,7 @@ static void free_clusters(BlockDriverState *bs,
 #ifdef DEBUG_ALLOC
 static void check_refcounts(BlockDriverState *bs);
 #endif
-static void scan_refcount(BlockDriverState *bs, int64_t *high);
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free);
 
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -283,7 +284,7 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)
     if (refcount_init(bs) < 0)
         goto fail;
 
-    scan_refcount(bs, &s->highest_alloc);
+    scan_refcount(bs, &s->highest_alloc, &s->nc_free);
 
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
@@ -1672,6 +1673,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->vm_state_offset = (int64_t)s->l1_vm_state_index <<
         (s->cluster_bits + s->l2_bits);
     bdi->highest_alloc = s->highest_alloc << s->cluster_bits;
+    bdi->num_free_bytes = s->nc_free  << s->cluster_bits;
     return 0;
 }
 
@@ -2214,25 +2216,35 @@ static int load_refcount_block(BlockDriverState *bs,
     return 0;
 }
 
-static void scan_refcount(BlockDriverState *bs, int64_t *high)
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free)
 {
     BDRVQcowState *s = bs->opaque;
-    int64_t refcnt_index, cluster_index, cluster_end, h = 0;
+    int64_t refcnt_index, cluster_index, cluster_end, h = 0, f = 0;
+    int64_t tail = 0; /* do not count last consecutive free entries */
 
     for (refcnt_index=0; refcnt_index < s->refcount_table_size; refcnt_index++){
         if (s->refcount_table[refcnt_index] == 0) {
+            f += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
+            tail += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
             continue;
         }
         cluster_index = refcnt_index << (s->cluster_bits - REFCOUNT_SHIFT);
         cluster_end = (refcnt_index + 1) << (s->cluster_bits - REFCOUNT_SHIFT);
         for ( ; cluster_index < cluster_end; cluster_index++) {
-            if (get_refcount(bs, cluster_index) == 0)
-                /* do nothing -- reserved for free counting */;
-            else
+            if (get_refcount(bs, cluster_index) == 0) {
+                f++;
+                tail++;
+            }
+            else {
                 h = cluster_index;
+                tail = 0;
+            }
         }
     }
 
+    f -= tail;
+    if (free)
+        *free = f;
     if (high)
         *high = (h+1);
 }
@@ -2278,8 +2290,10 @@ retry:
             (s->free_cluster_index - nb_clusters) << s->cluster_bits);
 #endif
 
-    if (s->highest_alloc < s->free_cluster_index)
+    if (s->highest_alloc < s->free_cluster_index) {
+        s->nc_free += (s->free_cluster_index - s->highest_alloc);
         s->highest_alloc = s->free_cluster_index;
+    }
 
     return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
 }
@@ -2456,6 +2470,12 @@ static int update_cluster_refcount(BlockDriverState *bs,
     block_index = cluster_index &
         ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
     refcount = be16_to_cpu(s->refcount_block_cache[block_index]);
+
+    if (refcount == 1 && addend == -1)
+        s->nc_free += 1;
+    else if (refcount == 0 && addend == 1)
+        s->nc_free -= 1;
+
     refcount += addend;
     if (refcount < 0 || refcount > 0xffff)
         return -EINVAL;
diff --git a/block.c b/block.c
index 0058531..589c67a 100644
--- a/block.c
+++ b/block.c
@@ -1090,8 +1090,9 @@ void bdrv_info_stats (void)
 		     bs->rd_bytes, bs->wr_bytes,
 		     bs->rd_ops, bs->wr_ops);
         if (bdrv_get_info(bs, &bdi) == 0)
-            term_printf(" high=%" PRIu64,
-                        bdi.highest_alloc);
+            term_printf(" high=%" PRId64
+                        " bytes_free=%" PRId64,
+                        bdi.highest_alloc, bdi.num_free_bytes);
         term_printf("\n");
     }
 }
diff --git a/block.h b/block.h
index 9c64af3..dca17db 100644
--- a/block.h
+++ b/block.h
@@ -26,6 +26,7 @@ typedef struct BlockDriverInfo {
     /* offset at which the VM state can be saved (0 if not possible) */
     int64_t vm_state_offset;
     int64_t highest_alloc; /* highest allocated block offset (in bytes) */
+    int64_t num_free_bytes; /* below highest_alloc  */
 } BlockDriverInfo;
 
 typedef struct QEMUSnapshotInfo {
-- 
1.6.0.6

[Qemu-devel] [PATCH 5/5 v2] qemu-img: info: show highest_alloc and num_free_bytes if exist

[Uri Lublin]

Signed-off-by: Uri Lublin <uril@redhat.com>
---
 qemu-img.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/qemu-img.c b/qemu-img.c
index 555ab5f..41ff13b 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -727,6 +727,10 @@ static int img_info(int argc, char **argv)
     if (bdrv_get_info(bs, &bdi) >= 0) {
         if (bdi.cluster_size != 0)
             printf("cluster_size: %d\n", bdi.cluster_size);
+        if (bdi.highest_alloc)
+            printf("highest_alloc: %ld\n", bdi.highest_alloc);
+        if (bdi.num_free_bytes)
+            printf("num_free_bytes: %ld\n", bdi.num_free_bytes);
     }
     bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename));
     if (backing_filename[0] != '\0') {
-- 
1.6.0.6

Re: [Qemu-devel] [PATCH 0/5 v2] qcow2 info: show highest-allocation and num-free-bytes

[Anthony Liguori]

Uri Lublin wrote:
> Hello,
>
> The following 5 enhance "info blockstats" for qcow2 images (and
> other formats that implement bdrv_get_info), to show 
> higest-allocated-offset and number-of-free-bytes (below
> the higest-allocated-offset) of the image.
>
> One of the main usage for this information is to know and act
> upon a case of end-of-storage-space. Instead of waiting for
> -ENOSPACE (which now may cause the VM to stop), one can
> define a threshold and extend the diskspace allocated for
> the image.
> This info is also good for management systems, so they
> can report the user disk space status.
>
> Also with highest_alloc and num_free_bytes, we can have a pretty 
> good idea about how fragmented a qcow2 images is, and try to
> defragment it later.
>
> Patches 1,2,3, Provides information about the highest allocated byte (offset)
> Patch 4, Provides information about the number of free bytes 
>          below highest_alloc.
> Patch 5, Provides info about highest_alloc and num_free_bytes through 
>          qemu-img info
>
>
> Changes from v1: better description of use cases.
>   

Applied all.  Thanks.

Regards,

Anthony Liguori

> Regards,
>     Uri.
>
>
>
>