[Qemu-devel] [PATCH 0/2] fixes for bdrv_invalidate_cache

[Qemu-devel] [PATCH 0/2] fixes for bdrv_invalidate_cache

[Denis V. Lunev]

Signed-off-by: Denis V. Lunev <den@openvz.org>
CC: Kevin Wolf <kwolf@redhat.com>
CC: Paolo Bonzini <pbonzini@redhat.com>

[Qemu-devel] [PATCH 1/2] block: fix assert in qcow2_get_specific_info

[Denis V. Lunev]

There is a possibility to hit assert qcow2_get_specific_info that
s->qcow_version is undefined. This happens when VM in starting from
suspended state, i.e. it processes incoming migration, and in the same
time 'info block' is called.

The problem is that in the qcow2_invalidate_cache closes and the image
and memsets BDRVQcowState in the middle.

The patch moves processing of qcow2_get_specific_info into coroutine
context and ensures that qcow2_invalidate_cache and qcow2_get_specific_info
can not run simultaneosly.

Signed-off-by: Denis V. Lunev <den@openvz.org>
CC: Kevin Wolf <kwolf@redhat.com>
CC: Paolo Bonzini <pbonzini@redhat.com>
---
 block/qcow2.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 block/qcow2.h |  2 ++
 2 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index 1789af4..12eda24 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1740,6 +1740,10 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
     Error *local_err = NULL;
     int ret;
 
+    qemu_co_mutex_lock(&s->lock);
+    s->in_transient_state = true;
+    qemu_co_mutex_unlock(&s->lock);
+
     /*
      * Backing files are read-only which makes all of their metadata immutable,
      * that means we don't have to worry about reopening them here.
@@ -1753,10 +1757,10 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
     bdrv_invalidate_cache(bs->file->bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto done;
     }
 
-    memset(s, 0, sizeof(BDRVQcow2State));
+    memset(s, 0, offsetof(BDRVQcow2State, in_transient_state));
     options = qdict_clone_shallow(bs->options);
 
     ret = qcow2_open(bs, options, flags, &local_err);
@@ -1765,13 +1769,18 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
         error_setg(errp, "Could not reopen qcow2 layer: %s",
                    error_get_pretty(local_err));
         error_free(local_err);
-        return;
+        goto done;
     } else if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not reopen qcow2 layer");
-        return;
+        goto done;
     }
 
     s->cipher = cipher;
+
+done:
+    qemu_co_mutex_lock(&s->lock);
+    s->in_transient_state = false;
+    qemu_co_mutex_unlock(&s->lock);
 }
 
 static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
@@ -2778,11 +2787,21 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     return 0;
 }
 
-static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
+
+static ImageInfoSpecific *qcow2_co_get_specific_info(BlockDriverState *bs)
 {
     BDRVQcow2State *s = bs->opaque;
+    AioContext *ctx = bdrv_get_aio_context(bs);
+
     ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
 
+    qemu_co_mutex_lock(&s->lock);
+    while (s->in_transient_state) {
+        qemu_co_mutex_unlock(&s->lock);
+        aio_poll(ctx, true);
+        qemu_co_mutex_lock(&s->lock);
+    }
+
     *spec_info = (ImageInfoSpecific){
         .type  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
         .u.qcow2 = g_new(ImageInfoSpecificQCow2, 1),
@@ -2808,10 +2827,45 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
          * added without having it covered here */
         assert(false);
     }
+    qemu_co_mutex_unlock(&s->lock);
 
     return spec_info;
 }
 
+struct InfoCo {
+    BlockDriverState *bs;
+    ImageInfoSpecific *info;
+};
+
+static void qcow2_co_get_specific_info_entry(void *opaque)
+{
+    struct InfoCo *ret = opaque;
+    ret->info = qcow2_co_get_specific_info(ret->bs);
+}
+
+static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
+{
+    Coroutine *co;
+    struct InfoCo info_co = {
+        .bs = bs,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        qcow2_co_get_specific_info_entry(&info_co);
+    } else {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        co = qemu_coroutine_create(qcow2_co_get_specific_info_entry);
+        qemu_coroutine_enter(co, &info_co);
+        while (info_co.info == NULL) {
+            aio_poll(aio_context, true);
+        }
+    }
+
+    return info_co.info;
+}
+
 #if 0
 static void dump_refcounts(BlockDriverState *bs)
 {
diff --git a/block/qcow2.h b/block/qcow2.h
index a063a3c..1114528 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -293,6 +293,8 @@ typedef struct BDRVQcow2State {
      * override) */
     char *image_backing_file;
     char *image_backing_format;
+
+    bool in_transient_state;
 } BDRVQcow2State;
 
 typedef struct Qcow2COWRegion {
-- 
2.5.0

Re: [Qemu-devel] [PATCH 1/2] block: fix assert in qcow2_get_specific_info

[Denis V. Lunev]

On 01/20/2016 10:12 AM, Denis V. Lunev wrote:
> There is a possibility to hit assert qcow2_get_specific_info that
> s->qcow_version is undefined. This happens when VM in starting from
> suspended state, i.e. it processes incoming migration, and in the same
> time 'info block' is called.
>
> The problem is that in the qcow2_invalidate_cache closes and the image
> and memsets BDRVQcowState in the middle.
>
> The patch moves processing of qcow2_get_specific_info into coroutine
> context and ensures that qcow2_invalidate_cache and qcow2_get_specific_info
> can not run simultaneosly.
>
> Signed-off-by: Denis V. Lunev <den@openvz.org>
> CC: Kevin Wolf <kwolf@redhat.com>
> CC: Paolo Bonzini <pbonzini@redhat.com>
> ---
>   block/qcow2.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
>   block/qcow2.h |  2 ++
>   2 files changed, 61 insertions(+), 5 deletions(-)
>
> diff --git a/block/qcow2.c b/block/qcow2.c
> index 1789af4..12eda24 100644
> --- a/block/qcow2.c
> +++ b/block/qcow2.c
> @@ -1740,6 +1740,10 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
>       Error *local_err = NULL;
>       int ret;
>   
> +    qemu_co_mutex_lock(&s->lock);
> +    s->in_transient_state = true;
> +    qemu_co_mutex_unlock(&s->lock);
> +
>       /*
>        * Backing files are read-only which makes all of their metadata immutable,
>        * that means we don't have to worry about reopening them here.
> @@ -1753,10 +1757,10 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
>       bdrv_invalidate_cache(bs->file->bs, &local_err);
>       if (local_err) {
>           error_propagate(errp, local_err);
> -        return;
> +        goto done;
>       }
>   
> -    memset(s, 0, sizeof(BDRVQcow2State));
> +    memset(s, 0, offsetof(BDRVQcow2State, in_transient_state));
>       options = qdict_clone_shallow(bs->options);
>   
>       ret = qcow2_open(bs, options, flags, &local_err);
> @@ -1765,13 +1769,18 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
>           error_setg(errp, "Could not reopen qcow2 layer: %s",
>                      error_get_pretty(local_err));
>           error_free(local_err);
> -        return;
> +        goto done;
>       } else if (ret < 0) {
>           error_setg_errno(errp, -ret, "Could not reopen qcow2 layer");
> -        return;
> +        goto done;
>       }
>   
>       s->cipher = cipher;
> +
> +done:
> +    qemu_co_mutex_lock(&s->lock);
> +    s->in_transient_state = false;
> +    qemu_co_mutex_unlock(&s->lock);
>   }
>   
>   static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
> @@ -2778,11 +2787,21 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
>       return 0;
>   }
>   
> -static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
> +
> +static ImageInfoSpecific *qcow2_co_get_specific_info(BlockDriverState *bs)
>   {
>       BDRVQcow2State *s = bs->opaque;
> +    AioContext *ctx = bdrv_get_aio_context(bs);
> +
>       ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
>   
> +    qemu_co_mutex_lock(&s->lock);
> +    while (s->in_transient_state) {
> +        qemu_co_mutex_unlock(&s->lock);
> +        aio_poll(ctx, true);
> +        qemu_co_mutex_lock(&s->lock);
> +    }
> +
>       *spec_info = (ImageInfoSpecific){
>           .type  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
>           .u.qcow2 = g_new(ImageInfoSpecificQCow2, 1),
> @@ -2808,10 +2827,45 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
>            * added without having it covered here */
>           assert(false);
>       }
> +    qemu_co_mutex_unlock(&s->lock);
>   
>       return spec_info;
>   }
>   
> +struct InfoCo {
> +    BlockDriverState *bs;
> +    ImageInfoSpecific *info;
> +};
> +
> +static void qcow2_co_get_specific_info_entry(void *opaque)
> +{
> +    struct InfoCo *ret = opaque;
> +    ret->info = qcow2_co_get_specific_info(ret->bs);
> +}
> +
> +static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
> +{
> +    Coroutine *co;
> +    struct InfoCo info_co = {
> +        .bs = bs,
> +    };
> +
> +    if (qemu_in_coroutine()) {
> +        /* Fast-path if already in coroutine context */
> +        qcow2_co_get_specific_info_entry(&info_co);
> +    } else {
> +        AioContext *aio_context = bdrv_get_aio_context(bs);
> +
> +        co = qemu_coroutine_create(qcow2_co_get_specific_info_entry);
> +        qemu_coroutine_enter(co, &info_co);
> +        while (info_co.info == NULL) {
> +            aio_poll(aio_context, true);
> +        }
> +    }
> +
> +    return info_co.info;
> +}
> +
>   #if 0
>   static void dump_refcounts(BlockDriverState *bs)
>   {
> diff --git a/block/qcow2.h b/block/qcow2.h
> index a063a3c..1114528 100644
> --- a/block/qcow2.h
> +++ b/block/qcow2.h
> @@ -293,6 +293,8 @@ typedef struct BDRVQcow2State {
>        * override) */
>       char *image_backing_file;
>       char *image_backing_format;
> +
> +    bool in_transient_state;
>   } BDRVQcow2State;
>   
>   typedef struct Qcow2COWRegion {
ping v2

Re: [Qemu-devel] [PATCH 1/2] block: fix assert in qcow2_get_specific_info

[Paolo Bonzini]

On 20/01/2016 08:12, Denis V. Lunev wrote:
> There is a possibility to hit assert qcow2_get_specific_info that
> s->qcow_version is undefined. This happens when VM in starting from
> suspended state, i.e. it processes incoming migration, and in the same
> time 'info block' is called.
> 
> The problem is that in the qcow2_invalidate_cache closes and the image
> and memsets BDRVQcowState in the middle.
> 
> The patch moves processing of qcow2_get_specific_info into coroutine
> context and ensures that qcow2_invalidate_cache and qcow2_get_specific_info
> can not run simultaneosly.

I'm sitting next to Kevin :) and this is not a qcow2 bug.

The problem is that qcow2_invalidate_cache is being called in coroutine
context.  The process_incoming_migration_co code starting with
bdrv_invalidate_cache_all should be moved out of the coroutine and into
the main loop.  You can use a bottom half to get out of coroutine context.

The result should be a much simpler patch, too.

Thanks, and sorry for the delay.  I saw qcow2 in the title and assumed
it was something I knew nothing about. :)

Paolo

Re: [Qemu-devel] [PATCH 1/2] block: fix assert in qcow2_get_specific_info

[Denis V. Lunev]

On 02/02/2016 04:20 PM, Paolo Bonzini wrote:
>
> On 20/01/2016 08:12, Denis V. Lunev wrote:
>> There is a possibility to hit assert qcow2_get_specific_info that
>> s->qcow_version is undefined. This happens when VM in starting from
>> suspended state, i.e. it processes incoming migration, and in the same
>> time 'info block' is called.
>>
>> The problem is that in the qcow2_invalidate_cache closes and the image
>> and memsets BDRVQcowState in the middle.
>>
>> The patch moves processing of qcow2_get_specific_info into coroutine
>> context and ensures that qcow2_invalidate_cache and qcow2_get_specific_info
>> can not run simultaneosly.
> I'm sitting next to Kevin :) and this is not a qcow2 bug.
>
> The problem is that qcow2_invalidate_cache is being called in coroutine
> context.  The process_incoming_migration_co code starting with
> bdrv_invalidate_cache_all should be moved out of the coroutine and into
> the main loop.  You can use a bottom half to get out of coroutine context.
>
> The result should be a much simpler patch, too.
>
> Thanks, and sorry for the delay.  I saw qcow2 in the title and assumed
> it was something I knew nothing about. :)
>
> Paolo
no prob. I'll check this and come with a patch if this
approach will work.

By the way, are you sitting next to Stefan too? :)
There is our set
   [PATCH v4 00/11] simplify usage of tracepoints, and connect them to 
logging
which was accepted by Stefan and still not merged.
We can have troubles as in 2.5 previously near the end of
the merge window.

Den

Re: [Qemu-devel] [PATCH 1/2] block: fix assert in qcow2_get_specific_info

[Paolo Bonzini]

On 02/02/2016 14:49, Denis V. Lunev wrote:
> 
> By the way, are you sitting next to Stefan too? :)

No, I am not. :)

> There is our set
>   [PATCH v4 00/11] simplify usage of tracepoints, and connect them to
> logging
> which was accepted by Stefan and still not merged.
> We can have troubles as in 2.5 previously near the end of
> the merge window.

I agree, I really want that series to go in.  I'll ping him.

Thanks!

Paolo

[Qemu-devel] [PATCH 2/2] nbd: add missed aio_context_acquire in nbd_export_new

[Denis V. Lunev]

blk_invalidate_cache() can call qcow2_invalidate_cache which performs
IO inside.

Signed-off-by: Denis V. Lunev <den@openvz.org>
CC: Kevin Wolf <kwolf@redhat.com>
CC: Paolo Bonzini <pbonzini@redhat.com>
---
 nbd.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nbd.c b/nbd.c
index b3d9654..b271db5 100644
--- a/nbd.c
+++ b/nbd.c
@@ -1080,7 +1080,9 @@ NBDExport *nbd_export_new(BlockBackend *blk, off_t dev_offset, off_t size,
      * that BDRV_O_INCOMING is cleared and the image is ready for write
      * access since the export could be available before migration handover.
      */
+    aio_context_acquire(exp->ctx);
     blk_invalidate_cache(blk, NULL);
+    aio_context_release(exp->ctx);
     return exp;
 
 fail:
-- 
2.5.0

Re: [Qemu-devel] [PATCH 2/2] nbd: add missed aio_context_acquire in nbd_export_new

[Paolo Bonzini]

On 20/01/2016 08:12, Denis V. Lunev wrote:
> blk_invalidate_cache() can call qcow2_invalidate_cache which performs
> IO inside.
> 
> Signed-off-by: Denis V. Lunev <den@openvz.org>
> CC: Kevin Wolf <kwolf@redhat.com>
> CC: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  nbd.c | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/nbd.c b/nbd.c
> index b3d9654..b271db5 100644
> --- a/nbd.c
> +++ b/nbd.c
> @@ -1080,7 +1080,9 @@ NBDExport *nbd_export_new(BlockBackend *blk, off_t dev_offset, off_t size,
>       * that BDRV_O_INCOMING is cleared and the image is ready for write
>       * access since the export could be available before migration handover.
>       */
> +    aio_context_acquire(exp->ctx);
>      blk_invalidate_cache(blk, NULL);
> +    aio_context_release(exp->ctx);
>      return exp;
>  
>  fail:
> 

Queuing this one myself.

Paolo

Re: [Qemu-devel] [PATCH 0/2] fixes for bdrv_invalidate_cache

[Denis V. Lunev]

On 01/20/2016 10:12 AM, Denis V. Lunev wrote:
> Signed-off-by: Denis V. Lunev <den@openvz.org>
> CC: Kevin Wolf <kwolf@redhat.com>
> CC: Paolo Bonzini <pbonzini@redhat.com>
>
ping!

This patchset fixes real crash in QEMU. Can you pls look?
The approach could be wrong actually thus your
feedback would be much appreciated.

Den

Re: [Qemu-devel] [PATCH 0/2] fixes for bdrv_invalidate_cache

[Denis V. Lunev]

On 01/20/2016 10:12 AM, Denis V. Lunev wrote:
> Signed-off-by: Denis V. Lunev <den@openvz.org>
> CC: Kevin Wolf <kwolf@redhat.com>
> CC: Paolo Bonzini <pbonzini@redhat.com>
>
guys?