Re: [Qemu-devel] [PATCH COLO-Frame v11 07/39] migration: Integrate COLO checkpoint process into loadvm

[zhanghailiang <zhang.zhanghailiang@huawei.com>]

On 2015/11/25 2:14, Dr. David Alan Gilbert wrote:
> * zhanghailiang (zhang.zhanghailiang@huawei.com) wrote:
>> Switch from normal migration loadvm process into COLO checkpoint process if
>> COLO mode is enabled.
>> We add three new members to struct MigrationIncomingState, 'have_colo_incoming_thread'
>> and 'colo_incoming_thread' record the colo related threads for secondary VM,
>> 'migration_incoming_co' records the original migration incoming coroutine.
>>
>> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
>> Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
>
> Minor comment that needs fixing, see below, but otherwise:
>
> Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
>
>> ---
>> v11:
>> - We moved the place of bdrv_invalidate_cache_all(), but done the deleting work
>>    in other patch. Fix it.
>> - Add documentation for colo in 'MigrationStatus' (Eric's review comment)
>> v10:
>> - fix a bug about fd leak which is found by Dave.
>> ---
>>   include/migration/colo.h      |  7 +++++++
>>   include/migration/migration.h |  7 +++++++
>>   migration/colo-comm.c         | 10 ++++++++++
>>   migration/colo.c              | 22 ++++++++++++++++++++++
>>   migration/migration.c         | 31 +++++++++++++++++++++----------
>>   qapi-schema.json              |  2 ++
>>   stubs/migration-colo.c        | 10 ++++++++++
>>   7 files changed, 79 insertions(+), 10 deletions(-)
>>
>> diff --git a/include/migration/colo.h b/include/migration/colo.h
>> index f462f06..2676c4a 100644
>> --- a/include/migration/colo.h
>> +++ b/include/migration/colo.h
>> @@ -15,6 +15,8 @@
>>
>>   #include "qemu-common.h"
>>   #include "migration/migration.h"
>> +#include "qemu/coroutine_int.h"
>> +#include "qemu/thread.h"
>>
>>   bool colo_supported(void);
>>   void colo_info_mig_init(void);
>> @@ -22,4 +24,9 @@ void colo_info_mig_init(void);
>>   void migrate_start_colo_process(MigrationState *s);
>>   bool migration_in_colo_state(void);
>>
>> +/* loadvm */
>> +bool migration_incoming_enable_colo(void);
>> +void migration_incoming_exit_colo(void);
>> +void *colo_process_incoming_thread(void *opaque);
>> +bool migration_incoming_in_colo_state(void);
>>   #endif
>> diff --git a/include/migration/migration.h b/include/migration/migration.h
>> index 99dfa92..a57a734 100644
>> --- a/include/migration/migration.h
>> +++ b/include/migration/migration.h
>> @@ -22,6 +22,7 @@
>>   #include "migration/vmstate.h"
>>   #include "qapi-types.h"
>>   #include "exec/cpu-common.h"
>> +#include "qemu/coroutine_int.h"
>>
>>   #define QEMU_VM_FILE_MAGIC           0x5145564d
>>   #define QEMU_VM_FILE_VERSION_COMPAT  0x00000002
>> @@ -106,6 +107,12 @@ struct MigrationIncomingState {
>>       void     *postcopy_tmp_page;
>>
>>       int state;
>> +
>> +    bool have_colo_incoming_thread;
>> +    QemuThread colo_incoming_thread;
>> +    /* The coroutine we should enter (back) after failover */
>> +    Coroutine *migration_incoming_co;
>> +
>>       /* See savevm.c */
>>       LoadStateEntry_Head loadvm_handlers;
>>   };
>> diff --git a/migration/colo-comm.c b/migration/colo-comm.c
>> index fb407e0..30df3d3 100644
>> --- a/migration/colo-comm.c
>> +++ b/migration/colo-comm.c
>> @@ -48,3 +48,13 @@ void colo_info_mig_init(void)
>>   {
>>       vmstate_register(NULL, 0, &colo_state, &colo_info);
>>   }
>> +
>> +bool migration_incoming_enable_colo(void)
>> +{
>> +    return colo_info.colo_requested;
>> +}
>> +
>> +void migration_incoming_exit_colo(void)
>> +{
>> +    colo_info.colo_requested = 0;
>> +}
>> diff --git a/migration/colo.c b/migration/colo.c
>> index cf0ccb8..6880aa0 100644
>> --- a/migration/colo.c
>> +++ b/migration/colo.c
>> @@ -13,6 +13,7 @@
>>   #include "sysemu/sysemu.h"
>>   #include "migration/colo.h"
>>   #include "trace.h"
>> +#include "qemu/error-report.h"
>>
>>   bool colo_supported(void)
>>   {
>> @@ -26,6 +27,13 @@ bool migration_in_colo_state(void)
>>       return (s->state == MIGRATION_STATUS_COLO);
>>   }
>>
>> +bool migration_incoming_in_colo_state(void)
>> +{
>> +    MigrationIncomingState *mis = migration_incoming_get_current();
>> +
>> +    return mis && (mis->state == MIGRATION_STATUS_COLO);
>> +}
>> +
>>   static void colo_process_checkpoint(MigrationState *s)
>>   {
>>       qemu_mutex_lock_iothread();
>> @@ -47,3 +55,17 @@ void migrate_start_colo_process(MigrationState *s)
>>       colo_process_checkpoint(s);
>>       qemu_mutex_lock_iothread();
>>   }
>> +
>> +void *colo_process_incoming_thread(void *opaque)
>> +{
>> +    MigrationIncomingState *mis = opaque;
>> +
>> +    migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
>> +                      MIGRATION_STATUS_COLO);
>> +
>> +    /* TODO: COLO checkpoint restore loop */
>> +
>> +    migration_incoming_exit_colo();
>> +
>> +    return NULL;
>> +}
>> diff --git a/migration/migration.c b/migration/migration.c
>> index 46fe8a9..41eac0d 100644
>> --- a/migration/migration.c
>> +++ b/migration/migration.c
>> @@ -358,6 +358,27 @@ static void process_incoming_migration_co(void *opaque)
>>           /* Else if something went wrong then just fall out of the normal exit */
>>       }
>>
>> +    if (!ret) {
>> +        /* Make sure all file formats flush their mutable metadata */
>> +        bdrv_invalidate_cache_all(&local_err);
>> +        if (local_err) {
>> +            error_report_err(local_err);
>> +            migrate_decompress_threads_join();
>> +            exit(EXIT_FAILURE);
>> +        }
>> +    }
>> +    /* we get colo info, and know if we are in colo mode */
>> +    if (!ret && migration_incoming_enable_colo()) {
>> +        mis->migration_incoming_co = qemu_coroutine_self();
>> +        qemu_thread_create(&mis->colo_incoming_thread, "colo incoming",
>> +             colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
>> +        mis->have_colo_incoming_thread = true;
>> +        qemu_coroutine_yield();
>> +
>> +        /* Wait checkpoint incoming thread exit before free resource */
>> +        qemu_thread_join(&mis->colo_incoming_thread);
>> +    }
>> +
>>       qemu_fclose(f);
>>       free_xbzrle_decoded_buf();
>>       migration_incoming_state_destroy();
>> @@ -370,16 +391,6 @@ static void process_incoming_migration_co(void *opaque)
>>           exit(EXIT_FAILURE);
>>       }
>>
>> -    /* Make sure all file formats flush their mutable metadata */
>> -    bdrv_invalidate_cache_all(&local_err);
>> -    if (local_err) {
>> -        migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
>> -                          MIGRATION_STATUS_FAILED);
>> -        error_report_err(local_err);
>> -        migrate_decompress_threads_join();
>> -        exit(EXIT_FAILURE);
>> -    }
>> -
>>       /*
>>        * This must happen after all error conditions are dealt with and
>>        * we're sure the VM is going to be running on this host.
>> diff --git a/qapi-schema.json b/qapi-schema.json
>> index 24b35f3..c2f3b63 100644
>> --- a/qapi-schema.json
>> +++ b/qapi-schema.json
>> @@ -438,6 +438,8 @@
>>   #
>>   # @failed: some error occurred during migration process.
>>   #
>> +# @colo: VM is in the process of fault tolerance. (since 2.6)
>> +#
>>   # Since: 2.3
>>   #
>>   ##
>
> That belongs in the previous patch I think; the previous patch
> added before the @failed, but this change is correct, it just
> needs to go with the actual addition.
>

Good catch, i will fix it in next version.

Thanks.
zhanghailiang

>
>> diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
>> index acddca6..c12516e 100644
>> --- a/stubs/migration-colo.c
>> +++ b/stubs/migration-colo.c
>> @@ -22,6 +22,16 @@ bool migration_in_colo_state(void)
>>       return false;
>>   }
>>
>> +bool migration_incoming_in_colo_state(void)
>> +{
>> +    return false;
>> +}
>> +
>>   void migrate_start_colo_process(MigrationState *s)
>>   {
>>   }
>> +
>> +void *colo_process_incoming_thread(void *opaque)
>> +{
>> +    return NULL;
>> +}
>> --
>> 1.8.3.1
>>
>>
> --
> Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
>
> .
>