Re: [Qemu-devel] [PATCH COLO-Frame v9 06/32] migration: Integrate COLO checkpoint process into loadvm

["Dr. David Alan Gilbert" <dgilbert@redhat.com>]

* zhanghailiang (zhang.zhanghailiang@huawei.com) wrote:
> Switch from normal migration loadvm process into COLO checkpoint process if
> COLO mode is enabled.
> We add three new members to struct MigrationIncomingState, 'have_colo_incoming_thread'
> and 'colo_incoming_thread' record the colo related threads for secondary VM,
> 'migration_incoming_co' records the original migration incoming coroutine.
> 
> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>

Mostly OK, some mostly minor comments, and one question below:

> ---
>  include/migration/colo.h      |  7 +++++++
>  include/migration/migration.h |  7 +++++++
>  migration/colo-comm.c         | 10 ++++++++++
>  migration/colo.c              | 22 ++++++++++++++++++++++
>  migration/migration.c         | 33 +++++++++++++++++++++++----------
>  stubs/migration-colo.c        | 10 ++++++++++
>  trace-events                  |  1 +
>  7 files changed, 80 insertions(+), 10 deletions(-)
> 
> diff --git a/include/migration/colo.h b/include/migration/colo.h
> index dface19..58849f7 100644
> --- a/include/migration/colo.h
> +++ b/include/migration/colo.h
> @@ -15,6 +15,8 @@
>  
>  #include "qemu-common.h"
>  #include "migration/migration.h"
> +#include "block/coroutine.h"
> +#include "qemu/thread.h"
>  
>  bool colo_supported(void);
>  void colo_info_mig_init(void);
> @@ -22,4 +24,9 @@ void colo_info_mig_init(void);
>  void colo_init_checkpointer(MigrationState *s);
>  bool migration_in_colo_state(void);
>  
> +/* loadvm */
> +bool migration_incoming_enable_colo(void);
> +void migration_incoming_exit_colo(void);
> +void *colo_process_incoming_thread(void *opaque);
> +bool migration_incoming_in_colo_state(void);
>  #endif
> diff --git a/include/migration/migration.h b/include/migration/migration.h
> index a62068f..9cdd6b6 100644
> --- a/include/migration/migration.h
> +++ b/include/migration/migration.h
> @@ -22,6 +22,7 @@
>  #include "migration/vmstate.h"
>  #include "qapi-types.h"
>  #include "exec/cpu-common.h"
> +#include "block/coroutine.h"
>  
>  #define QEMU_VM_FILE_MAGIC           0x5145564d
>  #define QEMU_VM_FILE_VERSION_COMPAT  0x00000002
> @@ -51,6 +52,12 @@ struct MigrationIncomingState {
>      QEMUFile *file;
>  
>      int state;
> +
> +    bool have_colo_incoming_thread;
> +    QemuThread colo_incoming_thread;
> +    /* The coroutine we should enter (back) after failover */
> +    Coroutine *migration_incoming_co;
> +
>      /* See savevm.c */
>      LoadStateEntry_Head loadvm_handlers;
>  };
> diff --git a/migration/colo-comm.c b/migration/colo-comm.c
> index 4330bd8..0808d6c 100644
> --- a/migration/colo-comm.c
> +++ b/migration/colo-comm.c
> @@ -52,3 +52,13 @@ void colo_info_mig_init(void)
>  {
>      vmstate_register(NULL, 0, &colo_state, &colo_info);
>  }
> +
> +bool migration_incoming_enable_colo(void)
> +{
> +    return colo_info.colo_requested;
> +}
> +
> +void migration_incoming_exit_colo(void)
> +{
> +    colo_info.colo_requested = 0;
> +}
> diff --git a/migration/colo.c b/migration/colo.c
> index 97e64a3..a341eee 100644
> --- a/migration/colo.c
> +++ b/migration/colo.c
> @@ -13,6 +13,7 @@
>  #include "sysemu/sysemu.h"
>  #include "migration/colo.h"
>  #include "trace.h"
> +#include "qemu/error-report.h"
>  
>  static QEMUBH *colo_bh;
>  
> @@ -28,6 +29,13 @@ bool migration_in_colo_state(void)
>      return (s->state == MIGRATION_STATUS_COLO);
>  }
>  
> +bool migration_incoming_in_colo_state(void)
> +{
> +    MigrationIncomingState *mis = migration_incoming_get_current();
> +
> +    return (mis && (mis->state == MIGRATION_STATUS_COLO));

Can remove outer brackets.

> +}
> +
>  static void *colo_thread(void *opaque)
>  {
>      MigrationState *s = opaque;
> @@ -74,3 +82,17 @@ void colo_init_checkpointer(MigrationState *s)
>      colo_bh = qemu_bh_new(colo_start_checkpointer, s);
>      qemu_bh_schedule(colo_bh);
>  }
> +
> +void *colo_process_incoming_thread(void *opaque)
> +{
> +    MigrationIncomingState *mis = opaque;
> +
> +    migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
> +                      MIGRATION_STATUS_COLO);
> +
> +    /* TODO: COLO checkpoint restore loop */
> +
> +    migration_incoming_exit_colo();
> +
> +    return NULL;
> +}
> diff --git a/migration/migration.c b/migration/migration.c
> index bee61aa..241689f 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -280,7 +280,28 @@ static void process_incoming_migration_co(void *opaque)
>                        MIGRATION_STATUS_ACTIVE);
>      ret = qemu_loadvm_state(f);
>  
> -    qemu_fclose(f);
> +    if (!ret) {
> +        /* Make sure all file formats flush their mutable metadata */
> +        bdrv_invalidate_cache_all(&local_err);
> +        if (local_err) {
> +            error_report_err(local_err);
> +            migrate_decompress_threads_join();
> +            exit(EXIT_FAILURE);
> +        }
> +    }
> +    /* we get colo info, and know if we are in colo mode */
> +    if (!ret && migration_incoming_enable_colo()) {
> +        mis->migration_incoming_co = qemu_coroutine_self();
> +        qemu_thread_create(&mis->colo_incoming_thread, "colo incoming",
> +             colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
> +        mis->have_colo_incoming_thread = true;
> +        qemu_coroutine_yield();
> +
> +        /* Wait checkpoint incoming thread exit before free resource */
> +        qemu_thread_join(&mis->colo_incoming_thread);
> +    } else {
> +        qemu_fclose(f);
> +    }

Why is that qemu_fclose(f) in the else ?  If you do colo, and do the colo
thread, when the incoming colo thread exits, don't we return here - and if
we return here then don't we want to close f?  (I guess this only happens
on a failover to secondary?)

>      free_xbzrle_decoded_buf();
>      migration_incoming_state_destroy();
>  
> @@ -295,18 +316,9 @@ static void process_incoming_migration_co(void *opaque)
>                        MIGRATION_STATUS_COMPLETED);
>      qemu_announce_self();
>  
> -    /* Make sure all file formats flush their mutable metadata */
> -    bdrv_invalidate_cache_all(&local_err);
> -    if (local_err) {
> -        error_report_err(local_err);
> -        migrate_decompress_threads_join();
> -        exit(EXIT_FAILURE);
> -    }
> -
>      /* If global state section was not received or we are in running
>         state, we need to obey autostart. Any other state is set with
>         runstate_set. */
> -
>      if (!global_state_received() ||
>          global_state_get_runstate() == RUN_STATE_RUNNING) {
>          if (autostart) {
> @@ -740,6 +752,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
>          error_setg(errp, QERR_MIGRATION_ACTIVE);
>          return;
>      }
> +
>      if (runstate_check(RUN_STATE_INMIGRATE)) {
>          error_setg(errp, "Guest is waiting for an incoming migration");
>          return;
> diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
> index 51b8f66..c49ee1a 100644
> --- a/stubs/migration-colo.c
> +++ b/stubs/migration-colo.c
> @@ -22,6 +22,16 @@ bool migration_in_colo_state(void)
>      return false;
>  }
>  
> +bool migration_incoming_in_colo_state(void)
> +{
> +    return false;
> +}
> +
>  void colo_init_checkpointer(MigrationState *s)
>  {
>  }
> +
> +void *colo_process_incoming_thread(void *opaque)
> +{
> +    return NULL;
> +}
> diff --git a/trace-events b/trace-events
> index 487d1c7..352e9c3 100644
> --- a/trace-events
> +++ b/trace-events
> @@ -1474,6 +1474,7 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) ""
>  
>  # migration/colo.c
>  colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
> +colo_receive_message(const char *msg) "Receive '%s'"

Should be in a different patch?

Dave

>  # kvm-all.c
>  kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
> -- 
> 1.8.3.1
> 
> 
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK