[Qemu-devel] [PATCH COLO-Frame v7 05/34] migration: Integrate COLO checkpoint process into migration

[zhanghailiang <zhang.zhanghailiang@huawei.com>]

Add a migrate state: MIGRATION_STATUS_COLO, enter this migration state
after the first live migration successfully finished.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
 include/migration/colo.h      |  3 +++
 include/migration/migration.h |  2 ++
 migration/colo.c              | 55 +++++++++++++++++++++++++++++++++++++++++++
 migration/migration.c         | 25 +++++++++++++++-----
 qapi-schema.json              |  2 +-
 stubs/migration-colo.c        |  9 +++++++
 trace-events                  |  3 +++
 7 files changed, 92 insertions(+), 7 deletions(-)

diff --git a/include/migration/colo.h b/include/migration/colo.h
index 9b6662d..2320594 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -19,4 +19,7 @@
 bool colo_supported(void);
 void colo_info_mig_init(void);
 
+void colo_init_checkpointer(MigrationState *s);
+bool migrate_in_colo_state(void);
+
 #endif
diff --git a/include/migration/migration.h b/include/migration/migration.h
index 1b23517..0fc85cc 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -82,6 +82,8 @@ struct MigrationState
     int64_t dirty_sync_count;
 };
 
+void migrate_set_state(MigrationState *s, int old_state, int new_state);
+
 void process_incoming_migration(QEMUFile *f);
 
 void qemu_start_incoming_migration(const char *uri, Error **errp);
diff --git a/migration/colo.c b/migration/colo.c
index 2c40d2e..33a5a48 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -10,9 +10,64 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "sysemu/sysemu.h"
 #include "migration/colo.h"
+#include "trace.h"
+
+static QEMUBH *colo_bh;
 
 bool colo_supported(void)
 {
     return true;
 }
+
+bool migrate_in_colo_state(void)
+{
+    MigrationState *s = migrate_get_current();
+    return (s->state == MIGRATION_STATUS_COLO);
+}
+
+static void *colo_thread(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    qemu_mutex_lock_iothread();
+    vm_start();
+    qemu_mutex_unlock_iothread();
+    trace_colo_vm_state_change("stop", "run");
+
+    /*TODO: COLO checkpoint savevm loop*/
+
+    migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+
+    qemu_mutex_lock_iothread();
+    qemu_bh_schedule(s->cleanup_bh);
+    qemu_mutex_unlock_iothread();
+
+    return NULL;
+}
+
+static void colo_start_checkpointer(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    if (colo_bh) {
+        qemu_bh_delete(colo_bh);
+        colo_bh = NULL;
+    }
+
+    qemu_mutex_unlock_iothread();
+    qemu_thread_join(&s->thread);
+    qemu_mutex_lock_iothread();
+
+    migrate_set_state(s, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_COLO);
+
+    qemu_thread_create(&s->thread, "colo", colo_thread, s,
+                       QEMU_THREAD_JOINABLE);
+}
+
+void colo_init_checkpointer(MigrationState *s)
+{
+    colo_bh = qemu_bh_new(colo_start_checkpointer, s);
+    qemu_bh_schedule(colo_bh);
+}
diff --git a/migration/migration.c b/migration/migration.c
index e34b2c9..fefa47b 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -428,6 +428,10 @@ MigrationInfo *qmp_query_migrate(Error **errp)
 
         get_xbzrle_cache_stats(info);
         break;
+    case MIGRATION_STATUS_COLO:
+        info->has_status = true;
+        /* TODO: display COLO specific information (checkpoint info etc.) */
+        break;
     case MIGRATION_STATUS_COMPLETED:
         get_xbzrle_cache_stats(info);
 
@@ -530,7 +534,7 @@ void qmp_migrate_set_parameters(bool has_compress_level,
 
 /* shared migration helpers */
 
-static void migrate_set_state(MigrationState *s, int old_state, int new_state)
+void migrate_set_state(MigrationState *s, int old_state, int new_state)
 {
     if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) {
         migrate_generate_event(new_state);
@@ -711,7 +715,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
 
     if (s->state == MIGRATION_STATUS_ACTIVE ||
         s->state == MIGRATION_STATUS_SETUP ||
-        s->state == MIGRATION_STATUS_CANCELLING) {
+        s->state == MIGRATION_STATUS_CANCELLING ||
+        s->state == MIGRATION_STATUS_COLO) {
         error_setg(errp, QERR_MIGRATION_ACTIVE);
         return;
     }
@@ -927,6 +932,7 @@ static void *migration_thread(void *opaque)
     int64_t max_size = 0;
     int64_t start_time = initial_time;
     bool old_vm_running = false;
+    bool enable_colo = migrate_enable_colo();
 
     qemu_savevm_state_header(s->file);
     qemu_savevm_state_begin(s->file, &s->params);
@@ -968,8 +974,10 @@ static void *migration_thread(void *opaque)
                 }
 
                 if (!qemu_file_get_error(s->file)) {
-                    migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
-                                      MIGRATION_STATUS_COMPLETED);
+                    if (!enable_colo) {
+                        migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
+                                          MIGRATION_STATUS_COMPLETED);
+                    }
                     break;
                 }
             }
@@ -1020,11 +1028,16 @@ static void *migration_thread(void *opaque)
         }
         runstate_set(RUN_STATE_POSTMIGRATE);
     } else {
-        if (old_vm_running) {
+        if (s->state == MIGRATION_STATUS_ACTIVE && enable_colo) {
+            colo_init_checkpointer(s);
+        } else if (old_vm_running) {
             vm_start();
         }
     }
-    qemu_bh_schedule(s->cleanup_bh);
+
+    if (!enable_colo) {
+        qemu_bh_schedule(s->cleanup_bh);
+    }
     qemu_mutex_unlock_iothread();
 
     return NULL;
diff --git a/qapi-schema.json b/qapi-schema.json
index 58c09f2..f16f482 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -433,7 +433,7 @@
 ##
 { 'enum': 'MigrationStatus',
   'data': [ 'none', 'setup', 'cancelling', 'cancelled',
-            'active', 'completed', 'failed' ] }
+            'active', 'completed', 'failed', 'colo' ] }
 
 ##
 # @MigrationInfo
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index 3d817df..96aa9d7 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -16,3 +16,12 @@ bool colo_supported(void)
 {
     return false;
 }
+
+bool migrate_in_colo_state(void)
+{
+    return false;
+}
+
+void colo_init_checkpointer(MigrationState *s)
+{
+}
diff --git a/trace-events b/trace-events
index 2d395c5..125a33b 100644
--- a/trace-events
+++ b/trace-events
@@ -1472,6 +1472,9 @@ rdma_start_incoming_migration_after_rdma_listen(void) ""
 rdma_start_outgoing_migration_after_rdma_connect(void) ""
 rdma_start_outgoing_migration_after_rdma_source_init(void) ""
 
+# migration/colo.c
+colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
+
 # kvm-all.c
 kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
 kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
-- 
1.7.12.4