[Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO primary/secondary vm failover work

[zhanghailiang <zhang.zhanghailiang@huawei.com>]

If there are some errors happen, we will give users(administrators) time to
get involved in failover verdict, which they can decide
which side should take over the work by using 'colo_lost_heartbeat' command.

Note: The default verdict is primary VM takes over work while secondary VM exit.
So if users choose secondary VM to take over work, please make sure that
Primary VM is dead, or there will be 'split-brain' problem.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
 include/migration/colo.h     |   4 ++
 include/migration/failover.h |   1 +
 migration/colo-failover.c    |  15 ++++-
 migration/colo.c             | 149 +++++++++++++++++++++++++++++++++++++++++--
 trace-events                 |   1 +
 5 files changed, 165 insertions(+), 5 deletions(-)

diff --git a/include/migration/colo.h b/include/migration/colo.h
index 815db2e..0364bab 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -43,4 +43,8 @@ int get_colo_mode(void);
 int create_and_init_ram_cache(void);
 void colo_flush_ram_cache(void);
 void release_ram_cache(void);
+
+/* failover */
+void colo_do_failover(MigrationState *s);
+
 #endif
diff --git a/include/migration/failover.h b/include/migration/failover.h
index 56d57d7..f85f97d 100644
--- a/include/migration/failover.h
+++ b/include/migration/failover.h
@@ -25,5 +25,6 @@ typedef enum COLOFailoverStatus {
 int failover_set_state(int old_state, int new_state);
 int failover_get_state(void);
 void failover_request_active(Error **errp);
+bool failover_request_is_active(void);
 
 #endif
diff --git a/migration/colo-failover.c b/migration/colo-failover.c
index 1bf22f5..7b0fa03 100644
--- a/migration/colo-failover.c
+++ b/migration/colo-failover.c
@@ -22,9 +22,17 @@ static COLOFailoverStatus failover_state;
 
 static void colo_failover_bh(void *opaque)
 {
+    int old_state;
+
     qemu_bh_delete(failover_bh);
     failover_bh = NULL;
-    /*TODO: Do failover work */
+    old_state = failover_set_state(FAILOVER_STATUS_REQUEST,
+                                   FAILOVER_STATUS_HANDLING);
+    if (old_state != FAILOVER_STATUS_REQUEST) {
+        error_report(" Unkown error for failover, old_state=%d", old_state);
+        return;
+    }
+    colo_do_failover(NULL);
 }
 
 void failover_request_active(Error **errp)
@@ -54,6 +62,11 @@ int failover_get_state(void)
     return atomic_read(&failover_state);
 }
 
+bool failover_request_is_active(void)
+{
+    return ((failover_get_state() != FAILOVER_STATUS_NONE));
+}
+
 void qmp_colo_lost_heartbeat(Error **errp)
 {
     if (get_colo_mode() == COLO_MODE_UNKNOWN) {
diff --git a/migration/colo.c b/migration/colo.c
index 6990c0c..243aa32 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -73,6 +73,83 @@ bool loadvm_in_colo_state(void)
     return colo != NULL;
 }
 
+static bool colo_runstate_is_stopped(void)
+{
+    return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
+}
+
+/*
+ * there are two way to entry this function
+ * 1. From colo checkpoint incoming thread, in this case
+ * we should protect it by iothread lock
+ * 2. From user command, because hmp/qmp command
+ * was happened in main loop, iothread lock will cause a
+ * dead lock.
+ */
+static void secondary_vm_do_failover(void)
+{
+    int old_state;
+
+    /* It means that VM exit from COLO state */
+    colo = NULL;
+
+    if (!autostart) {
+        error_report("\"-S\" qemu option will be ignored in secondary side");
+        /* recover runstate to normal migration finish state */
+        autostart = true;
+    }
+
+    old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+                                   FAILOVER_STATUS_COMPLETED);
+    if (old_state != FAILOVER_STATUS_HANDLING) {
+        error_report("Serious error while do failover for secondary VM,"
+                     "old_state: %d", old_state);
+        return;
+    }
+    /* For Secondary VM, jump to incoming co */
+    if (migration_incoming_co) {
+        qemu_coroutine_enter(migration_incoming_co, NULL);
+    }
+}
+
+static void primary_vm_do_failover(void)
+{
+    MigrationState *s = migrate_get_current();
+    int old_state;
+
+    if (!colo_runstate_is_stopped()) {
+        vm_stop_force_state(RUN_STATE_COLO);
+    }
+
+    if (s->state != MIGRATION_STATUS_FAILED) {
+        migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+    }
+
+    vm_start();
+
+    old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+                                   FAILOVER_STATUS_COMPLETED);
+    if (old_state != FAILOVER_STATUS_COMPLETED) {
+        error_report("Serious error while do failover for Primary VM,"
+                     "old_state: %d", old_state);
+        return;
+    }
+}
+
+void colo_do_failover(MigrationState *s)
+{
+    /* Make sure vm stopped while failover */
+    if (!colo_runstate_is_stopped()) {
+        vm_stop_force_state(RUN_STATE_COLO);
+    }
+
+    if (get_colo_mode() == COLO_MODE_SECONDARY) {
+        secondary_vm_do_failover();
+    } else {
+        primary_vm_do_failover();
+    }
+}
+
 /* colo checkpoint control helper */
 static int colo_ctl_put(QEMUFile *f, uint64_t request)
 {
@@ -144,11 +221,23 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
         goto out;
     }
 
+    if (failover_request_is_active()) {
+        ret = -1;
+        goto out;
+    }
     /* suspend and save vm state to colo buffer */
     qemu_mutex_lock_iothread();
     vm_stop_force_state(RUN_STATE_COLO);
     qemu_mutex_unlock_iothread();
     trace_colo_vm_state_change("run", "stop");
+    /*
+     * failover request bh could be called after
+     * vm_stop_force_state so we check failover_request_is_active() again.
+     */
+    if (failover_request_is_active()) {
+        ret = -1;
+        goto out;
+    }
 
     /* Disable block migration */
     s->params.blk = 0;
@@ -209,7 +298,7 @@ static void *colo_thread(void *opaque)
 {
     MigrationState *s = opaque;
     QEMUFile *colo_control = NULL;
-    int ret;
+    int i, ret;
 
     colo_control = qemu_fopen_socket(qemu_get_fd(s->file), "rb");
     if (!colo_control) {
@@ -239,6 +328,11 @@ static void *colo_thread(void *opaque)
     trace_colo_vm_state_change("stop", "run");
 
     while (s->state == MIGRATION_STATUS_COLO) {
+        if (failover_request_is_active()) {
+            error_report("failover request");
+            goto out;
+        }
+
         /* start a colo checkpoint */
         if (colo_do_checkpoint_transaction(s, colo_control)) {
             goto out;
@@ -246,7 +340,26 @@ static void *colo_thread(void *opaque)
     }
 
 out:
-    migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+    error_report("colo: some error happens in colo_thread");
+    /* Give users time (2s) to get involved in this verdict */
+    for (i = 0; i < 10; i++) {
+        if (failover_request_is_active()) {
+            error_report("Primary VM will take over work");
+            break;
+        }
+        usleep(200 * 1000);
+    }
+    qemu_mutex_lock_iothread();
+    if (!failover_request_is_active()) {
+        error_report("Primary VM will take over work in default");
+        failover_request_active(NULL);
+    }
+    qemu_mutex_unlock_iothread();
+
+    while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+        ;
+    }
+    failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
 
     qsb_free(colo_buffer);
     colo_buffer = NULL;
@@ -317,7 +430,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
     QEMUFile *f = colo_in->file;
     int fd = qemu_get_fd(f);
     QEMUFile *ctl = NULL, *fb = NULL;
-    int ret;
+    int i, ret;
     uint64_t total_size;
 
     colo = qemu_coroutine_self();
@@ -363,6 +476,11 @@ void *colo_process_incoming_checkpoints(void *opaque)
             }
         }
 
+        if (failover_request_is_active()) {
+            error_report("failover request");
+            goto out;
+        }
+
         /* suspend guest */
         qemu_mutex_lock_iothread();
         vm_stop_force_state(RUN_STATE_COLO);
@@ -431,7 +549,30 @@ void *colo_process_incoming_checkpoints(void *opaque)
     }
 
 out:
-    colo = NULL;
+    error_report("Detect some error or get a failover request");
+    /* Give users time (2s) to get involved in this verdict */
+    for (i = 0; i < 10; i++) {
+        if (failover_request_is_active()) {
+            error_report("Secondary VM will take over work");
+            break;
+        }
+        usleep(200*1000);
+    }
+    /* check flag again*/
+    if (!failover_request_is_active()) {
+        /*
+        * We assume that Primary VM is still alive according to heartbeat,
+        * just kill Secondary VM
+        */
+        error_report("SVM is going to exit in default!");
+        exit(1);
+    } else {
+        /* if we went here, means Primary VM may dead, we are doing failover */
+        while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+            ;
+        }
+        failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
+    }
 
     if (fb) {
         qemu_fclose(fb);
diff --git a/trace-events b/trace-events
index 20551c8..66bcd08 100644
--- a/trace-events
+++ b/trace-events
@@ -1476,6 +1476,7 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) ""
 colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
 colo_receive_message(const char *msg) "Receive '%s'"
 failover_set_state(int new_state) "new state %d"
+colo_rcv_pkt(int result) "Result of net packets comparing is different: %d"
 
 # kvm-all.c
 kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
-- 
1.7.12.4