[Qemu-devel] [PATCH v3 3/4] Curling: the sender

[Jules Wang <junqing.wang@cs2c.com.cn>]

By leveraging live migration feature, the sender simply starts a
new migration when the previous migration is completed.

We need to handle the variables related to live migration very
carefully. So the new migration does not restart from the very
begin of the migration, instead, it continues the previous
migration.

Signed-off-by: Jules Wang <junqing.wang@cs2c.com.cn>
---
 arch_init.c             | 25 ++++++++++++++++++++-----
 include/sysemu/sysemu.h |  3 ++-
 migration.c             | 25 +++++++++++++++++++++++--
 savevm.c                | 20 ++++++++++++++++----
 4 files changed, 61 insertions(+), 12 deletions(-)

diff --git a/arch_init.c b/arch_init.c
index 7545d96..f71dfc4 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -107,6 +107,7 @@ const uint32_t arch_type = QEMU_ARCH;
 static bool mig_throttle_on;
 static int dirty_rate_high_cnt;
 static void check_guest_throttling(void);
+static MigrationParams ram_mig_params;
 
 /***********************************************************/
 /* ram save/restore */
@@ -595,6 +596,11 @@ static void ram_migration_cancel(void *opaque)
     migration_end();
 }
 
+static void ram_set_params(const MigrationParams *params, void *opaque)
+{
+    ram_mig_params.ft = params->ft;
+}
+
 static void reset_ram_globals(void)
 {
     last_seen_block = NULL;
@@ -610,10 +616,14 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
 {
     RAMBlock *block;
     int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+    bool create = false;
 
-    migration_bitmap = bitmap_new(ram_pages);
-    bitmap_set(migration_bitmap, 0, ram_pages);
-    migration_dirty_pages = ram_pages;
+    if (!ram_mig_params.ft || !migration_bitmap)  {
+        migration_bitmap = bitmap_new(ram_pages);
+        bitmap_set(migration_bitmap, 0, ram_pages);
+        migration_dirty_pages = ram_pages;
+        create = true;
+    }
     mig_throttle_on = false;
     dirty_rate_high_cnt = 0;
 
@@ -633,7 +643,9 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     qemu_mutex_lock_iothread();
     qemu_mutex_lock_ramlist();
     bytes_transferred = 0;
-    reset_ram_globals();
+    if (!ram_mig_params.ft || create) {
+        reset_ram_globals();
+    }
 
     memory_global_dirty_log_start();
     migration_bitmap_sync();
@@ -748,7 +760,9 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
     }
 
     ram_control_after_iterate(f, RAM_CONTROL_FINISH);
-    migration_end();
+    if (!ram_mig_params.ft) {
+        migration_end();
+    }
 
     qemu_mutex_unlock_ramlist();
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
@@ -975,6 +989,7 @@ SaveVMHandlers savevm_ram_handlers = {
     .save_live_pending = ram_save_pending,
     .load_state = ram_load,
     .cancel = ram_migration_cancel,
+    .set_params = ram_set_params,
 };
 
 struct soundhw {
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index cd5791e..31d5e3f 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -82,7 +82,8 @@ bool qemu_savevm_state_blocked(Error **errp);
 void qemu_savevm_state_begin(QEMUFile *f,
                              const MigrationParams *params);
 int qemu_savevm_state_iterate(QEMUFile *f);
-void qemu_savevm_state_complete(QEMUFile *f);
+void qemu_savevm_state_complete(QEMUFile *f,
+                                const MigrationParams *params);
 void qemu_savevm_state_cancel(void);
 uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
 int qemu_loadvm_state(QEMUFile *f);
diff --git a/migration.c b/migration.c
index 08dcca0..28acd05 100644
--- a/migration.c
+++ b/migration.c
@@ -553,6 +553,7 @@ static void *migration_thread(void *opaque)
     int64_t max_size = 0;
     int64_t start_time = initial_time;
     bool old_vm_running = false;
+    int  time_window = 100;
 
     DPRINTF("beginning savevm\n");
     qemu_savevm_state_begin(s->file, &s->params);
@@ -564,6 +565,8 @@ static void *migration_thread(void *opaque)
 
     while (s->state == MIG_STATE_ACTIVE) {
         int64_t current_time;
+        int64_t time_spent;
+        int64_t migration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
         uint64_t pending_size;
 
         if (!qemu_file_rate_limit(s->file)) {
@@ -585,7 +588,7 @@ static void *migration_thread(void *opaque)
                 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
                 if (ret >= 0) {
                     qemu_file_set_rate_limit(s->file, INT_MAX);
-                    qemu_savevm_state_complete(s->file);
+                    qemu_savevm_state_complete(s->file, &s->params);
                 }
                 qemu_mutex_unlock_iothread();
 
@@ -594,10 +597,28 @@ static void *migration_thread(void *opaque)
                     break;
                 }
 
-                if (!qemu_file_get_error(s->file)) {
+                if (!qemu_file_get_error(s->file) && !s->params.ft) {
                     migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_COMPLETED);
                     break;
                 }
+
+                if (s->params.ft) {
+                    if (old_vm_running) {
+                        qemu_mutex_lock_iothread();
+                        vm_start();
+                        qemu_mutex_unlock_iothread();
+
+                        current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+                        time_spent = current_time - migration_start_time;
+                        DPRINTF("this migration lasts for %" PRId64 "ms\n",
+                                time_spent);
+                        if (time_spent < time_window) {
+                            g_usleep((time_window - time_spent)*1000);
+                            initial_time += time_window - time_spent;
+                        }
+                    }
+                    qemu_savevm_state_begin(s->file, &s->params);
+                }
             }
         }
 
diff --git a/savevm.c b/savevm.c
index 2f631d4..e75d5d4 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1822,6 +1822,7 @@ static void vmstate_save(QEMUFile *f, SaveStateEntry *se)
 }
 
 #define QEMU_VM_FILE_MAGIC           0x5145564d
+#define QEMU_VM_FILE_MAGIC_FT        0x51454654
 #define QEMU_VM_FILE_VERSION_COMPAT  0x00000002
 #define QEMU_VM_FILE_VERSION         0x00000003
 
@@ -1831,6 +1832,7 @@ static void vmstate_save(QEMUFile *f, SaveStateEntry *se)
 #define QEMU_VM_SECTION_END          0x03
 #define QEMU_VM_SECTION_FULL         0x04
 #define QEMU_VM_SUBSECTION           0x05
+#define QEMU_VM_EOF_MAGIC            0xFEEDCAFE
 
 bool qemu_savevm_state_blocked(Error **errp)
 {
@@ -1858,7 +1860,12 @@ void qemu_savevm_state_begin(QEMUFile *f,
         se->ops->set_params(params, se->opaque);
     }
     
-    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+    if (params->ft) {
+        qemu_put_be32(f, QEMU_VM_FILE_MAGIC_FT);
+    } else {
+        qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+    }
+
     qemu_put_be32(f, QEMU_VM_FILE_VERSION);
 
     QTAILQ_FOREACH(se, &savevm_handlers, entry) {
@@ -1937,7 +1944,8 @@ int qemu_savevm_state_iterate(QEMUFile *f)
     return ret;
 }
 
-void qemu_savevm_state_complete(QEMUFile *f)
+void qemu_savevm_state_complete(QEMUFile *f,
+                                const MigrationParams *params)
 {
     SaveStateEntry *se;
     int ret;
@@ -1990,6 +1998,9 @@ void qemu_savevm_state_complete(QEMUFile *f)
     }
 
     qemu_put_byte(f, QEMU_VM_EOF);
+    if (params->ft) {
+        qemu_put_be32(f, QEMU_VM_EOF_MAGIC);
+    }
     qemu_fflush(f);
 }
 
@@ -2028,7 +2039,8 @@ static int qemu_savevm_state(QEMUFile *f)
     int ret;
     MigrationParams params = {
         .blk = 0,
-        .shared = 0
+        .shared = 0,
+        .ft = 0
     };
 
     if (qemu_savevm_state_blocked(NULL)) {
@@ -2047,7 +2059,7 @@ static int qemu_savevm_state(QEMUFile *f)
 
     ret = qemu_file_get_error(f);
     if (ret == 0) {
-        qemu_savevm_state_complete(f);
+        qemu_savevm_state_complete(f, &params);
         ret = qemu_file_get_error(f);
     }
     if (ret != 0) {
-- 
1.8.0.1