[PATCH RFC v3 RESEND 07/12] Migration with Local Disks Mirroring: Added new libxl_read_stream and callbacks in restore flow

[Bruno Alvisio <bruno.alvisio@gmail.com>]

For migration with local disks mirroring a QEMU NBD server is started while
restoring the domain. This server will be responsible for receiving the disks
sent from the source node: The disks transfer will be triggered during the
domain save on the source using the QMP drive-mirror command.

A second libxl_read_stream is created (srs_mirror_disks): During domain restore
the first libxc stream transfers the necessary PFNs and params only so that the
QEMU process and NBD server can be started. After the disk mirroring jobs are
completed the second libxl_read_stream is started and the virtual RAM is
transferred.

In the migration with disks mirroring case:
1. domcreate_devmodel_deferred_started is the callback function for launch_dm

2. domcreate_stream_done is the callback function for the end of 1st
libxl_read_stream

3. domcreate_post_mirror_disks_stream_done is the callback function for the 2nd
libxl_read_stream

The original libxl_create calling sequence:
1. domcreate_bootloader_done
2. libxl/libxc stream
3. domcreate_stream_done
4. domcreate_pre_build
5. domcreate_rebuild_done
6. domcreate_multidev_begin
7. domcreate_launch_dm
8. domcreate_devmodel_started

In the case of migration with local disk mirroring the sequence becomes:

1. domcreate_bootloader_done
2. pre mirror disk libxl/libxc stream: Transfers PFNs/params necessary for
QEMU initilization
3. domcreate_stream_done
4. domcreate_pre_build
5. domcreate_rebuild_done
6. domcreate_multidev_begin
7. domcreate_launch_dm: Starts QEMU with "-incoming defer option"
8. domcreate_devmodel_deferred_started: Starts NBD server for disks mirroring
9. post mirror disks libxl/libxc stream: Transfers all virtual RAM: Similar to
original stream
10. domcreate_post_mirror_disks_steam_done: Resumes QEMU process by
executing QMP "migrate-incoming" command
11. domcreate_devmodel_started

Signed-off-by: Bruno Alvisio <bruno.alvisio@gmail.com>
---
 tools/libxl/libxl.h          |   6 +++
 tools/libxl/libxl_create.c   | 117 +++++++++++++++++++++++++++++++++++++++++--
 tools/libxl/libxl_internal.h |   2 +
 3 files changed, 120 insertions(+), 5 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 25245cc..af2aa9a 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1453,6 +1453,12 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int recv_fd,
 #define LIBXL_SUSPEND_LIVE 2
 #define LIBXL_SUSPEND_MIRROR_DISKS 4
 
+#define DRIVE_MIRROR_PORT "11000"
+#define DRIVE_MIRROR_DEVICE "ide0-hd0"
+
+static const char nbd_server_started_banner[]=
+    "nbd server started on source, start mirror job.\n";
+
 /* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
  *   If this parameter is true, use co-operative resume. The guest
  *   must support this.
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 95978a8..f834282 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -761,6 +761,9 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
 static void domcreate_devmodel_started(libxl__egc *egc,
                                        libxl__dm_spawn_state *dmss,
                                        int rc);
+static void domcreate_devmodel_deferred_started(libxl__egc *egc,
+                                                libxl__dm_spawn_state *dmss,
+                                                int ret);
 static void domcreate_bootloader_console_available(libxl__egc *egc,
                                                    libxl__bootloader_state *bl);
 static void domcreate_bootloader_done(libxl__egc *egc,
@@ -777,6 +780,10 @@ static void domcreate_stream_done(libxl__egc *egc,
                                   libxl__stream_read_state *srs,
                                   int ret);
 
+static void domcreate_post_mirror_disks_stream_done(libxl__egc *egc,
+                                                    libxl__stream_read_state *srs,
+                                                    int ret);
+
 static void domcreate_rebuild_done(libxl__egc *egc,
                                    libxl__domain_create_state *dcs,
                                    int ret);
@@ -1036,6 +1043,48 @@ static void libxl__colo_restore_setup_done(libxl__egc *egc,
     libxl__stream_read_start(egc, &dcs->srs);
 }
 
+static void domcreate_devmodel_deferred_started(libxl__egc *egc,
+                                                libxl__dm_spawn_state *dmss,
+                                                int ret)
+{
+    libxl__domain_create_state *dcs = CONTAINER_OF(dmss, *dcs, sdss.dm);
+    STATE_AO_GC(dmss->spawn.ao);
+    const uint32_t domid = dcs->guest_domid;
+    dcs->sdss.dm.guest_domid = domid;
+
+    if (ret) {
+        LOGD(ERROR, domid, "device model did not start: %d", ret);
+        goto error_out;
+    }
+
+    ret = libxl__qmp_nbd_server_start(gc, domid, "::", DRIVE_MIRROR_PORT);
+    if (ret) {
+        LOGD(ERROR, domid, "Failed to start NBD Server");
+        goto error_out;
+    }
+
+    ret = libxl__qmp_nbd_server_add(gc, domid, DRIVE_MIRROR_DEVICE);
+    if (ret) {
+        LOGD(ERROR, domid, "Failed to add NBD Server");
+        goto error_out;
+    }
+
+    ret = libxl_write_exactly(CTX, dcs->send_back_fd,
+                              nbd_server_started_banner,
+                              sizeof(nbd_server_started_banner)-1,
+                              "migration stream",
+                              "nbd server success/failure report");
+    if (ret)
+        goto error_out;
+
+    libxl__stream_read_start(egc, &dcs->srs);
+    return;
+
+error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
 static void domcreate_bootloader_done(libxl__egc *egc,
                                       libxl__bootloader_state *bl,
                                       int rc)
@@ -1053,6 +1102,8 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     libxl_domain_build_info *const info = &d_config->b_info;
     libxl__srm_restore_autogen_callbacks *const callbacks =
         &dcs->srs.shs.callbacks.restore.a;
+    libxl__srm_restore_autogen_callbacks *const callbacks_mirror_disks =
+        &dcs->srs_mirror_disks.shs.callbacks.restore.a;
 
     if (rc) {
         domcreate_rebuild_done(egc, dcs, rc);
@@ -1070,8 +1121,14 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->sdss.dm.spawn.ao = ao;
     dcs->sdss.dm.guest_config = dcs->guest_config;
     dcs->sdss.dm.build_state = &dcs->build_state;
-    dcs->sdss.dm.callback = domcreate_devmodel_started;
-    dcs->sdss.callback = domcreate_devmodel_started;
+    dcs->sdss.dm.mirror_disks = dcs->mirror_disks;
+    if (!dcs->mirror_disks) {
+        dcs->sdss.dm.callback = domcreate_devmodel_started;
+        dcs->sdss.callback = domcreate_devmodel_started;
+    } else {
+        dcs->sdss.dm.callback = domcreate_devmodel_deferred_started;
+        dcs->sdss.callback = domcreate_devmodel_deferred_started;
+    }
 
     if (restore_fd < 0 && dcs->domid_soft_reset == INVALID_DOMID) {
         rc = libxl__domain_build(gc, d_config, domid, state);
@@ -1081,6 +1138,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
 
     /* Restore */
     callbacks->restore_results = libxl__srm_callout_callback_restore_results;
+    callbacks_mirror_disks->restore_results = libxl__srm_callout_callback_restore_results;
 
     /* COLO only supports HVM now because it does not work very
      * well with pv drivers:
@@ -1106,7 +1164,21 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->srs.fd = restore_fd;
     dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
     dcs->srs.back_channel = false;
-    dcs->srs.completion_callback = domcreate_stream_done;
+    dcs->srs.mirror_disks = 0;
+
+    if (dcs->mirror_disks) {
+        dcs->srs_mirror_disks.ao = ao;
+        dcs->srs_mirror_disks.dcs = dcs;
+        dcs->srs_mirror_disks.fd = restore_fd;
+        dcs->srs_mirror_disks.legacy = (dcs->restore_params.stream_version == 1);
+        dcs->srs_mirror_disks.back_channel = false;
+        dcs->srs_mirror_disks.completion_callback = domcreate_stream_done;
+        dcs->srs_mirror_disks.mirror_disks = 1;
+
+        dcs->srs.completion_callback = domcreate_post_mirror_disks_stream_done;
+    } else {
+        dcs->srs.completion_callback = domcreate_stream_done;
+    }
 
     if (restore_fd >= 0) {
         switch (checkpointed_stream) {
@@ -1124,7 +1196,11 @@ static void domcreate_bootloader_done(libxl__egc *egc,
             libxl__remus_restore_setup(egc, dcs);
             /* fall through */
         case LIBXL_CHECKPOINTED_STREAM_NONE:
-            libxl__stream_read_start(egc, &dcs->srs);
+            if (dcs->mirror_disks) {
+                libxl__stream_read_start(egc, &dcs->srs_mirror_disks);
+            } else {
+                libxl__stream_read_start(egc, &dcs->srs);
+            }
         }
         return;
     }
@@ -1146,6 +1222,37 @@ void libxl__srm_callout_callback_restore_results(xen_pfn_t store_mfn,
     shs->need_results =           0;
 }
 
+static void domcreate_post_mirror_disks_stream_done(libxl__egc *egc,
+                                                    libxl__stream_read_state *srs,
+                                                    int ret)
+{
+    libxl__domain_create_state *dcs = srs->dcs;
+    STATE_AO_GC(dcs->ao);
+
+    const uint32_t domid = dcs->guest_domid;
+    const char* uri;
+    const char* state_file = GCSPRINTF(
+                             LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", domid);
+    if (ret)
+        goto error_out;
+
+    ret = libxl__qmp_nbd_server_stop(gc, domid);
+    if (ret){
+        LOGD(ERROR, domid, "Failed to stop NBD server");
+        goto error_out;
+    }
+    uri = GCSPRINTF("exec: /bin/cat %s", state_file);
+    ret = libxl__qmp_migrate_incoming(gc, domid, uri);
+    if (ret)
+        goto error_out;
+    domcreate_devmodel_started(egc, &dcs->sdss.dm, ret);
+    return;
+
+error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
 static void domcreate_stream_done(libxl__egc *egc,
                                   libxl__stream_read_state *srs,
                                   int ret)
@@ -1212,7 +1319,7 @@ static void domcreate_stream_done(libxl__egc *egc,
     if (ret)
         goto out;
 
-    if (info->type == LIBXL_DOMAIN_TYPE_HVM) {
+    if (info->type == LIBXL_DOMAIN_TYPE_HVM && !dcs->mirror_disks) {
         state->saved_state = GCSPRINTF(
                        LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", domid);
     }
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index ee62bfb..feb9370 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3118,6 +3118,7 @@ struct libxl__stream_read_state {
     int fd;
     bool legacy;
     bool back_channel;
+    int mirror_disks;
     void (*completion_callback)(libxl__egc *egc,
                                 libxl__stream_read_state *srs,
                                 int rc);
@@ -3774,6 +3775,7 @@ struct libxl__domain_create_state {
         /* If we're not doing stubdom, we use only dmss.dm,
          * for the non-stubdom device model. */
     libxl__stream_read_state srs;
+    libxl__stream_read_state srs_mirror_disks;
     /* necessary if the domain creation failed and we have to destroy it */
     libxl__domain_destroy_state dds;
     libxl__multidev multidev;
-- 
2.3.2 (Apple Git-55)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel