[PATCH 29/29] tools/[lib]xl: Alter libxl_domain_suspend() to write a v2 stream

[Andrew Cooper <andrew.cooper3@citrix.com>]

From: Ross Lagerwall <ross.lagerwall@citrix.com>

Note that for now, the xl header and device config blob at the beginning
of the stream is still written out since we don't have any domain JSON
yet.

Signed-off-by: Ross Lagerwall <ross.lagerwall@citrix.com>
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxl/libxl_dom.c      |  265 ++++++++++++++++++++++++++++++++++++------
 tools/libxl/libxl_internal.h |    5 +-
 tools/libxl/xl_cmdimpl.c     |    1 +
 3 files changed, 232 insertions(+), 39 deletions(-)

diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 4160695..1544378 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -19,6 +19,7 @@
 
 #include "libxl_internal.h"
 #include "libxl_arch.h"
+#include "libxl_saverestore.h"
 
 #include <xc_dom.h>
 #include <xen/hvm/hvm_info_table.h>
@@ -1066,7 +1067,9 @@ int libxl__domain_suspend_device_model(libxl__gc *gc,
     uint32_t const domid = dss->domid;
     const char *const filename = dss->dm_savefile;
 
-    switch (libxl__device_model_version_running(gc, domid)) {
+    dss->dm_version = libxl__device_model_version_running(gc, domid);
+
+    switch (dss->dm_version) {
     case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
         LOG(DEBUG, "Saving device model state to %s", filename);
         libxl__qemu_traditional_cmd(gc, domid, "save");
@@ -1410,10 +1413,9 @@ static inline char *physmap_path(libxl__gc *gc, uint32_t domid,
             domid, phys_offset, node);
 }
 
-int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
-        uint32_t *len, void *dss_void)
+static int libxl__toolstack_save(libxl__domain_suspend_state *dss,
+        uint8_t **buf, uint32_t *len)
 {
-    libxl__domain_suspend_state *dss = dss_void;
     STATE_AO_GC(dss->ao);
     int i = 0;
     char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
@@ -1423,6 +1425,9 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
     char **entries = NULL;
     struct libxl__physmap_info *pi;
 
+    /* Convenience aliases */
+    const uint32_t domid = dss->domid;
+
     entries = libxl__xs_directory(gc, 0, GCSPRINTF(
                 "/local/domain/0/device-model/%d/physmap", domid), &num);
     count = num;
@@ -1572,11 +1577,130 @@ static void remus_checkpoint_dm_saved(libxl__egc *egc,
 
 /*----- main code for suspending, in order of execution -----*/
 
+void libxl__save_write_header(libxl__egc *egc,
+                              libxl__domain_suspend_state *dss);
+
 void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
 {
     STATE_AO_GC(dss->ao);
-    int port;
+
+    libxl__save_write_header(egc, dss);
+}
+
+void libxl__save_write_end(libxl__egc *egc,
+                           libxl__domain_suspend_state *dss);
+
+static void domain_save_device_model_cb(libxl__egc *egc,
+                                        libxl__domain_suspend_state *dss,
+                                        int rc)
+{
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        domain_suspend_done(egc, dss, rc);
+    else
+        libxl__save_write_end(egc, dss);
+}
+
+static void write_toolstack_done(libxl__egc *egc,
+     libxl__datacopier_state *dc, int onwrite, int errnoval)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(dc, *dss, dc);
+    STATE_AO_GC(dss->ao);
+
+    int rc = ERROR_FAIL;
+
+    /* Convenience aliases */
+    const libxl_domain_type type = dss->type;
+
+    if (onwrite || errnoval)
+        goto out;
+
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
+        rc = libxl__domain_suspend_device_model(gc, dss);
+        if (rc) goto out;
+
+        libxl__domain_save_device_model(egc, dss, domain_save_device_model_cb);
+        return;
+    }
+
+    libxl__save_write_end(egc, dss);
+
+    return;
+
+out:
+    domain_suspend_done(egc, dss, rc);
+}
+
+void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
+                                int rc, int retval, int errnoval)
+{
+    libxl__domain_suspend_state *dss = dss_void;
+    STATE_AO_GC(dss->ao);
+    struct restore_rec_hdr rechdr;
+    uint8_t *buf;
+    uint32_t len;
+    unsigned char pad[8] = {0};
+
+    if (rc)
+        goto out;
+
+    if (retval) {
+        LOGEV(ERROR, errnoval, "saving domain: %s",
+                         dss->guest_responded ?
+                         "domain responded to suspend request" :
+                         "domain did not respond to suspend request");
+        if ( !dss->guest_responded )
+            rc = ERROR_GUEST_TIMEDOUT;
+        else
+            rc = ERROR_FAIL;
+        goto out;
+    }
+
+    libxl__datacopier_state *dc = &dss->dc;
+    memset(dc, 0, sizeof(*dc));
+    dc->readwhat = "";
+    dc->copywhat = "toolstack data";
+    dc->writewhat = "save/migration stream";
+    dc->ao = ao;
+    dc->readfd = -1;
+    dc->writefd = dss->fd;
+    dc->maxsz = INT_MAX;
+    dc->maxread = INT_MAX;
+    dc->callback = write_toolstack_done;
+
+    rc = libxl__datacopier_start(dc);
+    if (rc) goto out;
+
+    rc = libxl__toolstack_save(dss, &buf, &len);
+    fprintf(stderr, "toolstack_save returned %d, len = %u\n", rc, len);
+    if (rc) goto out;
+
+    rechdr.type = REC_TYPE_XENSTORE_DATA;
+    rechdr.length = len;
+    libxl__datacopier_prefixdata(egc, dc, &rechdr, sizeof(rechdr));
+    libxl__datacopier_prefixdata(egc, dc, buf, len);
+    free(buf);
+
+    len = ROUNDUP(len, REC_ALIGN_ORDER) - len;
+    assert(len >= 0 && len < 8);
+    if (len > 0)
+        libxl__datacopier_prefixdata(egc, dc, pad, len);
+
+    return;
+
+out:
+    domain_suspend_done(egc, dss, rc);
+}
+
+static void write_header_done(libxl__egc *egc,
+     libxl__datacopier_state *dc, int onwrite, int errnoval)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(dc, *dss, dc);
+    STATE_AO_GC(dss->ao);
+
     int rc = ERROR_FAIL;
+    int port;
 
     /* Convenience aliases */
     const uint32_t domid = dss->domid;
@@ -1587,6 +1711,9 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
     libxl__srm_save_autogen_callbacks *const callbacks =
         &dss->shs.callbacks.save.a;
 
+    if (onwrite || errnoval)
+        goto out;
+
     logdirty_init(&dss->logdirty);
     libxl__xswait_init(&dss->pvcontrol);
     libxl__ev_evtchn_init(&dss->guest_evtchn);
@@ -1643,50 +1770,97 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
         callbacks->suspend = libxl__domain_suspend_callback;
 
     callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
-    dss->shs.callbacks.save.toolstack_save = libxl__toolstack_save;
 
     libxl__xc_domain_save(egc, dss);
     return;
 
+out:
+    domain_suspend_done(egc, dss, rc);
+}
+
+void libxl__save_write_header(libxl__egc *egc,
+                              libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    struct restore_hdr hdr;
+    struct restore_rec_hdr rechdr;
+    int rc = ERROR_FAIL;
+
+    libxl__datacopier_state *dc = &dss->dc;
+    memset(dc, 0, sizeof(*dc));
+    dc->readwhat = "";
+    dc->copywhat = "suspend header";
+    dc->writewhat = "save/migration stream";
+    dc->ao = ao;
+    dc->readfd = -1;
+    dc->writefd = dss->fd;
+    dc->maxsz = INT_MAX;
+    dc->maxread = INT_MAX;
+    dc->callback = write_header_done;
+
+    rc = libxl__datacopier_start(dc);
+    if (rc) goto out;
+
+    hdr.ident = htobe64(RESTORE_STREAM_IDENT);
+    hdr.version = htobe32(RESTORE_STREAM_VERSION);
+    hdr.options = htobe32(0x0);
+    libxl__datacopier_prefixdata(egc, dc, &hdr, sizeof(hdr));
+
+    /* XXX need to write the domain config here. */
+
+    rechdr.type = REC_TYPE_LIBXC_CONTEXT;
+    rechdr.length = 0;
+    libxl__datacopier_prefixdata(egc, dc, &rechdr, sizeof(rechdr));
+
+    return;
+
  out:
     domain_suspend_done(egc, dss, rc);
 }
 
-void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
-                                int rc, int retval, int errnoval)
+static void write_end_writer_done(libxl__egc *egc,
+     libxl__datacopier_state *dc, int onwrite, int errnoval)
 {
-    libxl__domain_suspend_state *dss = dss_void;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(dc, *dss, dc);
     STATE_AO_GC(dss->ao);
 
-    /* Convenience aliases */
-    const libxl_domain_type type = dss->type;
+    int rc = 0;
 
-    if (rc)
-        goto out;
+    if (onwrite || errnoval)
+        rc = ERROR_FAIL;
 
-    if (retval) {
-        LOGEV(ERROR, errnoval, "saving domain: %s",
-                         dss->guest_responded ?
-                         "domain responded to suspend request" :
-                         "domain did not respond to suspend request");
-        if ( !dss->guest_responded )
-            rc = ERROR_GUEST_TIMEDOUT;
-        else
-            rc = ERROR_FAIL;
-        goto out;
-    }
+    domain_suspend_done(egc, dss, rc);
+}
 
-    if (type == LIBXL_DOMAIN_TYPE_HVM) {
-        rc = libxl__domain_suspend_device_model(gc, dss);
-        if (rc) goto out;
+void libxl__save_write_end(libxl__egc *egc,
+                           libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    struct restore_rec_hdr rechdr;
+    int rc = ERROR_FAIL;
 
-        libxl__domain_save_device_model(egc, dss, domain_suspend_done);
-        return;
-    }
+    libxl__datacopier_state *dc = &dss->dc;
+    memset(dc, 0, sizeof(*dc));
+    dc->readwhat = "";
+    dc->copywhat = "suspend footer";
+    dc->writewhat = "save/migration stream";
+    dc->ao = ao;
+    dc->readfd = -1;
+    dc->writefd = dss->fd;
+    dc->maxsz = INT_MAX;
+    dc->maxread = INT_MAX;
+    dc->callback = write_end_writer_done;
 
-    rc = 0;
+    rechdr.type = REC_TYPE_END;
+    rechdr.length = 0;
 
-out:
+    rc = libxl__datacopier_start(dc);
+    if (rc) goto out;
+
+    libxl__datacopier_prefixdata(egc, dc, &rechdr, sizeof(rechdr));
+    return;
+
+ out:
     domain_suspend_done(egc, dss, rc);
 }
 
@@ -1698,6 +1872,8 @@ void libxl__domain_save_device_model(libxl__egc *egc,
                                      libxl__save_device_model_cb *callback)
 {
     STATE_AO_GC(dss->ao);
+    struct restore_rec_hdr rechdr;
+    struct restore_emulator_hdr emuhdr;
     struct stat st;
     uint32_t qemu_state_len;
     int rc;
@@ -1707,8 +1883,9 @@ void libxl__domain_save_device_model(libxl__egc *egc,
     /* Convenience aliases */
     const char *const filename = dss->dm_savefile;
     const int fd = dss->fd;
+    const int dm_version = dss->dm_version;
 
-    libxl__datacopier_state *dc = &dss->save_dm_datacopier;
+    libxl__datacopier_state *dc = &dss->dc;
     memset(dc, 0, sizeof(*dc));
     dc->readwhat = GCSPRINTF("qemu save file %s", filename);
     dc->ao = ao;
@@ -1739,15 +1916,31 @@ void libxl__domain_save_device_model(libxl__egc *egc,
 
     qemu_state_len = st.st_size;
     LOG(DEBUG, "%s is %d bytes", dc->readwhat, qemu_state_len);
+    fprintf(stderr, "device model is %u\n", qemu_state_len);
 
     rc = libxl__datacopier_start(dc);
     if (rc) goto out;
 
+    rechdr.type = REC_TYPE_EMULATOR_CONTEXT;
+    rechdr.length = sizeof(emuhdr) + qemu_state_len;
     libxl__datacopier_prefixdata(egc, dc,
-                                 QEMU_SIGNATURE, strlen(QEMU_SIGNATURE));
+                                 &rechdr, sizeof(rechdr));
 
+    switch (dm_version) {
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
+        emuhdr.id = EMULATOR_QEMU_TRADITIONAL;
+        break;
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+        emuhdr.id = EMULATOR_QEMU_UPSTREAM;
+        break;
+    default:
+        emuhdr.id = EMULATOR_UNKNOWN;
+        break;
+    }
+    emuhdr.index = 0;
     libxl__datacopier_prefixdata(egc, dc,
-                                 &qemu_state_len, sizeof(qemu_state_len));
+                                 &emuhdr, sizeof(emuhdr));
+
     return;
 
  out:
@@ -1758,7 +1951,7 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
      libxl__datacopier_state *dc, int onwrite, int errnoval)
 {
     libxl__domain_suspend_state *dss =
-        CONTAINER_OF(dc, *dss, save_dm_datacopier);
+        CONTAINER_OF(dc, *dss, dc);
     STATE_AO_GC(dss->ao);
 
     /* Convenience aliases */
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index c56a167..10ab664 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2556,7 +2556,8 @@ struct libxl__domain_suspend_state {
                                  struct libxl__domain_suspend_state*, int ok);
     /* private for libxl__domain_save_device_model */
     libxl__save_device_model_cb *save_dm_callback;
-    libxl__datacopier_state save_dm_datacopier;
+    libxl__datacopier_state dc;
+    int dm_version;
 };
 
 
@@ -2851,8 +2852,6 @@ void libxl__xc_domain_saverestore_async_callback_done(libxl__egc *egc,
 
 _hidden void libxl__domain_suspend_common_switch_qemu_logdirty
                                (int domid, unsigned int enable, void *data);
-_hidden int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
-        uint32_t *len, void *data);
 
 
 /* calls libxl__xc_domain_restore_done when done */
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index d17e333..3193352 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -3414,6 +3414,7 @@ static void save_domain_core_writeconfig(int fd, const char *source,
     memset(&hdr, 0, sizeof(hdr));
     memcpy(hdr.magic, savefileheader_magic, sizeof(hdr.magic));
     hdr.byteorder = SAVEFILE_BYTEORDER_VALUE;
+    hdr.mandatory_flags = SAVEFILE_MANDATORY_STREAMV2;
 
     optdata_begin= 0;
 
-- 
1.7.10.4