[PATCH VERY RFC 5/5] tools/[lib]xl: Alter libxl_domain_suspend() to write a v2 stream

[Andrew Cooper <andrew.cooper3@citrix.com>]

From: Ross Lagerwall <ross.lagerwall@citrix.com>

Note that for now, the xl header and device config blob at the beginning
of the stream is still written out since we don't have any domain JSON
yet.

Note that because of the removal of legacy xc_domain_save() from libxc, there
is no possible way to support the legacy behaviour of libxl_domain_save() with
older LIBXL_API versions.  Bail with a clear error instead of substituing a
legacy stream with a v2 stream under the feet of an unsuspecting application.

Signed-off-by: Ross Lagerwall <ross.lagerwall@citrix.com>
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxl/libxl.h          |   37 ++++++++--
 tools/libxl/libxl_dom.c      |  160 +++++++++++++++++++++++++++++++++++-------
 tools/libxl/libxl_internal.h |    1 +
 tools/libxl/xl_cmdimpl.c     |    1 +
 4 files changed, 166 insertions(+), 33 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index fb14a91..98f586e 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -837,7 +837,15 @@ int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
                                 const libxl_asyncprogress_how *aop_console_how)
                                 LIBXL_EXTERNAL_CALLERS_ONLY;
 
-#if defined(LIBXL_API_VERSION) && LIBXL_API_VERSION < 0x040400
+#if defined(LIBXL_API_VERSION)
+
+/*
+ * The legacy xc_domain_restore() has disappeared, as it was completely unfit
+ * for purpose.  Instead of leaving a subtle surprise for libxl consumers using
+ * older API versions, fail early with a clear error.
+ */
+
+#if LIBXL_API_VERSION < 0x040400
 
 int static inline libxl_domain_create_restore_0x040200(
     libxl_ctx *ctx, libxl_domain_config *d_config,
@@ -846,15 +854,32 @@ int static inline libxl_domain_create_restore_0x040200(
     const libxl_asyncprogress_how *aop_console_how)
     LIBXL_EXTERNAL_CALLERS_ONLY
 {
-    libxl_domain_restore_params params;
-    params.checkpointed_stream = 0;
-
-    return libxl_domain_create_restore(
-        ctx, d_config, domid, restore_fd, &params, ao_how, aop_console_how);
+    LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+               "The legacy save/restore format is not supported. "
+               "Please update to the v2 interface");
+    return ERROR_FAIL;
 }
 
 #define libxl_domain_create_restore libxl_domain_create_restore_0x040200
 
+#elif LIBXL_API_VERSION < 0x040500
+
+int static inline libxl_domain_create_restore(
+    libxl_ctx *ctx, libxl_domain_config *d_config,
+    uint32_t *domid, int restore_fd,
+    const libxl_domain_restore_params *params,
+    const libxl_asyncop_how *ao_how,
+    const libxl_asyncprogress_how *aop_console_how)
+    LIBXL_EXTERNAL_CALLERS_ONLY
+{
+    LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+               "The legacy save/restore format is not supported. "
+               "Please update to the v2 interface");
+    return ERROR_FAIL;
+}
+
+#endif
+
 #endif
 
 /* Restore from a v2 stream
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index c944804..3a698f2 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -19,6 +19,7 @@
 
 #include "libxl_internal.h"
 #include "libxl_arch.h"
+#include "libxl_saverestore.h"
 
 #include <xc_dom.h>
 #include <xen/hvm/hvm_info_table.h>
@@ -1574,11 +1575,67 @@ static void remus_checkpoint_dm_saved(libxl__egc *egc,
 
 /*----- main code for suspending, in order of execution -----*/
 
+void libxl__save_write_header(libxl__egc *egc,
+                              libxl__domain_suspend_state *dss);
+
 void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
 {
     STATE_AO_GC(dss->ao);
-    int port;
+
+    libxl__save_write_header(egc, dss);
+}
+
+void libxl__save_write_end(libxl__egc *egc,
+                           libxl__domain_suspend_state *dss);
+
+void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
+                                int rc, int retval, int errnoval)
+{
+    libxl__domain_suspend_state *dss = dss_void;
+    STATE_AO_GC(dss->ao);
+
+    /* Convenience aliases */
+    const libxl_domain_type type = dss->type;
+
+    if (rc)
+        goto out;
+
+    if (retval) {
+        LOGEV(ERROR, errnoval, "saving domain: %s",
+                         dss->guest_responded ?
+                         "domain responded to suspend request" :
+                         "domain did not respond to suspend request");
+        if ( !dss->guest_responded )
+            rc = ERROR_GUEST_TIMEDOUT;
+        else
+            rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
+        rc = libxl__domain_suspend_device_model(gc, dss);
+        if (rc) goto out;
+
+        /* XXX need to writer footer after this */
+        libxl__domain_save_device_model(egc, dss, domain_suspend_done);
+        return;
+    }
+
+    libxl__save_write_end(egc, dss);
+    return;
+
+out:
+    domain_suspend_done(egc, dss, rc);
+}
+
+static void write_header_done(libxl__egc *egc,
+     libxl__writer_state *dw, int onwrite, int errnoval)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(dw, *dss, writer);
+    STATE_AO_GC(dss->ao);
+
     int rc = ERROR_FAIL;
+    int port;
 
     /* Convenience aliases */
     const uint32_t domid = dss->domid;
@@ -1589,6 +1646,11 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
     libxl__srm_save_autogen_callbacks *const callbacks =
         &dss->shs.callbacks.save.a;
 
+    libxl__writer_kill(dw);
+
+    if (onwrite || errnoval)
+        goto out;
+
     logdirty_init(&dss->logdirty);
     libxl__xswait_init(&dss->pvcontrol);
     libxl__ev_evtchn_init(&dss->guest_evtchn);
@@ -1650,45 +1712,89 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
     libxl__xc_domain_save(egc, dss);
     return;
 
+out:
+    domain_suspend_done(egc, dss, rc);
+}
+
+void libxl__save_write_header(libxl__egc *egc,
+                              libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    struct restore_hdr hdr;
+    struct restore_rec_hdr rechdr;
+    int rc = ERROR_FAIL;
+
+    libxl__writer_state *dw = &dss->writer;
+    memset(dw, 0, sizeof(*dw));
+    dw->ao = ao;
+    dw->writefd = dss->fd;
+    dw->maxsz = INT_MAX;
+    dw->towhat = "suspend stream";
+    dw->writewhat = "suspend header";
+    dw->callback = write_header_done;
+
+    rc = libxl__writer_start(dw);
+    if (rc) goto out;
+
+    hdr.ident = htobe64(RESTORE_STREAM_IDENT);
+    hdr.version = htobe32(RESTORE_STREAM_VERSION);
+    hdr.options = htobe32(0x0);
+    libxl__writer_append(egc, dw, &hdr, sizeof(hdr));
+
+    /* XXX need to write the domain config here. */
+
+    rechdr.type = REC_TYPE_LIBXC_CONTEXT;
+    rechdr.length = 0;
+    libxl__writer_append(egc, dw, &rechdr, sizeof(rechdr));
+
+    return;
+
  out:
     domain_suspend_done(egc, dss, rc);
 }
 
-void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
-                                int rc, int retval, int errnoval)
+static void write_end_writer_done(libxl__egc *egc,
+     libxl__writer_state *dw, int onwrite, int errnoval)
 {
-    libxl__domain_suspend_state *dss = dss_void;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(dw, *dss, writer);
     STATE_AO_GC(dss->ao);
 
-    /* Convenience aliases */
-    const libxl_domain_type type = dss->type;
+    int rc = 0;
 
-    if (rc)
-        goto out;
+    libxl__writer_kill(dw);
 
-    if (retval) {
-        LOGEV(ERROR, errnoval, "saving domain: %s",
-                         dss->guest_responded ?
-                         "domain responded to suspend request" :
-                         "domain did not respond to suspend request");
-        if ( !dss->guest_responded )
-            rc = ERROR_GUEST_TIMEDOUT;
-        else
-            rc = ERROR_FAIL;
-        goto out;
-    }
+    if (onwrite || errnoval)
+        rc = ERROR_FAIL;
 
-    if (type == LIBXL_DOMAIN_TYPE_HVM) {
-        rc = libxl__domain_suspend_device_model(gc, dss);
-        if (rc) goto out;
+    domain_suspend_done(egc, dss, rc);
+}
 
-        libxl__domain_save_device_model(egc, dss, domain_suspend_done);
-        return;
-    }
+void libxl__save_write_end(libxl__egc *egc,
+                           libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    struct restore_rec_hdr rechdr;
+    int rc = ERROR_FAIL;
 
-    rc = 0;
+    libxl__writer_state *dw = &dss->writer;
+    memset(dw, 0, sizeof(*dw));
+    dw->ao = ao;
+    dw->writefd = dss->fd;
+    dw->maxsz = INT_MAX;
+    dw->towhat = "suspend stream";
+    dw->writewhat = "suspend footer";
+    dw->callback = write_end_writer_done;
 
-out:
+    rechdr.type = REC_TYPE_END;
+    rechdr.length = 0;
+
+    rc = libxl__writer_start(dw);
+    if (rc) goto out;
+
+    libxl__writer_append(egc, dw, &rechdr, sizeof(rechdr));
+    return;
+
+ out:
     domain_suspend_done(egc, dss, rc);
 }
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 4128373..29ba2ac 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2562,6 +2562,7 @@ struct libxl__domain_suspend_state {
     /* private for libxl__domain_save_device_model */
     libxl__save_device_model_cb *save_dm_callback;
     libxl__datacopier_state save_dm_datacopier;
+    libxl__writer_state writer;
 };
 
 
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 8e0a6bb..691a2a0 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -3410,6 +3410,7 @@ static void save_domain_core_writeconfig(int fd, const char *source,
     memset(&hdr, 0, sizeof(hdr));
     memcpy(hdr.magic, savefileheader_magic, sizeof(hdr.magic));
     hdr.byteorder = SAVEFILE_BYTEORDER_VALUE;
+    hdr.mandatory_flags = SAVEFILE_MANDATORY_STREAMV2;
 
     optdata_begin= 0;
 
-- 
1.7.10.4