[PATCH v4 23/29] tools/libxc+libxl+xl: Save v2 streams

[Andrew Cooper <andrew.cooper3@citrix.com>]

This is a complicated set of changes which must be done together for
bisectability.

 * libxl-save-helper is updated to unconditionally use libxc migration v2.
 * libxl compatibility workarounds in libxc are disabled for save operations.
 * libxl__stream_write_start() is logically spliced into the event location
   where libxl__xc_domain_save() used to reside.
 * Ownership of the helper state moves into stream_write_state.
 * xl is updated to indicate that the stream is now v2

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>

---
v3: Many small changes, follwing similar review to the restore side
---
 tools/libxc/Makefile             |    2 --
 tools/libxl/libxl.h              |    2 ++
 tools/libxl/libxl_dom.c          |   69 ++++++++++++--------------------------
 tools/libxl/libxl_internal.h     |    3 +-
 tools/libxl/libxl_save_helper.c  |    2 +-
 tools/libxl/libxl_stream_write.c |   40 ++++++++++++++++++----
 tools/libxl/xl_cmdimpl.c         |    1 +
 7 files changed, 61 insertions(+), 58 deletions(-)

diff --git a/tools/libxc/Makefile b/tools/libxc/Makefile
index 2cd0b1a..1aec848 100644
--- a/tools/libxc/Makefile
+++ b/tools/libxc/Makefile
@@ -64,8 +64,6 @@ GUEST_SRCS-$(CONFIG_X86) += xc_sr_save_x86_hvm.c
 GUEST_SRCS-y += xc_sr_restore.c
 GUEST_SRCS-y += xc_sr_save.c
 GUEST_SRCS-y += xc_offline_page.c xc_compression.c
-xc_sr_save_x86_hvm.o: CFLAGS += -DXG_LIBXL_HVM_COMPAT
-xc_sr_save_x86_hvm.opic: CFLAGS += -DXG_LIBXL_HVM_COMPAT
 else
 GUEST_SRCS-y += xc_nomigrate.c
 endif
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 74b0829..5a7308d 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -812,6 +812,8 @@
  *
  * If this is defined, then the libxl_domain_create_restore() interface takes
  * a "stream_version" parameter and supports a value of 2.
+ *
+ * libxl_domain_suspend() will produce a v2 stream.
  */
 #define LIBXL_HAVE_SRM_V2 1
 
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index a685b77..cda3a7a 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1153,6 +1153,8 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *ptr,
 
 /*==================== Domain suspend (save) ====================*/
 
+static void stream_done(libxl__egc *egc,
+                        libxl__stream_write_state *sws, int rc);
 static void domain_suspend_done(libxl__egc *egc,
                         libxl__domain_suspend_state *dss, int rc);
 static void domain_suspend_callback_common_done(libxl__egc *egc,
@@ -1379,7 +1381,7 @@ static void switch_logdirty_done(libxl__egc *egc,
     } else {
         broke = 0;
     }
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, broke);
 }
 
 /*----- callbacks, called by xc_domain_save -----*/
@@ -1842,7 +1844,7 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
                                 libxl__domain_suspend_state *dss, int rc)
 {
     dss->rc = rc;
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, !rc);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
 }
 
 /*----- remus callbacks -----*/
@@ -1878,7 +1880,7 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
 
 out:
     dss->rc = rc;
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, !rc);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
 }
 
 static void remus_devices_postsuspend_cb(libxl__egc *egc,
@@ -1895,7 +1897,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
 out:
     if (rc)
         dss->rc = rc;
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, !rc);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
 }
 
 static void libxl__remus_domain_resume_callback(void *data)
@@ -1930,7 +1932,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
 out:
     if (rc)
         dss->rc = rc;
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, !rc);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
 }
 
 /*----- remus asynchronous checkpoint callback -----*/
@@ -1978,7 +1980,7 @@ static void remus_checkpoint_dm_saved(libxl__egc *egc,
     return;
 
 out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0);
 }
 
 static void remus_devices_commit_cb(libxl__egc *egc,
@@ -2013,7 +2015,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
     return;
 
 out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0);
 }
 
 static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
@@ -2034,7 +2036,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
     if (rc)
         dss->rc = rc;
 
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, !rc);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
 }
 
 /*----- main code for suspending, in order of execution -----*/
@@ -2052,7 +2054,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
     const int debug = dss->debug;
     const libxl_domain_remus_info *const r_info = dss->remus;
     libxl__srm_save_autogen_callbacks *const callbacks =
-        &dss->shs.callbacks.save.a;
+        &dss->sws.shs.callbacks.save.a;
 
     dss->rc = 0;
     logdirty_init(&dss->logdirty);
@@ -2115,53 +2117,24 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
         callbacks->suspend = libxl__domain_suspend_callback;
 
     callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
-    dss->shs.callbacks.save.toolstack_save = libxl__toolstack_save;
+    dss->sws.shs.callbacks.save.toolstack_save = libxl__toolstack_save;
 
-    libxl__xc_domain_save(egc, dss, &dss->shs);
+    dss->sws.ao  = dss->ao;
+    dss->sws.dss = dss;
+    dss->sws.fd  = dss->fd;
+    dss->sws.completion_callback = stream_done;
+
+    libxl__stream_write_start(egc, &dss->sws);
     return;
 
  out:
     domain_suspend_done(egc, dss, rc);
 }
 
-void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
-                                int rc, int retval, int errnoval)
+static void stream_done(libxl__egc *egc,
+                        libxl__stream_write_state *sws, int rc)
 {
-    libxl__domain_suspend_state *dss = dss_void;
-    STATE_AO_GC(dss->ao);
-
-    /* Convenience aliases */
-    const libxl_domain_type type = dss->type;
-
-    if (rc)
-        goto out;
-
-    if (retval) {
-        LOGEV(ERROR, errnoval, "saving domain: %s",
-                         dss->guest_responded ?
-                         "domain responded to suspend request" :
-                         "domain did not respond to suspend request");
-        if ( !dss->guest_responded )
-            rc = ERROR_GUEST_TIMEDOUT;
-        else if (dss->rc)
-            rc = dss->rc;
-        else
-            rc = ERROR_FAIL;
-        goto out;
-    }
-
-    if (type == LIBXL_DOMAIN_TYPE_HVM) {
-        rc = libxl__domain_suspend_device_model(gc, dss);
-        if (rc) goto out;
-
-        libxl__domain_save_device_model(egc, dss, domain_suspend_done);
-        return;
-    }
-
-    rc = 0;
-
-out:
-    domain_suspend_done(egc, dss, rc);
+    domain_suspend_done(egc, sws->dss, rc);
 }
 
 static void save_device_model_datacopier_done(libxl__egc *egc,
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 9ddb774..0697ba6 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2971,6 +2971,7 @@ struct libxl__stream_write_state {
     /* Private */
     int rc;
     bool running;
+    libxl__save_helper_state shs;
 
     /* Main stream-writing data. */
     libxl__datacopier_state dc;
@@ -3028,7 +3029,7 @@ struct libxl__domain_suspend_state {
     libxl__remus_devices_state rds;
     libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
     int interval; /* checkpoint interval (for Remus) */
-    libxl__save_helper_state shs;
+    libxl__stream_write_state sws;
     libxl__logdirty_switch logdirty;
     void (*callback_common_done)(libxl__egc*,
                                  struct libxl__domain_suspend_state*, int ok);
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
index efbe2eb..f196786 100644
--- a/tools/libxl/libxl_save_helper.c
+++ b/tools/libxl/libxl_save_helper.c
@@ -286,7 +286,7 @@ int main(int argc, char **argv)
         startup("save");
         setup_signals(save_signal_handler);
 
-        r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, flags,
+        r = xc_domain_save2(xch, io_fd, dom, max_iters, max_factor, flags,
                            &helper_save_callbacks, hvm);
         complete(r);
 
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index a324d30..d1343e0 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -218,11 +218,40 @@ static void stream_header_done(libxl__egc *egc,
 static void libxc_header_done(libxl__egc *egc,
                               libxl__stream_write_state *stream)
 {
-    libxl__xc_domain_save(egc, stream->dss, &stream->dss->shs);
+    libxl__xc_domain_save(egc, stream->dss, &stream->shs);
 }
 
-static void __attribute__((unused))
-write_toolstack_record(libxl__egc *egc,
+void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
+                                int rc, int retval, int errnoval)
+{
+    libxl__domain_suspend_state *dss = dss_void;
+    libxl__stream_write_state *stream = &dss->sws;
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        goto err;
+
+    if (retval) {
+        LOGEV(ERROR, errnoval, "saving domain: %s",
+              dss->guest_responded ?
+              "domain responded to suspend request" :
+              "domain did not respond to suspend request");
+        if (!dss->guest_responded)
+            rc = ERROR_GUEST_TIMEDOUT;
+        else if (dss->rc)
+            rc = dss->rc;
+        else
+            rc = ERROR_FAIL;
+        goto err;
+    }
+
+    write_toolstack_record(egc, stream);
+
+ err:
+    check_all_finished(egc, stream, rc);
+}
+
+static void write_toolstack_record(libxl__egc *egc,
                                    libxl__stream_write_state *stream)
 {
     libxl__domain_suspend_state *dss = stream->dss;
@@ -417,7 +446,6 @@ static void check_all_finished(libxl__egc *egc,
                                libxl__stream_write_state *stream,
                                int rc)
 {
-    libxl__domain_suspend_state *dss = stream->dss;
     STATE_AO_GC(stream->ao);
 
     if (!stream->rc && rc) {
@@ -425,12 +453,12 @@ static void check_all_finished(libxl__egc *egc,
         stream->rc = rc;
 
         libxl__stream_write_abort(egc, stream, rc);
-        libxl__save_helper_abort(egc, &dss->shs);
+        libxl__save_helper_abort(egc, &stream->shs);
     }
 
     /* Don't fire the callback until all our parallel tasks have stopped. */
     if (libxl__stream_write_inuse(stream) ||
-        libxl__save_helper_inuse(&dss->shs))
+        libxl__save_helper_inuse(&stream->shs))
         return;
 
     stream->completion_callback(egc, stream, stream->rc);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 71a22b8..13e154d 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -3922,6 +3922,7 @@ static void save_domain_core_writeconfig(int fd, const char *source,
     memset(&hdr, 0, sizeof(hdr));
     memcpy(hdr.magic, savefileheader_magic, sizeof(hdr.magic));
     hdr.byteorder = SAVEFILE_BYTEORDER_VALUE;
+    hdr.mandatory_flags = XL_MANDATORY_FLAG_STREAMv2;
 
     optdata_begin= 0;
 
-- 
1.7.10.4