[PATCH v2 23/27] tools/libxl: Write checkpoint records into the stream

[Andrew Cooper <andrew.cooper3@citrix.com>]

when signalled to do so by libxl__remus_domain_checkpoint_callback()

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>

---
This patch has changed substantially in v2 as a result of changes earlier in
the series.  No behavioural difference from v1.
---
 tools/libxl/libxl_dom.c          |   18 ++++-----
 tools/libxl/libxl_internal.h     |    7 ++++
 tools/libxl/libxl_stream_write.c |   80 ++++++++++++++++++++++++++++++++++++--
 3 files changed, 91 insertions(+), 14 deletions(-)

diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index de05124..9f5ddc9 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1937,8 +1937,8 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
 
 /*----- remus asynchronous checkpoint callback -----*/
 
-static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc);
+static void remus_checkpoint_stream_written(
+    libxl__egc *egc, libxl__stream_write_state *sws, int rc);
 static void remus_devices_commit_cb(libxl__egc *egc,
                                     libxl__remus_devices_state *rds,
                                     int rc);
@@ -1953,17 +1953,14 @@ static void libxl__remus_domain_checkpoint_callback(void *data)
     libxl__egc *egc = dss->shs.egc;
     STATE_AO_GC(dss->ao);
 
-    /* This would go into tailbuf. */
-    if (dss->hvm) {
-        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
-    } else {
-        remus_checkpoint_dm_saved(egc, dss, 0);
-    }
+    libxl__stream_write_start_checkpoint(egc, &dss->sws);
 }
 
-static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc)
+static void remus_checkpoint_stream_written(
+    libxl__egc *egc, libxl__stream_write_state *sws, int rc)
 {
+    libxl__domain_suspend_state *dss = CONTAINER_OF(sws, *dss, sws);
+
     /* Convenience aliases */
     libxl__remus_devices_state *const rds = &dss->rds;
 
@@ -2113,6 +2110,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
         callbacks->suspend = libxl__remus_domain_suspend_callback;
         callbacks->postcopy = libxl__remus_domain_resume_callback;
         callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
+        dss->sws.checkpoint_callback = remus_checkpoint_stream_written;
     } else
         callbacks->suspend = libxl__domain_suspend_callback;
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 2beb534..84e22c2 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2987,9 +2987,13 @@ struct libxl__stream_write_state {
     void (*completion_callback)(libxl__egc *egc,
                                 libxl__stream_write_state *sws,
                                 int rc);
+    void (*checkpoint_callback)(libxl__egc *egc,
+                                libxl__stream_write_state *sws,
+                                int rc);
     /* Private */
     int rc;
     bool running;
+    bool in_checkpoint;
 
     /* Active-stuff handling */
     int joined_rc;
@@ -3009,6 +3013,9 @@ struct libxl__stream_write_state {
 _hidden void libxl__stream_write_start(libxl__egc *egc,
                                        libxl__stream_write_state *stream);
 
+_hidden void libxl__stream_write_start_checkpoint(
+    libxl__egc *egc, libxl__stream_write_state *stream);
+
 _hidden void libxl__stream_write_abort(libxl__egc *egc,
                                        libxl__stream_write_state *stream,
                                        int rc);
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index 331173f..0f5216b 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -23,6 +23,9 @@
  *  - libxl__stream_write_start()
  *     - Start writing a stream from the start.
  *
+ *  - libxl__stream_write_start()
+ *     - Write the records which form a checkpoint into a stream.
+ *
  * In normal operation, there are two tasks running at once; this stream
  * processing, and the libxl-save-helper.  check_stream_finished() is used to
  * join all the tasks in both success and error cases.
@@ -39,6 +42,12 @@
  *  - Toolstack record
  *  - if (hvm), Qemu record
  *  - End record
+ *
+ * For checkpointed stream, there is a second loop which is triggered by a
+ * save-helper checkpoint callback.  It writes:
+ *  - Toolstack record
+ *  - if (hvm), Qemu record
+ *  - Checkpoint end record
  */
 
 static void stream_success(libxl__egc *egc,
@@ -73,6 +82,15 @@ static void emulator_record_done(libxl__egc *egc,
 static void write_end_record(libxl__egc *egc,
                              libxl__stream_write_state *stream);
 
+/* Event callbacks unique to checkpointed streams. */
+static void checkpoint_done(libxl__egc *egc,
+                            libxl__stream_write_state *stream,
+                            int rc);
+static void write_checkpoint_end_record(libxl__egc *egc,
+                                        libxl__stream_write_state *stream);
+static void checkpoint_end_record_done(libxl__egc *egc,
+                                       libxl__stream_write_state *stream);
+
 /* Helper to set up reading some data from the stream. */
 static void write_done(libxl__egc *egc,
                        libxl__datacopier_state *dc,
@@ -168,6 +186,16 @@ void libxl__stream_write_start(libxl__egc *egc,
     stream_failed(egc, stream, ret);
 }
 
+void libxl__stream_write_start_checkpoint(libxl__egc *egc,
+                                          libxl__stream_write_state *stream)
+{
+    assert(stream->running);
+    assert(!stream->in_checkpoint);
+    stream->in_checkpoint = true;
+
+    write_toolstack_record(egc, stream);
+}
+
 void libxl__stream_write_abort(libxl__egc *egc,
                                libxl__stream_write_state *stream, int rc)
 {
@@ -176,6 +204,8 @@ void libxl__stream_write_abort(libxl__egc *egc,
 
 static void stream_success(libxl__egc *egc, libxl__stream_write_state *stream)
 {
+    assert(!stream->in_checkpoint);
+
     stream->rc = 0;
 
     stream_done(egc, stream);
@@ -187,13 +217,24 @@ static void stream_failed(libxl__egc *egc,
     assert(rc);
     stream->rc = rc;
 
-    if (stream->running)
+    if (stream->running) {
+        /*
+         *If we are in a checkpoint, pass the failure to libxc, which will come
+         * back around to us via libxl__xc_domain_save_done().
+         */
+        if (stream->in_checkpoint) {
+            checkpoint_done(egc, stream, rc);
+            return;
+        }
+
         stream_done(egc, stream);
+    }
 }
 
 static void stream_done(libxl__egc *egc,
                         libxl__stream_write_state *stream)
 {
+    assert(!stream->in_checkpoint);
     assert(stream->running);
     stream->running = false;
 
@@ -345,8 +386,12 @@ static void toolstack_record_done(libxl__egc *egc,
 
     if (dss->type == LIBXL_DOMAIN_TYPE_HVM)
         write_emulator_record(egc, stream);
-    else
-        write_end_record(egc, stream);
+    else {
+        if (stream->in_checkpoint)
+            write_checkpoint_end_record(egc, stream);
+        else
+            write_end_record(egc, stream);
+    }
 }
 
 static void write_emulator_record(libxl__egc *egc,
@@ -461,7 +506,10 @@ static void emulator_record_done(libxl__egc *egc,
     free(stream->emu_body);
     stream->emu_body = NULL;
 
-    write_end_record(egc, stream);
+    if (stream->in_checkpoint)
+        write_checkpoint_end_record(egc, stream);
+    else
+        write_end_record(egc, stream);
 }
 
 static void write_end_record(libxl__egc *egc,
@@ -473,6 +521,30 @@ static void write_end_record(libxl__egc *egc,
                 &rec, NULL, stream_success);
 }
 
+static void checkpoint_done(libxl__egc *egc,
+                            libxl__stream_write_state *stream,
+                            int rc)
+{
+    assert(stream->in_checkpoint);
+    stream->in_checkpoint = false;
+    stream->checkpoint_callback(egc, stream, rc);
+}
+
+static void write_checkpoint_end_record(libxl__egc *egc,
+                                        libxl__stream_write_state *stream)
+{
+    struct libxl__sr_rec_hdr rec = { REC_TYPE_CHECKPOINT_END };
+
+    setup_write(egc, stream, "checkpoint end record",
+                &rec, NULL, checkpoint_end_record_done);
+}
+
+static void checkpoint_end_record_done(libxl__egc *egc,
+                                       libxl__stream_write_state *stream)
+{
+    checkpoint_done(egc, stream, 0);
+}
+
 /*
  * Local variables:
  * mode: C
-- 
1.7.10.4