[PATCH 25/27] tools/libxl: [RFC] Handle checkpoint records in a libxl migration v2 stream

[Andrew Cooper <andrew.cooper3@citrix.com>]

This is the final bit of untangling for Remus.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
---
 tools/libxl/libxl_create.c      |   25 ++++++++++++++++
 tools/libxl/libxl_internal.h    |    6 ++++
 tools/libxl/libxl_stream_read.c |   62 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 93 insertions(+)

diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 7dd7130..ac918bd 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -747,6 +747,27 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
         libxl_device_model_version_to_string(b_info->device_model_version));
 }
 
+/*----- remus asynchronous checkpoint callback -----*/
+
+static void remus_checkpoint_stream_done(
+    libxl__egc *egc, libxl__domain_create_state *dcs, int rc);
+
+static void libxl__remus_domain_checkpoint_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
+    libxl__egc *egc = dcs->shs.egc;
+    STATE_AO_GC(dcs->ao);
+
+    libxl__stream_read_start_checkpoint(egc, &dcs->srs);
+}
+
+static void remus_checkpoint_stream_done(
+    libxl__egc *egc, libxl__domain_create_state *dcs, int rc)
+{
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dcs->shs, rc);
+}
+
 /*----- main domain creation -----*/
 
 /* We have a linear control flow; only one event callback is
@@ -1008,6 +1029,8 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     libxl_domain_config *const d_config = dcs->guest_config;
     const int restore_fd = dcs->restore_fd;
     libxl__domain_build_state *const state = &dcs->build_state;
+    libxl__srm_restore_autogen_callbacks *const callbacks =
+        &dcs->shs.callbacks.restore.a;
 
     if (rc) {
         domcreate_rebuild_done(egc, dcs, rc);
@@ -1035,6 +1058,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     }
 
     /* Restore */
+    callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
 
     rc = libxl__build_pre(gc, domid, d_config, state);
     if (rc)
@@ -1044,6 +1068,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->srs.fd = restore_fd;
     dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
     dcs->srs.completion_callback = domcreate_stream_done;
+    dcs->srs.checkpoint_callback = remus_checkpoint_stream_done;
 
     libxl__stream_read_start(egc, &dcs->srs);
     return;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index bf1c377..e271a0b 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3205,11 +3205,15 @@ struct libxl__stream_read_state {
     void (*completion_callback)(libxl__egc *egc,
                                 libxl__domain_create_state *dcs,
                                 int rc);
+    void (*checkpoint_callback)(libxl__egc *egc,
+                                libxl__domain_create_state *dcs,
+                                int rc);
     /* Private */
     libxl__carefd *v2_carefd;
     int rc;
     int joined_rc;
     bool running;
+    bool in_checkpoint;
     libxl__datacopier_state dc;
     size_t expected_len;
     libxl_sr_hdr hdr;
@@ -3222,6 +3226,8 @@ _hidden void libxl__stream_read_start(libxl__egc *egc,
 
 _hidden void libxl__stream_read_continue(libxl__egc *egc,
                                          libxl__stream_read_state *stream);
+_hidden void libxl__stream_read_start_checkpoint(
+    libxl__egc *egc, libxl__stream_read_state *stream);
 
 _hidden void libxl__stream_read_abort(libxl__egc *egc,
                                       libxl__stream_read_state *stream, int rc);
diff --git a/tools/libxl/libxl_stream_read.c b/tools/libxl/libxl_stream_read.c
index a8cd2c3..09ef0aa 100644
--- a/tools/libxl/libxl_stream_read.c
+++ b/tools/libxl/libxl_stream_read.c
@@ -80,6 +80,10 @@ static void emulator_padding_done(libxl__egc *egc,
                                   libxl__datacopier_state *dc,
                                   int onwrite, int errnoval);
 
+/* Error handling for checkpoint mini-loop. */
+static void checkpoint_done(libxl__egc *egc,
+                            libxl__stream_read_state *stream, int rc);
+
 void libxl__stream_read_start(libxl__egc *egc,
                               libxl__stream_read_state *stream)
 {
@@ -162,6 +166,35 @@ void libxl__stream_read_continue(libxl__egc *egc,
     stream_failed(egc, stream, ret);
 }
 
+void libxl__stream_read_start_checkpoint(libxl__egc *egc,
+                                         libxl__stream_read_state *stream)
+{
+    libxl__datacopier_state *dc = &stream->dc;
+    int ret = 0;
+
+    assert(stream->running);
+    assert(!stream->in_checkpoint);
+    stream->in_checkpoint = true;
+
+    /* Read a record header. */
+    dc->readwhat = "record header";
+    dc->readbuf = &stream->rec_hdr;
+    stream->expected_len = dc->bytes_to_read = sizeof(stream->rec_hdr);
+    dc->used = 0;
+    dc->callback = record_header_done;
+
+    ret = libxl__datacopier_start(dc);
+    if (ret)
+        goto err;
+
+    assert(!ret);
+    return;
+
+ err:
+    assert(ret);
+    stream_failed(egc, stream, ret);
+}
+
 void libxl__stream_read_abort(libxl__egc *egc,
                               libxl__stream_read_state *stream, int rc)
 {
@@ -182,6 +215,15 @@ static void stream_failed(libxl__egc *egc,
     assert(rc);
     stream->rc = rc;
 
+    /*
+     *If we are in a checkpoint, pass the failure to libxc, which will come
+     * back around to us via libxl__xc_domain_restore_done().
+     */
+    if (stream->in_checkpoint) {
+        checkpoint_done(egc, stream, rc);
+        return;
+    }
+
     if (stream->running) {
         stream->running = false;
         stream_done(egc, stream);
@@ -194,6 +236,7 @@ static void stream_done(libxl__egc *egc,
     libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
 
     assert(!stream->running);
+    assert(!stream->in_checkpoint);
 
     if (stream->v2_carefd)
         libxl__carefd_close(stream->v2_carefd);
@@ -452,6 +495,15 @@ static void process_record(libxl__egc *egc,
         read_emulator_body(egc, stream);
         break;
 
+    case REC_TYPE_CHECKPOINT_END:
+        if (!stream->in_checkpoint) {
+            LOG(ERROR, "Unexpected CHECKPOINT_END record in stream");
+            ret = ERROR_FAIL;
+            goto err;
+        }
+        checkpoint_done(egc, stream, 0);
+        break;
+
     default:
         LOG(ERROR, "Unrecognised record 0x%08x", rec_hdr->type);
         ret = ERROR_FAIL;
@@ -592,6 +644,16 @@ static void emulator_padding_done(libxl__egc *egc,
     stream_failed(egc, stream, ret);
 }
 
+static void checkpoint_done(libxl__egc *egc,
+                            libxl__stream_read_state *stream, int rc)
+{
+    libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
+
+    assert(stream->in_checkpoint);
+    stream->in_checkpoint = false;
+    stream->checkpoint_callback(egc, dcs, rc);
+}
+
 /*
  * Local variables:
  * mode: C
-- 
1.7.10.4