[PATCH v2 0/6] Misc cleanups for libxl

[PATCH v2 0/6] Misc cleanups for libxl

[Yang Hongyang]

This patchset mainly focus on libxl save, most of the patches are
simply move codes out of libxl_dom.c, except a refactor patch.

Please see individual patch for detail.

Can get the whole patchset from:
    https://github.com/macrosheep/xen/tree/misc-libxl-v2

v1->v2:
  - use dsps for suspend_state and dss for save_state.
  - move resume code to libxl_dom_suspend.c
  - move toolstatck save/restore code to libxl_dom_save.c
  - move refactor pacth to the end so that rebase of the patchset easier.

Yang Hongyang (6):
  tools/libxl: rename libxl__domain_suspend to libxl__domain_save
  tools/libxl: move domain suspend code into libxl_dom_suspend.c
  tools/libxl: move domain resume code into libxl_dom_suspend.c
  tools/libxl: move remus code into libxl_remus.c
  tools/libxl: move save/restore code into libxl_dom_save.c
  libxl/save: Refactor libxl__domain_suspend_state

 tools/libxl/Makefile             |    5 +-
 tools/libxl/libxl.c              |  126 +---
 tools/libxl/libxl_dom.c          | 1202 --------------------------------------
 tools/libxl/libxl_dom_save.c     |  672 +++++++++++++++++++++
 tools/libxl/libxl_dom_suspend.c  |  465 +++++++++++++++
 tools/libxl/libxl_internal.h     |   65 ++-
 tools/libxl/libxl_netbuffer.c    |    2 +-
 tools/libxl/libxl_remus.c        |  307 ++++++++++
 tools/libxl/libxl_save_callout.c |    2 +-
 9 files changed, 1503 insertions(+), 1343 deletions(-)
 create mode 100644 tools/libxl/libxl_dom_save.c
 create mode 100644 tools/libxl/libxl_dom_suspend.c
 create mode 100644 tools/libxl/libxl_remus.c

-- 
1.9.1

[PATCH v2 1/6] tools/libxl: rename libxl__domain_suspend to libxl__domain_save

[Yang Hongyang]

Rename libxl__domain_suspend() to libxl__domain_save() since it
actually do the save domain work.

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxl/libxl.c          |  4 ++--
 tools/libxl/libxl_dom.c      | 16 ++++++++--------
 tools/libxl/libxl_internal.h |  4 ++--
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 9117b01..5a70062 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -911,7 +911,7 @@ static void libxl__remus_setup_done(libxl__egc *egc,
     STATE_AO_GC(dss->ao);
 
     if (!rc) {
-        libxl__domain_suspend(egc, dss);
+        libxl__domain_save(egc, dss);
         return;
     }
 
@@ -978,7 +978,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
     dss->live = flags & LIBXL_SUSPEND_LIVE;
     dss->debug = flags & LIBXL_SUSPEND_DEBUG;
 
-    libxl__domain_suspend(egc, dss);
+    libxl__domain_save(egc, dss);
     return AO_INPROGRESS;
 
  out_err:
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index a0c9850..cce04dd 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1103,8 +1103,8 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
 
 /*==================== Domain suspend (save) ====================*/
 
-static void domain_suspend_done(libxl__egc *egc,
-                        libxl__domain_suspend_state *dss, int rc);
+static void domain_save_done(libxl__egc *egc,
+                             libxl__domain_suspend_state *dss, int rc);
 static void domain_suspend_callback_common_done(libxl__egc *egc,
                                 libxl__domain_suspend_state *dss, int ok);
 
@@ -1959,7 +1959,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
 
 /*----- main code for suspending, in order of execution -----*/
 
-void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
 {
     STATE_AO_GC(dss->ao);
     int port;
@@ -2040,7 +2040,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
     return;
 
  out:
-    domain_suspend_done(egc, dss, rc);
+    domain_save_done(egc, dss, rc);
 }
 
 void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
@@ -2071,14 +2071,14 @@ void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
         rc = libxl__domain_suspend_device_model(gc, dss);
         if (rc) goto out;
 
-        libxl__domain_save_device_model(egc, dss, domain_suspend_done);
+        libxl__domain_save_device_model(egc, dss, domain_save_done);
         return;
     }
 
     rc = 0;
 
 out:
-    domain_suspend_done(egc, dss, rc);
+    domain_save_done(egc, dss, rc);
 }
 
 static void save_device_model_datacopier_done(libxl__egc *egc,
@@ -2177,8 +2177,8 @@ static void remus_teardown_done(libxl__egc *egc,
                                        libxl__remus_devices_state *rds,
                                        int rc);
 
-static void domain_suspend_done(libxl__egc *egc,
-                        libxl__domain_suspend_state *dss, int rc)
+static void domain_save_done(libxl__egc *egc,
+                             libxl__domain_suspend_state *dss, int rc)
 {
     STATE_AO_GC(dss->ao);
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index bb3a5c7..e765d68 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3140,8 +3140,8 @@ struct libxl__domain_create_state {
 /*----- Domain suspend (save) functions -----*/
 
 /* calls dss->callback when done */
-_hidden void libxl__domain_suspend(libxl__egc *egc,
-                                   libxl__domain_suspend_state *dss);
+_hidden void libxl__domain_save(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss);
 
 
 /* calls libxl__xc_domain_suspend_done when done */
-- 
1.9.1

Re: [PATCH v2 1/6] tools/libxl: rename libxl__domain_suspend to libxl__domain_save

[Ian Campbell]

On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
> Rename libxl__domain_suspend() to libxl__domain_save() since it
> actually do the save domain work.

This results in some strangeness in that some functions called *save*
are now passed a struct called *suspend*. I think this is probably
temporary and is all fixed up by the end of the series, is that true?

If so then this temporary state affairs is:
        Acked-by: Ian Campbell <ian.campbell@citrix.com>

Re: [PATCH v2 1/6] tools/libxl: rename libxl__domain_suspend to libxl__domain_save

[Yang Hongyang]

On 06/16/2015 08:59 PM, Ian Campbell wrote:
> On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
>> Rename libxl__domain_suspend() to libxl__domain_save() since it
>> actually do the save domain work.
>
> This results in some strangeness in that some functions called *save*
> are now passed a struct called *suspend*. I think this is probably
> temporary and is all fixed up by the end of the series, is that true?

Yes, it is fixed by the refactor of the suspend state.

>
> If so then this temporary state affairs is:
>          Acked-by: Ian Campbell <ian.campbell@citrix.com>
>
>
> .
>

-- 
Thanks,
Yang.

[PATCH v2 2/6] tools/libxl: move domain suspend code into libxl_dom_suspend.c

[Yang Hongyang]

Move domain suspend code into a separate file libxl_dom_suspend.c.
export an API libxl__domain_suspend() which wrappers the static
function domain_suspend_callback_common() for internal use.

Note that the newly added file libxl_dom_suspend.c is used for
suspend/resume code.

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
---
 tools/libxl/Makefile            |   3 +-
 tools/libxl/libxl_dom.c         | 350 +-----------------------------------
 tools/libxl/libxl_dom_suspend.c | 381 ++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_internal.h    |   6 +
 4 files changed, 393 insertions(+), 347 deletions(-)
 create mode 100644 tools/libxl/libxl_dom_suspend.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index cc9c152..3f98d62 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -95,7 +95,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
 			libxl_internal.o libxl_utils.o libxl_uuid.o \
 			libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \
 			libxl_save_callout.o _libxl_save_msgs_callout.o \
-			libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
+			libxl_qmp.o libxl_event.o libxl_fork.o libxl_dom_suspend.o \
+			$(LIBXL_OBJS-y)
 LIBXL_OBJS += libxl_genid.o
 LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
 
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index cce04dd..9444329 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1103,11 +1103,6 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
 
 /*==================== Domain suspend (save) ====================*/
 
-static void domain_save_done(libxl__egc *egc,
-                             libxl__domain_suspend_state *dss, int rc);
-static void domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok);
-
 /*----- complicated callback, called by xc_domain_save -----*/
 
 /*
@@ -1324,35 +1319,6 @@ static void switch_logdirty_done(libxl__egc *egc,
 
 /*----- callbacks, called by xc_domain_save -----*/
 
-int libxl__domain_suspend_device_model(libxl__gc *gc,
-                                       libxl__domain_suspend_state *dss)
-{
-    int ret = 0;
-    uint32_t const domid = dss->domid;
-    const char *const filename = dss->dm_savefile;
-
-    switch (libxl__device_model_version_running(gc, domid)) {
-    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
-        LOG(DEBUG, "Saving device model state to %s", filename);
-        libxl__qemu_traditional_cmd(gc, domid, "save");
-        libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
-        break;
-    }
-    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
-        if (libxl__qmp_stop(gc, domid))
-            return ERROR_FAIL;
-        /* Save DM state into filename */
-        ret = libxl__qmp_save(gc, domid, filename);
-        if (ret)
-            unlink(filename);
-        break;
-    default:
-        return ERROR_INVAL;
-    }
-
-    return ret;
-}
-
 int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
 {
 
@@ -1373,301 +1339,6 @@ int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
     return 0;
 }
 
-static void domain_suspend_common_wait_guest(libxl__egc *egc,
-                                             libxl__domain_suspend_state *dss);
-static void domain_suspend_common_guest_suspended(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss);
-
-static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
-      libxl__xswait_state *xswa, int rc, const char *state);
-static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
-        libxl__ev_evtchn *evev);
-static void suspend_common_wait_guest_watch(libxl__egc *egc,
-      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
-static void suspend_common_wait_guest_check(libxl__egc *egc,
-        libxl__domain_suspend_state *dss);
-static void suspend_common_wait_guest_timeout(libxl__egc *egc,
-      libxl__ev_time *ev, const struct timeval *requested_abs);
-
-static void domain_suspend_common_failed(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss);
-static void domain_suspend_common_done(libxl__egc *egc,
-                                       libxl__domain_suspend_state *dss,
-                                       bool ok);
-
-static bool domain_suspend_pvcontrol_acked(const char *state) {
-    /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
-    if (!state) return 1;
-    return strcmp(state,"suspend");
-}
-
-/* calls dss->callback_common_done when done */
-static void domain_suspend_callback_common(libxl__egc *egc,
-                                           libxl__domain_suspend_state *dss)
-{
-    STATE_AO_GC(dss->ao);
-    uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
-    int ret, rc;
-
-    /* Convenience aliases */
-    const uint32_t domid = dss->domid;
-
-    if (dss->hvm) {
-        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
-        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
-    }
-
-    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
-        LOG(DEBUG, "issuing %s suspend request via event channel",
-            dss->hvm ? "PVHVM" : "PV");
-        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
-        if (ret < 0) {
-            LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
-            goto err;
-        }
-
-        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
-        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
-        if (rc) goto err;
-
-        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
-                                         suspend_common_wait_guest_timeout,
-                                         60*1000);
-        if (rc) goto err;
-
-        return;
-    }
-
-    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
-        LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
-        ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
-        if (ret < 0) {
-            LOGE(ERROR, "xc_domain_shutdown failed");
-            goto err;
-        }
-        /* The guest does not (need to) respond to this sort of request. */
-        dss->guest_responded = 1;
-        domain_suspend_common_wait_guest(egc, dss);
-        return;
-    }
-
-    LOG(DEBUG, "issuing %s suspend request via XenBus control node",
-        dss->hvm ? "PVHVM" : "PV");
-
-    libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
-
-    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
-    if (!dss->pvcontrol.path) goto err;
-
-    dss->pvcontrol.ao = ao;
-    dss->pvcontrol.what = "guest acknowledgement of suspend request";
-    dss->pvcontrol.timeout_ms = 60 * 1000;
-    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
-    libxl__xswait_start(gc, &dss->pvcontrol);
-    return;
-
- err:
-    domain_suspend_common_failed(egc, dss);
-}
-
-static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
-        libxl__ev_evtchn *evev)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
-    STATE_AO_GC(dss->ao);
-    /* If we should be done waiting, suspend_common_wait_guest_check
-     * will end up calling domain_suspend_common_guest_suspended or
-     * domain_suspend_common_failed, both of which cancel the evtchn
-     * wait.  So re-enable it now. */
-    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
-    suspend_common_wait_guest_check(egc, dss);
-}
-
-static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
-      libxl__xswait_state *xswa, int rc, const char *state)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
-    STATE_AO_GC(dss->ao);
-    xs_transaction_t t = 0;
-
-    if (!rc && !domain_suspend_pvcontrol_acked(state))
-        /* keep waiting */
-        return;
-
-    libxl__xswait_stop(gc, &dss->pvcontrol);
-
-    if (rc == ERROR_TIMEDOUT) {
-        /*
-         * Guest appears to not be responding. Cancel the suspend
-         * request.
-         *
-         * We re-read the suspend node and clear it within a
-         * transaction in order to handle the case where we race
-         * against the guest catching up and acknowledging the request
-         * at the last minute.
-         */
-        for (;;) {
-            rc = libxl__xs_transaction_start(gc, &t);
-            if (rc) goto err;
-
-            rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
-            if (rc) goto err;
-
-            if (domain_suspend_pvcontrol_acked(state))
-                /* last minute ack */
-                break;
-
-            rc = libxl__xs_write_checked(gc, t, xswa->path, "");
-            if (rc) goto err;
-
-            rc = libxl__xs_transaction_commit(gc, &t);
-            if (!rc) {
-                LOG(ERROR,
-                    "guest didn't acknowledge suspend, cancelling request");
-                goto err;
-            }
-            if (rc<0) goto err;
-        }
-    } else if (rc) {
-        /* some error in xswait's read of xenstore, already logged */
-        goto err;
-    }
-
-    assert(domain_suspend_pvcontrol_acked(state));
-    LOG(DEBUG, "guest acknowledged suspend request");
-
-    libxl__xs_transaction_abort(gc, &t);
-    dss->guest_responded = 1;
-    domain_suspend_common_wait_guest(egc,dss);
-    return;
-
- err:
-    libxl__xs_transaction_abort(gc, &t);
-    domain_suspend_common_failed(egc, dss);
-    return;
-}
-
-static void domain_suspend_common_wait_guest(libxl__egc *egc,
-                                             libxl__domain_suspend_state *dss)
-{
-    STATE_AO_GC(dss->ao);
-    int rc;
-
-    LOG(DEBUG, "wait for the guest to suspend");
-
-    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
-                                    suspend_common_wait_guest_watch,
-                                    "@releaseDomain");
-    if (rc) goto err;
-
-    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
-                                     suspend_common_wait_guest_timeout,
-                                     60*1000);
-    if (rc) goto err;
-    return;
-
- err:
-    domain_suspend_common_failed(egc, dss);
-}
-
-static void suspend_common_wait_guest_watch(libxl__egc *egc,
-      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
-    suspend_common_wait_guest_check(egc, dss);
-}
-
-static void suspend_common_wait_guest_check(libxl__egc *egc,
-        libxl__domain_suspend_state *dss)
-{
-    STATE_AO_GC(dss->ao);
-    xc_domaininfo_t info;
-    int ret;
-    int shutdown_reason;
-
-    /* Convenience aliases */
-    const uint32_t domid = dss->domid;
-
-    ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
-    if (ret < 0) {
-        LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
-        goto err;
-    }
-
-    if (!(ret == 1 && info.domain == domid)) {
-        LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
-             domid);
-        goto err;
-    }
-
-    if (!(info.flags & XEN_DOMINF_shutdown))
-        /* keep waiting */
-        return;
-
-    shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
-        & XEN_DOMINF_shutdownmask;
-    if (shutdown_reason != SHUTDOWN_suspend) {
-        LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
-            " with unexpected reason code %d", domid, shutdown_reason);
-        goto err;
-    }
-
-    LOG(DEBUG, "guest has suspended");
-    domain_suspend_common_guest_suspended(egc, dss);
-    return;
-
- err:
-    domain_suspend_common_failed(egc, dss);
-}
-
-static void suspend_common_wait_guest_timeout(libxl__egc *egc,
-      libxl__ev_time *ev, const struct timeval *requested_abs)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
-    STATE_AO_GC(dss->ao);
-    LOG(ERROR, "guest did not suspend, timed out");
-    domain_suspend_common_failed(egc, dss);
-}
-
-static void domain_suspend_common_guest_suspended(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss)
-{
-    STATE_AO_GC(dss->ao);
-    int ret;
-
-    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
-    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
-    libxl__ev_time_deregister(gc, &dss->guest_timeout);
-
-    if (dss->hvm) {
-        ret = libxl__domain_suspend_device_model(gc, dss);
-        if (ret) {
-            LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
-            domain_suspend_common_failed(egc, dss);
-            return;
-        }
-    }
-    domain_suspend_common_done(egc, dss, 1);
-}
-
-static void domain_suspend_common_failed(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss)
-{
-    domain_suspend_common_done(egc, dss, 0);
-}
-
-static void domain_suspend_common_done(libxl__egc *egc,
-                                       libxl__domain_suspend_state *dss,
-                                       bool ok)
-{
-    EGC_GC;
-    assert(!libxl__xswait_inuse(&dss->pvcontrol));
-    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
-    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
-    libxl__ev_time_deregister(gc, &dss->guest_timeout);
-    dss->callback_common_done(egc, dss, ok);
-}
-
 static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
                                  uint32_t domid,
                                  char *phys_offset, char *node)
@@ -1758,22 +1429,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
     return 0;
 }
 
-static void libxl__domain_suspend_callback(void *data)
-{
-    libxl__save_helper_state *shs = data;
-    libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-
-    dss->callback_common_done = domain_suspend_callback_common_done;
-    domain_suspend_callback_common(egc, dss);
-}
-
-static void domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok)
-{
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
 /*----- remus callbacks -----*/
 static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
                                 libxl__domain_suspend_state *dss, int ok);
@@ -1791,7 +1446,7 @@ static void libxl__remus_domain_suspend_callback(void *data)
     libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
 
     dss->callback_common_done = remus_domain_suspend_callback_common_done;
-    domain_suspend_callback_common(egc, dss);
+    libxl__domain_suspend(egc, dss);
 }
 
 static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
@@ -1959,6 +1614,9 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
 
 /*----- main code for suspending, in order of execution -----*/
 
+static void domain_save_done(libxl__egc *egc,
+                             libxl__domain_suspend_state *dss, int rc);
+
 void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
 {
     STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
new file mode 100644
index 0000000..ef8d60b
--- /dev/null
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -0,0 +1,381 @@
+/*
+ * Copyright (C) 2009      Citrix Ltd.
+ * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+/*==================== Domain suspend ====================*/
+
+static void domain_suspend_callback_common_done(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss, int ok);
+
+/*----- callbacks, called by xc_domain_save -----*/
+
+int libxl__domain_suspend_device_model(libxl__gc *gc,
+                                       libxl__domain_suspend_state *dss)
+{
+    int ret = 0;
+    uint32_t const domid = dss->domid;
+    const char *const filename = dss->dm_savefile;
+
+    switch (libxl__device_model_version_running(gc, domid)) {
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
+        LOG(DEBUG, "Saving device model state to %s", filename);
+        libxl__qemu_traditional_cmd(gc, domid, "save");
+        libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
+        break;
+    }
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+        if (libxl__qmp_stop(gc, domid))
+            return ERROR_FAIL;
+        /* Save DM state into filename */
+        ret = libxl__qmp_save(gc, domid, filename);
+        if (ret)
+            unlink(filename);
+        break;
+    default:
+        return ERROR_INVAL;
+    }
+
+    return ret;
+}
+
+static void domain_suspend_common_wait_guest(libxl__egc *egc,
+                                             libxl__domain_suspend_state *dss);
+static void domain_suspend_common_guest_suspended(libxl__egc *egc,
+                                         libxl__domain_suspend_state *dss);
+
+static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
+      libxl__xswait_state *xswa, int rc, const char *state);
+static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
+        libxl__ev_evtchn *evev);
+static void suspend_common_wait_guest_watch(libxl__egc *egc,
+      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
+static void suspend_common_wait_guest_check(libxl__egc *egc,
+        libxl__domain_suspend_state *dss);
+static void suspend_common_wait_guest_timeout(libxl__egc *egc,
+      libxl__ev_time *ev, const struct timeval *requested_abs);
+
+static void domain_suspend_common_failed(libxl__egc *egc,
+                                         libxl__domain_suspend_state *dss);
+static void domain_suspend_common_done(libxl__egc *egc,
+                                       libxl__domain_suspend_state *dss,
+                                       bool ok);
+static void domain_suspend_callback_common(libxl__egc *egc,
+                                           libxl__domain_suspend_state *dss);
+
+static bool domain_suspend_pvcontrol_acked(const char *state) {
+    /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
+    if (!state) return 1;
+    return strcmp(state,"suspend");
+}
+
+/* calls dss->callback_common_done when done */
+void libxl__domain_suspend(libxl__egc *egc,
+                           libxl__domain_suspend_state *dss)
+{
+    domain_suspend_callback_common(egc, dss);
+}
+
+/* calls dss->callback_common_done when done */
+static void domain_suspend_callback_common(libxl__egc *egc,
+                                           libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
+    int ret, rc;
+
+    /* Convenience aliases */
+    const uint32_t domid = dss->domid;
+
+    if (dss->hvm) {
+        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
+        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
+    }
+
+    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
+        LOG(DEBUG, "issuing %s suspend request via event channel",
+            dss->hvm ? "PVHVM" : "PV");
+        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
+        if (ret < 0) {
+            LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
+            goto err;
+        }
+
+        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
+        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
+        if (rc) goto err;
+
+        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+                                         suspend_common_wait_guest_timeout,
+                                         60*1000);
+        if (rc) goto err;
+
+        return;
+    }
+
+    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
+        LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
+        ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
+        if (ret < 0) {
+            LOGE(ERROR, "xc_domain_shutdown failed");
+            goto err;
+        }
+        /* The guest does not (need to) respond to this sort of request. */
+        dss->guest_responded = 1;
+        domain_suspend_common_wait_guest(egc, dss);
+        return;
+    }
+
+    LOG(DEBUG, "issuing %s suspend request via XenBus control node",
+        dss->hvm ? "PVHVM" : "PV");
+
+    libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
+
+    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
+    if (!dss->pvcontrol.path) goto err;
+
+    dss->pvcontrol.ao = ao;
+    dss->pvcontrol.what = "guest acknowledgement of suspend request";
+    dss->pvcontrol.timeout_ms = 60 * 1000;
+    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
+    libxl__xswait_start(gc, &dss->pvcontrol);
+    return;
+
+ err:
+    domain_suspend_common_failed(egc, dss);
+}
+
+static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
+        libxl__ev_evtchn *evev)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
+    STATE_AO_GC(dss->ao);
+    /* If we should be done waiting, suspend_common_wait_guest_check
+     * will end up calling domain_suspend_common_guest_suspended or
+     * domain_suspend_common_failed, both of which cancel the evtchn
+     * wait.  So re-enable it now. */
+    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
+    suspend_common_wait_guest_check(egc, dss);
+}
+
+static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
+      libxl__xswait_state *xswa, int rc, const char *state)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
+    STATE_AO_GC(dss->ao);
+    xs_transaction_t t = 0;
+
+    if (!rc && !domain_suspend_pvcontrol_acked(state))
+        /* keep waiting */
+        return;
+
+    libxl__xswait_stop(gc, &dss->pvcontrol);
+
+    if (rc == ERROR_TIMEDOUT) {
+        /*
+         * Guest appears to not be responding. Cancel the suspend
+         * request.
+         *
+         * We re-read the suspend node and clear it within a
+         * transaction in order to handle the case where we race
+         * against the guest catching up and acknowledging the request
+         * at the last minute.
+         */
+        for (;;) {
+            rc = libxl__xs_transaction_start(gc, &t);
+            if (rc) goto err;
+
+            rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
+            if (rc) goto err;
+
+            if (domain_suspend_pvcontrol_acked(state))
+                /* last minute ack */
+                break;
+
+            rc = libxl__xs_write_checked(gc, t, xswa->path, "");
+            if (rc) goto err;
+
+            rc = libxl__xs_transaction_commit(gc, &t);
+            if (!rc) {
+                LOG(ERROR,
+                    "guest didn't acknowledge suspend, cancelling request");
+                goto err;
+            }
+            if (rc<0) goto err;
+        }
+    } else if (rc) {
+        /* some error in xswait's read of xenstore, already logged */
+        goto err;
+    }
+
+    assert(domain_suspend_pvcontrol_acked(state));
+    LOG(DEBUG, "guest acknowledged suspend request");
+
+    libxl__xs_transaction_abort(gc, &t);
+    dss->guest_responded = 1;
+    domain_suspend_common_wait_guest(egc,dss);
+    return;
+
+ err:
+    libxl__xs_transaction_abort(gc, &t);
+    domain_suspend_common_failed(egc, dss);
+    return;
+}
+
+static void domain_suspend_common_wait_guest(libxl__egc *egc,
+                                             libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    int rc;
+
+    LOG(DEBUG, "wait for the guest to suspend");
+
+    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
+                                    suspend_common_wait_guest_watch,
+                                    "@releaseDomain");
+    if (rc) goto err;
+
+    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+                                     suspend_common_wait_guest_timeout,
+                                     60*1000);
+    if (rc) goto err;
+    return;
+
+ err:
+    domain_suspend_common_failed(egc, dss);
+}
+
+static void suspend_common_wait_guest_watch(libxl__egc *egc,
+      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
+    suspend_common_wait_guest_check(egc, dss);
+}
+
+static void suspend_common_wait_guest_check(libxl__egc *egc,
+        libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    xc_domaininfo_t info;
+    int ret;
+    int shutdown_reason;
+
+    /* Convenience aliases */
+    const uint32_t domid = dss->domid;
+
+    ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
+    if (ret < 0) {
+        LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
+        goto err;
+    }
+
+    if (!(ret == 1 && info.domain == domid)) {
+        LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
+             domid);
+        goto err;
+    }
+
+    if (!(info.flags & XEN_DOMINF_shutdown))
+        /* keep waiting */
+        return;
+
+    shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
+        & XEN_DOMINF_shutdownmask;
+    if (shutdown_reason != SHUTDOWN_suspend) {
+        LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
+            " with unexpected reason code %d", domid, shutdown_reason);
+        goto err;
+    }
+
+    LOG(DEBUG, "guest has suspended");
+    domain_suspend_common_guest_suspended(egc, dss);
+    return;
+
+ err:
+    domain_suspend_common_failed(egc, dss);
+}
+
+static void suspend_common_wait_guest_timeout(libxl__egc *egc,
+      libxl__ev_time *ev, const struct timeval *requested_abs)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
+    STATE_AO_GC(dss->ao);
+    LOG(ERROR, "guest did not suspend, timed out");
+    domain_suspend_common_failed(egc, dss);
+}
+
+static void domain_suspend_common_guest_suspended(libxl__egc *egc,
+                                         libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    int ret;
+
+    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
+    libxl__ev_time_deregister(gc, &dss->guest_timeout);
+
+    if (dss->hvm) {
+        ret = libxl__domain_suspend_device_model(gc, dss);
+        if (ret) {
+            LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
+            domain_suspend_common_failed(egc, dss);
+            return;
+        }
+    }
+    domain_suspend_common_done(egc, dss, 1);
+}
+
+static void domain_suspend_common_failed(libxl__egc *egc,
+                                         libxl__domain_suspend_state *dss)
+{
+    domain_suspend_common_done(egc, dss, 0);
+}
+
+static void domain_suspend_common_done(libxl__egc *egc,
+                                       libxl__domain_suspend_state *dss,
+                                       bool ok)
+{
+    EGC_GC;
+    assert(!libxl__xswait_inuse(&dss->pvcontrol));
+    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
+    libxl__ev_time_deregister(gc, &dss->guest_timeout);
+    dss->callback_common_done(egc, dss, ok);
+}
+
+void libxl__domain_suspend_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__egc *egc = shs->egc;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+
+    dss->callback_common_done = domain_suspend_callback_common_done;
+    domain_suspend_callback_common(egc, dss);
+}
+
+static void domain_suspend_callback_common_done(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss, int ok)
+{
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index e765d68..8e59b98 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3185,6 +3185,12 @@ _hidden void libxl__domain_save_device_model(libxl__egc *egc,
 
 _hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t domid);
 
+/* calls dss->callback_common_done when done */
+_hidden void libxl__domain_suspend(libxl__egc *egc,
+                                   libxl__domain_suspend_state *dss);
+
+/* used by libxc to suspend the guest during migration */
+_hidden void libxl__domain_suspend_callback(void *data);
 
 /*
  * Convenience macros.
-- 
1.9.1

Re: [PATCH v2 2/6] tools/libxl: move domain suspend code into libxl_dom_suspend.c

[Ian Campbell]

On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
> Move domain suspend code into a separate file libxl_dom_suspend.c.
> export an API libxl__domain_suspend() which wrappers the static

just "..which wraps the..."

> function domain_suspend_callback_common() for internal use.
> 
> Note that the newly added file libxl_dom_suspend.c is used for
> suspend/resume code.
> 
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> CC: Ian Campbell <Ian.Campbell@citrix.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> CC: Andrew Cooper <andrew.cooper3@citrix.com>
> Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
> ---
>  tools/libxl/Makefile            |   3 +-
>  tools/libxl/libxl_dom.c         | 350 +-----------------------------------
>  tools/libxl/libxl_dom_suspend.c | 381 ++++++++++++++++++++++++++++++++++++++++
>  tools/libxl/libxl_internal.h    |   6 +
>  4 files changed, 393 insertions(+), 347 deletions(-)
>  create mode 100644 tools/libxl/libxl_dom_suspend.c
> 
> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
> index cc9c152..3f98d62 100644
> --- a/tools/libxl/Makefile
> +++ b/tools/libxl/Makefile
> @@ -95,7 +95,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
>  			libxl_internal.o libxl_utils.o libxl_uuid.o \
>  			libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \
>  			libxl_save_callout.o _libxl_save_msgs_callout.o \
> -			libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
> +			libxl_qmp.o libxl_event.o libxl_fork.o libxl_dom_suspend.o \
> +			$(LIBXL_OBJS-y)
>  LIBXL_OBJS += libxl_genid.o
>  LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
>  
> diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
> index cce04dd..9444329 100644
> --- a/tools/libxl/libxl_dom.c
> +++ b/tools/libxl/libxl_dom.c
> @@ -1103,11 +1103,6 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
>  
>  /*==================== Domain suspend (save) ====================*/
>  
> -static void domain_save_done(libxl__egc *egc,
> -                             libxl__domain_suspend_state *dss, int rc);
> -static void domain_suspend_callback_common_done(libxl__egc *egc,
> -                                libxl__domain_suspend_state *dss, int ok);
> -
>  /*----- complicated callback, called by xc_domain_save -----*/
>  
>  /*
> @@ -1324,35 +1319,6 @@ static void switch_logdirty_done(libxl__egc *egc,
>  
>  /*----- callbacks, called by xc_domain_save -----*/
>  
> -int libxl__domain_suspend_device_model(libxl__gc *gc,
> -                                       libxl__domain_suspend_state *dss)
> -{
> -    int ret = 0;
> -    uint32_t const domid = dss->domid;
> -    const char *const filename = dss->dm_savefile;
> -
> -    switch (libxl__device_model_version_running(gc, domid)) {
> -    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
> -        LOG(DEBUG, "Saving device model state to %s", filename);
> -        libxl__qemu_traditional_cmd(gc, domid, "save");
> -        libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
> -        break;
> -    }
> -    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
> -        if (libxl__qmp_stop(gc, domid))
> -            return ERROR_FAIL;
> -        /* Save DM state into filename */
> -        ret = libxl__qmp_save(gc, domid, filename);
> -        if (ret)
> -            unlink(filename);
> -        break;
> -    default:
> -        return ERROR_INVAL;
> -    }
> -
> -    return ret;
> -}
> -
>  int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>  {
>  
> @@ -1373,301 +1339,6 @@ int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>      return 0;
>  }
>  
> -static void domain_suspend_common_wait_guest(libxl__egc *egc,
> -                                             libxl__domain_suspend_state *dss);
> -static void domain_suspend_common_guest_suspended(libxl__egc *egc,
> -                                         libxl__domain_suspend_state *dss);
> -
> -static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
> -      libxl__xswait_state *xswa, int rc, const char *state);
> -static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
> -        libxl__ev_evtchn *evev);
> -static void suspend_common_wait_guest_watch(libxl__egc *egc,
> -      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
> -static void suspend_common_wait_guest_check(libxl__egc *egc,
> -        libxl__domain_suspend_state *dss);
> -static void suspend_common_wait_guest_timeout(libxl__egc *egc,
> -      libxl__ev_time *ev, const struct timeval *requested_abs);
> -
> -static void domain_suspend_common_failed(libxl__egc *egc,
> -                                         libxl__domain_suspend_state *dss);
> -static void domain_suspend_common_done(libxl__egc *egc,
> -                                       libxl__domain_suspend_state *dss,
> -                                       bool ok);
> -
> -static bool domain_suspend_pvcontrol_acked(const char *state) {
> -    /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
> -    if (!state) return 1;
> -    return strcmp(state,"suspend");
> -}
> -
> -/* calls dss->callback_common_done when done */
> -static void domain_suspend_callback_common(libxl__egc *egc,
> -                                           libxl__domain_suspend_state *dss)
> -{
> -    STATE_AO_GC(dss->ao);
> -    uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
> -    int ret, rc;
> -
> -    /* Convenience aliases */
> -    const uint32_t domid = dss->domid;
> -
> -    if (dss->hvm) {
> -        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
> -        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
> -    }
> -
> -    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
> -        LOG(DEBUG, "issuing %s suspend request via event channel",
> -            dss->hvm ? "PVHVM" : "PV");
> -        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
> -        if (ret < 0) {
> -            LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
> -            goto err;
> -        }
> -
> -        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
> -        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
> -        if (rc) goto err;
> -
> -        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
> -                                         suspend_common_wait_guest_timeout,
> -                                         60*1000);
> -        if (rc) goto err;
> -
> -        return;
> -    }
> -
> -    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
> -        LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
> -        ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
> -        if (ret < 0) {
> -            LOGE(ERROR, "xc_domain_shutdown failed");
> -            goto err;
> -        }
> -        /* The guest does not (need to) respond to this sort of request. */
> -        dss->guest_responded = 1;
> -        domain_suspend_common_wait_guest(egc, dss);
> -        return;
> -    }
> -
> -    LOG(DEBUG, "issuing %s suspend request via XenBus control node",
> -        dss->hvm ? "PVHVM" : "PV");
> -
> -    libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
> -
> -    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
> -    if (!dss->pvcontrol.path) goto err;
> -
> -    dss->pvcontrol.ao = ao;
> -    dss->pvcontrol.what = "guest acknowledgement of suspend request";
> -    dss->pvcontrol.timeout_ms = 60 * 1000;
> -    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
> -    libxl__xswait_start(gc, &dss->pvcontrol);
> -    return;
> -
> - err:
> -    domain_suspend_common_failed(egc, dss);
> -}
> -
> -static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
> -        libxl__ev_evtchn *evev)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
> -    STATE_AO_GC(dss->ao);
> -    /* If we should be done waiting, suspend_common_wait_guest_check
> -     * will end up calling domain_suspend_common_guest_suspended or
> -     * domain_suspend_common_failed, both of which cancel the evtchn
> -     * wait.  So re-enable it now. */
> -    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
> -    suspend_common_wait_guest_check(egc, dss);
> -}
> -
> -static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
> -      libxl__xswait_state *xswa, int rc, const char *state)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
> -    STATE_AO_GC(dss->ao);
> -    xs_transaction_t t = 0;
> -
> -    if (!rc && !domain_suspend_pvcontrol_acked(state))
> -        /* keep waiting */
> -        return;
> -
> -    libxl__xswait_stop(gc, &dss->pvcontrol);
> -
> -    if (rc == ERROR_TIMEDOUT) {
> -        /*
> -         * Guest appears to not be responding. Cancel the suspend
> -         * request.
> -         *
> -         * We re-read the suspend node and clear it within a
> -         * transaction in order to handle the case where we race
> -         * against the guest catching up and acknowledging the request
> -         * at the last minute.
> -         */
> -        for (;;) {
> -            rc = libxl__xs_transaction_start(gc, &t);
> -            if (rc) goto err;
> -
> -            rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
> -            if (rc) goto err;
> -
> -            if (domain_suspend_pvcontrol_acked(state))
> -                /* last minute ack */
> -                break;
> -
> -            rc = libxl__xs_write_checked(gc, t, xswa->path, "");
> -            if (rc) goto err;
> -
> -            rc = libxl__xs_transaction_commit(gc, &t);
> -            if (!rc) {
> -                LOG(ERROR,
> -                    "guest didn't acknowledge suspend, cancelling request");
> -                goto err;
> -            }
> -            if (rc<0) goto err;
> -        }
> -    } else if (rc) {
> -        /* some error in xswait's read of xenstore, already logged */
> -        goto err;
> -    }
> -
> -    assert(domain_suspend_pvcontrol_acked(state));
> -    LOG(DEBUG, "guest acknowledged suspend request");
> -
> -    libxl__xs_transaction_abort(gc, &t);
> -    dss->guest_responded = 1;
> -    domain_suspend_common_wait_guest(egc,dss);
> -    return;
> -
> - err:
> -    libxl__xs_transaction_abort(gc, &t);
> -    domain_suspend_common_failed(egc, dss);
> -    return;
> -}
> -
> -static void domain_suspend_common_wait_guest(libxl__egc *egc,
> -                                             libxl__domain_suspend_state *dss)
> -{
> -    STATE_AO_GC(dss->ao);
> -    int rc;
> -
> -    LOG(DEBUG, "wait for the guest to suspend");
> -
> -    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
> -                                    suspend_common_wait_guest_watch,
> -                                    "@releaseDomain");
> -    if (rc) goto err;
> -
> -    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
> -                                     suspend_common_wait_guest_timeout,
> -                                     60*1000);
> -    if (rc) goto err;
> -    return;
> -
> - err:
> -    domain_suspend_common_failed(egc, dss);
> -}
> -
> -static void suspend_common_wait_guest_watch(libxl__egc *egc,
> -      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
> -    suspend_common_wait_guest_check(egc, dss);
> -}
> -
> -static void suspend_common_wait_guest_check(libxl__egc *egc,
> -        libxl__domain_suspend_state *dss)
> -{
> -    STATE_AO_GC(dss->ao);
> -    xc_domaininfo_t info;
> -    int ret;
> -    int shutdown_reason;
> -
> -    /* Convenience aliases */
> -    const uint32_t domid = dss->domid;
> -
> -    ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
> -    if (ret < 0) {
> -        LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
> -        goto err;
> -    }
> -
> -    if (!(ret == 1 && info.domain == domid)) {
> -        LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
> -             domid);
> -        goto err;
> -    }
> -
> -    if (!(info.flags & XEN_DOMINF_shutdown))
> -        /* keep waiting */
> -        return;
> -
> -    shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
> -        & XEN_DOMINF_shutdownmask;
> -    if (shutdown_reason != SHUTDOWN_suspend) {
> -        LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
> -            " with unexpected reason code %d", domid, shutdown_reason);
> -        goto err;
> -    }
> -
> -    LOG(DEBUG, "guest has suspended");
> -    domain_suspend_common_guest_suspended(egc, dss);
> -    return;
> -
> - err:
> -    domain_suspend_common_failed(egc, dss);
> -}
> -
> -static void suspend_common_wait_guest_timeout(libxl__egc *egc,
> -      libxl__ev_time *ev, const struct timeval *requested_abs)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
> -    STATE_AO_GC(dss->ao);
> -    LOG(ERROR, "guest did not suspend, timed out");
> -    domain_suspend_common_failed(egc, dss);
> -}
> -
> -static void domain_suspend_common_guest_suspended(libxl__egc *egc,
> -                                         libxl__domain_suspend_state *dss)
> -{
> -    STATE_AO_GC(dss->ao);
> -    int ret;
> -
> -    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
> -    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
> -    libxl__ev_time_deregister(gc, &dss->guest_timeout);
> -
> -    if (dss->hvm) {
> -        ret = libxl__domain_suspend_device_model(gc, dss);
> -        if (ret) {
> -            LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
> -            domain_suspend_common_failed(egc, dss);
> -            return;
> -        }
> -    }
> -    domain_suspend_common_done(egc, dss, 1);
> -}
> -
> -static void domain_suspend_common_failed(libxl__egc *egc,
> -                                         libxl__domain_suspend_state *dss)
> -{
> -    domain_suspend_common_done(egc, dss, 0);
> -}
> -
> -static void domain_suspend_common_done(libxl__egc *egc,
> -                                       libxl__domain_suspend_state *dss,
> -                                       bool ok)
> -{
> -    EGC_GC;
> -    assert(!libxl__xswait_inuse(&dss->pvcontrol));
> -    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
> -    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
> -    libxl__ev_time_deregister(gc, &dss->guest_timeout);
> -    dss->callback_common_done(egc, dss, ok);
> -}
> -
>  static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
>                                   uint32_t domid,
>                                   char *phys_offset, char *node)
> @@ -1758,22 +1429,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
>      return 0;
>  }
>  
> -static void libxl__domain_suspend_callback(void *data)
> -{
> -    libxl__save_helper_state *shs = data;
> -    libxl__egc *egc = shs->egc;
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> -
> -    dss->callback_common_done = domain_suspend_callback_common_done;
> -    domain_suspend_callback_common(egc, dss);
> -}
> -
> -static void domain_suspend_callback_common_done(libxl__egc *egc,
> -                                libxl__domain_suspend_state *dss, int ok)
> -{
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> -}
> -
>  /*----- remus callbacks -----*/
>  static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>                                  libxl__domain_suspend_state *dss, int ok);
> @@ -1791,7 +1446,7 @@ static void libxl__remus_domain_suspend_callback(void *data)
>      libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>  
>      dss->callback_common_done = remus_domain_suspend_callback_common_done;
> -    domain_suspend_callback_common(egc, dss);
> +    libxl__domain_suspend(egc, dss);
>  }
>  
>  static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> @@ -1959,6 +1614,9 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>  
>  /*----- main code for suspending, in order of execution -----*/
>  
> +static void domain_save_done(libxl__egc *egc,
> +                             libxl__domain_suspend_state *dss, int rc);
> +
>  void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
>  {
>      STATE_AO_GC(dss->ao);
> diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
> new file mode 100644
> index 0000000..ef8d60b
> --- /dev/null
> +++ b/tools/libxl/libxl_dom_suspend.c
> @@ -0,0 +1,381 @@
> +/*
> + * Copyright (C) 2009      Citrix Ltd.
> + * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2.1 only. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#include "libxl_osdeps.h" /* must come before any other headers */
> +
> +#include "libxl_internal.h"
> +
> +/*==================== Domain suspend ====================*/
> +
> +static void domain_suspend_callback_common_done(libxl__egc *egc,
> +                                libxl__domain_suspend_state *dss, int ok);
> +
> +/*----- callbacks, called by xc_domain_save -----*/
> +
> +int libxl__domain_suspend_device_model(libxl__gc *gc,
> +                                       libxl__domain_suspend_state *dss)
> +{
> +    int ret = 0;
> +    uint32_t const domid = dss->domid;
> +    const char *const filename = dss->dm_savefile;
> +
> +    switch (libxl__device_model_version_running(gc, domid)) {
> +    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
> +        LOG(DEBUG, "Saving device model state to %s", filename);
> +        libxl__qemu_traditional_cmd(gc, domid, "save");
> +        libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
> +        break;
> +    }
> +    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
> +        if (libxl__qmp_stop(gc, domid))
> +            return ERROR_FAIL;
> +        /* Save DM state into filename */
> +        ret = libxl__qmp_save(gc, domid, filename);
> +        if (ret)
> +            unlink(filename);
> +        break;
> +    default:
> +        return ERROR_INVAL;
> +    }
> +
> +    return ret;
> +}
> +
> +static void domain_suspend_common_wait_guest(libxl__egc *egc,
> +                                             libxl__domain_suspend_state *dss);
> +static void domain_suspend_common_guest_suspended(libxl__egc *egc,
> +                                         libxl__domain_suspend_state *dss);
> +
> +static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
> +      libxl__xswait_state *xswa, int rc, const char *state);
> +static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
> +        libxl__ev_evtchn *evev);
> +static void suspend_common_wait_guest_watch(libxl__egc *egc,
> +      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
> +static void suspend_common_wait_guest_check(libxl__egc *egc,
> +        libxl__domain_suspend_state *dss);
> +static void suspend_common_wait_guest_timeout(libxl__egc *egc,
> +      libxl__ev_time *ev, const struct timeval *requested_abs);
> +
> +static void domain_suspend_common_failed(libxl__egc *egc,
> +                                         libxl__domain_suspend_state *dss);
> +static void domain_suspend_common_done(libxl__egc *egc,
> +                                       libxl__domain_suspend_state *dss,
> +                                       bool ok);
> +static void domain_suspend_callback_common(libxl__egc *egc,
> +                                           libxl__domain_suspend_state *dss);
> +
> +static bool domain_suspend_pvcontrol_acked(const char *state) {
> +    /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
> +    if (!state) return 1;
> +    return strcmp(state,"suspend");
> +}
> +
> +/* calls dss->callback_common_done when done */
> +void libxl__domain_suspend(libxl__egc *egc,
> +                           libxl__domain_suspend_state *dss)
> +{
> +    domain_suspend_callback_common(egc, dss);
> +}
> +
> +/* calls dss->callback_common_done when done */
> +static void domain_suspend_callback_common(libxl__egc *egc,
> +                                           libxl__domain_suspend_state *dss)
> +{
> +    STATE_AO_GC(dss->ao);
> +    uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
> +    int ret, rc;
> +
> +    /* Convenience aliases */
> +    const uint32_t domid = dss->domid;
> +
> +    if (dss->hvm) {
> +        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
> +        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
> +    }
> +
> +    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
> +        LOG(DEBUG, "issuing %s suspend request via event channel",
> +            dss->hvm ? "PVHVM" : "PV");
> +        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
> +        if (ret < 0) {
> +            LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
> +            goto err;
> +        }
> +
> +        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
> +        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
> +        if (rc) goto err;
> +
> +        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
> +                                         suspend_common_wait_guest_timeout,
> +                                         60*1000);
> +        if (rc) goto err;
> +
> +        return;
> +    }
> +
> +    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
> +        LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
> +        ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
> +        if (ret < 0) {
> +            LOGE(ERROR, "xc_domain_shutdown failed");
> +            goto err;
> +        }
> +        /* The guest does not (need to) respond to this sort of request. */
> +        dss->guest_responded = 1;
> +        domain_suspend_common_wait_guest(egc, dss);
> +        return;
> +    }
> +
> +    LOG(DEBUG, "issuing %s suspend request via XenBus control node",
> +        dss->hvm ? "PVHVM" : "PV");
> +
> +    libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
> +
> +    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
> +    if (!dss->pvcontrol.path) goto err;
> +
> +    dss->pvcontrol.ao = ao;
> +    dss->pvcontrol.what = "guest acknowledgement of suspend request";
> +    dss->pvcontrol.timeout_ms = 60 * 1000;
> +    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
> +    libxl__xswait_start(gc, &dss->pvcontrol);
> +    return;
> +
> + err:
> +    domain_suspend_common_failed(egc, dss);
> +}
> +
> +static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
> +        libxl__ev_evtchn *evev)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
> +    STATE_AO_GC(dss->ao);
> +    /* If we should be done waiting, suspend_common_wait_guest_check
> +     * will end up calling domain_suspend_common_guest_suspended or
> +     * domain_suspend_common_failed, both of which cancel the evtchn
> +     * wait.  So re-enable it now. */
> +    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
> +    suspend_common_wait_guest_check(egc, dss);
> +}
> +
> +static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
> +      libxl__xswait_state *xswa, int rc, const char *state)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
> +    STATE_AO_GC(dss->ao);
> +    xs_transaction_t t = 0;
> +
> +    if (!rc && !domain_suspend_pvcontrol_acked(state))
> +        /* keep waiting */
> +        return;
> +
> +    libxl__xswait_stop(gc, &dss->pvcontrol);
> +
> +    if (rc == ERROR_TIMEDOUT) {
> +        /*
> +         * Guest appears to not be responding. Cancel the suspend
> +         * request.
> +         *
> +         * We re-read the suspend node and clear it within a
> +         * transaction in order to handle the case where we race
> +         * against the guest catching up and acknowledging the request
> +         * at the last minute.
> +         */
> +        for (;;) {
> +            rc = libxl__xs_transaction_start(gc, &t);
> +            if (rc) goto err;
> +
> +            rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
> +            if (rc) goto err;
> +
> +            if (domain_suspend_pvcontrol_acked(state))
> +                /* last minute ack */
> +                break;
> +
> +            rc = libxl__xs_write_checked(gc, t, xswa->path, "");
> +            if (rc) goto err;
> +
> +            rc = libxl__xs_transaction_commit(gc, &t);
> +            if (!rc) {
> +                LOG(ERROR,
> +                    "guest didn't acknowledge suspend, cancelling request");
> +                goto err;
> +            }
> +            if (rc<0) goto err;
> +        }
> +    } else if (rc) {
> +        /* some error in xswait's read of xenstore, already logged */
> +        goto err;
> +    }
> +
> +    assert(domain_suspend_pvcontrol_acked(state));
> +    LOG(DEBUG, "guest acknowledged suspend request");
> +
> +    libxl__xs_transaction_abort(gc, &t);
> +    dss->guest_responded = 1;
> +    domain_suspend_common_wait_guest(egc,dss);
> +    return;
> +
> + err:
> +    libxl__xs_transaction_abort(gc, &t);
> +    domain_suspend_common_failed(egc, dss);
> +    return;
> +}
> +
> +static void domain_suspend_common_wait_guest(libxl__egc *egc,
> +                                             libxl__domain_suspend_state *dss)
> +{
> +    STATE_AO_GC(dss->ao);
> +    int rc;
> +
> +    LOG(DEBUG, "wait for the guest to suspend");
> +
> +    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
> +                                    suspend_common_wait_guest_watch,
> +                                    "@releaseDomain");
> +    if (rc) goto err;
> +
> +    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
> +                                     suspend_common_wait_guest_timeout,
> +                                     60*1000);
> +    if (rc) goto err;
> +    return;
> +
> + err:
> +    domain_suspend_common_failed(egc, dss);
> +}
> +
> +static void suspend_common_wait_guest_watch(libxl__egc *egc,
> +      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
> +    suspend_common_wait_guest_check(egc, dss);
> +}
> +
> +static void suspend_common_wait_guest_check(libxl__egc *egc,
> +        libxl__domain_suspend_state *dss)
> +{
> +    STATE_AO_GC(dss->ao);
> +    xc_domaininfo_t info;
> +    int ret;
> +    int shutdown_reason;
> +
> +    /* Convenience aliases */
> +    const uint32_t domid = dss->domid;
> +
> +    ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
> +    if (ret < 0) {
> +        LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
> +        goto err;
> +    }
> +
> +    if (!(ret == 1 && info.domain == domid)) {
> +        LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
> +             domid);
> +        goto err;
> +    }
> +
> +    if (!(info.flags & XEN_DOMINF_shutdown))
> +        /* keep waiting */
> +        return;
> +
> +    shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
> +        & XEN_DOMINF_shutdownmask;
> +    if (shutdown_reason != SHUTDOWN_suspend) {
> +        LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
> +            " with unexpected reason code %d", domid, shutdown_reason);
> +        goto err;
> +    }
> +
> +    LOG(DEBUG, "guest has suspended");
> +    domain_suspend_common_guest_suspended(egc, dss);
> +    return;
> +
> + err:
> +    domain_suspend_common_failed(egc, dss);
> +}
> +
> +static void suspend_common_wait_guest_timeout(libxl__egc *egc,
> +      libxl__ev_time *ev, const struct timeval *requested_abs)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
> +    STATE_AO_GC(dss->ao);
> +    LOG(ERROR, "guest did not suspend, timed out");
> +    domain_suspend_common_failed(egc, dss);
> +}
> +
> +static void domain_suspend_common_guest_suspended(libxl__egc *egc,
> +                                         libxl__domain_suspend_state *dss)
> +{
> +    STATE_AO_GC(dss->ao);
> +    int ret;
> +
> +    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
> +    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
> +    libxl__ev_time_deregister(gc, &dss->guest_timeout);
> +
> +    if (dss->hvm) {
> +        ret = libxl__domain_suspend_device_model(gc, dss);
> +        if (ret) {
> +            LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
> +            domain_suspend_common_failed(egc, dss);
> +            return;
> +        }
> +    }
> +    domain_suspend_common_done(egc, dss, 1);
> +}
> +
> +static void domain_suspend_common_failed(libxl__egc *egc,
> +                                         libxl__domain_suspend_state *dss)
> +{
> +    domain_suspend_common_done(egc, dss, 0);
> +}
> +
> +static void domain_suspend_common_done(libxl__egc *egc,
> +                                       libxl__domain_suspend_state *dss,
> +                                       bool ok)
> +{
> +    EGC_GC;
> +    assert(!libxl__xswait_inuse(&dss->pvcontrol));
> +    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
> +    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
> +    libxl__ev_time_deregister(gc, &dss->guest_timeout);
> +    dss->callback_common_done(egc, dss, ok);
> +}
> +
> +void libxl__domain_suspend_callback(void *data)
> +{
> +    libxl__save_helper_state *shs = data;
> +    libxl__egc *egc = shs->egc;
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> +
> +    dss->callback_common_done = domain_suspend_callback_common_done;
> +    domain_suspend_callback_common(egc, dss);
> +}
> +
> +static void domain_suspend_callback_common_done(libxl__egc *egc,
> +                                libxl__domain_suspend_state *dss, int ok)
> +{
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> +}
> +/*
> + * Local variables:
> + * mode: C
> + * c-basic-offset: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index e765d68..8e59b98 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -3185,6 +3185,12 @@ _hidden void libxl__domain_save_device_model(libxl__egc *egc,
>  
>  _hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t domid);
>  
> +/* calls dss->callback_common_done when done */
> +_hidden void libxl__domain_suspend(libxl__egc *egc,
> +                                   libxl__domain_suspend_state *dss);
> +
> +/* used by libxc to suspend the guest during migration */
> +_hidden void libxl__domain_suspend_callback(void *data);
>  
>  /*
>   * Convenience macros.

Re: [PATCH v2 2/6] tools/libxl: move domain suspend code into libxl_dom_suspend.c

[Yang Hongyang]

On 06/16/2015 09:00 PM, Ian Campbell wrote:
> On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
>> Move domain suspend code into a separate file libxl_dom_suspend.c.
>> export an API libxl__domain_suspend() which wrappers the static
>
> just "..which wraps the..."

will fix, thank you !

>
>> function domain_suspend_callback_common() for internal use.
>>
>> Note that the newly added file libxl_dom_suspend.c is used for
>> suspend/resume code.
>>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> CC: Ian Campbell <Ian.Campbell@citrix.com>
>> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
>> CC: Wei Liu <wei.liu2@citrix.com>
>> CC: Andrew Cooper <andrew.cooper3@citrix.com>
>> Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
>> ---
>>   tools/libxl/Makefile            |   3 +-
>>   tools/libxl/libxl_dom.c         | 350 +-----------------------------------
>>   tools/libxl/libxl_dom_suspend.c | 381 ++++++++++++++++++++++++++++++++++++++++
>>   tools/libxl/libxl_internal.h    |   6 +
>>   4 files changed, 393 insertions(+), 347 deletions(-)
>>   create mode 100644 tools/libxl/libxl_dom_suspend.c
>>
>> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
>> index cc9c152..3f98d62 100644
>> --- a/tools/libxl/Makefile
>> +++ b/tools/libxl/Makefile
>> @@ -95,7 +95,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
>>   			libxl_internal.o libxl_utils.o libxl_uuid.o \
>>   			libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \
>>   			libxl_save_callout.o _libxl_save_msgs_callout.o \
>> -			libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
>> +			libxl_qmp.o libxl_event.o libxl_fork.o libxl_dom_suspend.o \
>> +			$(LIBXL_OBJS-y)
>>   LIBXL_OBJS += libxl_genid.o
>>   LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
>>
>> diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
>> index cce04dd..9444329 100644
>> --- a/tools/libxl/libxl_dom.c
>> +++ b/tools/libxl/libxl_dom.c
>> @@ -1103,11 +1103,6 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
>>
>>   /*==================== Domain suspend (save) ====================*/
>>
>> -static void domain_save_done(libxl__egc *egc,
>> -                             libxl__domain_suspend_state *dss, int rc);
>> -static void domain_suspend_callback_common_done(libxl__egc *egc,
>> -                                libxl__domain_suspend_state *dss, int ok);
>> -
>>   /*----- complicated callback, called by xc_domain_save -----*/
>>
>>   /*
>> @@ -1324,35 +1319,6 @@ static void switch_logdirty_done(libxl__egc *egc,
>>
>>   /*----- callbacks, called by xc_domain_save -----*/
>>
>> -int libxl__domain_suspend_device_model(libxl__gc *gc,
>> -                                       libxl__domain_suspend_state *dss)
>> -{
>> -    int ret = 0;
>> -    uint32_t const domid = dss->domid;
>> -    const char *const filename = dss->dm_savefile;
>> -
>> -    switch (libxl__device_model_version_running(gc, domid)) {
>> -    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
>> -        LOG(DEBUG, "Saving device model state to %s", filename);
>> -        libxl__qemu_traditional_cmd(gc, domid, "save");
>> -        libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
>> -        break;
>> -    }
>> -    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
>> -        if (libxl__qmp_stop(gc, domid))
>> -            return ERROR_FAIL;
>> -        /* Save DM state into filename */
>> -        ret = libxl__qmp_save(gc, domid, filename);
>> -        if (ret)
>> -            unlink(filename);
>> -        break;
>> -    default:
>> -        return ERROR_INVAL;
>> -    }
>> -
>> -    return ret;
>> -}
>> -
>>   int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>>   {
>>
>> @@ -1373,301 +1339,6 @@ int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>>       return 0;
>>   }
>>
>> -static void domain_suspend_common_wait_guest(libxl__egc *egc,
>> -                                             libxl__domain_suspend_state *dss);
>> -static void domain_suspend_common_guest_suspended(libxl__egc *egc,
>> -                                         libxl__domain_suspend_state *dss);
>> -
>> -static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
>> -      libxl__xswait_state *xswa, int rc, const char *state);
>> -static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
>> -        libxl__ev_evtchn *evev);
>> -static void suspend_common_wait_guest_watch(libxl__egc *egc,
>> -      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
>> -static void suspend_common_wait_guest_check(libxl__egc *egc,
>> -        libxl__domain_suspend_state *dss);
>> -static void suspend_common_wait_guest_timeout(libxl__egc *egc,
>> -      libxl__ev_time *ev, const struct timeval *requested_abs);
>> -
>> -static void domain_suspend_common_failed(libxl__egc *egc,
>> -                                         libxl__domain_suspend_state *dss);
>> -static void domain_suspend_common_done(libxl__egc *egc,
>> -                                       libxl__domain_suspend_state *dss,
>> -                                       bool ok);
>> -
>> -static bool domain_suspend_pvcontrol_acked(const char *state) {
>> -    /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
>> -    if (!state) return 1;
>> -    return strcmp(state,"suspend");
>> -}
>> -
>> -/* calls dss->callback_common_done when done */
>> -static void domain_suspend_callback_common(libxl__egc *egc,
>> -                                           libxl__domain_suspend_state *dss)
>> -{
>> -    STATE_AO_GC(dss->ao);
>> -    uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
>> -    int ret, rc;
>> -
>> -    /* Convenience aliases */
>> -    const uint32_t domid = dss->domid;
>> -
>> -    if (dss->hvm) {
>> -        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
>> -        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
>> -    }
>> -
>> -    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
>> -        LOG(DEBUG, "issuing %s suspend request via event channel",
>> -            dss->hvm ? "PVHVM" : "PV");
>> -        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
>> -        if (ret < 0) {
>> -            LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
>> -            goto err;
>> -        }
>> -
>> -        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
>> -        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
>> -        if (rc) goto err;
>> -
>> -        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
>> -                                         suspend_common_wait_guest_timeout,
>> -                                         60*1000);
>> -        if (rc) goto err;
>> -
>> -        return;
>> -    }
>> -
>> -    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
>> -        LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
>> -        ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
>> -        if (ret < 0) {
>> -            LOGE(ERROR, "xc_domain_shutdown failed");
>> -            goto err;
>> -        }
>> -        /* The guest does not (need to) respond to this sort of request. */
>> -        dss->guest_responded = 1;
>> -        domain_suspend_common_wait_guest(egc, dss);
>> -        return;
>> -    }
>> -
>> -    LOG(DEBUG, "issuing %s suspend request via XenBus control node",
>> -        dss->hvm ? "PVHVM" : "PV");
>> -
>> -    libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
>> -
>> -    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
>> -    if (!dss->pvcontrol.path) goto err;
>> -
>> -    dss->pvcontrol.ao = ao;
>> -    dss->pvcontrol.what = "guest acknowledgement of suspend request";
>> -    dss->pvcontrol.timeout_ms = 60 * 1000;
>> -    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
>> -    libxl__xswait_start(gc, &dss->pvcontrol);
>> -    return;
>> -
>> - err:
>> -    domain_suspend_common_failed(egc, dss);
>> -}
>> -
>> -static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
>> -        libxl__ev_evtchn *evev)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
>> -    STATE_AO_GC(dss->ao);
>> -    /* If we should be done waiting, suspend_common_wait_guest_check
>> -     * will end up calling domain_suspend_common_guest_suspended or
>> -     * domain_suspend_common_failed, both of which cancel the evtchn
>> -     * wait.  So re-enable it now. */
>> -    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
>> -    suspend_common_wait_guest_check(egc, dss);
>> -}
>> -
>> -static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
>> -      libxl__xswait_state *xswa, int rc, const char *state)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
>> -    STATE_AO_GC(dss->ao);
>> -    xs_transaction_t t = 0;
>> -
>> -    if (!rc && !domain_suspend_pvcontrol_acked(state))
>> -        /* keep waiting */
>> -        return;
>> -
>> -    libxl__xswait_stop(gc, &dss->pvcontrol);
>> -
>> -    if (rc == ERROR_TIMEDOUT) {
>> -        /*
>> -         * Guest appears to not be responding. Cancel the suspend
>> -         * request.
>> -         *
>> -         * We re-read the suspend node and clear it within a
>> -         * transaction in order to handle the case where we race
>> -         * against the guest catching up and acknowledging the request
>> -         * at the last minute.
>> -         */
>> -        for (;;) {
>> -            rc = libxl__xs_transaction_start(gc, &t);
>> -            if (rc) goto err;
>> -
>> -            rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
>> -            if (rc) goto err;
>> -
>> -            if (domain_suspend_pvcontrol_acked(state))
>> -                /* last minute ack */
>> -                break;
>> -
>> -            rc = libxl__xs_write_checked(gc, t, xswa->path, "");
>> -            if (rc) goto err;
>> -
>> -            rc = libxl__xs_transaction_commit(gc, &t);
>> -            if (!rc) {
>> -                LOG(ERROR,
>> -                    "guest didn't acknowledge suspend, cancelling request");
>> -                goto err;
>> -            }
>> -            if (rc<0) goto err;
>> -        }
>> -    } else if (rc) {
>> -        /* some error in xswait's read of xenstore, already logged */
>> -        goto err;
>> -    }
>> -
>> -    assert(domain_suspend_pvcontrol_acked(state));
>> -    LOG(DEBUG, "guest acknowledged suspend request");
>> -
>> -    libxl__xs_transaction_abort(gc, &t);
>> -    dss->guest_responded = 1;
>> -    domain_suspend_common_wait_guest(egc,dss);
>> -    return;
>> -
>> - err:
>> -    libxl__xs_transaction_abort(gc, &t);
>> -    domain_suspend_common_failed(egc, dss);
>> -    return;
>> -}
>> -
>> -static void domain_suspend_common_wait_guest(libxl__egc *egc,
>> -                                             libxl__domain_suspend_state *dss)
>> -{
>> -    STATE_AO_GC(dss->ao);
>> -    int rc;
>> -
>> -    LOG(DEBUG, "wait for the guest to suspend");
>> -
>> -    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
>> -                                    suspend_common_wait_guest_watch,
>> -                                    "@releaseDomain");
>> -    if (rc) goto err;
>> -
>> -    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
>> -                                     suspend_common_wait_guest_timeout,
>> -                                     60*1000);
>> -    if (rc) goto err;
>> -    return;
>> -
>> - err:
>> -    domain_suspend_common_failed(egc, dss);
>> -}
>> -
>> -static void suspend_common_wait_guest_watch(libxl__egc *egc,
>> -      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
>> -    suspend_common_wait_guest_check(egc, dss);
>> -}
>> -
>> -static void suspend_common_wait_guest_check(libxl__egc *egc,
>> -        libxl__domain_suspend_state *dss)
>> -{
>> -    STATE_AO_GC(dss->ao);
>> -    xc_domaininfo_t info;
>> -    int ret;
>> -    int shutdown_reason;
>> -
>> -    /* Convenience aliases */
>> -    const uint32_t domid = dss->domid;
>> -
>> -    ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
>> -    if (ret < 0) {
>> -        LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
>> -        goto err;
>> -    }
>> -
>> -    if (!(ret == 1 && info.domain == domid)) {
>> -        LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
>> -             domid);
>> -        goto err;
>> -    }
>> -
>> -    if (!(info.flags & XEN_DOMINF_shutdown))
>> -        /* keep waiting */
>> -        return;
>> -
>> -    shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
>> -        & XEN_DOMINF_shutdownmask;
>> -    if (shutdown_reason != SHUTDOWN_suspend) {
>> -        LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
>> -            " with unexpected reason code %d", domid, shutdown_reason);
>> -        goto err;
>> -    }
>> -
>> -    LOG(DEBUG, "guest has suspended");
>> -    domain_suspend_common_guest_suspended(egc, dss);
>> -    return;
>> -
>> - err:
>> -    domain_suspend_common_failed(egc, dss);
>> -}
>> -
>> -static void suspend_common_wait_guest_timeout(libxl__egc *egc,
>> -      libxl__ev_time *ev, const struct timeval *requested_abs)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
>> -    STATE_AO_GC(dss->ao);
>> -    LOG(ERROR, "guest did not suspend, timed out");
>> -    domain_suspend_common_failed(egc, dss);
>> -}
>> -
>> -static void domain_suspend_common_guest_suspended(libxl__egc *egc,
>> -                                         libxl__domain_suspend_state *dss)
>> -{
>> -    STATE_AO_GC(dss->ao);
>> -    int ret;
>> -
>> -    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
>> -    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
>> -    libxl__ev_time_deregister(gc, &dss->guest_timeout);
>> -
>> -    if (dss->hvm) {
>> -        ret = libxl__domain_suspend_device_model(gc, dss);
>> -        if (ret) {
>> -            LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
>> -            domain_suspend_common_failed(egc, dss);
>> -            return;
>> -        }
>> -    }
>> -    domain_suspend_common_done(egc, dss, 1);
>> -}
>> -
>> -static void domain_suspend_common_failed(libxl__egc *egc,
>> -                                         libxl__domain_suspend_state *dss)
>> -{
>> -    domain_suspend_common_done(egc, dss, 0);
>> -}
>> -
>> -static void domain_suspend_common_done(libxl__egc *egc,
>> -                                       libxl__domain_suspend_state *dss,
>> -                                       bool ok)
>> -{
>> -    EGC_GC;
>> -    assert(!libxl__xswait_inuse(&dss->pvcontrol));
>> -    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
>> -    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
>> -    libxl__ev_time_deregister(gc, &dss->guest_timeout);
>> -    dss->callback_common_done(egc, dss, ok);
>> -}
>> -
>>   static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
>>                                    uint32_t domid,
>>                                    char *phys_offset, char *node)
>> @@ -1758,22 +1429,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
>>       return 0;
>>   }
>>
>> -static void libxl__domain_suspend_callback(void *data)
>> -{
>> -    libxl__save_helper_state *shs = data;
>> -    libxl__egc *egc = shs->egc;
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> -
>> -    dss->callback_common_done = domain_suspend_callback_common_done;
>> -    domain_suspend_callback_common(egc, dss);
>> -}
>> -
>> -static void domain_suspend_callback_common_done(libxl__egc *egc,
>> -                                libxl__domain_suspend_state *dss, int ok)
>> -{
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> -}
>> -
>>   /*----- remus callbacks -----*/
>>   static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>>                                   libxl__domain_suspend_state *dss, int ok);
>> @@ -1791,7 +1446,7 @@ static void libxl__remus_domain_suspend_callback(void *data)
>>       libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>>
>>       dss->callback_common_done = remus_domain_suspend_callback_common_done;
>> -    domain_suspend_callback_common(egc, dss);
>> +    libxl__domain_suspend(egc, dss);
>>   }
>>
>>   static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> @@ -1959,6 +1614,9 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>>
>>   /*----- main code for suspending, in order of execution -----*/
>>
>> +static void domain_save_done(libxl__egc *egc,
>> +                             libxl__domain_suspend_state *dss, int rc);
>> +
>>   void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
>>   {
>>       STATE_AO_GC(dss->ao);
>> diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
>> new file mode 100644
>> index 0000000..ef8d60b
>> --- /dev/null
>> +++ b/tools/libxl/libxl_dom_suspend.c
>> @@ -0,0 +1,381 @@
>> +/*
>> + * Copyright (C) 2009      Citrix Ltd.
>> + * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU Lesser General Public License as published
>> + * by the Free Software Foundation; version 2.1 only. with the special
>> + * exception on linking described in file LICENSE.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU Lesser General Public License for more details.
>> + */
>> +
>> +#include "libxl_osdeps.h" /* must come before any other headers */
>> +
>> +#include "libxl_internal.h"
>> +
>> +/*==================== Domain suspend ====================*/
>> +
>> +static void domain_suspend_callback_common_done(libxl__egc *egc,
>> +                                libxl__domain_suspend_state *dss, int ok);
>> +
>> +/*----- callbacks, called by xc_domain_save -----*/
>> +
>> +int libxl__domain_suspend_device_model(libxl__gc *gc,
>> +                                       libxl__domain_suspend_state *dss)
>> +{
>> +    int ret = 0;
>> +    uint32_t const domid = dss->domid;
>> +    const char *const filename = dss->dm_savefile;
>> +
>> +    switch (libxl__device_model_version_running(gc, domid)) {
>> +    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
>> +        LOG(DEBUG, "Saving device model state to %s", filename);
>> +        libxl__qemu_traditional_cmd(gc, domid, "save");
>> +        libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
>> +        break;
>> +    }
>> +    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
>> +        if (libxl__qmp_stop(gc, domid))
>> +            return ERROR_FAIL;
>> +        /* Save DM state into filename */
>> +        ret = libxl__qmp_save(gc, domid, filename);
>> +        if (ret)
>> +            unlink(filename);
>> +        break;
>> +    default:
>> +        return ERROR_INVAL;
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +static void domain_suspend_common_wait_guest(libxl__egc *egc,
>> +                                             libxl__domain_suspend_state *dss);
>> +static void domain_suspend_common_guest_suspended(libxl__egc *egc,
>> +                                         libxl__domain_suspend_state *dss);
>> +
>> +static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
>> +      libxl__xswait_state *xswa, int rc, const char *state);
>> +static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
>> +        libxl__ev_evtchn *evev);
>> +static void suspend_common_wait_guest_watch(libxl__egc *egc,
>> +      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
>> +static void suspend_common_wait_guest_check(libxl__egc *egc,
>> +        libxl__domain_suspend_state *dss);
>> +static void suspend_common_wait_guest_timeout(libxl__egc *egc,
>> +      libxl__ev_time *ev, const struct timeval *requested_abs);
>> +
>> +static void domain_suspend_common_failed(libxl__egc *egc,
>> +                                         libxl__domain_suspend_state *dss);
>> +static void domain_suspend_common_done(libxl__egc *egc,
>> +                                       libxl__domain_suspend_state *dss,
>> +                                       bool ok);
>> +static void domain_suspend_callback_common(libxl__egc *egc,
>> +                                           libxl__domain_suspend_state *dss);
>> +
>> +static bool domain_suspend_pvcontrol_acked(const char *state) {
>> +    /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
>> +    if (!state) return 1;
>> +    return strcmp(state,"suspend");
>> +}
>> +
>> +/* calls dss->callback_common_done when done */
>> +void libxl__domain_suspend(libxl__egc *egc,
>> +                           libxl__domain_suspend_state *dss)
>> +{
>> +    domain_suspend_callback_common(egc, dss);
>> +}
>> +
>> +/* calls dss->callback_common_done when done */
>> +static void domain_suspend_callback_common(libxl__egc *egc,
>> +                                           libxl__domain_suspend_state *dss)
>> +{
>> +    STATE_AO_GC(dss->ao);
>> +    uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
>> +    int ret, rc;
>> +
>> +    /* Convenience aliases */
>> +    const uint32_t domid = dss->domid;
>> +
>> +    if (dss->hvm) {
>> +        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
>> +        xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
>> +    }
>> +
>> +    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
>> +        LOG(DEBUG, "issuing %s suspend request via event channel",
>> +            dss->hvm ? "PVHVM" : "PV");
>> +        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
>> +        if (ret < 0) {
>> +            LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
>> +            goto err;
>> +        }
>> +
>> +        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
>> +        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
>> +        if (rc) goto err;
>> +
>> +        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
>> +                                         suspend_common_wait_guest_timeout,
>> +                                         60*1000);
>> +        if (rc) goto err;
>> +
>> +        return;
>> +    }
>> +
>> +    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
>> +        LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
>> +        ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
>> +        if (ret < 0) {
>> +            LOGE(ERROR, "xc_domain_shutdown failed");
>> +            goto err;
>> +        }
>> +        /* The guest does not (need to) respond to this sort of request. */
>> +        dss->guest_responded = 1;
>> +        domain_suspend_common_wait_guest(egc, dss);
>> +        return;
>> +    }
>> +
>> +    LOG(DEBUG, "issuing %s suspend request via XenBus control node",
>> +        dss->hvm ? "PVHVM" : "PV");
>> +
>> +    libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
>> +
>> +    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
>> +    if (!dss->pvcontrol.path) goto err;
>> +
>> +    dss->pvcontrol.ao = ao;
>> +    dss->pvcontrol.what = "guest acknowledgement of suspend request";
>> +    dss->pvcontrol.timeout_ms = 60 * 1000;
>> +    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
>> +    libxl__xswait_start(gc, &dss->pvcontrol);
>> +    return;
>> +
>> + err:
>> +    domain_suspend_common_failed(egc, dss);
>> +}
>> +
>> +static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
>> +        libxl__ev_evtchn *evev)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
>> +    STATE_AO_GC(dss->ao);
>> +    /* If we should be done waiting, suspend_common_wait_guest_check
>> +     * will end up calling domain_suspend_common_guest_suspended or
>> +     * domain_suspend_common_failed, both of which cancel the evtchn
>> +     * wait.  So re-enable it now. */
>> +    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
>> +    suspend_common_wait_guest_check(egc, dss);
>> +}
>> +
>> +static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
>> +      libxl__xswait_state *xswa, int rc, const char *state)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
>> +    STATE_AO_GC(dss->ao);
>> +    xs_transaction_t t = 0;
>> +
>> +    if (!rc && !domain_suspend_pvcontrol_acked(state))
>> +        /* keep waiting */
>> +        return;
>> +
>> +    libxl__xswait_stop(gc, &dss->pvcontrol);
>> +
>> +    if (rc == ERROR_TIMEDOUT) {
>> +        /*
>> +         * Guest appears to not be responding. Cancel the suspend
>> +         * request.
>> +         *
>> +         * We re-read the suspend node and clear it within a
>> +         * transaction in order to handle the case where we race
>> +         * against the guest catching up and acknowledging the request
>> +         * at the last minute.
>> +         */
>> +        for (;;) {
>> +            rc = libxl__xs_transaction_start(gc, &t);
>> +            if (rc) goto err;
>> +
>> +            rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
>> +            if (rc) goto err;
>> +
>> +            if (domain_suspend_pvcontrol_acked(state))
>> +                /* last minute ack */
>> +                break;
>> +
>> +            rc = libxl__xs_write_checked(gc, t, xswa->path, "");
>> +            if (rc) goto err;
>> +
>> +            rc = libxl__xs_transaction_commit(gc, &t);
>> +            if (!rc) {
>> +                LOG(ERROR,
>> +                    "guest didn't acknowledge suspend, cancelling request");
>> +                goto err;
>> +            }
>> +            if (rc<0) goto err;
>> +        }
>> +    } else if (rc) {
>> +        /* some error in xswait's read of xenstore, already logged */
>> +        goto err;
>> +    }
>> +
>> +    assert(domain_suspend_pvcontrol_acked(state));
>> +    LOG(DEBUG, "guest acknowledged suspend request");
>> +
>> +    libxl__xs_transaction_abort(gc, &t);
>> +    dss->guest_responded = 1;
>> +    domain_suspend_common_wait_guest(egc,dss);
>> +    return;
>> +
>> + err:
>> +    libxl__xs_transaction_abort(gc, &t);
>> +    domain_suspend_common_failed(egc, dss);
>> +    return;
>> +}
>> +
>> +static void domain_suspend_common_wait_guest(libxl__egc *egc,
>> +                                             libxl__domain_suspend_state *dss)
>> +{
>> +    STATE_AO_GC(dss->ao);
>> +    int rc;
>> +
>> +    LOG(DEBUG, "wait for the guest to suspend");
>> +
>> +    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
>> +                                    suspend_common_wait_guest_watch,
>> +                                    "@releaseDomain");
>> +    if (rc) goto err;
>> +
>> +    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
>> +                                     suspend_common_wait_guest_timeout,
>> +                                     60*1000);
>> +    if (rc) goto err;
>> +    return;
>> +
>> + err:
>> +    domain_suspend_common_failed(egc, dss);
>> +}
>> +
>> +static void suspend_common_wait_guest_watch(libxl__egc *egc,
>> +      libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
>> +    suspend_common_wait_guest_check(egc, dss);
>> +}
>> +
>> +static void suspend_common_wait_guest_check(libxl__egc *egc,
>> +        libxl__domain_suspend_state *dss)
>> +{
>> +    STATE_AO_GC(dss->ao);
>> +    xc_domaininfo_t info;
>> +    int ret;
>> +    int shutdown_reason;
>> +
>> +    /* Convenience aliases */
>> +    const uint32_t domid = dss->domid;
>> +
>> +    ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
>> +    if (ret < 0) {
>> +        LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
>> +        goto err;
>> +    }
>> +
>> +    if (!(ret == 1 && info.domain == domid)) {
>> +        LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
>> +             domid);
>> +        goto err;
>> +    }
>> +
>> +    if (!(info.flags & XEN_DOMINF_shutdown))
>> +        /* keep waiting */
>> +        return;
>> +
>> +    shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
>> +        & XEN_DOMINF_shutdownmask;
>> +    if (shutdown_reason != SHUTDOWN_suspend) {
>> +        LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
>> +            " with unexpected reason code %d", domid, shutdown_reason);
>> +        goto err;
>> +    }
>> +
>> +    LOG(DEBUG, "guest has suspended");
>> +    domain_suspend_common_guest_suspended(egc, dss);
>> +    return;
>> +
>> + err:
>> +    domain_suspend_common_failed(egc, dss);
>> +}
>> +
>> +static void suspend_common_wait_guest_timeout(libxl__egc *egc,
>> +      libxl__ev_time *ev, const struct timeval *requested_abs)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
>> +    STATE_AO_GC(dss->ao);
>> +    LOG(ERROR, "guest did not suspend, timed out");
>> +    domain_suspend_common_failed(egc, dss);
>> +}
>> +
>> +static void domain_suspend_common_guest_suspended(libxl__egc *egc,
>> +                                         libxl__domain_suspend_state *dss)
>> +{
>> +    STATE_AO_GC(dss->ao);
>> +    int ret;
>> +
>> +    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
>> +    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
>> +    libxl__ev_time_deregister(gc, &dss->guest_timeout);
>> +
>> +    if (dss->hvm) {
>> +        ret = libxl__domain_suspend_device_model(gc, dss);
>> +        if (ret) {
>> +            LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
>> +            domain_suspend_common_failed(egc, dss);
>> +            return;
>> +        }
>> +    }
>> +    domain_suspend_common_done(egc, dss, 1);
>> +}
>> +
>> +static void domain_suspend_common_failed(libxl__egc *egc,
>> +                                         libxl__domain_suspend_state *dss)
>> +{
>> +    domain_suspend_common_done(egc, dss, 0);
>> +}
>> +
>> +static void domain_suspend_common_done(libxl__egc *egc,
>> +                                       libxl__domain_suspend_state *dss,
>> +                                       bool ok)
>> +{
>> +    EGC_GC;
>> +    assert(!libxl__xswait_inuse(&dss->pvcontrol));
>> +    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
>> +    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
>> +    libxl__ev_time_deregister(gc, &dss->guest_timeout);
>> +    dss->callback_common_done(egc, dss, ok);
>> +}
>> +
>> +void libxl__domain_suspend_callback(void *data)
>> +{
>> +    libxl__save_helper_state *shs = data;
>> +    libxl__egc *egc = shs->egc;
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> +
>> +    dss->callback_common_done = domain_suspend_callback_common_done;
>> +    domain_suspend_callback_common(egc, dss);
>> +}
>> +
>> +static void domain_suspend_callback_common_done(libxl__egc *egc,
>> +                                libxl__domain_suspend_state *dss, int ok)
>> +{
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> +}
>> +/*
>> + * Local variables:
>> + * mode: C
>> + * c-basic-offset: 4
>> + * indent-tabs-mode: nil
>> + * End:
>> + */
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index e765d68..8e59b98 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
>> @@ -3185,6 +3185,12 @@ _hidden void libxl__domain_save_device_model(libxl__egc *egc,
>>
>>   _hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t domid);
>>
>> +/* calls dss->callback_common_done when done */
>> +_hidden void libxl__domain_suspend(libxl__egc *egc,
>> +                                   libxl__domain_suspend_state *dss);
>> +
>> +/* used by libxc to suspend the guest during migration */
>> +_hidden void libxl__domain_suspend_callback(void *data);
>>
>>   /*
>>    * Convenience macros.
>
>
> .
>

-- 
Thanks,
Yang.

[PATCH v2 3/6] tools/libxl: move domain resume code into libxl_dom_suspend.c

[Yang Hongyang]

move domain resume code into libxl_dom_suspend.c.

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxl/libxl.c             | 59 -----------------------------
 tools/libxl/libxl_dom.c         | 20 ----------
 tools/libxl/libxl_dom_suspend.c | 82 +++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_internal.h    |  1 +
 4 files changed, 83 insertions(+), 79 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 5a70062..77c6a36 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -510,39 +510,6 @@ int libxl_domain_rename(libxl_ctx *ctx, uint32_t domid,
     return rc;
 }
 
-int libxl__domain_resume(libxl__gc *gc, uint32_t domid, int suspend_cancel)
-{
-    int rc = 0;
-
-    if (xc_domain_resume(CTX->xch, domid, suspend_cancel)) {
-        LOGE(ERROR, "xc_domain_resume failed for domain %u", domid);
-        rc = ERROR_FAIL;
-        goto out;
-    }
-
-    libxl_domain_type type = libxl__domain_type(gc, domid);
-    if (type == LIBXL_DOMAIN_TYPE_INVALID) {
-        rc = ERROR_FAIL;
-        goto out;
-    }
-
-    if (type == LIBXL_DOMAIN_TYPE_HVM) {
-        rc = libxl__domain_resume_device_model(gc, domid);
-        if (rc) {
-            LOG(ERROR, "failed to resume device model for domain %u:%d",
-                domid, rc);
-            goto out;
-        }
-    }
-
-    if (!xs_resume_domain(CTX->xsh, domid)) {
-        LOGE(ERROR, "xs_resume_domain failed for domain %u", domid);
-        rc = ERROR_FAIL;
-    }
-out:
-    return rc;
-}
-
 int libxl_domain_resume(libxl_ctx *ctx, uint32_t domid, int suspend_cancel,
                         const libxl_asyncop_how *ao_how)
 {
@@ -5948,32 +5915,6 @@ int libxl_domain_sched_params_get(libxl_ctx *ctx, uint32_t domid,
     return ret;
 }
 
-static int libxl__domain_s3_resume(libxl__gc *gc, int domid)
-{
-    int rc = 0;
-
-    switch (libxl__domain_type(gc, domid)) {
-    case LIBXL_DOMAIN_TYPE_HVM:
-        switch (libxl__device_model_version_running(gc, domid)) {
-        case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
-            rc = xc_hvm_param_set(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, 0);
-            break;
-        case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
-            rc = libxl__qmp_system_wakeup(gc, domid);
-            break;
-        default:
-            rc = ERROR_INVAL;
-            break;
-        }
-        break;
-    default:
-        rc = ERROR_INVAL;
-        break;
-    }
-
-    return rc;
-}
-
 int libxl_send_trigger(libxl_ctx *ctx, uint32_t domid,
                        libxl_trigger trigger, uint32_t vcpuid)
 {
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 9444329..701e9f7 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1319,26 +1319,6 @@ static void switch_logdirty_done(libxl__egc *egc,
 
 /*----- callbacks, called by xc_domain_save -----*/
 
-int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
-{
-
-    switch (libxl__device_model_version_running(gc, domid)) {
-    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
-        libxl__qemu_traditional_cmd(gc, domid, "continue");
-        libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
-        break;
-    }
-    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
-        if (libxl__qmp_resume(gc, domid))
-            return ERROR_FAIL;
-        break;
-    default:
-        return ERROR_INVAL;
-    }
-
-    return 0;
-}
-
 static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
                                  uint32_t domid,
                                  char *phys_offset, char *node)
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
index ef8d60b..8b9dd6c 100644
--- a/tools/libxl/libxl_dom_suspend.c
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -372,6 +372,88 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
 {
     libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
 }
+
+/*==================== Domain resume ====================*/
+
+int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
+{
+
+    switch (libxl__device_model_version_running(gc, domid)) {
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
+        libxl__qemu_traditional_cmd(gc, domid, "continue");
+        libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
+        break;
+    }
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+        if (libxl__qmp_resume(gc, domid))
+            return ERROR_FAIL;
+        break;
+    default:
+        return ERROR_INVAL;
+    }
+
+    return 0;
+}
+
+int libxl__domain_s3_resume(libxl__gc *gc, int domid)
+{
+    int rc = 0;
+
+    switch (libxl__domain_type(gc, domid)) {
+    case LIBXL_DOMAIN_TYPE_HVM:
+        switch (libxl__device_model_version_running(gc, domid)) {
+        case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
+            rc = xc_hvm_param_set(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, 0);
+            break;
+        case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+            rc = libxl__qmp_system_wakeup(gc, domid);
+            break;
+        default:
+            rc = ERROR_INVAL;
+            break;
+        }
+        break;
+    default:
+        rc = ERROR_INVAL;
+        break;
+    }
+
+    return rc;
+}
+
+int libxl__domain_resume(libxl__gc *gc, uint32_t domid, int suspend_cancel)
+{
+    int rc = 0;
+
+    if (xc_domain_resume(CTX->xch, domid, suspend_cancel)) {
+        LOGE(ERROR, "xc_domain_resume failed for domain %u", domid);
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    libxl_domain_type type = libxl__domain_type(gc, domid);
+    if (type == LIBXL_DOMAIN_TYPE_INVALID) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
+        rc = libxl__domain_resume_device_model(gc, domid);
+        if (rc) {
+            LOG(ERROR, "failed to resume device model for domain %u:%d",
+                domid, rc);
+            goto out;
+        }
+    }
+
+    if (!xs_resume_domain(CTX->xsh, domid)) {
+        LOGE(ERROR, "xs_resume_domain failed for domain %u", domid);
+        rc = ERROR_FAIL;
+    }
+out:
+    return rc;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 8e59b98..164c448 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1041,6 +1041,7 @@ _hidden int libxl__userdata_store(libxl__gc *gc, uint32_t domid,
 
 _hidden int libxl__domain_resume(libxl__gc *gc, uint32_t domid,
                                  int suspend_cancel);
+_hidden int libxl__domain_s3_resume(libxl__gc *gc, int domid);
 
 /* returns 0 or 1, or a libxl error code */
 _hidden int libxl__domain_pvcontrol_available(libxl__gc *gc, uint32_t domid);
-- 
1.9.1

Re: [PATCH v2 3/6] tools/libxl: move domain resume code into libxl_dom_suspend.c

[Ian Campbell]

On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
> move domain resume code into libxl_dom_suspend.c.

Even though it has "resume" in the name, I'm not sure that
libxl__domain_s3_resume is a good candidate for moving to the suspend
code, it's called only from libxl_send_trigger and IIRC we don't really
implement s3, just a fake version where the domain is paused/unpaused
(rather the destroyed and resumed, say.

Having moved libxl__domain_resume and libxl__domain_resume_device_model
into the same file I think the latter could become static. (If you do
that in this patch please say something along the lines of "pure code
motion except for making libxl__domain_resume_device_model static" in
the commit message)

Re: [PATCH v2 3/6] tools/libxl: move domain resume code into libxl_dom_suspend.c

[Yang Hongyang]

On 06/16/2015 09:04 PM, Ian Campbell wrote:
> On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
>> move domain resume code into libxl_dom_suspend.c.
>
> Even though it has "resume" in the name, I'm not sure that
> libxl__domain_s3_resume is a good candidate for moving to the suspend
> code, it's called only from libxl_send_trigger and IIRC we don't really
> implement s3, just a fake version where the domain is paused/unpaused
> (rather the destroyed and resumed, say.
>
> Having moved libxl__domain_resume and libxl__domain_resume_device_model
> into the same file I think the latter could become static. (If you do
> that in this patch please say something along the lines of "pure code
> motion except for making libxl__domain_resume_device_model static" in
> the commit message)

ok, will fix this in the next version, thank you for the review!

>
>
> .
>

-- 
Thanks,
Yang.

[PATCH v2 4/6] tools/libxl: move remus code into libxl_remus.c

[Yang Hongyang]

move remus code into libxl_remus.c.

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
---
 tools/libxl/Makefile         |   2 +-
 tools/libxl/libxl.c          |  55 +-------
 tools/libxl/libxl_dom.c      | 206 +----------------------------
 tools/libxl/libxl_internal.h |  11 ++
 tools/libxl/libxl_remus.c    | 304 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 318 insertions(+), 260 deletions(-)
 create mode 100644 tools/libxl/libxl_remus.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 3f98d62..8535eaa 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -56,7 +56,7 @@ else
 LIBXL_OBJS-y += libxl_nonetbuffer.o
 endif
 
-LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o
+LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
 
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 77c6a36..0f9248e 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -792,10 +792,6 @@ out:
     return ptr;
 }
 
-static void libxl__remus_setup_done(libxl__egc *egc,
-                                    libxl__remus_devices_state *rds, int rc);
-static void libxl__remus_setup_failed(libxl__egc *egc,
-                                      libxl__remus_devices_state *rds, int rc);
 static void remus_failover_cb(libxl__egc *egc,
                               libxl__domain_suspend_state *dss, int rc);
 
@@ -844,63 +840,14 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
 
     assert(info);
 
-    /* Convenience aliases */
-    libxl__remus_devices_state *const rds = &dss->rds;
-
-    if (libxl_defbool_val(info->netbuf)) {
-        if (!libxl__netbuffer_enabled(gc)) {
-            LOG(ERROR, "Remus: No support for network buffering");
-            rc = ERROR_FAIL;
-            goto out;
-        }
-        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
-    }
-
-    if (libxl_defbool_val(info->diskbuf))
-        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
-
-    rds->ao = ao;
-    rds->domid = domid;
-    rds->callback = libxl__remus_setup_done;
-
     /* Point of no return */
-    libxl__remus_devices_setup(egc, rds);
+    libxl__remus_setup(egc, dss);
     return AO_INPROGRESS;
 
  out:
     return AO_ABORT(rc);
 }
 
-static void libxl__remus_setup_done(libxl__egc *egc,
-                                    libxl__remus_devices_state *rds, int rc)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-    STATE_AO_GC(dss->ao);
-
-    if (!rc) {
-        libxl__domain_save(egc, dss);
-        return;
-    }
-
-    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
-        dss->domid, rc);
-    rds->callback = libxl__remus_setup_failed;
-    libxl__remus_devices_teardown(egc, rds);
-}
-
-static void libxl__remus_setup_failed(libxl__egc *egc,
-                                      libxl__remus_devices_state *rds, int rc)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-    STATE_AO_GC(dss->ao);
-
-    if (rc)
-        LOG(ERROR, "Remus: failed to teardown device after setup failed"
-            " for guest with domid %u, rc %d", dss->domid, rc);
-
-    dss->callback(egc, dss, rc);
-}
-
 static void remus_failover_cb(libxl__egc *egc,
                               libxl__domain_suspend_state *dss, int rc)
 {
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 701e9f7..0f81081 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1409,189 +1409,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
     return 0;
 }
 
-/*----- remus callbacks -----*/
-static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok);
-static void remus_devices_postsuspend_cb(libxl__egc *egc,
-                                         libxl__remus_devices_state *rds,
-                                         int rc);
-static void remus_devices_preresume_cb(libxl__egc *egc,
-                                       libxl__remus_devices_state *rds,
-                                       int rc);
-
-static void libxl__remus_domain_suspend_callback(void *data)
-{
-    libxl__save_helper_state *shs = data;
-    libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-
-    dss->callback_common_done = remus_domain_suspend_callback_common_done;
-    libxl__domain_suspend(egc, dss);
-}
-
-static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok)
-{
-    if (!ok)
-        goto out;
-
-    libxl__remus_devices_state *const rds = &dss->rds;
-    rds->callback = remus_devices_postsuspend_cb;
-    libxl__remus_devices_postsuspend(egc, rds);
-    return;
-
-out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
-static void remus_devices_postsuspend_cb(libxl__egc *egc,
-                                         libxl__remus_devices_state *rds,
-                                         int rc)
-{
-    int ok = 0;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-
-    if (rc)
-        goto out;
-
-    ok = 1;
-
-out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
-static void libxl__remus_domain_resume_callback(void *data)
-{
-    libxl__save_helper_state *shs = data;
-    libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-    STATE_AO_GC(dss->ao);
-
-    libxl__remus_devices_state *const rds = &dss->rds;
-    rds->callback = remus_devices_preresume_cb;
-    libxl__remus_devices_preresume(egc, rds);
-}
-
-static void remus_devices_preresume_cb(libxl__egc *egc,
-                                       libxl__remus_devices_state *rds,
-                                       int rc)
-{
-    int ok = 0;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-    STATE_AO_GC(dss->ao);
-
-    if (rc)
-        goto out;
-
-    /* Resumes the domain and the device model */
-    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
-    if (rc)
-        goto out;
-
-    ok = 1;
-
-out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
-/*----- remus asynchronous checkpoint callback -----*/
-
-static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc);
-static void remus_devices_commit_cb(libxl__egc *egc,
-                                    libxl__remus_devices_state *rds,
-                                    int rc);
-static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
-                                  const struct timeval *requested_abs);
-
-static void libxl__remus_domain_checkpoint_callback(void *data)
-{
-    libxl__save_helper_state *shs = data;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-    libxl__egc *egc = dss->shs.egc;
-    STATE_AO_GC(dss->ao);
-
-    /* This would go into tailbuf. */
-    if (dss->hvm) {
-        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
-    } else {
-        remus_checkpoint_dm_saved(egc, dss, 0);
-    }
-}
-
-static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc)
-{
-    /* Convenience aliases */
-    libxl__remus_devices_state *const rds = &dss->rds;
-
-    STATE_AO_GC(dss->ao);
-
-    if (rc) {
-        LOG(ERROR, "Failed to save device model. Terminating Remus..");
-        goto out;
-    }
-
-    rds->callback = remus_devices_commit_cb;
-    libxl__remus_devices_commit(egc, rds);
-
-    return;
-
-out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
-}
-
-static void remus_devices_commit_cb(libxl__egc *egc,
-                                    libxl__remus_devices_state *rds,
-                                    int rc)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-
-    STATE_AO_GC(dss->ao);
-
-    if (rc) {
-        LOG(ERROR, "Failed to do device commit op."
-            " Terminating Remus..");
-        goto out;
-    }
-
-    /*
-     * At this point, we have successfully checkpointed the guest and
-     * committed it at the backup. We'll come back after the checkpoint
-     * interval to checkpoint the guest again. Until then, let the guest
-     * continue execution.
-     */
-
-    /* Set checkpoint interval timeout */
-    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
-                                     remus_next_checkpoint,
-                                     dss->interval);
-
-    if (rc)
-        goto out;
-
-    return;
-
-out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
-}
-
-static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
-                                  const struct timeval *requested_abs)
-{
-    libxl__domain_suspend_state *dss =
-                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
-
-    STATE_AO_GC(dss->ao);
-
-    /*
-     * Time to checkpoint the guest again. We return 1 to libxc
-     * (xc_domain_save.c). in order to continue executing the infinite loop
-     * (suspend, checkpoint, resume) in xc_domain_save().
-     */
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
-}
-
 /*----- main code for suspending, in order of execution -----*/
 
 static void domain_save_done(libxl__egc *egc,
@@ -1811,10 +1628,6 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
     dss->save_dm_callback(egc, dss, our_rc);
 }
 
-static void remus_teardown_done(libxl__egc *egc,
-                                       libxl__remus_devices_state *rds,
-                                       int rc);
-
 static void domain_save_done(libxl__egc *egc,
                              libxl__domain_suspend_state *dss, int rc)
 {
@@ -1840,24 +1653,7 @@ static void domain_save_done(libxl__egc *egc,
      * from sending checkpoints. Teardown the network buffers and
      * release netlink resources.  This is an async op.
      */
-    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
-        " teardown Remus devices...", rc);
-    dss->rds.callback = remus_teardown_done;
-    libxl__remus_devices_teardown(egc, &dss->rds);
-}
-
-static void remus_teardown_done(libxl__egc *egc,
-                                       libxl__remus_devices_state *rds,
-                                       int rc)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-    STATE_AO_GC(dss->ao);
-
-    if (rc)
-        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
-            " rc %d", dss->domid, rc);
-
-    dss->callback(egc, dss, rc);
+    libxl__remus_teardown(egc, dss, rc);
 }
 
 /*==================== Miscellaneous ====================*/
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 164c448..dc6b62b 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3193,6 +3193,17 @@ _hidden void libxl__domain_suspend(libxl__egc *egc,
 /* used by libxc to suspend the guest during migration */
 _hidden void libxl__domain_suspend_callback(void *data);
 
+/* Remus callbacks for save */
+_hidden void libxl__remus_domain_suspend_callback(void *data);
+_hidden void libxl__remus_domain_resume_callback(void *data);
+_hidden void libxl__remus_domain_checkpoint_callback(void *data);
+/* Remus setup and teardown*/
+_hidden void libxl__remus_setup(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss);
+_hidden void libxl__remus_teardown(libxl__egc *egc,
+                                   libxl__domain_suspend_state *dss,
+                                   int rc);
+
 /*
  * Convenience macros.
  */
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
new file mode 100644
index 0000000..51261e0
--- /dev/null
+++ b/tools/libxl/libxl_remus.c
@@ -0,0 +1,304 @@
+/*
+ * Copyright (C) 2009      Citrix Ltd.
+ * Author Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2 and later. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+/*----- Remus setup and teardown -----*/
+
+static void remus_setup_done(libxl__egc *egc,
+                             libxl__remus_devices_state *rds, int rc);
+static void remus_setup_failed(libxl__egc *egc,
+                               libxl__remus_devices_state *rds, int rc);
+
+void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
+{
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds = &dss->rds;
+    const libxl_domain_remus_info *const info = dss->remus;
+
+    STATE_AO_GC(dss->ao);
+
+    if (libxl_defbool_val(info->netbuf)) {
+        if (!libxl__netbuffer_enabled(gc)) {
+            LOG(ERROR, "Remus: No support for network buffering");
+            goto out;
+        }
+        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
+    }
+
+    if (libxl_defbool_val(info->diskbuf))
+        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
+
+    rds->ao = ao;
+    rds->domid = dss->domid;
+    rds->callback = remus_setup_done;
+
+    libxl__remus_devices_setup(egc, rds);
+    return;
+
+out:
+    dss->callback(egc, dss, ERROR_FAIL);
+}
+
+static void remus_setup_done(libxl__egc *egc,
+                             libxl__remus_devices_state *rds, int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (!rc) {
+        libxl__domain_save(egc, dss);
+        return;
+    }
+
+    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
+        dss->domid, rc);
+    rds->callback = remus_setup_failed;
+    libxl__remus_devices_teardown(egc, rds);
+}
+
+static void remus_setup_failed(libxl__egc *egc,
+                               libxl__remus_devices_state *rds, int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        LOG(ERROR, "Remus: failed to teardown device after setup failed"
+            " for guest with domid %u, rc %d", dss->domid, rc);
+
+    dss->callback(egc, dss, rc);
+}
+
+static void remus_teardown_done(libxl__egc *egc,
+                                libxl__remus_devices_state *rds,
+                                int rc);
+void libxl__remus_teardown(libxl__egc *egc,
+                           libxl__domain_suspend_state *dss,
+                           int rc)
+{
+    EGC_GC;
+
+    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
+        " teardown Remus devices...", rc);
+    dss->rds.callback = remus_teardown_done;
+    libxl__remus_devices_teardown(egc, &dss->rds);
+}
+
+static void remus_teardown_done(libxl__egc *egc,
+                                libxl__remus_devices_state *rds,
+                                int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
+            " rc %d", dss->domid, rc);
+
+    dss->callback(egc, dss, rc);
+}
+
+/*----- remus callbacks -----*/
+static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss, int ok);
+static void remus_devices_postsuspend_cb(libxl__egc *egc,
+                                         libxl__remus_devices_state *rds,
+                                         int rc);
+static void remus_devices_preresume_cb(libxl__egc *egc,
+                                       libxl__remus_devices_state *rds,
+                                       int rc);
+
+void libxl__remus_domain_suspend_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__egc *egc = shs->egc;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+
+    dss->callback_common_done = remus_domain_suspend_callback_common_done;
+    libxl__domain_suspend(egc, dss);
+}
+
+static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss, int ok)
+{
+    if (!ok)
+        goto out;
+
+    libxl__remus_devices_state *const rds = &dss->rds;
+    rds->callback = remus_devices_postsuspend_cb;
+    libxl__remus_devices_postsuspend(egc, rds);
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+static void remus_devices_postsuspend_cb(libxl__egc *egc,
+                                         libxl__remus_devices_state *rds,
+                                         int rc)
+{
+    int ok = 0;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+
+    if (rc)
+        goto out;
+
+    ok = 1;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+void libxl__remus_domain_resume_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__egc *egc = shs->egc;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    STATE_AO_GC(dss->ao);
+
+    libxl__remus_devices_state *const rds = &dss->rds;
+    rds->callback = remus_devices_preresume_cb;
+    libxl__remus_devices_preresume(egc, rds);
+}
+
+static void remus_devices_preresume_cb(libxl__egc *egc,
+                                       libxl__remus_devices_state *rds,
+                                       int rc)
+{
+    int ok = 0;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        goto out;
+
+    /* Resumes the domain and the device model */
+    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
+    if (rc)
+        goto out;
+
+    ok = 1;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+/*----- remus asynchronous checkpoint callback -----*/
+
+static void remus_checkpoint_dm_saved(libxl__egc *egc,
+                                      libxl__domain_suspend_state *dss, int rc);
+static void remus_devices_commit_cb(libxl__egc *egc,
+                                    libxl__remus_devices_state *rds,
+                                    int rc);
+static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
+                                  const struct timeval *requested_abs);
+
+void libxl__remus_domain_checkpoint_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__egc *egc = dss->shs.egc;
+    STATE_AO_GC(dss->ao);
+
+    /* This would go into tailbuf. */
+    if (dss->hvm) {
+        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
+    } else {
+        remus_checkpoint_dm_saved(egc, dss, 0);
+    }
+}
+
+static void remus_checkpoint_dm_saved(libxl__egc *egc,
+                                      libxl__domain_suspend_state *dss, int rc)
+{
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds = &dss->rds;
+
+    STATE_AO_GC(dss->ao);
+
+    if (rc) {
+        LOG(ERROR, "Failed to save device model. Terminating Remus..");
+        goto out;
+    }
+
+    rds->callback = remus_devices_commit_cb;
+    libxl__remus_devices_commit(egc, rds);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+}
+
+static void remus_devices_commit_cb(libxl__egc *egc,
+                                    libxl__remus_devices_state *rds,
+                                    int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+
+    STATE_AO_GC(dss->ao);
+
+    if (rc) {
+        LOG(ERROR, "Failed to do device commit op."
+            " Terminating Remus..");
+        goto out;
+    }
+
+    /*
+     * At this point, we have successfully checkpointed the guest and
+     * committed it at the backup. We'll come back after the checkpoint
+     * interval to checkpoint the guest again. Until then, let the guest
+     * continue execution.
+     */
+
+    /* Set checkpoint interval timeout */
+    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
+                                     remus_next_checkpoint,
+                                     dss->interval);
+
+    if (rc)
+        goto out;
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+}
+
+static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
+                                  const struct timeval *requested_abs)
+{
+    libxl__domain_suspend_state *dss =
+                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
+
+    STATE_AO_GC(dss->ao);
+
+    /*
+     * Time to checkpoint the guest again. We return 1 to libxc
+     * (xc_domain_save.c). in order to continue executing the infinite loop
+     * (suspend, checkpoint, resume) in xc_domain_save().
+     */
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
+}
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
-- 
1.9.1

Re: [PATCH v2 4/6] tools/libxl: move remus code into libxl_remus.c

[Ian Campbell]

On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
> move remus code into libxl_remus.c.

Please say something like "... by refactoring bits of
libxl_domain_remus_start and domain_save_done into X and Y and moving
the remaining functionality unchanged into the new file".

I gave two examples of functions which changed there, but please make
sure the list is complete+accurate.

> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> CC: Ian Campbell <Ian.Campbell@citrix.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> ---
>  tools/libxl/Makefile         |   2 +-
>  tools/libxl/libxl.c          |  55 +-------
>  tools/libxl/libxl_dom.c      | 206 +----------------------------
>  tools/libxl/libxl_internal.h |  11 ++
>  tools/libxl/libxl_remus.c    | 304 +++++++++++++++++++++++++++++++++++++++++++
>  5 files changed, 318 insertions(+), 260 deletions(-)
>  create mode 100644 tools/libxl/libxl_remus.c
> 
> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
> index 3f98d62..8535eaa 100644
> --- a/tools/libxl/Makefile
> +++ b/tools/libxl/Makefile
> @@ -56,7 +56,7 @@ else
>  LIBXL_OBJS-y += libxl_nonetbuffer.o
>  endif
>  
> -LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o
> +LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
>  
>  LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
>  LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> index 77c6a36..0f9248e 100644
> --- a/tools/libxl/libxl.c
> +++ b/tools/libxl/libxl.c
> @@ -792,10 +792,6 @@ out:
>      return ptr;
>  }
>  
> -static void libxl__remus_setup_done(libxl__egc *egc,
> -                                    libxl__remus_devices_state *rds, int rc);
> -static void libxl__remus_setup_failed(libxl__egc *egc,
> -                                      libxl__remus_devices_state *rds, int rc);
>  static void remus_failover_cb(libxl__egc *egc,
>                                libxl__domain_suspend_state *dss, int rc);
>  
> @@ -844,63 +840,14 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
>  
>      assert(info);
>  
> -    /* Convenience aliases */
> -    libxl__remus_devices_state *const rds = &dss->rds;
> -
> -    if (libxl_defbool_val(info->netbuf)) {
> -        if (!libxl__netbuffer_enabled(gc)) {
> -            LOG(ERROR, "Remus: No support for network buffering");
> -            rc = ERROR_FAIL;
> -            goto out;
> -        }
> -        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
> -    }
> -
> -    if (libxl_defbool_val(info->diskbuf))
> -        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
> -
> -    rds->ao = ao;
> -    rds->domid = domid;
> -    rds->callback = libxl__remus_setup_done;
> -
>      /* Point of no return */
> -    libxl__remus_devices_setup(egc, rds);
> +    libxl__remus_setup(egc, dss);
>      return AO_INPROGRESS;
>  
>   out:
>      return AO_ABORT(rc);
>  }
>  
> -static void libxl__remus_setup_done(libxl__egc *egc,
> -                                    libxl__remus_devices_state *rds, int rc)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> -    STATE_AO_GC(dss->ao);
> -
> -    if (!rc) {
> -        libxl__domain_save(egc, dss);
> -        return;
> -    }
> -
> -    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
> -        dss->domid, rc);
> -    rds->callback = libxl__remus_setup_failed;
> -    libxl__remus_devices_teardown(egc, rds);
> -}
> -
> -static void libxl__remus_setup_failed(libxl__egc *egc,
> -                                      libxl__remus_devices_state *rds, int rc)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> -    STATE_AO_GC(dss->ao);
> -
> -    if (rc)
> -        LOG(ERROR, "Remus: failed to teardown device after setup failed"
> -            " for guest with domid %u, rc %d", dss->domid, rc);
> -
> -    dss->callback(egc, dss, rc);
> -}
> -
>  static void remus_failover_cb(libxl__egc *egc,
>                                libxl__domain_suspend_state *dss, int rc)
>  {
> diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
> index 701e9f7..0f81081 100644
> --- a/tools/libxl/libxl_dom.c
> +++ b/tools/libxl/libxl_dom.c
> @@ -1409,189 +1409,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
>      return 0;
>  }
>  
> -/*----- remus callbacks -----*/
> -static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> -                                libxl__domain_suspend_state *dss, int ok);
> -static void remus_devices_postsuspend_cb(libxl__egc *egc,
> -                                         libxl__remus_devices_state *rds,
> -                                         int rc);
> -static void remus_devices_preresume_cb(libxl__egc *egc,
> -                                       libxl__remus_devices_state *rds,
> -                                       int rc);
> -
> -static void libxl__remus_domain_suspend_callback(void *data)
> -{
> -    libxl__save_helper_state *shs = data;
> -    libxl__egc *egc = shs->egc;
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> -
> -    dss->callback_common_done = remus_domain_suspend_callback_common_done;
> -    libxl__domain_suspend(egc, dss);
> -}
> -
> -static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> -                                libxl__domain_suspend_state *dss, int ok)
> -{
> -    if (!ok)
> -        goto out;
> -
> -    libxl__remus_devices_state *const rds = &dss->rds;
> -    rds->callback = remus_devices_postsuspend_cb;
> -    libxl__remus_devices_postsuspend(egc, rds);
> -    return;
> -
> -out:
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> -}
> -
> -static void remus_devices_postsuspend_cb(libxl__egc *egc,
> -                                         libxl__remus_devices_state *rds,
> -                                         int rc)
> -{
> -    int ok = 0;
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> -
> -    if (rc)
> -        goto out;
> -
> -    ok = 1;
> -
> -out:
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> -}
> -
> -static void libxl__remus_domain_resume_callback(void *data)
> -{
> -    libxl__save_helper_state *shs = data;
> -    libxl__egc *egc = shs->egc;
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> -    STATE_AO_GC(dss->ao);
> -
> -    libxl__remus_devices_state *const rds = &dss->rds;
> -    rds->callback = remus_devices_preresume_cb;
> -    libxl__remus_devices_preresume(egc, rds);
> -}
> -
> -static void remus_devices_preresume_cb(libxl__egc *egc,
> -                                       libxl__remus_devices_state *rds,
> -                                       int rc)
> -{
> -    int ok = 0;
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> -    STATE_AO_GC(dss->ao);
> -
> -    if (rc)
> -        goto out;
> -
> -    /* Resumes the domain and the device model */
> -    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
> -    if (rc)
> -        goto out;
> -
> -    ok = 1;
> -
> -out:
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> -}
> -
> -/*----- remus asynchronous checkpoint callback -----*/
> -
> -static void remus_checkpoint_dm_saved(libxl__egc *egc,
> -                                      libxl__domain_suspend_state *dss, int rc);
> -static void remus_devices_commit_cb(libxl__egc *egc,
> -                                    libxl__remus_devices_state *rds,
> -                                    int rc);
> -static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
> -                                  const struct timeval *requested_abs);
> -
> -static void libxl__remus_domain_checkpoint_callback(void *data)
> -{
> -    libxl__save_helper_state *shs = data;
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> -    libxl__egc *egc = dss->shs.egc;
> -    STATE_AO_GC(dss->ao);
> -
> -    /* This would go into tailbuf. */
> -    if (dss->hvm) {
> -        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
> -    } else {
> -        remus_checkpoint_dm_saved(egc, dss, 0);
> -    }
> -}
> -
> -static void remus_checkpoint_dm_saved(libxl__egc *egc,
> -                                      libxl__domain_suspend_state *dss, int rc)
> -{
> -    /* Convenience aliases */
> -    libxl__remus_devices_state *const rds = &dss->rds;
> -
> -    STATE_AO_GC(dss->ao);
> -
> -    if (rc) {
> -        LOG(ERROR, "Failed to save device model. Terminating Remus..");
> -        goto out;
> -    }
> -
> -    rds->callback = remus_devices_commit_cb;
> -    libxl__remus_devices_commit(egc, rds);
> -
> -    return;
> -
> -out:
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
> -}
> -
> -static void remus_devices_commit_cb(libxl__egc *egc,
> -                                    libxl__remus_devices_state *rds,
> -                                    int rc)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> -
> -    STATE_AO_GC(dss->ao);
> -
> -    if (rc) {
> -        LOG(ERROR, "Failed to do device commit op."
> -            " Terminating Remus..");
> -        goto out;
> -    }
> -
> -    /*
> -     * At this point, we have successfully checkpointed the guest and
> -     * committed it at the backup. We'll come back after the checkpoint
> -     * interval to checkpoint the guest again. Until then, let the guest
> -     * continue execution.
> -     */
> -
> -    /* Set checkpoint interval timeout */
> -    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
> -                                     remus_next_checkpoint,
> -                                     dss->interval);
> -
> -    if (rc)
> -        goto out;
> -
> -    return;
> -
> -out:
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
> -}
> -
> -static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
> -                                  const struct timeval *requested_abs)
> -{
> -    libxl__domain_suspend_state *dss =
> -                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
> -
> -    STATE_AO_GC(dss->ao);
> -
> -    /*
> -     * Time to checkpoint the guest again. We return 1 to libxc
> -     * (xc_domain_save.c). in order to continue executing the infinite loop
> -     * (suspend, checkpoint, resume) in xc_domain_save().
> -     */
> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
> -}
> -
>  /*----- main code for suspending, in order of execution -----*/
>  
>  static void domain_save_done(libxl__egc *egc,
> @@ -1811,10 +1628,6 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
>      dss->save_dm_callback(egc, dss, our_rc);
>  }
>  
> -static void remus_teardown_done(libxl__egc *egc,
> -                                       libxl__remus_devices_state *rds,
> -                                       int rc);
> -
>  static void domain_save_done(libxl__egc *egc,
>                               libxl__domain_suspend_state *dss, int rc)
>  {
> @@ -1840,24 +1653,7 @@ static void domain_save_done(libxl__egc *egc,
>       * from sending checkpoints. Teardown the network buffers and
>       * release netlink resources.  This is an async op.
>       */
> -    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
> -        " teardown Remus devices...", rc);
> -    dss->rds.callback = remus_teardown_done;
> -    libxl__remus_devices_teardown(egc, &dss->rds);
> -}
> -
> -static void remus_teardown_done(libxl__egc *egc,
> -                                       libxl__remus_devices_state *rds,
> -                                       int rc)
> -{
> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> -    STATE_AO_GC(dss->ao);
> -
> -    if (rc)
> -        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
> -            " rc %d", dss->domid, rc);
> -
> -    dss->callback(egc, dss, rc);
> +    libxl__remus_teardown(egc, dss, rc);
>  }
>  
>  /*==================== Miscellaneous ====================*/
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 164c448..dc6b62b 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -3193,6 +3193,17 @@ _hidden void libxl__domain_suspend(libxl__egc *egc,
>  /* used by libxc to suspend the guest during migration */
>  _hidden void libxl__domain_suspend_callback(void *data);
>  
> +/* Remus callbacks for save */
> +_hidden void libxl__remus_domain_suspend_callback(void *data);
> +_hidden void libxl__remus_domain_resume_callback(void *data);
> +_hidden void libxl__remus_domain_checkpoint_callback(void *data);
> +/* Remus setup and teardown*/
> +_hidden void libxl__remus_setup(libxl__egc *egc,
> +                                libxl__domain_suspend_state *dss);
> +_hidden void libxl__remus_teardown(libxl__egc *egc,
> +                                   libxl__domain_suspend_state *dss,
> +                                   int rc);
> +
>  /*
>   * Convenience macros.
>   */
> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
> new file mode 100644
> index 0000000..51261e0
> --- /dev/null
> +++ b/tools/libxl/libxl_remus.c
> @@ -0,0 +1,304 @@
> +/*
> + * Copyright (C) 2009      Citrix Ltd.
> + * Author Yang Hongyang <yanghy@cn.fujitsu.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2 and later. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#include "libxl_osdeps.h" /* must come before any other headers */
> +
> +#include "libxl_internal.h"
> +
> +/*----- Remus setup and teardown -----*/
> +
> +static void remus_setup_done(libxl__egc *egc,
> +                             libxl__remus_devices_state *rds, int rc);
> +static void remus_setup_failed(libxl__egc *egc,
> +                               libxl__remus_devices_state *rds, int rc);
> +
> +void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
> +{
> +    /* Convenience aliases */
> +    libxl__remus_devices_state *const rds = &dss->rds;
> +    const libxl_domain_remus_info *const info = dss->remus;
> +
> +    STATE_AO_GC(dss->ao);
> +
> +    if (libxl_defbool_val(info->netbuf)) {
> +        if (!libxl__netbuffer_enabled(gc)) {
> +            LOG(ERROR, "Remus: No support for network buffering");
> +            goto out;
> +        }
> +        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
> +    }
> +
> +    if (libxl_defbool_val(info->diskbuf))
> +        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
> +
> +    rds->ao = ao;
> +    rds->domid = dss->domid;
> +    rds->callback = remus_setup_done;
> +
> +    libxl__remus_devices_setup(egc, rds);
> +    return;
> +
> +out:
> +    dss->callback(egc, dss, ERROR_FAIL);
> +}
> +
> +static void remus_setup_done(libxl__egc *egc,
> +                             libxl__remus_devices_state *rds, int rc)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> +    STATE_AO_GC(dss->ao);
> +
> +    if (!rc) {
> +        libxl__domain_save(egc, dss);
> +        return;
> +    }
> +
> +    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
> +        dss->domid, rc);
> +    rds->callback = remus_setup_failed;
> +    libxl__remus_devices_teardown(egc, rds);
> +}
> +
> +static void remus_setup_failed(libxl__egc *egc,
> +                               libxl__remus_devices_state *rds, int rc)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> +    STATE_AO_GC(dss->ao);
> +
> +    if (rc)
> +        LOG(ERROR, "Remus: failed to teardown device after setup failed"
> +            " for guest with domid %u, rc %d", dss->domid, rc);
> +
> +    dss->callback(egc, dss, rc);
> +}
> +
> +static void remus_teardown_done(libxl__egc *egc,
> +                                libxl__remus_devices_state *rds,
> +                                int rc);
> +void libxl__remus_teardown(libxl__egc *egc,
> +                           libxl__domain_suspend_state *dss,
> +                           int rc)
> +{
> +    EGC_GC;
> +
> +    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
> +        " teardown Remus devices...", rc);
> +    dss->rds.callback = remus_teardown_done;
> +    libxl__remus_devices_teardown(egc, &dss->rds);
> +}
> +
> +static void remus_teardown_done(libxl__egc *egc,
> +                                libxl__remus_devices_state *rds,
> +                                int rc)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> +    STATE_AO_GC(dss->ao);
> +
> +    if (rc)
> +        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
> +            " rc %d", dss->domid, rc);
> +
> +    dss->callback(egc, dss, rc);
> +}
> +
> +/*----- remus callbacks -----*/
> +static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> +                                libxl__domain_suspend_state *dss, int ok);
> +static void remus_devices_postsuspend_cb(libxl__egc *egc,
> +                                         libxl__remus_devices_state *rds,
> +                                         int rc);
> +static void remus_devices_preresume_cb(libxl__egc *egc,
> +                                       libxl__remus_devices_state *rds,
> +                                       int rc);
> +
> +void libxl__remus_domain_suspend_callback(void *data)
> +{
> +    libxl__save_helper_state *shs = data;
> +    libxl__egc *egc = shs->egc;
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> +
> +    dss->callback_common_done = remus_domain_suspend_callback_common_done;
> +    libxl__domain_suspend(egc, dss);
> +}
> +
> +static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> +                                libxl__domain_suspend_state *dss, int ok)
> +{
> +    if (!ok)
> +        goto out;
> +
> +    libxl__remus_devices_state *const rds = &dss->rds;
> +    rds->callback = remus_devices_postsuspend_cb;
> +    libxl__remus_devices_postsuspend(egc, rds);
> +    return;
> +
> +out:
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> +}
> +
> +static void remus_devices_postsuspend_cb(libxl__egc *egc,
> +                                         libxl__remus_devices_state *rds,
> +                                         int rc)
> +{
> +    int ok = 0;
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> +
> +    if (rc)
> +        goto out;
> +
> +    ok = 1;
> +
> +out:
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> +}
> +
> +void libxl__remus_domain_resume_callback(void *data)
> +{
> +    libxl__save_helper_state *shs = data;
> +    libxl__egc *egc = shs->egc;
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> +    STATE_AO_GC(dss->ao);
> +
> +    libxl__remus_devices_state *const rds = &dss->rds;
> +    rds->callback = remus_devices_preresume_cb;
> +    libxl__remus_devices_preresume(egc, rds);
> +}
> +
> +static void remus_devices_preresume_cb(libxl__egc *egc,
> +                                       libxl__remus_devices_state *rds,
> +                                       int rc)
> +{
> +    int ok = 0;
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> +    STATE_AO_GC(dss->ao);
> +
> +    if (rc)
> +        goto out;
> +
> +    /* Resumes the domain and the device model */
> +    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
> +    if (rc)
> +        goto out;
> +
> +    ok = 1;
> +
> +out:
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
> +}
> +
> +/*----- remus asynchronous checkpoint callback -----*/
> +
> +static void remus_checkpoint_dm_saved(libxl__egc *egc,
> +                                      libxl__domain_suspend_state *dss, int rc);
> +static void remus_devices_commit_cb(libxl__egc *egc,
> +                                    libxl__remus_devices_state *rds,
> +                                    int rc);
> +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
> +                                  const struct timeval *requested_abs);
> +
> +void libxl__remus_domain_checkpoint_callback(void *data)
> +{
> +    libxl__save_helper_state *shs = data;
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
> +    libxl__egc *egc = dss->shs.egc;
> +    STATE_AO_GC(dss->ao);
> +
> +    /* This would go into tailbuf. */
> +    if (dss->hvm) {
> +        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
> +    } else {
> +        remus_checkpoint_dm_saved(egc, dss, 0);
> +    }
> +}
> +
> +static void remus_checkpoint_dm_saved(libxl__egc *egc,
> +                                      libxl__domain_suspend_state *dss, int rc)
> +{
> +    /* Convenience aliases */
> +    libxl__remus_devices_state *const rds = &dss->rds;
> +
> +    STATE_AO_GC(dss->ao);
> +
> +    if (rc) {
> +        LOG(ERROR, "Failed to save device model. Terminating Remus..");
> +        goto out;
> +    }
> +
> +    rds->callback = remus_devices_commit_cb;
> +    libxl__remus_devices_commit(egc, rds);
> +
> +    return;
> +
> +out:
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
> +}
> +
> +static void remus_devices_commit_cb(libxl__egc *egc,
> +                                    libxl__remus_devices_state *rds,
> +                                    int rc)
> +{
> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
> +
> +    STATE_AO_GC(dss->ao);
> +
> +    if (rc) {
> +        LOG(ERROR, "Failed to do device commit op."
> +            " Terminating Remus..");
> +        goto out;
> +    }
> +
> +    /*
> +     * At this point, we have successfully checkpointed the guest and
> +     * committed it at the backup. We'll come back after the checkpoint
> +     * interval to checkpoint the guest again. Until then, let the guest
> +     * continue execution.
> +     */
> +
> +    /* Set checkpoint interval timeout */
> +    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
> +                                     remus_next_checkpoint,
> +                                     dss->interval);
> +
> +    if (rc)
> +        goto out;
> +
> +    return;
> +
> +out:
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
> +}
> +
> +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
> +                                  const struct timeval *requested_abs)
> +{
> +    libxl__domain_suspend_state *dss =
> +                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
> +
> +    STATE_AO_GC(dss->ao);
> +
> +    /*
> +     * Time to checkpoint the guest again. We return 1 to libxc
> +     * (xc_domain_save.c). in order to continue executing the infinite loop
> +     * (suspend, checkpoint, resume) in xc_domain_save().
> +     */
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
> +}
> +/*
> + * Local variables:
> + * mode: C
> + * c-basic-offset: 4
> + * indent-tabs-mode: nil
> + * End:
> + */

Re: [PATCH v2 4/6] tools/libxl: move remus code into libxl_remus.c

[Yang Hongyang]

On 06/16/2015 09:08 PM, Ian Campbell wrote:
> On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
>> move remus code into libxl_remus.c.
>
> Please say something like "... by refactoring bits of
> libxl_domain_remus_start and domain_save_done into X and Y and moving
> the remaining functionality unchanged into the new file".
>
> I gave two examples of functions which changed there, but please make
> sure the list is complete+accurate.

Okay, will fix the commit message next time, thank you!

>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> CC: Ian Campbell <Ian.Campbell@citrix.com>
>> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
>> CC: Wei Liu <wei.liu2@citrix.com>
>> ---
>>   tools/libxl/Makefile         |   2 +-
>>   tools/libxl/libxl.c          |  55 +-------
>>   tools/libxl/libxl_dom.c      | 206 +----------------------------
>>   tools/libxl/libxl_internal.h |  11 ++
>>   tools/libxl/libxl_remus.c    | 304 +++++++++++++++++++++++++++++++++++++++++++
>>   5 files changed, 318 insertions(+), 260 deletions(-)
>>   create mode 100644 tools/libxl/libxl_remus.c
>>
>> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
>> index 3f98d62..8535eaa 100644
>> --- a/tools/libxl/Makefile
>> +++ b/tools/libxl/Makefile
>> @@ -56,7 +56,7 @@ else
>>   LIBXL_OBJS-y += libxl_nonetbuffer.o
>>   endif
>>
>> -LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o
>> +LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
>>
>>   LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
>>   LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
>> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
>> index 77c6a36..0f9248e 100644
>> --- a/tools/libxl/libxl.c
>> +++ b/tools/libxl/libxl.c
>> @@ -792,10 +792,6 @@ out:
>>       return ptr;
>>   }
>>
>> -static void libxl__remus_setup_done(libxl__egc *egc,
>> -                                    libxl__remus_devices_state *rds, int rc);
>> -static void libxl__remus_setup_failed(libxl__egc *egc,
>> -                                      libxl__remus_devices_state *rds, int rc);
>>   static void remus_failover_cb(libxl__egc *egc,
>>                                 libxl__domain_suspend_state *dss, int rc);
>>
>> @@ -844,63 +840,14 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
>>
>>       assert(info);
>>
>> -    /* Convenience aliases */
>> -    libxl__remus_devices_state *const rds = &dss->rds;
>> -
>> -    if (libxl_defbool_val(info->netbuf)) {
>> -        if (!libxl__netbuffer_enabled(gc)) {
>> -            LOG(ERROR, "Remus: No support for network buffering");
>> -            rc = ERROR_FAIL;
>> -            goto out;
>> -        }
>> -        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
>> -    }
>> -
>> -    if (libxl_defbool_val(info->diskbuf))
>> -        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
>> -
>> -    rds->ao = ao;
>> -    rds->domid = domid;
>> -    rds->callback = libxl__remus_setup_done;
>> -
>>       /* Point of no return */
>> -    libxl__remus_devices_setup(egc, rds);
>> +    libxl__remus_setup(egc, dss);
>>       return AO_INPROGRESS;
>>
>>    out:
>>       return AO_ABORT(rc);
>>   }
>>
>> -static void libxl__remus_setup_done(libxl__egc *egc,
>> -                                    libxl__remus_devices_state *rds, int rc)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    if (!rc) {
>> -        libxl__domain_save(egc, dss);
>> -        return;
>> -    }
>> -
>> -    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
>> -        dss->domid, rc);
>> -    rds->callback = libxl__remus_setup_failed;
>> -    libxl__remus_devices_teardown(egc, rds);
>> -}
>> -
>> -static void libxl__remus_setup_failed(libxl__egc *egc,
>> -                                      libxl__remus_devices_state *rds, int rc)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    if (rc)
>> -        LOG(ERROR, "Remus: failed to teardown device after setup failed"
>> -            " for guest with domid %u, rc %d", dss->domid, rc);
>> -
>> -    dss->callback(egc, dss, rc);
>> -}
>> -
>>   static void remus_failover_cb(libxl__egc *egc,
>>                                 libxl__domain_suspend_state *dss, int rc)
>>   {
>> diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
>> index 701e9f7..0f81081 100644
>> --- a/tools/libxl/libxl_dom.c
>> +++ b/tools/libxl/libxl_dom.c
>> @@ -1409,189 +1409,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
>>       return 0;
>>   }
>>
>> -/*----- remus callbacks -----*/
>> -static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> -                                libxl__domain_suspend_state *dss, int ok);
>> -static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> -                                         libxl__remus_devices_state *rds,
>> -                                         int rc);
>> -static void remus_devices_preresume_cb(libxl__egc *egc,
>> -                                       libxl__remus_devices_state *rds,
>> -                                       int rc);
>> -
>> -static void libxl__remus_domain_suspend_callback(void *data)
>> -{
>> -    libxl__save_helper_state *shs = data;
>> -    libxl__egc *egc = shs->egc;
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> -
>> -    dss->callback_common_done = remus_domain_suspend_callback_common_done;
>> -    libxl__domain_suspend(egc, dss);
>> -}
>> -
>> -static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> -                                libxl__domain_suspend_state *dss, int ok)
>> -{
>> -    if (!ok)
>> -        goto out;
>> -
>> -    libxl__remus_devices_state *const rds = &dss->rds;
>> -    rds->callback = remus_devices_postsuspend_cb;
>> -    libxl__remus_devices_postsuspend(egc, rds);
>> -    return;
>> -
>> -out:
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> -}
>> -
>> -static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> -                                         libxl__remus_devices_state *rds,
>> -                                         int rc)
>> -{
>> -    int ok = 0;
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> -
>> -    if (rc)
>> -        goto out;
>> -
>> -    ok = 1;
>> -
>> -out:
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> -}
>> -
>> -static void libxl__remus_domain_resume_callback(void *data)
>> -{
>> -    libxl__save_helper_state *shs = data;
>> -    libxl__egc *egc = shs->egc;
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    libxl__remus_devices_state *const rds = &dss->rds;
>> -    rds->callback = remus_devices_preresume_cb;
>> -    libxl__remus_devices_preresume(egc, rds);
>> -}
>> -
>> -static void remus_devices_preresume_cb(libxl__egc *egc,
>> -                                       libxl__remus_devices_state *rds,
>> -                                       int rc)
>> -{
>> -    int ok = 0;
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    if (rc)
>> -        goto out;
>> -
>> -    /* Resumes the domain and the device model */
>> -    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
>> -    if (rc)
>> -        goto out;
>> -
>> -    ok = 1;
>> -
>> -out:
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> -}
>> -
>> -/*----- remus asynchronous checkpoint callback -----*/
>> -
>> -static void remus_checkpoint_dm_saved(libxl__egc *egc,
>> -                                      libxl__domain_suspend_state *dss, int rc);
>> -static void remus_devices_commit_cb(libxl__egc *egc,
>> -                                    libxl__remus_devices_state *rds,
>> -                                    int rc);
>> -static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>> -                                  const struct timeval *requested_abs);
>> -
>> -static void libxl__remus_domain_checkpoint_callback(void *data)
>> -{
>> -    libxl__save_helper_state *shs = data;
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> -    libxl__egc *egc = dss->shs.egc;
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    /* This would go into tailbuf. */
>> -    if (dss->hvm) {
>> -        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
>> -    } else {
>> -        remus_checkpoint_dm_saved(egc, dss, 0);
>> -    }
>> -}
>> -
>> -static void remus_checkpoint_dm_saved(libxl__egc *egc,
>> -                                      libxl__domain_suspend_state *dss, int rc)
>> -{
>> -    /* Convenience aliases */
>> -    libxl__remus_devices_state *const rds = &dss->rds;
>> -
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    if (rc) {
>> -        LOG(ERROR, "Failed to save device model. Terminating Remus..");
>> -        goto out;
>> -    }
>> -
>> -    rds->callback = remus_devices_commit_cb;
>> -    libxl__remus_devices_commit(egc, rds);
>> -
>> -    return;
>> -
>> -out:
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
>> -}
>> -
>> -static void remus_devices_commit_cb(libxl__egc *egc,
>> -                                    libxl__remus_devices_state *rds,
>> -                                    int rc)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> -
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    if (rc) {
>> -        LOG(ERROR, "Failed to do device commit op."
>> -            " Terminating Remus..");
>> -        goto out;
>> -    }
>> -
>> -    /*
>> -     * At this point, we have successfully checkpointed the guest and
>> -     * committed it at the backup. We'll come back after the checkpoint
>> -     * interval to checkpoint the guest again. Until then, let the guest
>> -     * continue execution.
>> -     */
>> -
>> -    /* Set checkpoint interval timeout */
>> -    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
>> -                                     remus_next_checkpoint,
>> -                                     dss->interval);
>> -
>> -    if (rc)
>> -        goto out;
>> -
>> -    return;
>> -
>> -out:
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
>> -}
>> -
>> -static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>> -                                  const struct timeval *requested_abs)
>> -{
>> -    libxl__domain_suspend_state *dss =
>> -                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
>> -
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    /*
>> -     * Time to checkpoint the guest again. We return 1 to libxc
>> -     * (xc_domain_save.c). in order to continue executing the infinite loop
>> -     * (suspend, checkpoint, resume) in xc_domain_save().
>> -     */
>> -    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
>> -}
>> -
>>   /*----- main code for suspending, in order of execution -----*/
>>
>>   static void domain_save_done(libxl__egc *egc,
>> @@ -1811,10 +1628,6 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
>>       dss->save_dm_callback(egc, dss, our_rc);
>>   }
>>
>> -static void remus_teardown_done(libxl__egc *egc,
>> -                                       libxl__remus_devices_state *rds,
>> -                                       int rc);
>> -
>>   static void domain_save_done(libxl__egc *egc,
>>                                libxl__domain_suspend_state *dss, int rc)
>>   {
>> @@ -1840,24 +1653,7 @@ static void domain_save_done(libxl__egc *egc,
>>        * from sending checkpoints. Teardown the network buffers and
>>        * release netlink resources.  This is an async op.
>>        */
>> -    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
>> -        " teardown Remus devices...", rc);
>> -    dss->rds.callback = remus_teardown_done;
>> -    libxl__remus_devices_teardown(egc, &dss->rds);
>> -}
>> -
>> -static void remus_teardown_done(libxl__egc *egc,
>> -                                       libxl__remus_devices_state *rds,
>> -                                       int rc)
>> -{
>> -    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> -    STATE_AO_GC(dss->ao);
>> -
>> -    if (rc)
>> -        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
>> -            " rc %d", dss->domid, rc);
>> -
>> -    dss->callback(egc, dss, rc);
>> +    libxl__remus_teardown(egc, dss, rc);
>>   }
>>
>>   /*==================== Miscellaneous ====================*/
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index 164c448..dc6b62b 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
>> @@ -3193,6 +3193,17 @@ _hidden void libxl__domain_suspend(libxl__egc *egc,
>>   /* used by libxc to suspend the guest during migration */
>>   _hidden void libxl__domain_suspend_callback(void *data);
>>
>> +/* Remus callbacks for save */
>> +_hidden void libxl__remus_domain_suspend_callback(void *data);
>> +_hidden void libxl__remus_domain_resume_callback(void *data);
>> +_hidden void libxl__remus_domain_checkpoint_callback(void *data);
>> +/* Remus setup and teardown*/
>> +_hidden void libxl__remus_setup(libxl__egc *egc,
>> +                                libxl__domain_suspend_state *dss);
>> +_hidden void libxl__remus_teardown(libxl__egc *egc,
>> +                                   libxl__domain_suspend_state *dss,
>> +                                   int rc);
>> +
>>   /*
>>    * Convenience macros.
>>    */
>> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
>> new file mode 100644
>> index 0000000..51261e0
>> --- /dev/null
>> +++ b/tools/libxl/libxl_remus.c
>> @@ -0,0 +1,304 @@
>> +/*
>> + * Copyright (C) 2009      Citrix Ltd.
>> + * Author Yang Hongyang <yanghy@cn.fujitsu.com>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU Lesser General Public License as published
>> + * by the Free Software Foundation; version 2 and later. with the special
>> + * exception on linking described in file LICENSE.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU Lesser General Public License for more details.
>> + */
>> +
>> +#include "libxl_osdeps.h" /* must come before any other headers */
>> +
>> +#include "libxl_internal.h"
>> +
>> +/*----- Remus setup and teardown -----*/
>> +
>> +static void remus_setup_done(libxl__egc *egc,
>> +                             libxl__remus_devices_state *rds, int rc);
>> +static void remus_setup_failed(libxl__egc *egc,
>> +                               libxl__remus_devices_state *rds, int rc);
>> +
>> +void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
>> +{
>> +    /* Convenience aliases */
>> +    libxl__remus_devices_state *const rds = &dss->rds;
>> +    const libxl_domain_remus_info *const info = dss->remus;
>> +
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (libxl_defbool_val(info->netbuf)) {
>> +        if (!libxl__netbuffer_enabled(gc)) {
>> +            LOG(ERROR, "Remus: No support for network buffering");
>> +            goto out;
>> +        }
>> +        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
>> +    }
>> +
>> +    if (libxl_defbool_val(info->diskbuf))
>> +        rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
>> +
>> +    rds->ao = ao;
>> +    rds->domid = dss->domid;
>> +    rds->callback = remus_setup_done;
>> +
>> +    libxl__remus_devices_setup(egc, rds);
>> +    return;
>> +
>> +out:
>> +    dss->callback(egc, dss, ERROR_FAIL);
>> +}
>> +
>> +static void remus_setup_done(libxl__egc *egc,
>> +                             libxl__remus_devices_state *rds, int rc)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (!rc) {
>> +        libxl__domain_save(egc, dss);
>> +        return;
>> +    }
>> +
>> +    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
>> +        dss->domid, rc);
>> +    rds->callback = remus_setup_failed;
>> +    libxl__remus_devices_teardown(egc, rds);
>> +}
>> +
>> +static void remus_setup_failed(libxl__egc *egc,
>> +                               libxl__remus_devices_state *rds, int rc)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (rc)
>> +        LOG(ERROR, "Remus: failed to teardown device after setup failed"
>> +            " for guest with domid %u, rc %d", dss->domid, rc);
>> +
>> +    dss->callback(egc, dss, rc);
>> +}
>> +
>> +static void remus_teardown_done(libxl__egc *egc,
>> +                                libxl__remus_devices_state *rds,
>> +                                int rc);
>> +void libxl__remus_teardown(libxl__egc *egc,
>> +                           libxl__domain_suspend_state *dss,
>> +                           int rc)
>> +{
>> +    EGC_GC;
>> +
>> +    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
>> +        " teardown Remus devices...", rc);
>> +    dss->rds.callback = remus_teardown_done;
>> +    libxl__remus_devices_teardown(egc, &dss->rds);
>> +}
>> +
>> +static void remus_teardown_done(libxl__egc *egc,
>> +                                libxl__remus_devices_state *rds,
>> +                                int rc)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (rc)
>> +        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
>> +            " rc %d", dss->domid, rc);
>> +
>> +    dss->callback(egc, dss, rc);
>> +}
>> +
>> +/*----- remus callbacks -----*/
>> +static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> +                                libxl__domain_suspend_state *dss, int ok);
>> +static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> +                                         libxl__remus_devices_state *rds,
>> +                                         int rc);
>> +static void remus_devices_preresume_cb(libxl__egc *egc,
>> +                                       libxl__remus_devices_state *rds,
>> +                                       int rc);
>> +
>> +void libxl__remus_domain_suspend_callback(void *data)
>> +{
>> +    libxl__save_helper_state *shs = data;
>> +    libxl__egc *egc = shs->egc;
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> +
>> +    dss->callback_common_done = remus_domain_suspend_callback_common_done;
>> +    libxl__domain_suspend(egc, dss);
>> +}
>> +
>> +static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> +                                libxl__domain_suspend_state *dss, int ok)
>> +{
>> +    if (!ok)
>> +        goto out;
>> +
>> +    libxl__remus_devices_state *const rds = &dss->rds;
>> +    rds->callback = remus_devices_postsuspend_cb;
>> +    libxl__remus_devices_postsuspend(egc, rds);
>> +    return;
>> +
>> +out:
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> +}
>> +
>> +static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> +                                         libxl__remus_devices_state *rds,
>> +                                         int rc)
>> +{
>> +    int ok = 0;
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> +
>> +    if (rc)
>> +        goto out;
>> +
>> +    ok = 1;
>> +
>> +out:
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> +}
>> +
>> +void libxl__remus_domain_resume_callback(void *data)
>> +{
>> +    libxl__save_helper_state *shs = data;
>> +    libxl__egc *egc = shs->egc;
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    libxl__remus_devices_state *const rds = &dss->rds;
>> +    rds->callback = remus_devices_preresume_cb;
>> +    libxl__remus_devices_preresume(egc, rds);
>> +}
>> +
>> +static void remus_devices_preresume_cb(libxl__egc *egc,
>> +                                       libxl__remus_devices_state *rds,
>> +                                       int rc)
>> +{
>> +    int ok = 0;
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (rc)
>> +        goto out;
>> +
>> +    /* Resumes the domain and the device model */
>> +    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
>> +    if (rc)
>> +        goto out;
>> +
>> +    ok = 1;
>> +
>> +out:
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
>> +}
>> +
>> +/*----- remus asynchronous checkpoint callback -----*/
>> +
>> +static void remus_checkpoint_dm_saved(libxl__egc *egc,
>> +                                      libxl__domain_suspend_state *dss, int rc);
>> +static void remus_devices_commit_cb(libxl__egc *egc,
>> +                                    libxl__remus_devices_state *rds,
>> +                                    int rc);
>> +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>> +                                  const struct timeval *requested_abs);
>> +
>> +void libxl__remus_domain_checkpoint_callback(void *data)
>> +{
>> +    libxl__save_helper_state *shs = data;
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
>> +    libxl__egc *egc = dss->shs.egc;
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    /* This would go into tailbuf. */
>> +    if (dss->hvm) {
>> +        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
>> +    } else {
>> +        remus_checkpoint_dm_saved(egc, dss, 0);
>> +    }
>> +}
>> +
>> +static void remus_checkpoint_dm_saved(libxl__egc *egc,
>> +                                      libxl__domain_suspend_state *dss, int rc)
>> +{
>> +    /* Convenience aliases */
>> +    libxl__remus_devices_state *const rds = &dss->rds;
>> +
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (rc) {
>> +        LOG(ERROR, "Failed to save device model. Terminating Remus..");
>> +        goto out;
>> +    }
>> +
>> +    rds->callback = remus_devices_commit_cb;
>> +    libxl__remus_devices_commit(egc, rds);
>> +
>> +    return;
>> +
>> +out:
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
>> +}
>> +
>> +static void remus_devices_commit_cb(libxl__egc *egc,
>> +                                    libxl__remus_devices_state *rds,
>> +                                    int rc)
>> +{
>> +    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
>> +
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    if (rc) {
>> +        LOG(ERROR, "Failed to do device commit op."
>> +            " Terminating Remus..");
>> +        goto out;
>> +    }
>> +
>> +    /*
>> +     * At this point, we have successfully checkpointed the guest and
>> +     * committed it at the backup. We'll come back after the checkpoint
>> +     * interval to checkpoint the guest again. Until then, let the guest
>> +     * continue execution.
>> +     */
>> +
>> +    /* Set checkpoint interval timeout */
>> +    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
>> +                                     remus_next_checkpoint,
>> +                                     dss->interval);
>> +
>> +    if (rc)
>> +        goto out;
>> +
>> +    return;
>> +
>> +out:
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
>> +}
>> +
>> +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>> +                                  const struct timeval *requested_abs)
>> +{
>> +    libxl__domain_suspend_state *dss =
>> +                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
>> +
>> +    STATE_AO_GC(dss->ao);
>> +
>> +    /*
>> +     * Time to checkpoint the guest again. We return 1 to libxc
>> +     * (xc_domain_save.c). in order to continue executing the infinite loop
>> +     * (suspend, checkpoint, resume) in xc_domain_save().
>> +     */
>> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
>> +}
>> +/*
>> + * Local variables:
>> + * mode: C
>> + * c-basic-offset: 4
>> + * indent-tabs-mode: nil
>> + * End:
>> + */
>
>
> .
>

-- 
Thanks,
Yang.

[PATCH v2 5/6] tools/libxl: move save/restore code into libxl_dom_save.c

[Yang Hongyang]

move save/restore code into libxl_dom_save.c.

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxl/Makefile         |   2 +-
 tools/libxl/libxl_dom.c      | 636 -----------------------------------------
 tools/libxl/libxl_dom_save.c | 664 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 665 insertions(+), 637 deletions(-)
 create mode 100644 tools/libxl/libxl_dom_save.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 8535eaa..df51b22 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -96,7 +96,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
 			libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \
 			libxl_save_callout.o _libxl_save_msgs_callout.o \
 			libxl_qmp.o libxl_event.o libxl_fork.o libxl_dom_suspend.o \
-			$(LIBXL_OBJS-y)
+			libxl_dom_save.o $(LIBXL_OBJS-y)
 LIBXL_OBJS += libxl_genid.o
 LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
 
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 0f81081..d536475 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1020,642 +1020,6 @@ int libxl__qemu_traditional_cmd(libxl__gc *gc, uint32_t domid,
     return libxl__xs_write(gc, XBT_NULL, path, "%s", cmd);
 }
 
-struct libxl__physmap_info {
-    uint64_t phys_offset;
-    uint64_t start_addr;
-    uint64_t size;
-    uint32_t namelen;
-    char name[];
-};
-
-#define TOOLSTACK_SAVE_VERSION 1
-
-static inline char *restore_helper(libxl__gc *gc, uint32_t dm_domid,
-                                   uint32_t domid,
-                                   uint64_t phys_offset, char *node)
-{
-    return libxl__device_model_xs_path(gc, dm_domid, domid,
-                                       "/physmap/%"PRIx64"/%s",
-                                       phys_offset, node);
-}
-
-int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
-                             uint32_t size, void *user)
-{
-    libxl__save_helper_state *shs = user;
-    libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
-    STATE_AO_GC(dcs->ao);
-    int i, ret;
-    const uint8_t *ptr = buf;
-    uint32_t count = 0, version = 0;
-    struct libxl__physmap_info* pi;
-    char *xs_path;
-    uint32_t dm_domid;
-
-    LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, size);
-
-    if (size < sizeof(version) + sizeof(count)) {
-        LOG(ERROR, "wrong size");
-        return -1;
-    }
-
-    memcpy(&version, ptr, sizeof(version));
-    ptr += sizeof(version);
-
-    if (version != TOOLSTACK_SAVE_VERSION) {
-        LOG(ERROR, "wrong version");
-        return -1;
-    }
-
-    memcpy(&count, ptr, sizeof(count));
-    ptr += sizeof(count);
-
-    if (size < sizeof(version) + sizeof(count) +
-            count * (sizeof(struct libxl__physmap_info))) {
-        LOG(ERROR, "wrong size");
-        return -1;
-    }
-
-    dm_domid = libxl_get_stubdom_id(CTX, domid);
-    for (i = 0; i < count; i++) {
-        pi = (struct libxl__physmap_info*) ptr;
-        ptr += sizeof(struct libxl__physmap_info) + pi->namelen;
-
-        xs_path = restore_helper(gc, dm_domid, domid,
-                                 pi->phys_offset, "start_addr");
-        ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->start_addr);
-        if (ret)
-            return -1;
-        xs_path = restore_helper(gc, dm_domid, domid, pi->phys_offset, "size");
-        ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->size);
-        if (ret)
-            return -1;
-        if (pi->namelen > 0) {
-            xs_path = restore_helper(gc, dm_domid, domid,
-                                     pi->phys_offset, "name");
-            ret = libxl__xs_write(gc, 0, xs_path, "%s", pi->name);
-            if (ret)
-                return -1;
-        }
-    }
-    return 0;
-}
-
-/*==================== Domain suspend (save) ====================*/
-
-/*----- complicated callback, called by xc_domain_save -----*/
-
-/*
- * We implement the other end of protocol for controlling qemu-dm's
- * logdirty.  There is no documentation for this protocol, but our
- * counterparty's implementation is in
- * qemu-xen-traditional.git:xenstore.c in the function
- * xenstore_process_logdirty_event
- */
-
-static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
-                                    const struct timeval *requested_abs);
-static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
-                            const char *watch_path, const char *event_path);
-static void switch_logdirty_done(libxl__egc *egc,
-                                 libxl__domain_suspend_state *dss, int ok);
-
-static void logdirty_init(libxl__logdirty_switch *lds)
-{
-    lds->cmd_path = 0;
-    libxl__ev_xswatch_init(&lds->watch);
-    libxl__ev_time_init(&lds->timeout);
-}
-
-static void domain_suspend_switch_qemu_xen_traditional_logdirty
-                               (int domid, unsigned enable,
-                                libxl__save_helper_state *shs)
-{
-    libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-    libxl__logdirty_switch *lds = &dss->logdirty;
-    STATE_AO_GC(dss->ao);
-    int rc;
-    xs_transaction_t t = 0;
-    const char *got;
-
-    if (!lds->cmd_path) {
-        uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
-        lds->cmd_path = libxl__device_model_xs_path(gc, dm_domid, domid,
-                                                    "/logdirty/cmd");
-        lds->ret_path = libxl__device_model_xs_path(gc, dm_domid, domid,
-                                                    "/logdirty/ret");
-    }
-    lds->cmd = enable ? "enable" : "disable";
-
-    rc = libxl__ev_xswatch_register(gc, &lds->watch,
-                                switch_logdirty_xswatch, lds->ret_path);
-    if (rc) goto out;
-
-    rc = libxl__ev_time_register_rel(gc, &lds->timeout,
-                                switch_logdirty_timeout, 10*1000);
-    if (rc) goto out;
-
-    for (;;) {
-        rc = libxl__xs_transaction_start(gc, &t);
-        if (rc) goto out;
-
-        rc = libxl__xs_read_checked(gc, t, lds->cmd_path, &got);
-        if (rc) goto out;
-
-        if (got) {
-            const char *got_ret;
-            rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got_ret);
-            if (rc) goto out;
-
-            if (!got_ret || strcmp(got, got_ret)) {
-                LOG(ERROR,"controlling logdirty: qemu was already sent"
-                    " command `%s' (xenstore path `%s') but result is `%s'",
-                    got, lds->cmd_path, got_ret ? got_ret : "<none>");
-                rc = ERROR_FAIL;
-                goto out;
-            }
-            rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
-            if (rc) goto out;
-        }
-
-        rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
-        if (rc) goto out;
-
-        rc = libxl__xs_write_checked(gc, t, lds->cmd_path, lds->cmd);
-        if (rc) goto out;
-
-        rc = libxl__xs_transaction_commit(gc, &t);
-        if (!rc) break;
-        if (rc<0) goto out;
-    }
-
-    /* OK, wait for some callback */
-    return;
-
- out:
-    LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
-    libxl__xs_transaction_abort(gc, &t);
-    switch_logdirty_done(egc,dss,-1);
-}
-
-static void domain_suspend_switch_qemu_xen_logdirty
-                               (int domid, unsigned enable,
-                                libxl__save_helper_state *shs)
-{
-    libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-    STATE_AO_GC(dss->ao);
-    int rc;
-
-    rc = libxl__qmp_set_global_dirty_log(gc, domid, enable);
-    if (!rc) {
-        libxl__xc_domain_saverestore_async_callback_done(egc, shs, 0);
-    } else {
-        LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
-        libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
-    }
-}
-
-void libxl__domain_suspend_common_switch_qemu_logdirty
-                               (int domid, unsigned enable, void *user)
-{
-    libxl__save_helper_state *shs = user;
-    libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-    STATE_AO_GC(dss->ao);
-
-    switch (libxl__device_model_version_running(gc, domid)) {
-    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
-        domain_suspend_switch_qemu_xen_traditional_logdirty(domid, enable, shs);
-        break;
-    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
-        domain_suspend_switch_qemu_xen_logdirty(domid, enable, shs);
-        break;
-    default:
-        LOG(ERROR,"logdirty switch failed"
-            ", no valid device model version found, aborting suspend");
-        libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
-    }
-}
-static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
-                                    const struct timeval *requested_abs)
-{
-    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
-    STATE_AO_GC(dss->ao);
-    LOG(ERROR,"logdirty switch: wait for device model timed out");
-    switch_logdirty_done(egc,dss,-1);
-}
-
-static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
-                            const char *watch_path, const char *event_path)
-{
-    libxl__domain_suspend_state *dss =
-        CONTAINER_OF(watch, *dss, logdirty.watch);
-    libxl__logdirty_switch *lds = &dss->logdirty;
-    STATE_AO_GC(dss->ao);
-    const char *got;
-    xs_transaction_t t = 0;
-    int rc;
-
-    for (;;) {
-        rc = libxl__xs_transaction_start(gc, &t);
-        if (rc) goto out;
-
-        rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got);
-        if (rc) goto out;
-
-        if (!got) {
-            rc = +1;
-            goto out;
-        }
-
-        if (strcmp(got, lds->cmd)) {
-            LOG(ERROR,"logdirty switch: sent command `%s' but got reply `%s'"
-                " (xenstore paths `%s' / `%s')", lds->cmd, got,
-                lds->cmd_path, lds->ret_path);
-            rc = ERROR_FAIL;
-            goto out;
-        }
-
-        rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
-        if (rc) goto out;
-
-        rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
-        if (rc) goto out;
-
-        rc = libxl__xs_transaction_commit(gc, &t);
-        if (!rc) break;
-        if (rc<0) goto out;
-    }
-
- out:
-    /* rc < 0: error
-     * rc == 0: ok, we are done
-     * rc == +1: need to keep waiting
-     */
-    libxl__xs_transaction_abort(gc, &t);
-
-    if (!rc) {
-        switch_logdirty_done(egc,dss,0);
-    } else if (rc < 0) {
-        LOG(ERROR,"logdirty switch: failed (rc=%d)",rc);
-        switch_logdirty_done(egc,dss,-1);
-    }
-}
-
-static void switch_logdirty_done(libxl__egc *egc,
-                                 libxl__domain_suspend_state *dss,
-                                 int broke)
-{
-    STATE_AO_GC(dss->ao);
-    libxl__logdirty_switch *lds = &dss->logdirty;
-
-    libxl__ev_xswatch_deregister(gc, &lds->watch);
-    libxl__ev_time_deregister(gc, &lds->timeout);
-
-    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
-}
-
-/*----- callbacks, called by xc_domain_save -----*/
-
-static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
-                                 uint32_t domid,
-                                 char *phys_offset, char *node)
-{
-    return libxl__device_model_xs_path(gc, dm_domid, domid,
-                                       "/physmap/%s/%s",
-                                       phys_offset, node);
-}
-
-int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
-        uint32_t *len, void *dss_void)
-{
-    libxl__domain_suspend_state *dss = dss_void;
-    STATE_AO_GC(dss->ao);
-    int i = 0;
-    char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
-    unsigned int num = 0;
-    uint32_t count = 0, version = TOOLSTACK_SAVE_VERSION, namelen = 0;
-    uint8_t *ptr = NULL;
-    char **entries = NULL;
-    struct libxl__physmap_info *pi;
-    uint32_t dm_domid;
-
-    dm_domid = libxl_get_stubdom_id(CTX, domid);
-
-    entries = libxl__xs_directory(gc, 0,
-                libxl__device_model_xs_path(gc, dm_domid, domid, "/physmap"),
-                &num);
-    count = num;
-
-    *len = sizeof(version) + sizeof(count);
-    *buf = calloc(1, *len);
-    ptr = *buf;
-    if (*buf == NULL)
-        return -1;
-
-    memcpy(ptr, &version, sizeof(version));
-    ptr += sizeof(version);
-    memcpy(ptr, &count, sizeof(count));
-    ptr += sizeof(count);
-
-    for (i = 0; i < count; i++) {
-        unsigned long offset;
-        char *xs_path;
-        phys_offset = entries[i];
-        if (phys_offset == NULL) {
-            LOG(ERROR, "phys_offset %d is NULL", i);
-            return -1;
-        }
-
-        xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "start_addr");
-        start_addr = libxl__xs_read(gc, 0, xs_path);
-        if (start_addr == NULL) {
-            LOG(ERROR, "%s is NULL", xs_path);
-            return -1;
-        }
-
-        xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "size");
-        size = libxl__xs_read(gc, 0, xs_path);
-        if (size == NULL) {
-            LOG(ERROR, "%s is NULL", xs_path);
-            return -1;
-        }
-
-        xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "name");
-        name = libxl__xs_read(gc, 0, xs_path);
-        if (name == NULL)
-            namelen = 0;
-        else
-            namelen = strlen(name) + 1;
-        *len += namelen + sizeof(struct libxl__physmap_info);
-        offset = ptr - (*buf);
-        *buf = realloc(*buf, *len);
-        if (*buf == NULL)
-            return -1;
-        ptr = (*buf) + offset;
-        pi = (struct libxl__physmap_info *) ptr;
-        pi->phys_offset = strtoll(phys_offset, NULL, 16);
-        pi->start_addr = strtoll(start_addr, NULL, 16);
-        pi->size = strtoll(size, NULL, 16);
-        pi->namelen = namelen;
-        memcpy(pi->name, name, namelen);
-        ptr += sizeof(struct libxl__physmap_info) + namelen;
-    }
-
-    LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, *len);
-
-    return 0;
-}
-
-/*----- main code for suspending, in order of execution -----*/
-
-static void domain_save_done(libxl__egc *egc,
-                             libxl__domain_suspend_state *dss, int rc);
-
-void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
-{
-    STATE_AO_GC(dss->ao);
-    int port;
-    int rc = ERROR_FAIL;
-
-    /* Convenience aliases */
-    const uint32_t domid = dss->domid;
-    const libxl_domain_type type = dss->type;
-    const int live = dss->live;
-    const int debug = dss->debug;
-    const libxl_domain_remus_info *const r_info = dss->remus;
-    libxl__srm_save_autogen_callbacks *const callbacks =
-        &dss->shs.callbacks.save.a;
-
-    logdirty_init(&dss->logdirty);
-    libxl__xswait_init(&dss->pvcontrol);
-    libxl__ev_evtchn_init(&dss->guest_evtchn);
-    libxl__ev_xswatch_init(&dss->guest_watch);
-    libxl__ev_time_init(&dss->guest_timeout);
-
-    switch (type) {
-    case LIBXL_DOMAIN_TYPE_HVM: {
-        dss->hvm = 1;
-        break;
-    }
-    case LIBXL_DOMAIN_TYPE_PV:
-        dss->hvm = 0;
-        break;
-    default:
-        abort();
-    }
-
-    dss->xcflags = (live ? XCFLAGS_LIVE : 0)
-          | (debug ? XCFLAGS_DEBUG : 0)
-          | (dss->hvm ? XCFLAGS_HVM : 0);
-
-    dss->guest_evtchn.port = -1;
-    dss->guest_evtchn_lockfd = -1;
-    dss->guest_responded = 0;
-    dss->dm_savefile = libxl__device_model_savefile(gc, domid);
-
-    if (r_info != NULL) {
-        dss->interval = r_info->interval;
-        dss->xcflags |= XCFLAGS_CHECKPOINTED;
-        if (libxl_defbool_val(r_info->compression))
-            dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
-    }
-
-    port = xs_suspend_evtchn_port(dss->domid);
-
-    if (port >= 0) {
-        rc = libxl__ctx_evtchn_init(gc);
-        if (rc) goto out;
-
-        dss->guest_evtchn.port =
-            xc_suspend_evtchn_init_exclusive(CTX->xch, CTX->xce,
-                                  dss->domid, port, &dss->guest_evtchn_lockfd);
-
-        if (dss->guest_evtchn.port < 0) {
-            LOG(WARN, "Suspend event channel initialization failed");
-            rc = ERROR_FAIL;
-            goto out;
-        }
-    }
-
-    memset(callbacks, 0, sizeof(*callbacks));
-    if (r_info != NULL) {
-        callbacks->suspend = libxl__remus_domain_suspend_callback;
-        callbacks->postcopy = libxl__remus_domain_resume_callback;
-        callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
-    } else
-        callbacks->suspend = libxl__domain_suspend_callback;
-
-    callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
-    dss->shs.callbacks.save.toolstack_save = libxl__toolstack_save;
-
-    libxl__xc_domain_save(egc, dss);
-    return;
-
- out:
-    domain_save_done(egc, dss, rc);
-}
-
-void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
-                                int rc, int retval, int errnoval)
-{
-    libxl__domain_suspend_state *dss = dss_void;
-    STATE_AO_GC(dss->ao);
-
-    /* Convenience aliases */
-    const libxl_domain_type type = dss->type;
-
-    if (rc)
-        goto out;
-
-    if (retval) {
-        LOGEV(ERROR, errnoval, "saving domain: %s",
-                         dss->guest_responded ?
-                         "domain responded to suspend request" :
-                         "domain did not respond to suspend request");
-        if ( !dss->guest_responded )
-            rc = ERROR_GUEST_TIMEDOUT;
-        else
-            rc = ERROR_FAIL;
-        goto out;
-    }
-
-    if (type == LIBXL_DOMAIN_TYPE_HVM) {
-        rc = libxl__domain_suspend_device_model(gc, dss);
-        if (rc) goto out;
-
-        libxl__domain_save_device_model(egc, dss, domain_save_done);
-        return;
-    }
-
-    rc = 0;
-
-out:
-    domain_save_done(egc, dss, rc);
-}
-
-static void save_device_model_datacopier_done(libxl__egc *egc,
-     libxl__datacopier_state *dc, int onwrite, int errnoval);
-
-void libxl__domain_save_device_model(libxl__egc *egc,
-                                     libxl__domain_suspend_state *dss,
-                                     libxl__save_device_model_cb *callback)
-{
-    STATE_AO_GC(dss->ao);
-    struct stat st;
-    uint32_t qemu_state_len;
-    int rc;
-
-    dss->save_dm_callback = callback;
-
-    /* Convenience aliases */
-    const char *const filename = dss->dm_savefile;
-    const int fd = dss->fd;
-
-    libxl__datacopier_state *dc = &dss->save_dm_datacopier;
-    memset(dc, 0, sizeof(*dc));
-    dc->readwhat = GCSPRINTF("qemu save file %s", filename);
-    dc->ao = ao;
-    dc->readfd = -1;
-    dc->writefd = fd;
-    dc->maxsz = INT_MAX;
-    dc->bytes_to_read = -1;
-    dc->copywhat = GCSPRINTF("qemu save file for domain %"PRIu32, dss->domid);
-    dc->writewhat = "save/migration stream";
-    dc->callback = save_device_model_datacopier_done;
-
-    dc->readfd = open(filename, O_RDONLY);
-    if (dc->readfd < 0) {
-        LOGE(ERROR, "unable to open %s", dc->readwhat);
-        goto out;
-    }
-
-    if (fstat(dc->readfd, &st))
-    {
-        LOGE(ERROR, "unable to fstat %s", dc->readwhat);
-        goto out;
-    }
-
-    if (!S_ISREG(st.st_mode)) {
-        LOG(ERROR, "%s is not a plain file!", dc->readwhat);
-        goto out;
-    }
-
-    qemu_state_len = st.st_size;
-    LOG(DEBUG, "%s is %d bytes", dc->readwhat, qemu_state_len);
-
-    rc = libxl__datacopier_start(dc);
-    if (rc) goto out;
-
-    libxl__datacopier_prefixdata(egc, dc,
-                                 QEMU_SIGNATURE, strlen(QEMU_SIGNATURE));
-
-    libxl__datacopier_prefixdata(egc, dc,
-                                 &qemu_state_len, sizeof(qemu_state_len));
-    return;
-
- out:
-    save_device_model_datacopier_done(egc, dc, -1, 0);
-}
-
-static void save_device_model_datacopier_done(libxl__egc *egc,
-     libxl__datacopier_state *dc, int onwrite, int errnoval)
-{
-    libxl__domain_suspend_state *dss =
-        CONTAINER_OF(dc, *dss, save_dm_datacopier);
-    STATE_AO_GC(dss->ao);
-
-    /* Convenience aliases */
-    const char *const filename = dss->dm_savefile;
-    int our_rc = 0;
-    int rc;
-
-    libxl__datacopier_kill(dc);
-
-    if (onwrite || errnoval)
-        our_rc = ERROR_FAIL;
-
-    if (dc->readfd >= 0) {
-        close(dc->readfd);
-        dc->readfd = -1;
-    }
-
-    rc = libxl__remove_file(gc, filename);
-    if (!our_rc) our_rc = rc;
-
-    dss->save_dm_callback(egc, dss, our_rc);
-}
-
-static void domain_save_done(libxl__egc *egc,
-                             libxl__domain_suspend_state *dss, int rc)
-{
-    STATE_AO_GC(dss->ao);
-
-    /* Convenience aliases */
-    const uint32_t domid = dss->domid;
-
-    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
-
-    if (dss->guest_evtchn.port > 0)
-        xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
-                           dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
-
-    if (!dss->remus) {
-        dss->callback(egc, dss, rc);
-        return;
-    }
-
-    /*
-     * With Remus, if we reach this point, it means either
-     * backup died or some network error occurred preventing us
-     * from sending checkpoints. Teardown the network buffers and
-     * release netlink resources.  This is an async op.
-     */
-    libxl__remus_teardown(egc, dss, rc);
-}
-
 /*==================== Miscellaneous ====================*/
 
 char *libxl__uuid2string(libxl__gc *gc, const libxl_uuid uuid)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
new file mode 100644
index 0000000..2081892
--- /dev/null
+++ b/tools/libxl/libxl_dom_save.c
@@ -0,0 +1,664 @@
+/*
+ * Copyright (C) 2009      Citrix Ltd.
+ * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+struct libxl__physmap_info {
+    uint64_t phys_offset;
+    uint64_t start_addr;
+    uint64_t size;
+    uint32_t namelen;
+    char name[];
+};
+
+#define TOOLSTACK_SAVE_VERSION 1
+
+/*==================== Domain save ====================*/
+
+/*----- complicated callback, called by xc_domain_save -----*/
+
+/*
+ * We implement the other end of protocol for controlling qemu-dm's
+ * logdirty.  There is no documentation for this protocol, but our
+ * counterparty's implementation is in
+ * qemu-xen-traditional.git:xenstore.c in the function
+ * xenstore_process_logdirty_event
+ */
+
+static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
+                                    const struct timeval *requested_abs);
+static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
+                            const char *watch_path, const char *event_path);
+static void switch_logdirty_done(libxl__egc *egc,
+                                 libxl__domain_suspend_state *dss, int ok);
+
+static void logdirty_init(libxl__logdirty_switch *lds)
+{
+    lds->cmd_path = 0;
+    libxl__ev_xswatch_init(&lds->watch);
+    libxl__ev_time_init(&lds->timeout);
+}
+
+static void domain_suspend_switch_qemu_xen_traditional_logdirty
+                               (int domid, unsigned enable,
+                                libxl__save_helper_state *shs)
+{
+    libxl__egc *egc = shs->egc;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__logdirty_switch *lds = &dss->logdirty;
+    STATE_AO_GC(dss->ao);
+    int rc;
+    xs_transaction_t t = 0;
+    const char *got;
+
+    if (!lds->cmd_path) {
+        uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
+        lds->cmd_path = libxl__device_model_xs_path(gc, dm_domid, domid,
+                                                    "/logdirty/cmd");
+        lds->ret_path = libxl__device_model_xs_path(gc, dm_domid, domid,
+                                                    "/logdirty/ret");
+    }
+    lds->cmd = enable ? "enable" : "disable";
+
+    rc = libxl__ev_xswatch_register(gc, &lds->watch,
+                                switch_logdirty_xswatch, lds->ret_path);
+    if (rc) goto out;
+
+    rc = libxl__ev_time_register_rel(gc, &lds->timeout,
+                                switch_logdirty_timeout, 10*1000);
+    if (rc) goto out;
+
+    for (;;) {
+        rc = libxl__xs_transaction_start(gc, &t);
+        if (rc) goto out;
+
+        rc = libxl__xs_read_checked(gc, t, lds->cmd_path, &got);
+        if (rc) goto out;
+
+        if (got) {
+            const char *got_ret;
+            rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got_ret);
+            if (rc) goto out;
+
+            if (!got_ret || strcmp(got, got_ret)) {
+                LOG(ERROR,"controlling logdirty: qemu was already sent"
+                    " command `%s' (xenstore path `%s') but result is `%s'",
+                    got, lds->cmd_path, got_ret ? got_ret : "<none>");
+                rc = ERROR_FAIL;
+                goto out;
+            }
+            rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
+            if (rc) goto out;
+        }
+
+        rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
+        if (rc) goto out;
+
+        rc = libxl__xs_write_checked(gc, t, lds->cmd_path, lds->cmd);
+        if (rc) goto out;
+
+        rc = libxl__xs_transaction_commit(gc, &t);
+        if (!rc) break;
+        if (rc<0) goto out;
+    }
+
+    /* OK, wait for some callback */
+    return;
+
+ out:
+    LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
+    libxl__xs_transaction_abort(gc, &t);
+    switch_logdirty_done(egc,dss,-1);
+}
+
+static void domain_suspend_switch_qemu_xen_logdirty
+                               (int domid, unsigned enable,
+                                libxl__save_helper_state *shs)
+{
+    libxl__egc *egc = shs->egc;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    STATE_AO_GC(dss->ao);
+    int rc;
+
+    rc = libxl__qmp_set_global_dirty_log(gc, domid, enable);
+    if (!rc) {
+        libxl__xc_domain_saverestore_async_callback_done(egc, shs, 0);
+    } else {
+        LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
+        libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
+    }
+}
+
+void libxl__domain_suspend_common_switch_qemu_logdirty
+                               (int domid, unsigned enable, void *user)
+{
+    libxl__save_helper_state *shs = user;
+    libxl__egc *egc = shs->egc;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    STATE_AO_GC(dss->ao);
+
+    switch (libxl__device_model_version_running(gc, domid)) {
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
+        domain_suspend_switch_qemu_xen_traditional_logdirty(domid, enable, shs);
+        break;
+    case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+        domain_suspend_switch_qemu_xen_logdirty(domid, enable, shs);
+        break;
+    default:
+        LOG(ERROR,"logdirty switch failed"
+            ", no valid device model version found, aborting suspend");
+        libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
+    }
+}
+static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
+                                    const struct timeval *requested_abs)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
+    STATE_AO_GC(dss->ao);
+    LOG(ERROR,"logdirty switch: wait for device model timed out");
+    switch_logdirty_done(egc,dss,-1);
+}
+
+static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
+                            const char *watch_path, const char *event_path)
+{
+    libxl__domain_suspend_state *dss =
+        CONTAINER_OF(watch, *dss, logdirty.watch);
+    libxl__logdirty_switch *lds = &dss->logdirty;
+    STATE_AO_GC(dss->ao);
+    const char *got;
+    xs_transaction_t t = 0;
+    int rc;
+
+    for (;;) {
+        rc = libxl__xs_transaction_start(gc, &t);
+        if (rc) goto out;
+
+        rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got);
+        if (rc) goto out;
+
+        if (!got) {
+            rc = +1;
+            goto out;
+        }
+
+        if (strcmp(got, lds->cmd)) {
+            LOG(ERROR,"logdirty switch: sent command `%s' but got reply `%s'"
+                " (xenstore paths `%s' / `%s')", lds->cmd, got,
+                lds->cmd_path, lds->ret_path);
+            rc = ERROR_FAIL;
+            goto out;
+        }
+
+        rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
+        if (rc) goto out;
+
+        rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
+        if (rc) goto out;
+
+        rc = libxl__xs_transaction_commit(gc, &t);
+        if (!rc) break;
+        if (rc<0) goto out;
+    }
+
+ out:
+    /* rc < 0: error
+     * rc == 0: ok, we are done
+     * rc == +1: need to keep waiting
+     */
+    libxl__xs_transaction_abort(gc, &t);
+
+    if (!rc) {
+        switch_logdirty_done(egc,dss,0);
+    } else if (rc < 0) {
+        LOG(ERROR,"logdirty switch: failed (rc=%d)",rc);
+        switch_logdirty_done(egc,dss,-1);
+    }
+}
+
+static void switch_logdirty_done(libxl__egc *egc,
+                                 libxl__domain_suspend_state *dss,
+                                 int broke)
+{
+    STATE_AO_GC(dss->ao);
+    libxl__logdirty_switch *lds = &dss->logdirty;
+
+    libxl__ev_xswatch_deregister(gc, &lds->watch);
+    libxl__ev_time_deregister(gc, &lds->timeout);
+
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
+}
+
+/*----- callbacks, called by xc_domain_save -----*/
+
+static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
+                                 uint32_t domid,
+                                 char *phys_offset, char *node)
+{
+    return libxl__device_model_xs_path(gc, dm_domid, domid,
+                                       "/physmap/%s/%s",
+                                       phys_offset, node);
+}
+
+int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
+        uint32_t *len, void *dss_void)
+{
+    libxl__domain_suspend_state *dss = dss_void;
+    STATE_AO_GC(dss->ao);
+    int i = 0;
+    char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
+    unsigned int num = 0;
+    uint32_t count = 0, version = TOOLSTACK_SAVE_VERSION, namelen = 0;
+    uint8_t *ptr = NULL;
+    char **entries = NULL;
+    struct libxl__physmap_info *pi;
+    uint32_t dm_domid;
+
+    dm_domid = libxl_get_stubdom_id(CTX, domid);
+
+    entries = libxl__xs_directory(gc, 0,
+                libxl__device_model_xs_path(gc, dm_domid, domid, "/physmap"),
+                &num);
+    count = num;
+
+    *len = sizeof(version) + sizeof(count);
+    *buf = calloc(1, *len);
+    ptr = *buf;
+    if (*buf == NULL)
+        return -1;
+
+    memcpy(ptr, &version, sizeof(version));
+    ptr += sizeof(version);
+    memcpy(ptr, &count, sizeof(count));
+    ptr += sizeof(count);
+
+    for (i = 0; i < count; i++) {
+        unsigned long offset;
+        char *xs_path;
+        phys_offset = entries[i];
+        if (phys_offset == NULL) {
+            LOG(ERROR, "phys_offset %d is NULL", i);
+            return -1;
+        }
+
+        xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "start_addr");
+        start_addr = libxl__xs_read(gc, 0, xs_path);
+        if (start_addr == NULL) {
+            LOG(ERROR, "%s is NULL", xs_path);
+            return -1;
+        }
+
+        xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "size");
+        size = libxl__xs_read(gc, 0, xs_path);
+        if (size == NULL) {
+            LOG(ERROR, "%s is NULL", xs_path);
+            return -1;
+        }
+
+        xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "name");
+        name = libxl__xs_read(gc, 0, xs_path);
+        if (name == NULL)
+            namelen = 0;
+        else
+            namelen = strlen(name) + 1;
+        *len += namelen + sizeof(struct libxl__physmap_info);
+        offset = ptr - (*buf);
+        *buf = realloc(*buf, *len);
+        if (*buf == NULL)
+            return -1;
+        ptr = (*buf) + offset;
+        pi = (struct libxl__physmap_info *) ptr;
+        pi->phys_offset = strtoll(phys_offset, NULL, 16);
+        pi->start_addr = strtoll(start_addr, NULL, 16);
+        pi->size = strtoll(size, NULL, 16);
+        pi->namelen = namelen;
+        memcpy(pi->name, name, namelen);
+        ptr += sizeof(struct libxl__physmap_info) + namelen;
+    }
+
+    LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, *len);
+
+    return 0;
+}
+
+/*----- main code for saving, in order of execution -----*/
+
+static void domain_save_done(libxl__egc *egc,
+                             libxl__domain_suspend_state *dss, int rc);
+
+void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
+{
+    STATE_AO_GC(dss->ao);
+    int port;
+    int rc = ERROR_FAIL;
+
+    /* Convenience aliases */
+    const uint32_t domid = dss->domid;
+    const libxl_domain_type type = dss->type;
+    const int live = dss->live;
+    const int debug = dss->debug;
+    const libxl_domain_remus_info *const r_info = dss->remus;
+    libxl__srm_save_autogen_callbacks *const callbacks =
+        &dss->shs.callbacks.save.a;
+
+    logdirty_init(&dss->logdirty);
+    libxl__xswait_init(&dss->pvcontrol);
+    libxl__ev_evtchn_init(&dss->guest_evtchn);
+    libxl__ev_xswatch_init(&dss->guest_watch);
+    libxl__ev_time_init(&dss->guest_timeout);
+
+    switch (type) {
+    case LIBXL_DOMAIN_TYPE_HVM: {
+        dss->hvm = 1;
+        break;
+    }
+    case LIBXL_DOMAIN_TYPE_PV:
+        dss->hvm = 0;
+        break;
+    default:
+        abort();
+    }
+
+    dss->xcflags = (live ? XCFLAGS_LIVE : 0)
+          | (debug ? XCFLAGS_DEBUG : 0)
+          | (dss->hvm ? XCFLAGS_HVM : 0);
+
+    dss->guest_evtchn.port = -1;
+    dss->guest_evtchn_lockfd = -1;
+    dss->guest_responded = 0;
+    dss->dm_savefile = libxl__device_model_savefile(gc, domid);
+
+    if (r_info != NULL) {
+        dss->interval = r_info->interval;
+        dss->xcflags |= XCFLAGS_CHECKPOINTED;
+        if (libxl_defbool_val(r_info->compression))
+            dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
+    }
+
+    port = xs_suspend_evtchn_port(dss->domid);
+
+    if (port >= 0) {
+        rc = libxl__ctx_evtchn_init(gc);
+        if (rc) goto out;
+
+        dss->guest_evtchn.port =
+            xc_suspend_evtchn_init_exclusive(CTX->xch, CTX->xce,
+                                  dss->domid, port, &dss->guest_evtchn_lockfd);
+
+        if (dss->guest_evtchn.port < 0) {
+            LOG(WARN, "Suspend event channel initialization failed");
+            rc = ERROR_FAIL;
+            goto out;
+        }
+    }
+
+    memset(callbacks, 0, sizeof(*callbacks));
+    if (r_info != NULL) {
+        callbacks->suspend = libxl__remus_domain_suspend_callback;
+        callbacks->postcopy = libxl__remus_domain_resume_callback;
+        callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
+    } else
+        callbacks->suspend = libxl__domain_suspend_callback;
+
+    callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
+    dss->shs.callbacks.save.toolstack_save = libxl__toolstack_save;
+
+    libxl__xc_domain_save(egc, dss);
+    return;
+
+ out:
+    domain_save_done(egc, dss, rc);
+}
+
+void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
+                                int rc, int retval, int errnoval)
+{
+    libxl__domain_suspend_state *dss = dss_void;
+    STATE_AO_GC(dss->ao);
+
+    /* Convenience aliases */
+    const libxl_domain_type type = dss->type;
+
+    if (rc)
+        goto out;
+
+    if (retval) {
+        LOGEV(ERROR, errnoval, "saving domain: %s",
+                         dss->guest_responded ?
+                         "domain responded to suspend request" :
+                         "domain did not respond to suspend request");
+        if ( !dss->guest_responded )
+            rc = ERROR_GUEST_TIMEDOUT;
+        else
+            rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
+        rc = libxl__domain_suspend_device_model(gc, dss);
+        if (rc) goto out;
+
+        libxl__domain_save_device_model(egc, dss, domain_save_done);
+        return;
+    }
+
+    rc = 0;
+
+out:
+    domain_save_done(egc, dss, rc);
+}
+
+static void save_device_model_datacopier_done(libxl__egc *egc,
+     libxl__datacopier_state *dc, int onwrite, int errnoval);
+
+void libxl__domain_save_device_model(libxl__egc *egc,
+                                     libxl__domain_suspend_state *dss,
+                                     libxl__save_device_model_cb *callback)
+{
+    STATE_AO_GC(dss->ao);
+    struct stat st;
+    uint32_t qemu_state_len;
+    int rc;
+
+    dss->save_dm_callback = callback;
+
+    /* Convenience aliases */
+    const char *const filename = dss->dm_savefile;
+    const int fd = dss->fd;
+
+    libxl__datacopier_state *dc = &dss->save_dm_datacopier;
+    memset(dc, 0, sizeof(*dc));
+    dc->readwhat = GCSPRINTF("qemu save file %s", filename);
+    dc->ao = ao;
+    dc->readfd = -1;
+    dc->writefd = fd;
+    dc->maxsz = INT_MAX;
+    dc->bytes_to_read = -1;
+    dc->copywhat = GCSPRINTF("qemu save file for domain %"PRIu32, dss->domid);
+    dc->writewhat = "save/migration stream";
+    dc->callback = save_device_model_datacopier_done;
+
+    dc->readfd = open(filename, O_RDONLY);
+    if (dc->readfd < 0) {
+        LOGE(ERROR, "unable to open %s", dc->readwhat);
+        goto out;
+    }
+
+    if (fstat(dc->readfd, &st))
+    {
+        LOGE(ERROR, "unable to fstat %s", dc->readwhat);
+        goto out;
+    }
+
+    if (!S_ISREG(st.st_mode)) {
+        LOG(ERROR, "%s is not a plain file!", dc->readwhat);
+        goto out;
+    }
+
+    qemu_state_len = st.st_size;
+    LOG(DEBUG, "%s is %d bytes", dc->readwhat, qemu_state_len);
+
+    rc = libxl__datacopier_start(dc);
+    if (rc) goto out;
+
+    libxl__datacopier_prefixdata(egc, dc,
+                                 QEMU_SIGNATURE, strlen(QEMU_SIGNATURE));
+
+    libxl__datacopier_prefixdata(egc, dc,
+                                 &qemu_state_len, sizeof(qemu_state_len));
+    return;
+
+ out:
+    save_device_model_datacopier_done(egc, dc, -1, 0);
+}
+
+static void save_device_model_datacopier_done(libxl__egc *egc,
+     libxl__datacopier_state *dc, int onwrite, int errnoval)
+{
+    libxl__domain_suspend_state *dss =
+        CONTAINER_OF(dc, *dss, save_dm_datacopier);
+    STATE_AO_GC(dss->ao);
+
+    /* Convenience aliases */
+    const char *const filename = dss->dm_savefile;
+    int our_rc = 0;
+    int rc;
+
+    libxl__datacopier_kill(dc);
+
+    if (onwrite || errnoval)
+        our_rc = ERROR_FAIL;
+
+    if (dc->readfd >= 0) {
+        close(dc->readfd);
+        dc->readfd = -1;
+    }
+
+    rc = libxl__remove_file(gc, filename);
+    if (!our_rc) our_rc = rc;
+
+    dss->save_dm_callback(egc, dss, our_rc);
+}
+
+static void domain_save_done(libxl__egc *egc,
+                             libxl__domain_suspend_state *dss, int rc)
+{
+    STATE_AO_GC(dss->ao);
+
+    /* Convenience aliases */
+    const uint32_t domid = dss->domid;
+
+    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+
+    if (dss->guest_evtchn.port > 0)
+        xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
+                           dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
+
+    if (!dss->remus) {
+        dss->callback(egc, dss, rc);
+        return;
+    }
+
+    /*
+     * With Remus, if we reach this point, it means either
+     * backup died or some network error occurred preventing us
+     * from sending checkpoints. Teardown the network buffers and
+     * release netlink resources.  This is an async op.
+     */
+    libxl__remus_teardown(egc, dss, rc);
+}
+
+/*==================== Domain restore ====================*/
+
+static inline char *restore_helper(libxl__gc *gc, uint32_t dm_domid,
+                                   uint32_t domid,
+                                   uint64_t phys_offset, char *node)
+{
+    return libxl__device_model_xs_path(gc, dm_domid, domid,
+                                       "/physmap/%"PRIx64"/%s",
+                                       phys_offset, node);
+}
+
+int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
+                             uint32_t size, void *user)
+{
+    libxl__save_helper_state *shs = user;
+    libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
+    STATE_AO_GC(dcs->ao);
+    int i, ret;
+    const uint8_t *ptr = buf;
+    uint32_t count = 0, version = 0;
+    struct libxl__physmap_info* pi;
+    char *xs_path;
+    uint32_t dm_domid;
+
+    LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, size);
+
+    if (size < sizeof(version) + sizeof(count)) {
+        LOG(ERROR, "wrong size");
+        return -1;
+    }
+
+    memcpy(&version, ptr, sizeof(version));
+    ptr += sizeof(version);
+
+    if (version != TOOLSTACK_SAVE_VERSION) {
+        LOG(ERROR, "wrong version");
+        return -1;
+    }
+
+    memcpy(&count, ptr, sizeof(count));
+    ptr += sizeof(count);
+
+    if (size < sizeof(version) + sizeof(count) +
+            count * (sizeof(struct libxl__physmap_info))) {
+        LOG(ERROR, "wrong size");
+        return -1;
+    }
+
+    dm_domid = libxl_get_stubdom_id(CTX, domid);
+    for (i = 0; i < count; i++) {
+        pi = (struct libxl__physmap_info*) ptr;
+        ptr += sizeof(struct libxl__physmap_info) + pi->namelen;
+
+        xs_path = restore_helper(gc, dm_domid, domid,
+                                 pi->phys_offset, "start_addr");
+        ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->start_addr);
+        if (ret)
+            return -1;
+        xs_path = restore_helper(gc, dm_domid, domid, pi->phys_offset, "size");
+        ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->size);
+        if (ret)
+            return -1;
+        if (pi->namelen > 0) {
+            xs_path = restore_helper(gc, dm_domid, domid,
+                                     pi->phys_offset, "name");
+            ret = libxl__xs_write(gc, 0, xs_path, "%s", pi->name);
+            if (ret)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
-- 
1.9.1

Re: [PATCH v2 5/6] tools/libxl: move save/restore code into libxl_dom_save.c

[Ian Campbell]

On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
> move save/restore code into libxl_dom_save.c.

If this (unlike other patches in the series) is purely code motion
please indicate that this is the case.

You might also like to consider refactoring things such that all patches
are pure motion.

Re: [PATCH v2 5/6] tools/libxl: move save/restore code into libxl_dom_save.c

[Yang Hongyang]

On 06/16/2015 09:09 PM, Ian Campbell wrote:
> On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
>> move save/restore code into libxl_dom_save.c.
>
> If this (unlike other patches in the series) is purely code motion
> please indicate that this is the case.
>
> You might also like to consider refactoring things such that all patches
> are pure motion.

Yes, this is only code move, no refactoring or other thing.

>
>
> .
>

-- 
Thanks,
Yang.

[PATCH v2 6/6] libxl/save: Refactor libxl__domain_suspend_state

[Yang Hongyang]

Currently struct libxl__domain_suspend_state contains 2 type of states,
one is save state, another is suspend state. This patch separate it out.
The motivation of this is that COLO will need to do suspend/resume
continuesly, we need a more common suspend state.

After this change, dss stands for libxl__domain_save_state,
dsps stands for libxl__domain_suspend_state.

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxl/libxl.c              |  10 +--
 tools/libxl/libxl_dom_save.c     |  76 ++++++++++--------
 tools/libxl/libxl_dom_suspend.c  | 166 ++++++++++++++++++++-------------------
 tools/libxl/libxl_internal.h     |  51 +++++++-----
 tools/libxl/libxl_netbuffer.c    |   2 +-
 tools/libxl/libxl_remus.c        |  39 ++++-----
 tools/libxl/libxl_save_callout.c |   2 +-
 7 files changed, 185 insertions(+), 161 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 0f9248e..ba2da92 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -793,7 +793,7 @@ out:
 }
 
 static void remus_failover_cb(libxl__egc *egc,
-                              libxl__domain_suspend_state *dss, int rc);
+                              libxl__domain_save_state *dss, int rc);
 
 /* TODO: Explicit Checkpoint acknowledgements via recv_fd. */
 int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
@@ -801,7 +801,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
                              const libxl_asyncop_how *ao_how)
 {
     AO_CREATE(ctx, domid, ao_how);
-    libxl__domain_suspend_state *dss;
+    libxl__domain_save_state *dss;
     int rc;
 
     libxl_domain_type type = libxl__domain_type(gc, domid);
@@ -849,7 +849,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
 }
 
 static void remus_failover_cb(libxl__egc *egc,
-                              libxl__domain_suspend_state *dss, int rc)
+                              libxl__domain_save_state *dss, int rc)
 {
     STATE_AO_GC(dss->ao);
     /*
@@ -861,7 +861,7 @@ static void remus_failover_cb(libxl__egc *egc,
 }
 
 static void domain_suspend_cb(libxl__egc *egc,
-                              libxl__domain_suspend_state *dss, int rc)
+                              libxl__domain_save_state *dss, int rc)
 {
     STATE_AO_GC(dss->ao);
     libxl__ao_complete(egc,ao,rc);
@@ -880,7 +880,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
         goto out_err;
     }
 
-    libxl__domain_suspend_state *dss;
+    libxl__domain_save_state *dss;
     GCNEW(dss);
 
     dss->ao = ao;
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 2081892..74a6bae 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -44,7 +44,7 @@ static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
 static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
                             const char *watch_path, const char *event_path);
 static void switch_logdirty_done(libxl__egc *egc,
-                                 libxl__domain_suspend_state *dss, int ok);
+                                 libxl__domain_save_state *dss, int ok);
 
 static void logdirty_init(libxl__logdirty_switch *lds)
 {
@@ -58,7 +58,7 @@ static void domain_suspend_switch_qemu_xen_traditional_logdirty
                                 libxl__save_helper_state *shs)
 {
     libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
     libxl__logdirty_switch *lds = &dss->logdirty;
     STATE_AO_GC(dss->ao);
     int rc;
@@ -130,7 +130,7 @@ static void domain_suspend_switch_qemu_xen_logdirty
                                 libxl__save_helper_state *shs)
 {
     libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
     STATE_AO_GC(dss->ao);
     int rc;
 
@@ -148,7 +148,7 @@ void libxl__domain_suspend_common_switch_qemu_logdirty
 {
     libxl__save_helper_state *shs = user;
     libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
     STATE_AO_GC(dss->ao);
 
     switch (libxl__device_model_version_running(gc, domid)) {
@@ -167,7 +167,7 @@ void libxl__domain_suspend_common_switch_qemu_logdirty
 static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
                                     const struct timeval *requested_abs)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
+    libxl__domain_save_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
     STATE_AO_GC(dss->ao);
     LOG(ERROR,"logdirty switch: wait for device model timed out");
     switch_logdirty_done(egc,dss,-1);
@@ -176,7 +176,7 @@ static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
 static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
                             const char *watch_path, const char *event_path)
 {
-    libxl__domain_suspend_state *dss =
+    libxl__domain_save_state *dss =
         CONTAINER_OF(watch, *dss, logdirty.watch);
     libxl__logdirty_switch *lds = &dss->logdirty;
     STATE_AO_GC(dss->ao);
@@ -231,7 +231,7 @@ static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
 }
 
 static void switch_logdirty_done(libxl__egc *egc,
-                                 libxl__domain_suspend_state *dss,
+                                 libxl__domain_save_state *dss,
                                  int broke)
 {
     STATE_AO_GC(dss->ao);
@@ -257,7 +257,7 @@ static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
 int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
         uint32_t *len, void *dss_void)
 {
-    libxl__domain_suspend_state *dss = dss_void;
+    libxl__domain_save_state *dss = dss_void;
     STATE_AO_GC(dss->ao);
     int i = 0;
     char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
@@ -338,9 +338,9 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
 /*----- main code for saving, in order of execution -----*/
 
 static void domain_save_done(libxl__egc *egc,
-                             libxl__domain_suspend_state *dss, int rc);
+                             libxl__domain_save_state *dss, int rc);
 
-void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
 {
     STATE_AO_GC(dss->ao);
     int port;
@@ -354,20 +354,23 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
     const libxl_domain_remus_info *const r_info = dss->remus;
     libxl__srm_save_autogen_callbacks *const callbacks =
         &dss->shs.callbacks.save.a;
+    libxl__domain_suspend_state *dsps = &dss->dsps;
 
     logdirty_init(&dss->logdirty);
-    libxl__xswait_init(&dss->pvcontrol);
-    libxl__ev_evtchn_init(&dss->guest_evtchn);
-    libxl__ev_xswatch_init(&dss->guest_watch);
-    libxl__ev_time_init(&dss->guest_timeout);
+    libxl__xswait_init(&dsps->pvcontrol);
+    libxl__ev_evtchn_init(&dsps->guest_evtchn);
+    libxl__ev_xswatch_init(&dsps->guest_watch);
+    libxl__ev_time_init(&dsps->guest_timeout);
 
     switch (type) {
     case LIBXL_DOMAIN_TYPE_HVM: {
         dss->hvm = 1;
+        dsps->hvm = 1;
         break;
     }
     case LIBXL_DOMAIN_TYPE_PV:
         dss->hvm = 0;
+        dsps->hvm = 0;
         break;
     default:
         abort();
@@ -377,10 +380,12 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
           | (debug ? XCFLAGS_DEBUG : 0)
           | (dss->hvm ? XCFLAGS_HVM : 0);
 
-    dss->guest_evtchn.port = -1;
-    dss->guest_evtchn_lockfd = -1;
-    dss->guest_responded = 0;
-    dss->dm_savefile = libxl__device_model_savefile(gc, domid);
+    dsps->ao = ao;
+    dsps->domid = domid;
+    dsps->guest_evtchn.port = -1;
+    dsps->guest_evtchn_lockfd = -1;
+    dsps->guest_responded = 0;
+    dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
 
     if (r_info != NULL) {
         dss->interval = r_info->interval;
@@ -389,17 +394,18 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
             dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
     }
 
-    port = xs_suspend_evtchn_port(dss->domid);
+    port = xs_suspend_evtchn_port(dsps->domid);
 
     if (port >= 0) {
         rc = libxl__ctx_evtchn_init(gc);
         if (rc) goto out;
 
-        dss->guest_evtchn.port =
+        dsps->guest_evtchn.port =
             xc_suspend_evtchn_init_exclusive(CTX->xch, CTX->xce,
-                                  dss->domid, port, &dss->guest_evtchn_lockfd);
+                                             dsps->domid, port,
+                                             &dsps->guest_evtchn_lockfd);
 
-        if (dss->guest_evtchn.port < 0) {
+        if (dsps->guest_evtchn.port < 0) {
             LOG(WARN, "Suspend event channel initialization failed");
             rc = ERROR_FAIL;
             goto out;
@@ -427,21 +433,22 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
 void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
                                 int rc, int retval, int errnoval)
 {
-    libxl__domain_suspend_state *dss = dss_void;
+    libxl__domain_save_state *dss = dss_void;
     STATE_AO_GC(dss->ao);
 
     /* Convenience aliases */
     const libxl_domain_type type = dss->type;
+    libxl__domain_suspend_state *dsps = &dss->dsps;
 
     if (rc)
         goto out;
 
     if (retval) {
         LOGEV(ERROR, errnoval, "saving domain: %s",
-                         dss->guest_responded ?
+                         dsps->guest_responded ?
                          "domain responded to suspend request" :
                          "domain did not respond to suspend request");
-        if ( !dss->guest_responded )
+        if ( !dsps->guest_responded )
             rc = ERROR_GUEST_TIMEDOUT;
         else
             rc = ERROR_FAIL;
@@ -449,7 +456,7 @@ void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
     }
 
     if (type == LIBXL_DOMAIN_TYPE_HVM) {
-        rc = libxl__domain_suspend_device_model(gc, dss);
+        rc = libxl__domain_suspend_device_model(gc, dsps);
         if (rc) goto out;
 
         libxl__domain_save_device_model(egc, dss, domain_save_done);
@@ -466,7 +473,7 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
      libxl__datacopier_state *dc, int onwrite, int errnoval);
 
 void libxl__domain_save_device_model(libxl__egc *egc,
-                                     libxl__domain_suspend_state *dss,
+                                     libxl__domain_save_state *dss,
                                      libxl__save_device_model_cb *callback)
 {
     STATE_AO_GC(dss->ao);
@@ -477,7 +484,7 @@ void libxl__domain_save_device_model(libxl__egc *egc,
     dss->save_dm_callback = callback;
 
     /* Convenience aliases */
-    const char *const filename = dss->dm_savefile;
+    const char *const filename = dss->dsps.dm_savefile;
     const int fd = dss->fd;
 
     libxl__datacopier_state *dc = &dss->save_dm_datacopier;
@@ -529,12 +536,12 @@ void libxl__domain_save_device_model(libxl__egc *egc,
 static void save_device_model_datacopier_done(libxl__egc *egc,
      libxl__datacopier_state *dc, int onwrite, int errnoval)
 {
-    libxl__domain_suspend_state *dss =
+    libxl__domain_save_state *dss =
         CONTAINER_OF(dc, *dss, save_dm_datacopier);
     STATE_AO_GC(dss->ao);
 
     /* Convenience aliases */
-    const char *const filename = dss->dm_savefile;
+    const char *const filename = dss->dsps.dm_savefile;
     int our_rc = 0;
     int rc;
 
@@ -555,18 +562,19 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
 }
 
 static void domain_save_done(libxl__egc *egc,
-                             libxl__domain_suspend_state *dss, int rc)
+                             libxl__domain_save_state *dss, int rc)
 {
     STATE_AO_GC(dss->ao);
 
     /* Convenience aliases */
     const uint32_t domid = dss->domid;
+    libxl__domain_suspend_state *dsps = &dss->dsps;
 
-    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+    libxl__ev_evtchn_cancel(gc, &dsps->guest_evtchn);
 
-    if (dss->guest_evtchn.port > 0)
+    if (dsps->guest_evtchn.port > 0)
         xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
-                           dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
+                           dsps->guest_evtchn.port, &dsps->guest_evtchn_lockfd);
 
     if (!dss->remus) {
         dss->callback(egc, dss, rc);
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
index 8b9dd6c..e2703bc 100644
--- a/tools/libxl/libxl_dom_suspend.c
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -20,16 +20,16 @@
 /*==================== Domain suspend ====================*/
 
 static void domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok);
+                                libxl__domain_suspend_state *dsps, int ok);
 
 /*----- callbacks, called by xc_domain_save -----*/
 
 int libxl__domain_suspend_device_model(libxl__gc *gc,
-                                       libxl__domain_suspend_state *dss)
+                                       libxl__domain_suspend_state *dsps)
 {
     int ret = 0;
-    uint32_t const domid = dss->domid;
-    const char *const filename = dss->dm_savefile;
+    uint32_t const domid = dsps->domid;
+    const char *const filename = dsps->dm_savefile;
 
     switch (libxl__device_model_version_running(gc, domid)) {
     case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
@@ -54,9 +54,9 @@ int libxl__domain_suspend_device_model(libxl__gc *gc,
 }
 
 static void domain_suspend_common_wait_guest(libxl__egc *egc,
-                                             libxl__domain_suspend_state *dss);
+                                             libxl__domain_suspend_state *dsps);
 static void domain_suspend_common_guest_suspended(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss);
+                                         libxl__domain_suspend_state *dsps);
 
 static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
       libxl__xswait_state *xswa, int rc, const char *state);
@@ -65,17 +65,17 @@ static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
 static void suspend_common_wait_guest_watch(libxl__egc *egc,
       libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
 static void suspend_common_wait_guest_check(libxl__egc *egc,
-        libxl__domain_suspend_state *dss);
+        libxl__domain_suspend_state *dsps);
 static void suspend_common_wait_guest_timeout(libxl__egc *egc,
       libxl__ev_time *ev, const struct timeval *requested_abs);
 
 static void domain_suspend_common_failed(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss);
+                                         libxl__domain_suspend_state *dsps);
 static void domain_suspend_common_done(libxl__egc *egc,
-                                       libxl__domain_suspend_state *dss,
+                                       libxl__domain_suspend_state *dsps,
                                        bool ok);
 static void domain_suspend_callback_common(libxl__egc *egc,
-                                           libxl__domain_suspend_state *dss);
+                                           libxl__domain_suspend_state *dsps);
 
 static bool domain_suspend_pvcontrol_acked(const char *state) {
     /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
@@ -83,43 +83,43 @@ static bool domain_suspend_pvcontrol_acked(const char *state) {
     return strcmp(state,"suspend");
 }
 
-/* calls dss->callback_common_done when done */
+/* calls dsps->callback_common_done when done */
 void libxl__domain_suspend(libxl__egc *egc,
-                           libxl__domain_suspend_state *dss)
+                           libxl__domain_suspend_state *dsps)
 {
-    domain_suspend_callback_common(egc, dss);
+    domain_suspend_callback_common(egc, dsps);
 }
 
-/* calls dss->callback_common_done when done */
+/* calls dsps->callback_common_done when done */
 static void domain_suspend_callback_common(libxl__egc *egc,
-                                           libxl__domain_suspend_state *dss)
+                                           libxl__domain_suspend_state *dsps)
 {
-    STATE_AO_GC(dss->ao);
+    STATE_AO_GC(dsps->ao);
     uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
     int ret, rc;
 
     /* Convenience aliases */
-    const uint32_t domid = dss->domid;
+    const uint32_t domid = dsps->domid;
 
-    if (dss->hvm) {
+    if (dsps->hvm) {
         xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
         xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
     }
 
-    if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
+    if ((hvm_s_state == 0) && (dsps->guest_evtchn.port >= 0)) {
         LOG(DEBUG, "issuing %s suspend request via event channel",
-            dss->hvm ? "PVHVM" : "PV");
-        ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
+            dsps->hvm ? "PVHVM" : "PV");
+        ret = xc_evtchn_notify(CTX->xce, dsps->guest_evtchn.port);
         if (ret < 0) {
             LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
             goto err;
         }
 
-        dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
-        rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
+        dsps->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
+        rc = libxl__ev_evtchn_wait(gc, &dsps->guest_evtchn);
         if (rc) goto err;
 
-        rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+        rc = libxl__ev_time_register_rel(gc, &dsps->guest_timeout,
                                          suspend_common_wait_guest_timeout,
                                          60*1000);
         if (rc) goto err;
@@ -127,7 +127,7 @@ static void domain_suspend_callback_common(libxl__egc *egc,
         return;
     }
 
-    if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
+    if (dsps->hvm && (!hvm_pvdrv || hvm_s_state)) {
         LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
         ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
         if (ret < 0) {
@@ -135,55 +135,55 @@ static void domain_suspend_callback_common(libxl__egc *egc,
             goto err;
         }
         /* The guest does not (need to) respond to this sort of request. */
-        dss->guest_responded = 1;
-        domain_suspend_common_wait_guest(egc, dss);
+        dsps->guest_responded = 1;
+        domain_suspend_common_wait_guest(egc, dsps);
         return;
     }
 
     LOG(DEBUG, "issuing %s suspend request via XenBus control node",
-        dss->hvm ? "PVHVM" : "PV");
+        dsps->hvm ? "PVHVM" : "PV");
 
     libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
 
-    dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
-    if (!dss->pvcontrol.path) goto err;
+    dsps->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
+    if (!dsps->pvcontrol.path) goto err;
 
-    dss->pvcontrol.ao = ao;
-    dss->pvcontrol.what = "guest acknowledgement of suspend request";
-    dss->pvcontrol.timeout_ms = 60 * 1000;
-    dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
-    libxl__xswait_start(gc, &dss->pvcontrol);
+    dsps->pvcontrol.ao = ao;
+    dsps->pvcontrol.what = "guest acknowledgement of suspend request";
+    dsps->pvcontrol.timeout_ms = 60 * 1000;
+    dsps->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
+    libxl__xswait_start(gc, &dsps->pvcontrol);
     return;
 
  err:
-    domain_suspend_common_failed(egc, dss);
+    domain_suspend_common_failed(egc, dsps);
 }
 
 static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
         libxl__ev_evtchn *evev)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
-    STATE_AO_GC(dss->ao);
+    libxl__domain_suspend_state *dsps = CONTAINER_OF(evev, *dsps, guest_evtchn);
+    STATE_AO_GC(dsps->ao);
     /* If we should be done waiting, suspend_common_wait_guest_check
      * will end up calling domain_suspend_common_guest_suspended or
      * domain_suspend_common_failed, both of which cancel the evtchn
      * wait.  So re-enable it now. */
-    libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
-    suspend_common_wait_guest_check(egc, dss);
+    libxl__ev_evtchn_wait(gc, &dsps->guest_evtchn);
+    suspend_common_wait_guest_check(egc, dsps);
 }
 
 static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
       libxl__xswait_state *xswa, int rc, const char *state)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
-    STATE_AO_GC(dss->ao);
+    libxl__domain_suspend_state *dsps = CONTAINER_OF(xswa, *dsps, pvcontrol);
+    STATE_AO_GC(dsps->ao);
     xs_transaction_t t = 0;
 
     if (!rc && !domain_suspend_pvcontrol_acked(state))
         /* keep waiting */
         return;
 
-    libxl__xswait_stop(gc, &dss->pvcontrol);
+    libxl__xswait_stop(gc, &dsps->pvcontrol);
 
     if (rc == ERROR_TIMEDOUT) {
         /*
@@ -226,56 +226,56 @@ static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
     LOG(DEBUG, "guest acknowledged suspend request");
 
     libxl__xs_transaction_abort(gc, &t);
-    dss->guest_responded = 1;
-    domain_suspend_common_wait_guest(egc,dss);
+    dsps->guest_responded = 1;
+    domain_suspend_common_wait_guest(egc,dsps);
     return;
 
  err:
     libxl__xs_transaction_abort(gc, &t);
-    domain_suspend_common_failed(egc, dss);
+    domain_suspend_common_failed(egc, dsps);
     return;
 }
 
 static void domain_suspend_common_wait_guest(libxl__egc *egc,
-                                             libxl__domain_suspend_state *dss)
+                                             libxl__domain_suspend_state *dsps)
 {
-    STATE_AO_GC(dss->ao);
+    STATE_AO_GC(dsps->ao);
     int rc;
 
     LOG(DEBUG, "wait for the guest to suspend");
 
-    rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
+    rc = libxl__ev_xswatch_register(gc, &dsps->guest_watch,
                                     suspend_common_wait_guest_watch,
                                     "@releaseDomain");
     if (rc) goto err;
 
-    rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+    rc = libxl__ev_time_register_rel(gc, &dsps->guest_timeout,
                                      suspend_common_wait_guest_timeout,
                                      60*1000);
     if (rc) goto err;
     return;
 
  err:
-    domain_suspend_common_failed(egc, dss);
+    domain_suspend_common_failed(egc, dsps);
 }
 
 static void suspend_common_wait_guest_watch(libxl__egc *egc,
       libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
-    suspend_common_wait_guest_check(egc, dss);
+    libxl__domain_suspend_state *dsps = CONTAINER_OF(xsw, *dsps, guest_watch);
+    suspend_common_wait_guest_check(egc, dsps);
 }
 
 static void suspend_common_wait_guest_check(libxl__egc *egc,
-        libxl__domain_suspend_state *dss)
+        libxl__domain_suspend_state *dsps)
 {
-    STATE_AO_GC(dss->ao);
+    STATE_AO_GC(dsps->ao);
     xc_domaininfo_t info;
     int ret;
     int shutdown_reason;
 
     /* Convenience aliases */
-    const uint32_t domid = dss->domid;
+    const uint32_t domid = dsps->domid;
 
     ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
     if (ret < 0) {
@@ -302,74 +302,76 @@ static void suspend_common_wait_guest_check(libxl__egc *egc,
     }
 
     LOG(DEBUG, "guest has suspended");
-    domain_suspend_common_guest_suspended(egc, dss);
+    domain_suspend_common_guest_suspended(egc, dsps);
     return;
 
  err:
-    domain_suspend_common_failed(egc, dss);
+    domain_suspend_common_failed(egc, dsps);
 }
 
 static void suspend_common_wait_guest_timeout(libxl__egc *egc,
       libxl__ev_time *ev, const struct timeval *requested_abs)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
-    STATE_AO_GC(dss->ao);
+    libxl__domain_suspend_state *dsps = CONTAINER_OF(ev, *dsps, guest_timeout);
+    STATE_AO_GC(dsps->ao);
     LOG(ERROR, "guest did not suspend, timed out");
-    domain_suspend_common_failed(egc, dss);
+    domain_suspend_common_failed(egc, dsps);
 }
 
 static void domain_suspend_common_guest_suspended(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss)
+                                         libxl__domain_suspend_state *dsps)
 {
-    STATE_AO_GC(dss->ao);
+    STATE_AO_GC(dsps->ao);
     int ret;
 
-    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
-    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
-    libxl__ev_time_deregister(gc, &dss->guest_timeout);
+    libxl__ev_evtchn_cancel(gc, &dsps->guest_evtchn);
+    libxl__ev_xswatch_deregister(gc, &dsps->guest_watch);
+    libxl__ev_time_deregister(gc, &dsps->guest_timeout);
 
-    if (dss->hvm) {
-        ret = libxl__domain_suspend_device_model(gc, dss);
+    if (dsps->hvm) {
+        ret = libxl__domain_suspend_device_model(gc, dsps);
         if (ret) {
             LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
-            domain_suspend_common_failed(egc, dss);
+            domain_suspend_common_failed(egc, dsps);
             return;
         }
     }
-    domain_suspend_common_done(egc, dss, 1);
+    domain_suspend_common_done(egc, dsps, 1);
 }
 
 static void domain_suspend_common_failed(libxl__egc *egc,
-                                         libxl__domain_suspend_state *dss)
+                                         libxl__domain_suspend_state *dsps)
 {
-    domain_suspend_common_done(egc, dss, 0);
+    domain_suspend_common_done(egc, dsps, 0);
 }
 
 static void domain_suspend_common_done(libxl__egc *egc,
-                                       libxl__domain_suspend_state *dss,
+                                       libxl__domain_suspend_state *dsps,
                                        bool ok)
 {
     EGC_GC;
-    assert(!libxl__xswait_inuse(&dss->pvcontrol));
-    libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
-    libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
-    libxl__ev_time_deregister(gc, &dss->guest_timeout);
-    dss->callback_common_done(egc, dss, ok);
+    assert(!libxl__xswait_inuse(&dsps->pvcontrol));
+    libxl__ev_evtchn_cancel(gc, &dsps->guest_evtchn);
+    libxl__ev_xswatch_deregister(gc, &dsps->guest_watch);
+    libxl__ev_time_deregister(gc, &dsps->guest_timeout);
+    dsps->callback_common_done(egc, dsps, ok);
 }
 
 void libxl__domain_suspend_callback(void *data)
 {
     libxl__save_helper_state *shs = data;
     libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_suspend_state *dsps = &dss->dsps;
 
-    dss->callback_common_done = domain_suspend_callback_common_done;
-    domain_suspend_callback_common(egc, dss);
+    dsps->callback_common_done = domain_suspend_callback_common_done;
+    domain_suspend_callback_common(egc, dsps);
 }
 
 static void domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok)
+                                libxl__domain_suspend_state *dsps, int ok)
 {
+    libxl__domain_save_state *dss = CONTAINER_OF(dsps, *dss, dsps);
     libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
 }
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index dc6b62b..1905195 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2824,11 +2824,12 @@ _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
 /*----- Domain suspend (save) state structure -----*/
 
 typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
+typedef struct libxl__domain_save_state libxl__domain_save_state;
 
-typedef void libxl__domain_suspend_cb(libxl__egc*,
-                                      libxl__domain_suspend_state*, int rc);
+typedef void libxl__domain_save_cb(libxl__egc*,
+                                   libxl__domain_save_state*, int rc);
 typedef void libxl__save_device_model_cb(libxl__egc*,
-                                         libxl__domain_suspend_state*, int rc);
+                                         libxl__domain_save_state*, int rc);
 
 typedef struct libxl__logdirty_switch {
     const char *cmd;
@@ -2839,9 +2840,27 @@ typedef struct libxl__logdirty_switch {
 } libxl__logdirty_switch;
 
 struct libxl__domain_suspend_state {
+    /* set by caller of domain_suspend_callback_common */
+    libxl__ao *ao;
+
+    uint32_t domid;
+    int hvm;
+    /* private */
+    libxl__ev_evtchn guest_evtchn;
+    int guest_evtchn_lockfd;
+    int guest_responded;
+    libxl__xswait_state pvcontrol;
+    libxl__ev_xswatch guest_watch;
+    libxl__ev_time guest_timeout;
+    const char *dm_savefile;
+    void (*callback_common_done)(libxl__egc*,
+                                 struct libxl__domain_suspend_state*, int ok);
+};
+
+struct libxl__domain_save_state {
     /* set by caller of libxl__domain_suspend */
     libxl__ao *ao;
-    libxl__domain_suspend_cb *callback;
+    libxl__domain_save_cb *callback;
 
     uint32_t domid;
     int fd;
@@ -2850,22 +2869,14 @@ struct libxl__domain_suspend_state {
     int debug;
     const libxl_domain_remus_info *remus;
     /* private */
-    libxl__ev_evtchn guest_evtchn;
-    int guest_evtchn_lockfd;
+    libxl__domain_suspend_state dsps;
     int hvm;
     int xcflags;
-    int guest_responded;
-    libxl__xswait_state pvcontrol;
-    libxl__ev_xswatch guest_watch;
-    libxl__ev_time guest_timeout;
-    const char *dm_savefile;
     libxl__remus_devices_state rds;
     libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
     int interval; /* checkpoint interval (for Remus) */
     libxl__save_helper_state shs;
     libxl__logdirty_switch logdirty;
-    void (*callback_common_done)(libxl__egc*,
-                                 struct libxl__domain_suspend_state*, int ok);
     /* private for libxl__domain_save_device_model */
     libxl__save_device_model_cb *save_dm_callback;
     libxl__datacopier_state save_dm_datacopier;
@@ -3142,11 +3153,11 @@ struct libxl__domain_create_state {
 
 /* calls dss->callback when done */
 _hidden void libxl__domain_save(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss);
+                                libxl__domain_save_state *dss);
 
 
 /* calls libxl__xc_domain_suspend_done when done */
-_hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_suspend_state*);
+_hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_save_state*);
 /* If rc==0 then retval is the return value from xc_domain_save
  * and errnoval is the errno value it provided.
  * If rc!=0, retval and errnoval are undefined. */
@@ -3179,16 +3190,16 @@ _hidden void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
 
 /* Each time the dm needs to be saved, we must call suspend and then save */
 _hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
-                                           libxl__domain_suspend_state *dss);
+                                           libxl__domain_suspend_state *dsps);
 _hidden void libxl__domain_save_device_model(libxl__egc *egc,
-                                     libxl__domain_suspend_state *dss,
+                                     libxl__domain_save_state *dss,
                                      libxl__save_device_model_cb *callback);
 
 _hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t domid);
 
 /* calls dss->callback_common_done when done */
 _hidden void libxl__domain_suspend(libxl__egc *egc,
-                                   libxl__domain_suspend_state *dss);
+                                   libxl__domain_suspend_state *dsps);
 
 /* used by libxc to suspend the guest during migration */
 _hidden void libxl__domain_suspend_callback(void *data);
@@ -3199,9 +3210,9 @@ _hidden void libxl__remus_domain_resume_callback(void *data);
 _hidden void libxl__remus_domain_checkpoint_callback(void *data);
 /* Remus setup and teardown*/
 _hidden void libxl__remus_setup(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss);
+                                libxl__domain_save_state *dss);
 _hidden void libxl__remus_teardown(libxl__egc *egc,
-                                   libxl__domain_suspend_state *dss,
+                                   libxl__domain_save_state *dss,
                                    int rc);
 
 /*
diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
index edc6843..71c6531 100644
--- a/tools/libxl/libxl_netbuffer.c
+++ b/tools/libxl/libxl_netbuffer.c
@@ -41,7 +41,7 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
 int init_subkind_nic(libxl__remus_devices_state *rds)
 {
     int rc, ret;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
 
     STATE_AO_GC(rds->ao);
 
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index 51261e0..d6cb832 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -24,7 +24,7 @@ static void remus_setup_done(libxl__egc *egc,
 static void remus_setup_failed(libxl__egc *egc,
                                libxl__remus_devices_state *rds, int rc);
 
-void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
 {
     /* Convenience aliases */
     libxl__remus_devices_state *const rds = &dss->rds;
@@ -57,7 +57,7 @@ out:
 static void remus_setup_done(libxl__egc *egc,
                              libxl__remus_devices_state *rds, int rc)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
     STATE_AO_GC(dss->ao);
 
     if (!rc) {
@@ -74,7 +74,7 @@ static void remus_setup_done(libxl__egc *egc,
 static void remus_setup_failed(libxl__egc *egc,
                                libxl__remus_devices_state *rds, int rc)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
     STATE_AO_GC(dss->ao);
 
     if (rc)
@@ -88,7 +88,7 @@ static void remus_teardown_done(libxl__egc *egc,
                                 libxl__remus_devices_state *rds,
                                 int rc);
 void libxl__remus_teardown(libxl__egc *egc,
-                           libxl__domain_suspend_state *dss,
+                           libxl__domain_save_state *dss,
                            int rc)
 {
     EGC_GC;
@@ -103,7 +103,7 @@ static void remus_teardown_done(libxl__egc *egc,
                                 libxl__remus_devices_state *rds,
                                 int rc)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
     STATE_AO_GC(dss->ao);
 
     if (rc)
@@ -115,7 +115,7 @@ static void remus_teardown_done(libxl__egc *egc,
 
 /*----- remus callbacks -----*/
 static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok);
+                                libxl__domain_suspend_state *dsps, int ok);
 static void remus_devices_postsuspend_cb(libxl__egc *egc,
                                          libxl__remus_devices_state *rds,
                                          int rc);
@@ -127,15 +127,18 @@ void libxl__remus_domain_suspend_callback(void *data)
 {
     libxl__save_helper_state *shs = data;
     libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_suspend_state *dsps = &dss->dsps;
 
-    dss->callback_common_done = remus_domain_suspend_callback_common_done;
-    libxl__domain_suspend(egc, dss);
+    dsps->callback_common_done = remus_domain_suspend_callback_common_done;
+    libxl__domain_suspend(egc, dsps);
 }
 
 static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok)
+                                libxl__domain_suspend_state *dsps, int ok)
 {
+    libxl__domain_save_state *dss = CONTAINER_OF(dsps, *dss, dsps);
+
     if (!ok)
         goto out;
 
@@ -153,7 +156,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
                                          int rc)
 {
     int ok = 0;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
 
     if (rc)
         goto out;
@@ -168,7 +171,7 @@ void libxl__remus_domain_resume_callback(void *data)
 {
     libxl__save_helper_state *shs = data;
     libxl__egc *egc = shs->egc;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
     STATE_AO_GC(dss->ao);
 
     libxl__remus_devices_state *const rds = &dss->rds;
@@ -181,7 +184,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
                                        int rc)
 {
     int ok = 0;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
     STATE_AO_GC(dss->ao);
 
     if (rc)
@@ -201,7 +204,7 @@ out:
 /*----- remus asynchronous checkpoint callback -----*/
 
 static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc);
+                                      libxl__domain_save_state *dss, int rc);
 static void remus_devices_commit_cb(libxl__egc *egc,
                                     libxl__remus_devices_state *rds,
                                     int rc);
@@ -211,7 +214,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
 void libxl__remus_domain_checkpoint_callback(void *data)
 {
     libxl__save_helper_state *shs = data;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
     libxl__egc *egc = dss->shs.egc;
     STATE_AO_GC(dss->ao);
 
@@ -224,7 +227,7 @@ void libxl__remus_domain_checkpoint_callback(void *data)
 }
 
 static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc)
+                                      libxl__domain_save_state *dss, int rc)
 {
     /* Convenience aliases */
     libxl__remus_devices_state *const rds = &dss->rds;
@@ -249,7 +252,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
                                     libxl__remus_devices_state *rds,
                                     int rc)
 {
-    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
 
     STATE_AO_GC(dss->ao);
 
@@ -283,7 +286,7 @@ out:
 static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
                                   const struct timeval *requested_abs)
 {
-    libxl__domain_suspend_state *dss =
+    libxl__domain_save_state *dss =
                             CONTAINER_OF(ev, *dss, checkpoint_timeout);
 
     STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 40b25e4..cd342b9 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -74,7 +74,7 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
                argnums, ARRAY_SIZE(argnums));
 }
 
-void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
 {
     STATE_AO_GC(dss->ao);
     int r, rc, toolstack_data_fd = -1;
-- 
1.9.1

Re: [PATCH v2 6/6] libxl/save: Refactor libxl__domain_suspend_state

[Ian Campbell]

On Wed, 2015-06-03 at 16:01 +0800, Yang Hongyang wrote:
> Currently struct libxl__domain_suspend_state contains 2 type of states,
> one is save state, another is suspend state. This patch separate it out.

"This patch separates those two out".

> The motivation of this is that COLO will need to do suspend/resume
> continuesly, we need a more common suspend state.

"continuously"

> After this change, dss stands for libxl__domain_save_state,
> dsps stands for libxl__domain_suspend_state.
> 
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> CC: Ian Campbell <Ian.Campbell@citrix.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> CC: Andrew Cooper <andrew.cooper3@citrix.com>

I presume this is almost entirely mechanical and the compiler mostly
took care of identifying everywhere to be changed:

Acked-by: Ian Campbell <ian.campbell@citrix.com>

[...]
> @@ -2839,9 +2840,27 @@ typedef struct libxl__logdirty_switch {
>  } libxl__logdirty_switch;
>  
>  struct libxl__domain_suspend_state {
[...]
> +struct libxl__domain_save_state {

Please could you add a little comment before each of these explaining
their scope for the hard of thinking (i.e. me). One is the live phase
and one is the final stop-and-copy phase, I think.

Ian.

Re: [PATCH v2 6/6] libxl/save: Refactor libxl__domain_suspend_state

[Ian Jackson]

Yang Hongyang writes ("[PATCH v2 6/6] libxl/save: Refactor libxl__domain_suspend_state"):
> Currently struct libxl__domain_suspend_state contains 2 type of states,
> one is save state, another is suspend state. This patch separate it out.
> The motivation of this is that COLO will need to do suspend/resume
> continuesly, we need a more common suspend state.

Currently in libxl/libxc/etc.  "suspend" and "save" have referred to
the same thing, but different terminology has been used at different
layers.  Ie both "suspend" and "save" each refer to either or both of
"save to disk" or "suspend for live migrate", or to the relevant
underlying mechanisms.

So I'm not sure introducing a distinction between those two terms in
libxl is really helpful.  If it is to be done there should be a clear
explanation of what the difference is.

On IRC you said:

14:21 <yanghy> Diziet, currently, in libxl, suspend is used as 2
               means, one is save(corrspond to libxc save), another is
               suspend the guest(related to suspend callback)

That's rather different, I think.  Or, at least, I'm not sure that I
understand this distinction as you are making it.  The suspend
callback is part of the implementation of what at higher layers we
save/suspend/restore/migration.

AIUI this callback is related to pausing the guest, or manipulating
its VCPUs ?  Perhaps we should rename this callback ?  Maybe Andrew
Cooper can suggest a name ?

Forgive me if I'm confused and going off in the wrong direction...

Thanks,
Ian.

Re: [PATCH v2 6/6] libxl/save: Refactor libxl__domain_suspend_state

[Andrew Cooper]

On 16/06/15 14:26, Ian Jackson wrote:
> Yang Hongyang writes ("[PATCH v2 6/6] libxl/save: Refactor libxl__domain_suspend_state"):
>> Currently struct libxl__domain_suspend_state contains 2 type of states,
>> one is save state, another is suspend state. This patch separate it out.
>> The motivation of this is that COLO will need to do suspend/resume
>> continuesly, we need a more common suspend state.
> Currently in libxl/libxc/etc.  "suspend" and "save" have referred to
> the same thing, but different terminology has been used at different
> layers.  Ie both "suspend" and "save" each refer to either or both of
> "save to disk" or "suspend for live migrate", or to the relevant
> underlying mechanisms.
>
> So I'm not sure introducing a distinction between those two terms in
> libxl is really helpful.  If it is to be done there should be a clear
> explanation of what the difference is.
>
> On IRC you said:
>
> 14:21 <yanghy> Diziet, currently, in libxl, suspend is used as 2
>                means, one is save(corrspond to libxc save), another is
>                suspend the guest(related to suspend callback)
>
> That's rather different, I think.  Or, at least, I'm not sure that I
> understand this distinction as you are making it.  The suspend
> callback is part of the implementation of what at higher layers we
> save/suspend/restore/migration.
>
> AIUI this callback is related to pausing the guest, or manipulating
> its VCPUs ?  Perhaps we should rename this callback ?  Maybe Andrew
> Cooper can suggest a name ?
>
> Forgive me if I'm confused and going off in the wrong direction...

The terminology used by libxc is more consistent in this area.

"suspend" refers to quiescing the VM, so pausing qemu, making a
remote_shutdown(SHUTDOWN_suspend) hypercall etc.

"save" refers to the actions involved in actually shuffling the state of
the VM, so xc_domain_save() etc.

I believe that these are useful distinctions to maintain.  libxl
currently uses "suspend" to encapsulate both.

~Andrew

Re: [PATCH v2 0/6] Misc cleanups for libxl

[Andrew Cooper]

On 03/06/15 09:01, Yang Hongyang wrote:
> This patchset mainly focus on libxl save, most of the patches are
> simply move codes out of libxl_dom.c, except a refactor patch.
>
> Please see individual patch for detail.
>
> Can get the whole patchset from:
>     https://github.com/macrosheep/xen/tree/misc-libxl-v2

Overall, these look like good changes, although I have not reviewed them
in detail.

~Andrew

>
> v1->v2:
>   - use dsps for suspend_state and dss for save_state.
>   - move resume code to libxl_dom_suspend.c
>   - move toolstatck save/restore code to libxl_dom_save.c
>   - move refactor pacth to the end so that rebase of the patchset easier.
>
> Yang Hongyang (6):
>   tools/libxl: rename libxl__domain_suspend to libxl__domain_save
>   tools/libxl: move domain suspend code into libxl_dom_suspend.c
>   tools/libxl: move domain resume code into libxl_dom_suspend.c
>   tools/libxl: move remus code into libxl_remus.c
>   tools/libxl: move save/restore code into libxl_dom_save.c
>   libxl/save: Refactor libxl__domain_suspend_state
>
>  tools/libxl/Makefile             |    5 +-
>  tools/libxl/libxl.c              |  126 +---
>  tools/libxl/libxl_dom.c          | 1202 --------------------------------------
>  tools/libxl/libxl_dom_save.c     |  672 +++++++++++++++++++++
>  tools/libxl/libxl_dom_suspend.c  |  465 +++++++++++++++
>  tools/libxl/libxl_internal.h     |   65 ++-
>  tools/libxl/libxl_netbuffer.c    |    2 +-
>  tools/libxl/libxl_remus.c        |  307 ++++++++++
>  tools/libxl/libxl_save_callout.c |    2 +-
>  9 files changed, 1503 insertions(+), 1343 deletions(-)
>  create mode 100644 tools/libxl/libxl_dom_save.c
>  create mode 100644 tools/libxl/libxl_dom_suspend.c
>  create mode 100644 tools/libxl/libxl_remus.c
>

Re: [PATCH v2 0/6] Misc cleanups for libxl

[Yang Hongyang]

On 06/03/2015 06:51 PM, Andrew Cooper wrote:
> On 03/06/15 09:01, Yang Hongyang wrote:
>> This patchset mainly focus on libxl save, most of the patches are
>> simply move codes out of libxl_dom.c, except a refactor patch.
>>
>> Please see individual patch for detail.
>>
>> Can get the whole patchset from:
>>      https://github.com/macrosheep/xen/tree/misc-libxl-v2
>
> Overall, these look like good changes, although I have not reviewed them
> in detail.

Thank you! hope it won't affect migration v2 too much.

>
> ~Andrew
>
>>
>> v1->v2:
>>    - use dsps for suspend_state and dss for save_state.
>>    - move resume code to libxl_dom_suspend.c
>>    - move toolstatck save/restore code to libxl_dom_save.c
>>    - move refactor pacth to the end so that rebase of the patchset easier.
>>
>> Yang Hongyang (6):
>>    tools/libxl: rename libxl__domain_suspend to libxl__domain_save
>>    tools/libxl: move domain suspend code into libxl_dom_suspend.c
>>    tools/libxl: move domain resume code into libxl_dom_suspend.c
>>    tools/libxl: move remus code into libxl_remus.c
>>    tools/libxl: move save/restore code into libxl_dom_save.c
>>    libxl/save: Refactor libxl__domain_suspend_state
>>
>>   tools/libxl/Makefile             |    5 +-
>>   tools/libxl/libxl.c              |  126 +---
>>   tools/libxl/libxl_dom.c          | 1202 --------------------------------------
>>   tools/libxl/libxl_dom_save.c     |  672 +++++++++++++++++++++
>>   tools/libxl/libxl_dom_suspend.c  |  465 +++++++++++++++
>>   tools/libxl/libxl_internal.h     |   65 ++-
>>   tools/libxl/libxl_netbuffer.c    |    2 +-
>>   tools/libxl/libxl_remus.c        |  307 ++++++++++
>>   tools/libxl/libxl_save_callout.c |    2 +-
>>   9 files changed, 1503 insertions(+), 1343 deletions(-)
>>   create mode 100644 tools/libxl/libxl_dom_save.c
>>   create mode 100644 tools/libxl/libxl_dom_suspend.c
>>   create mode 100644 tools/libxl/libxl_remus.c
>>
>
> .
>

-- 
Thanks,
Yang.