Re: [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device

[Ian Campbell <ian.campbell@citrix.com>]

On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Checkpoint device is an abstract layer to do checkpoint.
> COLO can also use it to do checkpoint. But there are
> still some codes in checkpoint device which touch remus:
> 1. remus_ops: we use remus ops directly in checkpoint
>    device. Store it in checkpoint device state.
> 2. concrete layer's private member: add a new structure
>    remus state, and move them to remus state.
> 3. init/cleanup device subkind: we call (init|cleanup)_subkind_nic
>    and (init|cleanup)_subkind_drbd_disk directly in checkpoint
>    device. Call them before calling libxl__checkpoint_devices_setup()
>    or after calling libxl__checkpoint_devices_teardown().

This is quite a lot to think about in one go, can any of it be split up?

> it is pure refactoring and no functional changes.
> 
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
>  tools/libxl/libxl.c                   |  2 +-
>  tools/libxl/libxl_checkpoint_device.c | 52 ++-------------------
>  tools/libxl/libxl_dom_save.c          |  3 +-
>  tools/libxl/libxl_internal.h          | 40 ++++++++++------
>  tools/libxl/libxl_netbuffer.c         | 51 +++++++++++---------
>  tools/libxl/libxl_remus.c             | 88 ++++++++++++++++++++++++++++-------
>  tools/libxl/libxl_remus_disk_drbd.c   |  8 ++--
>  7 files changed, 135 insertions(+), 109 deletions(-)
> 
> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> index a837ff2..f851957 100644
> --- a/tools/libxl/libxl.c
> +++ b/tools/libxl/libxl.c
> @@ -842,7 +842,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
>      assert(info);
>  
>      /* Point of no return */
> -    libxl__remus_setup(egc, dss);
> +    libxl__remus_setup(egc, &dss->rs);
>      return AO_INPROGRESS;
>  
>   out:
> diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
> index 226f159..0a16dbb 100644
> --- a/tools/libxl/libxl_checkpoint_device.c
> +++ b/tools/libxl/libxl_checkpoint_device.c
> @@ -17,46 +17,6 @@
>  
>  #include "libxl_internal.h"
>  
> -extern const libxl__checkpoint_device_instance_ops remus_device_nic;
> -extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
> -static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
> -    &remus_device_nic,
> -    &remus_device_drbd_disk,
> -    NULL,
> -};
> -
> -/*----- helper functions -----*/
> -
> -static int init_device_subkind(libxl__checkpoint_devices_state *cds)
> -{
> -    /* init device subkind-specific state in the libxl ctx */
> -    int rc;
> -    STATE_AO_GC(cds->ao);
> -
> -    if (libxl__netbuffer_enabled(gc)) {
> -        rc = init_subkind_nic(cds);
> -        if (rc) goto out;
> -    }
> -
> -    rc = init_subkind_drbd_disk(cds);
> -    if (rc) goto out;
> -
> -    rc = 0;
> -out:
> -    return rc;
> -}
> -
> -static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
> -{
> -    /* cleanup device subkind-specific state in the libxl ctx */
> -    STATE_AO_GC(cds->ao);
> -
> -    if (libxl__netbuffer_enabled(gc))
> -        cleanup_subkind_nic(cds);
> -
> -    cleanup_subkind_drbd_disk(cds);
> -}
> -
>  /*----- setup() and teardown() -----*/
>  
>  /* callbacks */
> @@ -94,14 +54,10 @@ static void checkpoint_devices_setup(libxl__egc *egc,
>  void libxl__checkpoint_devices_setup(libxl__egc *egc,
>                                       libxl__checkpoint_devices_state *cds)
>  {
> -    int i, rc;
> +    int i;
>  
>      STATE_AO_GC(cds->ao);
>  
> -    rc = init_device_subkind(cds);
> -    if (rc)
> -        goto out;
> -
>      cds->num_devices = 0;
>      cds->num_nics = 0;
>      cds->num_disks = 0;
> @@ -134,7 +90,7 @@ void libxl__checkpoint_devices_setup(libxl__egc *egc,
>      return;
>  
>  out:
> -    cds->callback(egc, cds, rc);
> +    cds->callback(egc, cds, 0);
>  }
>  
>  static void checkpoint_devices_setup(libxl__egc *egc,
> @@ -172,7 +128,7 @@ static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
>          goto out;
>  
>      do {
> -        dev->ops = remus_ops[++dev->ops_index];
> +        dev->ops = dev->cds->ops[++dev->ops_index];
>          if (!dev->ops) {
>              libxl_device_nic * nic = NULL;
>              libxl_device_disk * disk = NULL;
> @@ -271,8 +227,6 @@ static void devices_teardown_cb(libxl__egc *egc,
>      cds->disks = NULL;
>      cds->num_disks = 0;
>  
> -    cleanup_device_subkind(cds);
> -
>      cds->callback(egc, cds, rc);
>  }
>  
> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
> index 9a3b33d..9a3d009 100644
> --- a/tools/libxl/libxl_dom_save.c
> +++ b/tools/libxl/libxl_dom_save.c
> @@ -410,7 +410,6 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>      dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
>  
>      if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
> -        dss->interval = r_info->interval;
>          if (libxl_defbool_val(r_info->compression))
>              dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
>      }
> @@ -581,7 +580,7 @@ static void domain_save_done(libxl__egc *egc,
>       * from sending checkpoints. Teardown the network buffers and
>       * release netlink resources.  This is an async op.
>       */
> -    libxl__remus_teardown(egc, dss, rc);
> +    libxl__remus_teardown(egc, &dss->rs, rc);
>  }
>  
>  /*========================= Domain restore ============================*/
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index f3a139b..840734d 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -2752,6 +2752,8 @@ struct libxl__checkpoint_devices_state {
>      uint32_t domid;
>      libxl__checkpoint_callback *callback;
>      int device_kind_flags;
> +    /* The ops must be pointer array, and the last ops must be NULL */
> +    const libxl__checkpoint_device_instance_ops **ops;
>  
>      /*----- private for abstract layer only -----*/
>  
> @@ -2772,16 +2774,6 @@ struct libxl__checkpoint_devices_state {
>      int num_disks;
>  
>      libxl__multidev multidev;
> -
> -    /*----- private for concrete (device-specific) layer only -----*/
> -
> -    /* private for nic device subkind ops */
> -    char *netbufscript;
> -    struct nl_sock *nlsock;
> -    struct nl_cache *qdisc_cache;
> -
> -    /* private for drbd disk subkind ops */
> -    char *drbd_probe_script;
>  };
>  
>  /*
> @@ -2829,6 +2821,26 @@ _hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
>                                          libxl__checkpoint_devices_state *cds);
>  _hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
>                                          libxl__checkpoint_devices_state *cds);
> +
> +/*----- Remus related state structure -----*/
> +typedef struct libxl__remus_state libxl__remus_state;
> +struct libxl__remus_state {
> +    /* private */
> +    libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
> +    int interval; /* checkpoint interval */
> +
> +    /* abstract layer */
> +    libxl__checkpoint_devices_state cds;
> +
> +    /*----- private for concrete (device-specific) layer only -----*/
> +    /* private for nic device subkind ops */
> +    char *netbufscript;
> +    struct nl_sock *nlsock;
> +    struct nl_cache *qdisc_cache;
> +
> +    /* private for drbd disk subkind ops */
> +    char *drbd_probe_script;
> +};
>  _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
>  
>  /*----- Legacy conversion helper -----*/
> @@ -2962,9 +2974,7 @@ struct libxl__domain_save_state {
>      libxl__domain_suspend_state dsps;
>      int hvm;
>      int xcflags;
> -    libxl__checkpoint_devices_state cds;
> -    libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
> -    int interval; /* checkpoint interval (for Remus) */
> +    libxl__remus_state rs;
>      libxl__save_helper_state shs;
>      libxl__logdirty_switch logdirty;
>      /* private for libxl__domain_save_device_model */
> @@ -3364,9 +3374,9 @@ _hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
>  _hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
>  /* Remus setup and teardown*/
>  _hidden void libxl__remus_setup(libxl__egc *egc,
> -                                libxl__domain_save_state *dss);
> +                                libxl__remus_state *rs);
>  _hidden void libxl__remus_teardown(libxl__egc *egc,
> -                                   libxl__domain_save_state *dss,
> +                                   libxl__remus_state *rs,
>                                     int rc);
>  
>  /*
> diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
> index 86afba6..1d01e10 100644
> --- a/tools/libxl/libxl_netbuffer.c
> +++ b/tools/libxl/libxl_netbuffer.c
> @@ -41,18 +41,19 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
>  int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>  {
>      int rc, ret;
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
>  
>      STATE_AO_GC(cds->ao);
>  
> -    cds->nlsock = nl_socket_alloc();
> -    if (!cds->nlsock) {
> +    rs->nlsock = nl_socket_alloc();
> +    if (!rs->nlsock) {
>          LOG(ERROR, "cannot allocate nl socket");
>          rc = ERROR_FAIL;
>          goto out;
>      }
>  
> -    ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
> +    ret = nl_connect(rs->nlsock, NETLINK_ROUTE);
>      if (ret) {
>          LOG(ERROR, "failed to open netlink socket: %s",
>              nl_geterror(ret));
> @@ -61,7 +62,7 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>      }
>  
>      /* get list of all qdiscs installed on network devs. */
> -    ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
> +    ret = rtnl_qdisc_alloc_cache(rs->nlsock, &rs->qdisc_cache);
>      if (ret) {
>          LOG(ERROR, "failed to allocate qdisc cache: %s",
>              nl_geterror(ret));
> @@ -70,10 +71,10 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>      }
>  
>      if (dss->remus->netbufscript) {
> -        cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
> +        rs->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
>      } else {
> -        cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
> -                                      libxl__xen_script_dir_path());
> +        rs->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
> +                                     libxl__xen_script_dir_path());
>      }
>  
>      rc = 0;
> @@ -84,20 +85,22 @@ out:
>  
>  void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
>  {
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
> +
>      STATE_AO_GC(cds->ao);
>  
>      /* free qdisc cache */
> -    if (cds->qdisc_cache) {
> -        nl_cache_clear(cds->qdisc_cache);
> -        nl_cache_free(cds->qdisc_cache);
> -        cds->qdisc_cache = NULL;
> +    if (rs->qdisc_cache) {
> +        nl_cache_clear(rs->qdisc_cache);
> +        nl_cache_free(rs->qdisc_cache);
> +        rs->qdisc_cache = NULL;
>      }
>  
>      /* close & free nlsock */
> -    if (cds->nlsock) {
> -        nl_close(cds->nlsock);
> -        nl_socket_free(cds->nlsock);
> -        cds->nlsock = NULL;
> +    if (rs->nlsock) {
> +        nl_close(rs->nlsock);
> +        nl_socket_free(rs->nlsock);
> +        rs->nlsock = NULL;
>      }
>  }
>  
> @@ -150,13 +153,14 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
>      int rc, ret, ifindex;
>      struct rtnl_link *ifb = NULL;
>      struct rtnl_qdisc *qdisc = NULL;
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>  
>      STATE_AO_GC(cds->ao);
>  
>      /* Now that we have brought up REMUS_IFB device with plug qdisc for
>       * this vif, so we need to refill the qdisc cache.
>       */
> -    ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
> +    ret = nl_cache_refill(rs->nlsock, rs->qdisc_cache);
>      if (ret) {
>          LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
>          rc = ERROR_FAIL;
> @@ -164,7 +168,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
>      }
>  
>      /* get a handle to the REMUS_IFB interface */
> -    ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
> +    ret = rtnl_link_get_kernel(rs->nlsock, 0, remus_nic->ifb, &ifb);
>      if (ret) {
>          LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
>              nl_geterror(ret));
> @@ -187,7 +191,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
>       * There is no need to explicitly free this qdisc as its just a
>       * reference from the qdisc cache we allocated earlier.
>       */
> -    qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
> +    qdisc = rtnl_qdisc_get_by_parent(rs->qdisc_cache, ifindex, TC_H_ROOT);
>      if (qdisc) {
>          const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
>          /* Sanity check: Ensure that the root qdisc is a plug qdisc. */
> @@ -238,11 +242,12 @@ static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
>      libxl__remus_device_nic *remus_nic = dev->concrete_data;
>      libxl__checkpoint_devices_state *cds = dev->cds;
>      libxl__async_exec_state *aes = &dev->aodev.aes;
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>  
>      STATE_AO_GC(cds->ao);
>  
>      /* Convenience aliases */
> -    char *const script = libxl__strdup(gc, cds->netbufscript);
> +    char *const script = libxl__strdup(gc, rs->netbufscript);
>      const uint32_t domid = cds->domid;
>      const int dev_id = remus_nic->devid;
>      const char *const vif = remus_nic->vif;
> @@ -333,6 +338,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>      libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>      libxl__remus_device_nic *remus_nic = dev->concrete_data;
>      libxl__checkpoint_devices_state *cds = dev->cds;
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>      const char *out_path_base, *hotplug_error = NULL;
>      int rc;
>  
> @@ -373,7 +379,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>  
>      if (hotplug_error) {
>          LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
> -            cds->netbufscript, vif, hotplug_error);
> +            rs->netbufscript, vif, hotplug_error);
>          rc = ERROR_FAIL;
>          goto out;
>      }
> @@ -444,6 +450,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
>                             int buffer_op)
>  {
>      int rc, ret;
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>  
>      STATE_AO_GC(cds->ao);
>  
> @@ -457,7 +464,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
>          goto out;
>      }
>  
> -    ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
> +    ret = rtnl_qdisc_add(rs->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
>      if (ret) {
>          rc = ERROR_FAIL;
>          goto out;
> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
> index fe4acc8..2a427a4 100644
> --- a/tools/libxl/libxl_remus.c
> +++ b/tools/libxl/libxl_remus.c
> @@ -17,6 +17,46 @@
>  
>  #include "libxl_internal.h"
>  
> +extern const libxl__checkpoint_device_instance_ops remus_device_nic;
> +extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
> +static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
> +    &remus_device_nic,
> +    &remus_device_drbd_disk,
> +    NULL,
> +};
> +
> +/*----- helper functions -----*/
> +
> +static int init_device_subkind(libxl__checkpoint_devices_state *cds)
> +{
> +    /* init device subkind-specific state in the libxl ctx */
> +    int rc;
> +    STATE_AO_GC(cds->ao);
> +
> +    if (libxl__netbuffer_enabled(gc)) {
> +        rc = init_subkind_nic(cds);
> +        if (rc) goto out;
> +    }
> +
> +    rc = init_subkind_drbd_disk(cds);
> +    if (rc) goto out;
> +
> +    rc = 0;
> +out:
> +    return rc;
> +}
> +
> +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
> +{
> +    /* cleanup device subkind-specific state in the libxl ctx */
> +    STATE_AO_GC(cds->ao);
> +
> +    if (libxl__netbuffer_enabled(gc))
> +        cleanup_subkind_nic(cds);
> +
> +    cleanup_subkind_drbd_disk(cds);
> +}
> +
>  /*----- Remus setup and teardown -----*/
>  
>  static void remus_setup_done(libxl__egc *egc,
> @@ -24,10 +64,12 @@ static void remus_setup_done(libxl__egc *egc,
>  static void remus_setup_failed(libxl__egc *egc,
>                                 libxl__checkpoint_devices_state *cds, int rc);
>  
> -void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
> +void libxl__remus_setup(libxl__egc *egc, libxl__remus_state *rs)
>  {
> +    libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
> +
>      /* Convenience aliases */
> -    libxl__checkpoint_devices_state *const cds = &dss->cds;
> +    libxl__checkpoint_devices_state *const cds = &rs->cds;
>      const libxl_domain_remus_info *const info = dss->remus;
>  
>      STATE_AO_GC(dss->ao);
> @@ -46,6 +88,14 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
>      cds->ao = ao;
>      cds->domid = dss->domid;
>      cds->callback = remus_setup_done;
> +    cds->ops = remus_ops;
> +    rs->interval = info->interval;
> +
> +    if (init_device_subkind(cds)) {
> +        LOG(ERROR, "Remus: failed to init device subkind for guest %u",
> +            dss->domid);
> +        goto out;
> +    }
>  
>      libxl__checkpoint_devices_setup(egc, cds);
>      return;
> @@ -57,7 +107,7 @@ out:
>  static void remus_setup_done(libxl__egc *egc,
>                                     libxl__checkpoint_devices_state *cds, int rc)
>  {
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>      STATE_AO_GC(dss->ao);
>  
>      if (!rc) {
> @@ -74,13 +124,15 @@ static void remus_setup_done(libxl__egc *egc,
>  static void remus_setup_failed(libxl__egc *egc,
>                                 libxl__checkpoint_devices_state *cds, int rc)
>  {
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>      STATE_AO_GC(dss->ao);
>  
>      if (rc)
>          LOG(ERROR, "Remus: failed to teardown device after setup failed"
>              " for guest with domid %u, rc %d", dss->domid, rc);
>  
> +    cleanup_device_subkind(cds);
> +
>      dss->callback(egc, dss, rc);
>  }
>  
> @@ -88,28 +140,30 @@ static void remus_teardown_done(libxl__egc *egc,
>                                  libxl__checkpoint_devices_state *cds,
>                                  int rc);
>  void libxl__remus_teardown(libxl__egc *egc,
> -                           libxl__domain_save_state *dss,
> +                           libxl__remus_state *rs,
>                             int rc)
>  {
>      EGC_GC;
>  
>      LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
>          " teardown Remus devices...", rc);
> -    dss->cds.callback = remus_teardown_done;
> -    libxl__checkpoint_devices_teardown(egc, &dss->cds);
> +    rs->cds.callback = remus_teardown_done;
> +    libxl__checkpoint_devices_teardown(egc, &rs->cds);
>  }
>  
>  static void remus_teardown_done(libxl__egc *egc,
>                                  libxl__checkpoint_devices_state *cds,
>                                  int rc)
>  {
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>      STATE_AO_GC(dss->ao);
>  
>      if (rc)
>          LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
>              " rc %d", dss->domid, rc);
>  
> +    cleanup_device_subkind(cds);
> +
>      dss->callback(egc, dss, rc);
>  }
>  
> @@ -143,7 +197,7 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>      if (!ok)
>          goto out;
>  
> -    libxl__checkpoint_devices_state *const cds = &dss->cds;
> +    libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>      cds->callback = remus_devices_postsuspend_cb;
>      libxl__checkpoint_devices_postsuspend(egc, cds);
>      return;
> @@ -157,7 +211,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
>                                           int rc)
>  {
>      int ok = 0;
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>  
>      if (rc)
>          goto out;
> @@ -175,7 +229,7 @@ void libxl__remus_domain_resume_callback(void *data)
>      libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
>      STATE_AO_GC(dss->ao);
>  
> -    libxl__checkpoint_devices_state *const cds = &dss->cds;
> +    libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>      cds->callback = remus_devices_preresume_cb;
>      libxl__checkpoint_devices_preresume(egc, cds);
>  }
> @@ -185,7 +239,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
>                                         int rc)
>  {
>      int ok = 0;
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>      STATE_AO_GC(dss->ao);
>  
>      if (rc)
> @@ -229,7 +283,7 @@ static void remus_checkpoint_stream_written(
>      libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
>  
>      /* Convenience aliases */
> -    libxl__checkpoint_devices_state *const cds = &dss->cds;
> +    libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>  
>      STATE_AO_GC(dss->ao);
>  
> @@ -251,7 +305,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
>                                      libxl__checkpoint_devices_state *cds,
>                                      int rc)
>  {
> -    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> +    libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>  
>      STATE_AO_GC(dss->ao);
>  
> @@ -269,9 +323,9 @@ static void remus_devices_commit_cb(libxl__egc *egc,
>       */
>  
>      /* Set checkpoint interval timeout */
> -    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
> +    rc = libxl__ev_time_register_rel(gc, &dss->rs.checkpoint_timeout,
>                                       remus_next_checkpoint,
> -                                     dss->interval);
> +                                     dss->rs.interval);
>  
>      if (rc)
>          goto out;
> @@ -286,7 +340,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>                                    const struct timeval *requested_abs)
>  {
>      libxl__domain_save_state *dss =
> -                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
> +                            CONTAINER_OF(ev, *dss, rs.checkpoint_timeout);
>  
>      STATE_AO_GC(dss->ao);
>  
> diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
> index 50b897d..a8d8949 100644
> --- a/tools/libxl/libxl_remus_disk_drbd.c
> +++ b/tools/libxl/libxl_remus_disk_drbd.c
> @@ -28,10 +28,11 @@ typedef struct libxl__remus_drbd_disk {
>  
>  int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
>  {
> +    libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>      STATE_AO_GC(cds->ao);
>  
> -    cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
> -                                       libxl__xen_script_dir_path());
> +    rs->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
> +                                      libxl__xen_script_dir_path());
>  
>      return 0;
>  }
> @@ -96,6 +97,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
>      int arraysize, nr = 0, rc;
>      const libxl_device_disk *disk = dev->backend_dev;
>      libxl__async_exec_state *aes = &dev->aodev.aes;
> +    libxl__remus_state *rs = CONTAINER_OF(dev->cds, *rs, cds);
>      STATE_AO_GC(dev->cds->ao);
>  
>      /* setup env & args */
> @@ -107,7 +109,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
>      arraysize = 3;
>      nr = 0;
>      GCNEW_ARRAY(aes->args, arraysize);
> -    aes->args[nr++] = dev->cds->drbd_probe_script;
> +    aes->args[nr++] = rs->drbd_probe_script;
>      aes->args[nr++] = disk->pdev_path;
>      aes->args[nr++] = NULL;
>      assert(nr <= arraysize);