From mboxrd@z Thu Jan 1 00:00:00 1970 From: Yang Hongyang Subject: Re: [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device Date: Wed, 1 Jul 2015 11:11:47 +0800 Message-ID: <55935A73.8060106@cn.fujitsu.com> References: <1435213552-10556-1-git-send-email-yanghy@cn.fujitsu.com> <1435213552-10556-27-git-send-email-yanghy@cn.fujitsu.com> <1435661439.21469.87.camel@citrix.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii"; Format="flowed" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1435661439.21469.87.camel@citrix.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Ian Campbell Cc: wei.liu2@citrix.com, wency@cn.fujitsu.com, andrew.cooper3@citrix.com, yunhong.jiang@intel.com, eddie.dong@intel.com, xen-devel@lists.xen.org, guijianfeng@cn.fujitsu.com, rshriram@cs.ubc.ca, ian.jackson@eu.citrix.com List-Id: xen-devel@lists.xenproject.org On 06/30/2015 06:50 PM, Ian Campbell wrote: > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote: >> Checkpoint device is an abstract layer to do checkpoint. >> COLO can also use it to do checkpoint. But there are >> still some codes in checkpoint device which touch remus: >> 1. remus_ops: we use remus ops directly in checkpoint >> device. Store it in checkpoint device state. >> 2. concrete layer's private member: add a new structure >> remus state, and move them to remus state. >> 3. init/cleanup device subkind: we call (init|cleanup)_subkind_nic >> and (init|cleanup)_subkind_drbd_disk directly in checkpoint >> device. Call them before calling libxl__checkpoint_devices_setup() >> or after calling libxl__checkpoint_devices_teardown(). > > This is quite a lot to think about in one go, can any of it be split up? I will try to split this patch in the next version... > >> it is pure refactoring and no functional changes. >> >> Signed-off-by: Wen Congyang >> Signed-off-by: Yang Hongyang >> --- >> tools/libxl/libxl.c | 2 +- >> tools/libxl/libxl_checkpoint_device.c | 52 ++------------------- >> tools/libxl/libxl_dom_save.c | 3 +- >> tools/libxl/libxl_internal.h | 40 ++++++++++------ >> tools/libxl/libxl_netbuffer.c | 51 +++++++++++--------- >> tools/libxl/libxl_remus.c | 88 ++++++++++++++++++++++++++++------- >> tools/libxl/libxl_remus_disk_drbd.c | 8 ++-- >> 7 files changed, 135 insertions(+), 109 deletions(-) >> >> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c >> index a837ff2..f851957 100644 >> --- a/tools/libxl/libxl.c >> +++ b/tools/libxl/libxl.c >> @@ -842,7 +842,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, >> assert(info); >> >> /* Point of no return */ >> - libxl__remus_setup(egc, dss); >> + libxl__remus_setup(egc, &dss->rs); >> return AO_INPROGRESS; >> >> out: >> diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c >> index 226f159..0a16dbb 100644 >> --- a/tools/libxl/libxl_checkpoint_device.c >> +++ b/tools/libxl/libxl_checkpoint_device.c >> @@ -17,46 +17,6 @@ >> >> #include "libxl_internal.h" >> >> -extern const libxl__checkpoint_device_instance_ops remus_device_nic; >> -extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk; >> -static const libxl__checkpoint_device_instance_ops *remus_ops[] = { >> - &remus_device_nic, >> - &remus_device_drbd_disk, >> - NULL, >> -}; >> - >> -/*----- helper functions -----*/ >> - >> -static int init_device_subkind(libxl__checkpoint_devices_state *cds) >> -{ >> - /* init device subkind-specific state in the libxl ctx */ >> - int rc; >> - STATE_AO_GC(cds->ao); >> - >> - if (libxl__netbuffer_enabled(gc)) { >> - rc = init_subkind_nic(cds); >> - if (rc) goto out; >> - } >> - >> - rc = init_subkind_drbd_disk(cds); >> - if (rc) goto out; >> - >> - rc = 0; >> -out: >> - return rc; >> -} >> - >> -static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds) >> -{ >> - /* cleanup device subkind-specific state in the libxl ctx */ >> - STATE_AO_GC(cds->ao); >> - >> - if (libxl__netbuffer_enabled(gc)) >> - cleanup_subkind_nic(cds); >> - >> - cleanup_subkind_drbd_disk(cds); >> -} >> - >> /*----- setup() and teardown() -----*/ >> >> /* callbacks */ >> @@ -94,14 +54,10 @@ static void checkpoint_devices_setup(libxl__egc *egc, >> void libxl__checkpoint_devices_setup(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds) >> { >> - int i, rc; >> + int i; >> >> STATE_AO_GC(cds->ao); >> >> - rc = init_device_subkind(cds); >> - if (rc) >> - goto out; >> - >> cds->num_devices = 0; >> cds->num_nics = 0; >> cds->num_disks = 0; >> @@ -134,7 +90,7 @@ void libxl__checkpoint_devices_setup(libxl__egc *egc, >> return; >> >> out: >> - cds->callback(egc, cds, rc); >> + cds->callback(egc, cds, 0); >> } >> >> static void checkpoint_devices_setup(libxl__egc *egc, >> @@ -172,7 +128,7 @@ static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev) >> goto out; >> >> do { >> - dev->ops = remus_ops[++dev->ops_index]; >> + dev->ops = dev->cds->ops[++dev->ops_index]; >> if (!dev->ops) { >> libxl_device_nic * nic = NULL; >> libxl_device_disk * disk = NULL; >> @@ -271,8 +227,6 @@ static void devices_teardown_cb(libxl__egc *egc, >> cds->disks = NULL; >> cds->num_disks = 0; >> >> - cleanup_device_subkind(cds); >> - >> cds->callback(egc, cds, rc); >> } >> >> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c >> index 9a3b33d..9a3d009 100644 >> --- a/tools/libxl/libxl_dom_save.c >> +++ b/tools/libxl/libxl_dom_save.c >> @@ -410,7 +410,6 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss) >> dsps->dm_savefile = libxl__device_model_savefile(gc, domid); >> >> if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) { >> - dss->interval = r_info->interval; >> if (libxl_defbool_val(r_info->compression)) >> dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS; >> } >> @@ -581,7 +580,7 @@ static void domain_save_done(libxl__egc *egc, >> * from sending checkpoints. Teardown the network buffers and >> * release netlink resources. This is an async op. >> */ >> - libxl__remus_teardown(egc, dss, rc); >> + libxl__remus_teardown(egc, &dss->rs, rc); >> } >> >> /*========================= Domain restore ============================*/ >> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h >> index f3a139b..840734d 100644 >> --- a/tools/libxl/libxl_internal.h >> +++ b/tools/libxl/libxl_internal.h >> @@ -2752,6 +2752,8 @@ struct libxl__checkpoint_devices_state { >> uint32_t domid; >> libxl__checkpoint_callback *callback; >> int device_kind_flags; >> + /* The ops must be pointer array, and the last ops must be NULL */ >> + const libxl__checkpoint_device_instance_ops **ops; >> >> /*----- private for abstract layer only -----*/ >> >> @@ -2772,16 +2774,6 @@ struct libxl__checkpoint_devices_state { >> int num_disks; >> >> libxl__multidev multidev; >> - >> - /*----- private for concrete (device-specific) layer only -----*/ >> - >> - /* private for nic device subkind ops */ >> - char *netbufscript; >> - struct nl_sock *nlsock; >> - struct nl_cache *qdisc_cache; >> - >> - /* private for drbd disk subkind ops */ >> - char *drbd_probe_script; >> }; >> >> /* >> @@ -2829,6 +2821,26 @@ _hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds); >> _hidden void libxl__checkpoint_devices_commit(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds); >> + >> +/*----- Remus related state structure -----*/ >> +typedef struct libxl__remus_state libxl__remus_state; >> +struct libxl__remus_state { >> + /* private */ >> + libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */ >> + int interval; /* checkpoint interval */ >> + >> + /* abstract layer */ >> + libxl__checkpoint_devices_state cds; >> + >> + /*----- private for concrete (device-specific) layer only -----*/ >> + /* private for nic device subkind ops */ >> + char *netbufscript; >> + struct nl_sock *nlsock; >> + struct nl_cache *qdisc_cache; >> + >> + /* private for drbd disk subkind ops */ >> + char *drbd_probe_script; >> +}; >> _hidden int libxl__netbuffer_enabled(libxl__gc *gc); >> >> /*----- Legacy conversion helper -----*/ >> @@ -2962,9 +2974,7 @@ struct libxl__domain_save_state { >> libxl__domain_suspend_state dsps; >> int hvm; >> int xcflags; >> - libxl__checkpoint_devices_state cds; >> - libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */ >> - int interval; /* checkpoint interval (for Remus) */ >> + libxl__remus_state rs; >> libxl__save_helper_state shs; >> libxl__logdirty_switch logdirty; >> /* private for libxl__domain_save_device_model */ >> @@ -3364,9 +3374,9 @@ _hidden void libxl__remus_domain_save_checkpoint_callback(void *data); >> _hidden void libxl__remus_domain_restore_checkpoint_callback(void *data); >> /* Remus setup and teardown*/ >> _hidden void libxl__remus_setup(libxl__egc *egc, >> - libxl__domain_save_state *dss); >> + libxl__remus_state *rs); >> _hidden void libxl__remus_teardown(libxl__egc *egc, >> - libxl__domain_save_state *dss, >> + libxl__remus_state *rs, >> int rc); >> >> /* >> diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c >> index 86afba6..1d01e10 100644 >> --- a/tools/libxl/libxl_netbuffer.c >> +++ b/tools/libxl/libxl_netbuffer.c >> @@ -41,18 +41,19 @@ int libxl__netbuffer_enabled(libxl__gc *gc) >> int init_subkind_nic(libxl__checkpoint_devices_state *cds) >> { >> int rc, ret; >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs); >> >> STATE_AO_GC(cds->ao); >> >> - cds->nlsock = nl_socket_alloc(); >> - if (!cds->nlsock) { >> + rs->nlsock = nl_socket_alloc(); >> + if (!rs->nlsock) { >> LOG(ERROR, "cannot allocate nl socket"); >> rc = ERROR_FAIL; >> goto out; >> } >> >> - ret = nl_connect(cds->nlsock, NETLINK_ROUTE); >> + ret = nl_connect(rs->nlsock, NETLINK_ROUTE); >> if (ret) { >> LOG(ERROR, "failed to open netlink socket: %s", >> nl_geterror(ret)); >> @@ -61,7 +62,7 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds) >> } >> >> /* get list of all qdiscs installed on network devs. */ >> - ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache); >> + ret = rtnl_qdisc_alloc_cache(rs->nlsock, &rs->qdisc_cache); >> if (ret) { >> LOG(ERROR, "failed to allocate qdisc cache: %s", >> nl_geterror(ret)); >> @@ -70,10 +71,10 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds) >> } >> >> if (dss->remus->netbufscript) { >> - cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript); >> + rs->netbufscript = libxl__strdup(gc, dss->remus->netbufscript); >> } else { >> - cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup", >> - libxl__xen_script_dir_path()); >> + rs->netbufscript = GCSPRINTF("%s/remus-netbuf-setup", >> + libxl__xen_script_dir_path()); >> } >> >> rc = 0; >> @@ -84,20 +85,22 @@ out: >> >> void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds) >> { >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> + >> STATE_AO_GC(cds->ao); >> >> /* free qdisc cache */ >> - if (cds->qdisc_cache) { >> - nl_cache_clear(cds->qdisc_cache); >> - nl_cache_free(cds->qdisc_cache); >> - cds->qdisc_cache = NULL; >> + if (rs->qdisc_cache) { >> + nl_cache_clear(rs->qdisc_cache); >> + nl_cache_free(rs->qdisc_cache); >> + rs->qdisc_cache = NULL; >> } >> >> /* close & free nlsock */ >> - if (cds->nlsock) { >> - nl_close(cds->nlsock); >> - nl_socket_free(cds->nlsock); >> - cds->nlsock = NULL; >> + if (rs->nlsock) { >> + nl_close(rs->nlsock); >> + nl_socket_free(rs->nlsock); >> + rs->nlsock = NULL; >> } >> } >> >> @@ -150,13 +153,14 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds, >> int rc, ret, ifindex; >> struct rtnl_link *ifb = NULL; >> struct rtnl_qdisc *qdisc = NULL; >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> >> STATE_AO_GC(cds->ao); >> >> /* Now that we have brought up REMUS_IFB device with plug qdisc for >> * this vif, so we need to refill the qdisc cache. >> */ >> - ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache); >> + ret = nl_cache_refill(rs->nlsock, rs->qdisc_cache); >> if (ret) { >> LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret)); >> rc = ERROR_FAIL; >> @@ -164,7 +168,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds, >> } >> >> /* get a handle to the REMUS_IFB interface */ >> - ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb); >> + ret = rtnl_link_get_kernel(rs->nlsock, 0, remus_nic->ifb, &ifb); >> if (ret) { >> LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb, >> nl_geterror(ret)); >> @@ -187,7 +191,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds, >> * There is no need to explicitly free this qdisc as its just a >> * reference from the qdisc cache we allocated earlier. >> */ >> - qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT); >> + qdisc = rtnl_qdisc_get_by_parent(rs->qdisc_cache, ifindex, TC_H_ROOT); >> if (qdisc) { >> const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc)); >> /* Sanity check: Ensure that the root qdisc is a plug qdisc. */ >> @@ -238,11 +242,12 @@ static void setup_async_exec(libxl__checkpoint_device *dev, char *op) >> libxl__remus_device_nic *remus_nic = dev->concrete_data; >> libxl__checkpoint_devices_state *cds = dev->cds; >> libxl__async_exec_state *aes = &dev->aodev.aes; >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> >> STATE_AO_GC(cds->ao); >> >> /* Convenience aliases */ >> - char *const script = libxl__strdup(gc, cds->netbufscript); >> + char *const script = libxl__strdup(gc, rs->netbufscript); >> const uint32_t domid = cds->domid; >> const int dev_id = remus_nic->devid; >> const char *const vif = remus_nic->vif; >> @@ -333,6 +338,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc, >> libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev); >> libxl__remus_device_nic *remus_nic = dev->concrete_data; >> libxl__checkpoint_devices_state *cds = dev->cds; >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> const char *out_path_base, *hotplug_error = NULL; >> int rc; >> >> @@ -373,7 +379,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc, >> >> if (hotplug_error) { >> LOG(ERROR, "netbuf script %s setup failed for vif %s: %s", >> - cds->netbufscript, vif, hotplug_error); >> + rs->netbufscript, vif, hotplug_error); >> rc = ERROR_FAIL; >> goto out; >> } >> @@ -444,6 +450,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic, >> int buffer_op) >> { >> int rc, ret; >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> >> STATE_AO_GC(cds->ao); >> >> @@ -457,7 +464,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic, >> goto out; >> } >> >> - ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST); >> + ret = rtnl_qdisc_add(rs->nlsock, remus_nic->qdisc, NLM_F_REQUEST); >> if (ret) { >> rc = ERROR_FAIL; >> goto out; >> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c >> index fe4acc8..2a427a4 100644 >> --- a/tools/libxl/libxl_remus.c >> +++ b/tools/libxl/libxl_remus.c >> @@ -17,6 +17,46 @@ >> >> #include "libxl_internal.h" >> >> +extern const libxl__checkpoint_device_instance_ops remus_device_nic; >> +extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk; >> +static const libxl__checkpoint_device_instance_ops *remus_ops[] = { >> + &remus_device_nic, >> + &remus_device_drbd_disk, >> + NULL, >> +}; >> + >> +/*----- helper functions -----*/ >> + >> +static int init_device_subkind(libxl__checkpoint_devices_state *cds) >> +{ >> + /* init device subkind-specific state in the libxl ctx */ >> + int rc; >> + STATE_AO_GC(cds->ao); >> + >> + if (libxl__netbuffer_enabled(gc)) { >> + rc = init_subkind_nic(cds); >> + if (rc) goto out; >> + } >> + >> + rc = init_subkind_drbd_disk(cds); >> + if (rc) goto out; >> + >> + rc = 0; >> +out: >> + return rc; >> +} >> + >> +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds) >> +{ >> + /* cleanup device subkind-specific state in the libxl ctx */ >> + STATE_AO_GC(cds->ao); >> + >> + if (libxl__netbuffer_enabled(gc)) >> + cleanup_subkind_nic(cds); >> + >> + cleanup_subkind_drbd_disk(cds); >> +} >> + >> /*----- Remus setup and teardown -----*/ >> >> static void remus_setup_done(libxl__egc *egc, >> @@ -24,10 +64,12 @@ static void remus_setup_done(libxl__egc *egc, >> static void remus_setup_failed(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds, int rc); >> >> -void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss) >> +void libxl__remus_setup(libxl__egc *egc, libxl__remus_state *rs) >> { >> + libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs); >> + >> /* Convenience aliases */ >> - libxl__checkpoint_devices_state *const cds = &dss->cds; >> + libxl__checkpoint_devices_state *const cds = &rs->cds; >> const libxl_domain_remus_info *const info = dss->remus; >> >> STATE_AO_GC(dss->ao); >> @@ -46,6 +88,14 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss) >> cds->ao = ao; >> cds->domid = dss->domid; >> cds->callback = remus_setup_done; >> + cds->ops = remus_ops; >> + rs->interval = info->interval; >> + >> + if (init_device_subkind(cds)) { >> + LOG(ERROR, "Remus: failed to init device subkind for guest %u", >> + dss->domid); >> + goto out; >> + } >> >> libxl__checkpoint_devices_setup(egc, cds); >> return; >> @@ -57,7 +107,7 @@ out: >> static void remus_setup_done(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds, int rc) >> { >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds); >> STATE_AO_GC(dss->ao); >> >> if (!rc) { >> @@ -74,13 +124,15 @@ static void remus_setup_done(libxl__egc *egc, >> static void remus_setup_failed(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds, int rc) >> { >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds); >> STATE_AO_GC(dss->ao); >> >> if (rc) >> LOG(ERROR, "Remus: failed to teardown device after setup failed" >> " for guest with domid %u, rc %d", dss->domid, rc); >> >> + cleanup_device_subkind(cds); >> + >> dss->callback(egc, dss, rc); >> } >> >> @@ -88,28 +140,30 @@ static void remus_teardown_done(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds, >> int rc); >> void libxl__remus_teardown(libxl__egc *egc, >> - libxl__domain_save_state *dss, >> + libxl__remus_state *rs, >> int rc) >> { >> EGC_GC; >> >> LOG(WARN, "Remus: Domain suspend terminated with rc %d," >> " teardown Remus devices...", rc); >> - dss->cds.callback = remus_teardown_done; >> - libxl__checkpoint_devices_teardown(egc, &dss->cds); >> + rs->cds.callback = remus_teardown_done; >> + libxl__checkpoint_devices_teardown(egc, &rs->cds); >> } >> >> static void remus_teardown_done(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds, >> int rc) >> { >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds); >> STATE_AO_GC(dss->ao); >> >> if (rc) >> LOG(ERROR, "Remus: failed to teardown device for guest with domid %u," >> " rc %d", dss->domid, rc); >> >> + cleanup_device_subkind(cds); >> + >> dss->callback(egc, dss, rc); >> } >> >> @@ -143,7 +197,7 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc, >> if (!ok) >> goto out; >> >> - libxl__checkpoint_devices_state *const cds = &dss->cds; >> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds; >> cds->callback = remus_devices_postsuspend_cb; >> libxl__checkpoint_devices_postsuspend(egc, cds); >> return; >> @@ -157,7 +211,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc, >> int rc) >> { >> int ok = 0; >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds); >> >> if (rc) >> goto out; >> @@ -175,7 +229,7 @@ void libxl__remus_domain_resume_callback(void *data) >> libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs); >> STATE_AO_GC(dss->ao); >> >> - libxl__checkpoint_devices_state *const cds = &dss->cds; >> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds; >> cds->callback = remus_devices_preresume_cb; >> libxl__checkpoint_devices_preresume(egc, cds); >> } >> @@ -185,7 +239,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc, >> int rc) >> { >> int ok = 0; >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds); >> STATE_AO_GC(dss->ao); >> >> if (rc) >> @@ -229,7 +283,7 @@ static void remus_checkpoint_stream_written( >> libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws); >> >> /* Convenience aliases */ >> - libxl__checkpoint_devices_state *const cds = &dss->cds; >> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds; >> >> STATE_AO_GC(dss->ao); >> >> @@ -251,7 +305,7 @@ static void remus_devices_commit_cb(libxl__egc *egc, >> libxl__checkpoint_devices_state *cds, >> int rc) >> { >> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds); >> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds); >> >> STATE_AO_GC(dss->ao); >> >> @@ -269,9 +323,9 @@ static void remus_devices_commit_cb(libxl__egc *egc, >> */ >> >> /* Set checkpoint interval timeout */ >> - rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout, >> + rc = libxl__ev_time_register_rel(gc, &dss->rs.checkpoint_timeout, >> remus_next_checkpoint, >> - dss->interval); >> + dss->rs.interval); >> >> if (rc) >> goto out; >> @@ -286,7 +340,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, >> const struct timeval *requested_abs) >> { >> libxl__domain_save_state *dss = >> - CONTAINER_OF(ev, *dss, checkpoint_timeout); >> + CONTAINER_OF(ev, *dss, rs.checkpoint_timeout); >> >> STATE_AO_GC(dss->ao); >> >> diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c >> index 50b897d..a8d8949 100644 >> --- a/tools/libxl/libxl_remus_disk_drbd.c >> +++ b/tools/libxl/libxl_remus_disk_drbd.c >> @@ -28,10 +28,11 @@ typedef struct libxl__remus_drbd_disk { >> >> int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds) >> { >> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds); >> STATE_AO_GC(cds->ao); >> >> - cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe", >> - libxl__xen_script_dir_path()); >> + rs->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe", >> + libxl__xen_script_dir_path()); >> >> return 0; >> } >> @@ -96,6 +97,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev) >> int arraysize, nr = 0, rc; >> const libxl_device_disk *disk = dev->backend_dev; >> libxl__async_exec_state *aes = &dev->aodev.aes; >> + libxl__remus_state *rs = CONTAINER_OF(dev->cds, *rs, cds); >> STATE_AO_GC(dev->cds->ao); >> >> /* setup env & args */ >> @@ -107,7 +109,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev) >> arraysize = 3; >> nr = 0; >> GCNEW_ARRAY(aes->args, arraysize); >> - aes->args[nr++] = dev->cds->drbd_probe_script; >> + aes->args[nr++] = rs->drbd_probe_script; >> aes->args[nr++] = disk->pdev_path; >> aes->args[nr++] = NULL; >> assert(nr <= arraysize); > > > . > -- Thanks, Yang.