* [PATCH v3 COLOPre 01/26] tools/libxl: rename libxl__domain_suspend to libxl__domain_save
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 15:43 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 02/26] tools/libxl: move domain suspend code into libxl_dom_suspend.c Yang Hongyang
` (24 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
Rename libxl__domain_suspend() to libxl__domain_save() since it
actually do the save domain work.
This results in some strangeness in that some functions called *save*
are now passed a struct called *suspend*, this is temporary and is all
fixed up later by the refactoring of the suspend_state.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/libxl/libxl.c | 4 ++--
tools/libxl/libxl_dom.c | 14 +++++++-------
tools/libxl/libxl_internal.h | 4 ++--
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 9117b01..5a70062 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -911,7 +911,7 @@ static void libxl__remus_setup_done(libxl__egc *egc,
STATE_AO_GC(dss->ao);
if (!rc) {
- libxl__domain_suspend(egc, dss);
+ libxl__domain_save(egc, dss);
return;
}
@@ -978,7 +978,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
dss->live = flags & LIBXL_SUSPEND_LIVE;
dss->debug = flags & LIBXL_SUSPEND_DEBUG;
- libxl__domain_suspend(egc, dss);
+ libxl__domain_save(egc, dss);
return AO_INPROGRESS;
out_err:
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 43915a2..9d9e409 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1109,8 +1109,8 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
static void stream_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc);
-static void domain_suspend_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
+static void domain_save_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc);
static void domain_suspend_callback_common_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int ok);
@@ -1960,7 +1960,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
/*----- main code for suspending, in order of execution -----*/
-void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
{
STATE_AO_GC(dss->ao);
int port;
@@ -2045,13 +2045,13 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
return;
out:
- domain_suspend_done(egc, dss, rc);
+ domain_save_done(egc, dss, rc);
}
static void stream_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc)
{
- domain_suspend_done(egc, dss, rc);
+ domain_save_done(egc, dss, rc);
}
static void save_device_model_datacopier_done(libxl__egc *egc,
@@ -2150,8 +2150,8 @@ static void remus_teardown_done(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc);
-static void domain_suspend_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
+static void domain_save_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc)
{
STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index e0f6e09..19ebaab 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3264,8 +3264,8 @@ struct libxl__domain_create_state {
/*----- Domain suspend (save) functions -----*/
/* calls dss->callback when done */
-_hidden void libxl__domain_suspend(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+_hidden void libxl__domain_save(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
/* calls libxl__xc_domain_suspend_done when done */
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 01/26] tools/libxl: rename libxl__domain_suspend to libxl__domain_save
2015-06-25 6:25 ` [PATCH v3 COLOPre 01/26] tools/libxl: rename libxl__domain_suspend to libxl__domain_save Yang Hongyang
@ 2015-06-29 15:43 ` Ian Campbell
2015-06-30 9:32 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 15:43 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, eddie.dong, wency, andrew.cooper3, yunhong.jiang,
ian.jackson, xen-devel, guijianfeng, rshriram
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Rename libxl__domain_suspend() to libxl__domain_save() since it
> actually do the save domain work.
In response to Ian J's query on the last version Andrew said:
The terminology used by libxc is more consistent in this area.
"suspend" refers to quiescing the VM, so pausing qemu, making a
remote_shutdown(SHUTDOWN_suspend) hypercall etc.
"save" refers to the actions involved in actually shuffling the
state of the VM, so xc_domain_save() etc.
I believe that these are useful distinctions to maintain. libxl
currently uses "suspend" to encapsulate both.
Are your changes here staying true to those definitions?
I think incorporating some of that (in particular the second and third
paragraphs) into a code comment somewhere (likely libxl_internal.h)
would make it much clearer why you were doing this, and give people like
me a reminder of the distinction each time they trip over it in this
series too.
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index e0f6e09..19ebaab 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -3264,8 +3264,8 @@ struct libxl__domain_create_state {
> /*----- Domain suspend (save) functions -----*/
... here might be a good place to explain why/how suspend and save are
different.
>
> /* calls dss->callback when done */
> -_hidden void libxl__domain_suspend(libxl__egc *egc,
> - libxl__domain_suspend_state *dss);
> +_hidden void libxl__domain_save(libxl__egc *egc,
> + libxl__domain_suspend_state *dss);
>
>
> /* calls libxl__xc_domain_suspend_done when done */
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 01/26] tools/libxl: rename libxl__domain_suspend to libxl__domain_save
2015-06-29 15:43 ` Ian Campbell
@ 2015-06-30 9:32 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 9:32 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, eddie.dong, wency, andrew.cooper3, yunhong.jiang,
ian.jackson, xen-devel, guijianfeng, rshriram
On 06/29/2015 11:43 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Rename libxl__domain_suspend() to libxl__domain_save() since it
>> actually do the save domain work.
>
> In response to Ian J's query on the last version Andrew said:
>
> The terminology used by libxc is more consistent in this area.
>
> "suspend" refers to quiescing the VM, so pausing qemu, making a
> remote_shutdown(SHUTDOWN_suspend) hypercall etc.
>
> "save" refers to the actions involved in actually shuffling the
> state of the VM, so xc_domain_save() etc.
>
> I believe that these are useful distinctions to maintain. libxl
> currently uses "suspend" to encapsulate both.
>
> Are your changes here staying true to those definitions?
>
> I think incorporating some of that (in particular the second and third
> paragraphs) into a code comment somewhere (likely libxl_internal.h)
> would make it much clearer why you were doing this, and give people like
> me a reminder of the distinction each time they trip over it in this
> series too.
Ok, will make the comment above into a code comment in the next version.
>
>
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index e0f6e09..19ebaab 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
>> @@ -3264,8 +3264,8 @@ struct libxl__domain_create_state {
>> /*----- Domain suspend (save) functions -----*/
>
> ... here might be a good place to explain why/how suspend and save are
> different.
>
>>
>> /* calls dss->callback when done */
>> -_hidden void libxl__domain_suspend(libxl__egc *egc,
>> - libxl__domain_suspend_state *dss);
>> +_hidden void libxl__domain_save(libxl__egc *egc,
>> + libxl__domain_suspend_state *dss);
>>
>>
>> /* calls libxl__xc_domain_suspend_done when done */
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 02/26] tools/libxl: move domain suspend code into libxl_dom_suspend.c
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 01/26] tools/libxl: rename libxl__domain_suspend to libxl__domain_save Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 03/26] tools/libxl: move domain resume " Yang Hongyang
` (23 subsequent siblings)
25 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
Move domain suspend code into a separate file libxl_dom_suspend.c.
Add an API libxl__domain_suspend() which wraps the static
function domain_suspend_callback_common() for internal use.
Export the existing API libxl__domain_suspend_callback() used by
libxc to suspend the guest during migration.
Note that the newly added file libxl_dom_suspend.c is used for
suspend/resume code.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
---
tools/libxl/Makefile | 2 +-
tools/libxl/libxl_dom.c | 346 +-----------------------------------
tools/libxl/libxl_dom_suspend.c | 381 ++++++++++++++++++++++++++++++++++++++++
tools/libxl/libxl_internal.h | 6 +
4 files changed, 389 insertions(+), 346 deletions(-)
create mode 100644 tools/libxl/libxl_dom_suspend.c
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 63e32f7..e98e26f 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -96,7 +96,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \
libxl_stream_read.o libxl_stream_write.o \
libxl_save_callout.o _libxl_save_msgs_callout.o \
- libxl_convert_callout.o \
+ libxl_convert_callout.o libxl_dom_suspend.o \
libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
LIBXL_OBJS += libxl_genid.o
LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 9d9e409..3b02562 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1111,8 +1111,6 @@ static void stream_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc);
static void domain_save_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc);
-static void domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok);
/*----- complicated callback, called by xc_domain_save -----*/
@@ -1328,37 +1326,6 @@ static void switch_logdirty_done(libxl__egc *egc,
libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
}
-/*----- callbacks, called by xc_domain_save -----*/
-
-int libxl__domain_suspend_device_model(libxl__gc *gc,
- libxl__domain_suspend_state *dss)
-{
- int ret = 0;
- uint32_t const domid = dss->domid;
- const char *const filename = dss->dm_savefile;
-
- switch (libxl__device_model_version_running(gc, domid)) {
- case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
- LOG(DEBUG, "Saving device model state to %s", filename);
- libxl__qemu_traditional_cmd(gc, domid, "save");
- libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
- break;
- }
- case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
- if (libxl__qmp_stop(gc, domid))
- return ERROR_FAIL;
- /* Save DM state into filename */
- ret = libxl__qmp_save(gc, domid, filename);
- if (ret)
- unlink(filename);
- break;
- default:
- return ERROR_INVAL;
- }
-
- return ret;
-}
-
int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
{
@@ -1379,301 +1346,6 @@ int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
return 0;
}
-static void domain_suspend_common_wait_guest(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
-static void domain_suspend_common_guest_suspended(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
-
-static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
- libxl__xswait_state *xswa, int rc, const char *state);
-static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
- libxl__ev_evtchn *evev);
-static void suspend_common_wait_guest_watch(libxl__egc *egc,
- libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
-static void suspend_common_wait_guest_check(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
-static void suspend_common_wait_guest_timeout(libxl__egc *egc,
- libxl__ev_time *ev, const struct timeval *requested_abs);
-
-static void domain_suspend_common_failed(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
-static void domain_suspend_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
- bool ok);
-
-static bool domain_suspend_pvcontrol_acked(const char *state) {
- /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
- if (!state) return 1;
- return strcmp(state,"suspend");
-}
-
-/* calls dss->callback_common_done when done */
-static void domain_suspend_callback_common(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
-{
- STATE_AO_GC(dss->ao);
- uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
- int ret, rc;
-
- /* Convenience aliases */
- const uint32_t domid = dss->domid;
-
- if (dss->hvm) {
- xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
- xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
- }
-
- if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
- LOG(DEBUG, "issuing %s suspend request via event channel",
- dss->hvm ? "PVHVM" : "PV");
- ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
- if (ret < 0) {
- LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
- goto err;
- }
-
- dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
- rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
- if (rc) goto err;
-
- rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
- suspend_common_wait_guest_timeout,
- 60*1000);
- if (rc) goto err;
-
- return;
- }
-
- if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
- LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
- ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
- if (ret < 0) {
- LOGE(ERROR, "xc_domain_shutdown failed");
- goto err;
- }
- /* The guest does not (need to) respond to this sort of request. */
- dss->guest_responded = 1;
- domain_suspend_common_wait_guest(egc, dss);
- return;
- }
-
- LOG(DEBUG, "issuing %s suspend request via XenBus control node",
- dss->hvm ? "PVHVM" : "PV");
-
- libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
-
- dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
- if (!dss->pvcontrol.path) goto err;
-
- dss->pvcontrol.ao = ao;
- dss->pvcontrol.what = "guest acknowledgement of suspend request";
- dss->pvcontrol.timeout_ms = 60 * 1000;
- dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
- libxl__xswait_start(gc, &dss->pvcontrol);
- return;
-
- err:
- domain_suspend_common_failed(egc, dss);
-}
-
-static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
- libxl__ev_evtchn *evev)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
- STATE_AO_GC(dss->ao);
- /* If we should be done waiting, suspend_common_wait_guest_check
- * will end up calling domain_suspend_common_guest_suspended or
- * domain_suspend_common_failed, both of which cancel the evtchn
- * wait. So re-enable it now. */
- libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
- suspend_common_wait_guest_check(egc, dss);
-}
-
-static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
- libxl__xswait_state *xswa, int rc, const char *state)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
- STATE_AO_GC(dss->ao);
- xs_transaction_t t = 0;
-
- if (!rc && !domain_suspend_pvcontrol_acked(state))
- /* keep waiting */
- return;
-
- libxl__xswait_stop(gc, &dss->pvcontrol);
-
- if (rc == ERROR_TIMEDOUT) {
- /*
- * Guest appears to not be responding. Cancel the suspend
- * request.
- *
- * We re-read the suspend node and clear it within a
- * transaction in order to handle the case where we race
- * against the guest catching up and acknowledging the request
- * at the last minute.
- */
- for (;;) {
- rc = libxl__xs_transaction_start(gc, &t);
- if (rc) goto err;
-
- rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
- if (rc) goto err;
-
- if (domain_suspend_pvcontrol_acked(state))
- /* last minute ack */
- break;
-
- rc = libxl__xs_write_checked(gc, t, xswa->path, "");
- if (rc) goto err;
-
- rc = libxl__xs_transaction_commit(gc, &t);
- if (!rc) {
- LOG(ERROR,
- "guest didn't acknowledge suspend, cancelling request");
- goto err;
- }
- if (rc<0) goto err;
- }
- } else if (rc) {
- /* some error in xswait's read of xenstore, already logged */
- goto err;
- }
-
- assert(domain_suspend_pvcontrol_acked(state));
- LOG(DEBUG, "guest acknowledged suspend request");
-
- libxl__xs_transaction_abort(gc, &t);
- dss->guest_responded = 1;
- domain_suspend_common_wait_guest(egc,dss);
- return;
-
- err:
- libxl__xs_transaction_abort(gc, &t);
- domain_suspend_common_failed(egc, dss);
- return;
-}
-
-static void domain_suspend_common_wait_guest(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
-{
- STATE_AO_GC(dss->ao);
- int rc;
-
- LOG(DEBUG, "wait for the guest to suspend");
-
- rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
- suspend_common_wait_guest_watch,
- "@releaseDomain");
- if (rc) goto err;
-
- rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
- suspend_common_wait_guest_timeout,
- 60*1000);
- if (rc) goto err;
- return;
-
- err:
- domain_suspend_common_failed(egc, dss);
-}
-
-static void suspend_common_wait_guest_watch(libxl__egc *egc,
- libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
- suspend_common_wait_guest_check(egc, dss);
-}
-
-static void suspend_common_wait_guest_check(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
-{
- STATE_AO_GC(dss->ao);
- xc_domaininfo_t info;
- int ret;
- int shutdown_reason;
-
- /* Convenience aliases */
- const uint32_t domid = dss->domid;
-
- ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
- if (ret < 0) {
- LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
- goto err;
- }
-
- if (!(ret == 1 && info.domain == domid)) {
- LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
- domid);
- goto err;
- }
-
- if (!(info.flags & XEN_DOMINF_shutdown))
- /* keep waiting */
- return;
-
- shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
- & XEN_DOMINF_shutdownmask;
- if (shutdown_reason != SHUTDOWN_suspend) {
- LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
- " with unexpected reason code %d", domid, shutdown_reason);
- goto err;
- }
-
- LOG(DEBUG, "guest has suspended");
- domain_suspend_common_guest_suspended(egc, dss);
- return;
-
- err:
- domain_suspend_common_failed(egc, dss);
-}
-
-static void suspend_common_wait_guest_timeout(libxl__egc *egc,
- libxl__ev_time *ev, const struct timeval *requested_abs)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
- STATE_AO_GC(dss->ao);
- LOG(ERROR, "guest did not suspend, timed out");
- domain_suspend_common_failed(egc, dss);
-}
-
-static void domain_suspend_common_guest_suspended(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
-{
- STATE_AO_GC(dss->ao);
- int ret;
-
- libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
- libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
- libxl__ev_time_deregister(gc, &dss->guest_timeout);
-
- if (dss->hvm) {
- ret = libxl__domain_suspend_device_model(gc, dss);
- if (ret) {
- LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
- domain_suspend_common_failed(egc, dss);
- return;
- }
- }
- domain_suspend_common_done(egc, dss, 1);
-}
-
-static void domain_suspend_common_failed(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
-{
- domain_suspend_common_done(egc, dss, 0);
-}
-
-static void domain_suspend_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
- bool ok)
-{
- EGC_GC;
- assert(!libxl__xswait_inuse(&dss->pvcontrol));
- libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
- libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
- libxl__ev_time_deregister(gc, &dss->guest_timeout);
- dss->callback_common_done(egc, dss, ok);
-}
-
static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
uint32_t domid,
char *phys_offset, char *node)
@@ -1764,22 +1436,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
return 0;
}
-static void libxl__domain_suspend_callback(void *data)
-{
- libxl__save_helper_state *shs = data;
- libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-
- dss->callback_common_done = domain_suspend_callback_common_done;
- domain_suspend_callback_common(egc, dss);
-}
-
-static void domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok)
-{
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
/*----- remus callbacks -----*/
static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int ok);
@@ -1797,7 +1453,7 @@ static void libxl__remus_domain_suspend_callback(void *data)
libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
dss->callback_common_done = remus_domain_suspend_callback_common_done;
- domain_suspend_callback_common(egc, dss);
+ libxl__domain_suspend(egc, dss);
}
static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
new file mode 100644
index 0000000..4edf936
--- /dev/null
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -0,0 +1,381 @@
+/*
+ * Copyright (C) 2009 Citrix Ltd.
+ * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+/*======================= Domain suspend ========================*/
+
+/*----- callbacks, called by xc_domain_save -----*/
+
+int libxl__domain_suspend_device_model(libxl__gc *gc,
+ libxl__domain_suspend_state *dss)
+{
+ int ret = 0;
+ uint32_t const domid = dss->domid;
+ const char *const filename = dss->dm_savefile;
+
+ switch (libxl__device_model_version_running(gc, domid)) {
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
+ LOG(DEBUG, "Saving device model state to %s", filename);
+ libxl__qemu_traditional_cmd(gc, domid, "save");
+ libxl__wait_for_device_model_deprecated(gc, domid, "paused", NULL, NULL, NULL);
+ break;
+ }
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+ if (libxl__qmp_stop(gc, domid))
+ return ERROR_FAIL;
+ /* Save DM state into filename */
+ ret = libxl__qmp_save(gc, domid, filename);
+ if (ret)
+ unlink(filename);
+ break;
+ default:
+ return ERROR_INVAL;
+ }
+
+ return ret;
+}
+
+static void domain_suspend_common_wait_guest(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+static void domain_suspend_common_guest_suspended(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+
+static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
+ libxl__xswait_state *xswa, int rc, const char *state);
+static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
+ libxl__ev_evtchn *evev);
+static void suspend_common_wait_guest_watch(libxl__egc *egc,
+ libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
+static void suspend_common_wait_guest_check(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+static void suspend_common_wait_guest_timeout(libxl__egc *egc,
+ libxl__ev_time *ev, const struct timeval *requested_abs);
+
+static void domain_suspend_common_failed(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+static void domain_suspend_common_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ bool ok);
+static void domain_suspend_callback_common(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+static void domain_suspend_callback_common_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int ok);
+
+/* calls dss->callback_common_done when done */
+void libxl__domain_suspend(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ domain_suspend_callback_common(egc, dss);
+}
+
+
+static bool domain_suspend_pvcontrol_acked(const char *state) {
+ /* any value other than "suspend", including ENOENT (i.e. !state), is OK */
+ if (!state) return 1;
+ return strcmp(state,"suspend");
+}
+
+/* calls dss->callback_common_done when done */
+static void domain_suspend_callback_common(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ STATE_AO_GC(dss->ao);
+ uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
+ int ret, rc;
+
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+
+ if (dss->hvm) {
+ xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
+ xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
+ }
+
+ if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
+ LOG(DEBUG, "issuing %s suspend request via event channel",
+ dss->hvm ? "PVHVM" : "PV");
+ ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
+ if (ret < 0) {
+ LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
+ goto err;
+ }
+
+ dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
+ rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
+ if (rc) goto err;
+
+ rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+ suspend_common_wait_guest_timeout,
+ 60*1000);
+ if (rc) goto err;
+
+ return;
+ }
+
+ if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
+ LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
+ ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
+ if (ret < 0) {
+ LOGE(ERROR, "xc_domain_shutdown failed");
+ goto err;
+ }
+ /* The guest does not (need to) respond to this sort of request. */
+ dss->guest_responded = 1;
+ domain_suspend_common_wait_guest(egc, dss);
+ return;
+ }
+
+ LOG(DEBUG, "issuing %s suspend request via XenBus control node",
+ dss->hvm ? "PVHVM" : "PV");
+
+ libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
+
+ dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
+ if (!dss->pvcontrol.path) goto err;
+
+ dss->pvcontrol.ao = ao;
+ dss->pvcontrol.what = "guest acknowledgement of suspend request";
+ dss->pvcontrol.timeout_ms = 60 * 1000;
+ dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
+ libxl__xswait_start(gc, &dss->pvcontrol);
+ return;
+
+ err:
+ domain_suspend_common_failed(egc, dss);
+}
+
+static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
+ libxl__ev_evtchn *evev)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
+ STATE_AO_GC(dss->ao);
+ /* If we should be done waiting, suspend_common_wait_guest_check
+ * will end up calling domain_suspend_common_guest_suspended or
+ * domain_suspend_common_failed, both of which cancel the evtchn
+ * wait. So re-enable it now. */
+ libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
+ suspend_common_wait_guest_check(egc, dss);
+}
+
+static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
+ libxl__xswait_state *xswa, int rc, const char *state)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
+ STATE_AO_GC(dss->ao);
+ xs_transaction_t t = 0;
+
+ if (!rc && !domain_suspend_pvcontrol_acked(state))
+ /* keep waiting */
+ return;
+
+ libxl__xswait_stop(gc, &dss->pvcontrol);
+
+ if (rc == ERROR_TIMEDOUT) {
+ /*
+ * Guest appears to not be responding. Cancel the suspend
+ * request.
+ *
+ * We re-read the suspend node and clear it within a
+ * transaction in order to handle the case where we race
+ * against the guest catching up and acknowledging the request
+ * at the last minute.
+ */
+ for (;;) {
+ rc = libxl__xs_transaction_start(gc, &t);
+ if (rc) goto err;
+
+ rc = libxl__xs_read_checked(gc, t, xswa->path, &state);
+ if (rc) goto err;
+
+ if (domain_suspend_pvcontrol_acked(state))
+ /* last minute ack */
+ break;
+
+ rc = libxl__xs_write_checked(gc, t, xswa->path, "");
+ if (rc) goto err;
+
+ rc = libxl__xs_transaction_commit(gc, &t);
+ if (!rc) {
+ LOG(ERROR,
+ "guest didn't acknowledge suspend, cancelling request");
+ goto err;
+ }
+ if (rc<0) goto err;
+ }
+ } else if (rc) {
+ /* some error in xswait's read of xenstore, already logged */
+ goto err;
+ }
+
+ assert(domain_suspend_pvcontrol_acked(state));
+ LOG(DEBUG, "guest acknowledged suspend request");
+
+ libxl__xs_transaction_abort(gc, &t);
+ dss->guest_responded = 1;
+ domain_suspend_common_wait_guest(egc,dss);
+ return;
+
+ err:
+ libxl__xs_transaction_abort(gc, &t);
+ domain_suspend_common_failed(egc, dss);
+ return;
+}
+
+static void domain_suspend_common_wait_guest(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ STATE_AO_GC(dss->ao);
+ int rc;
+
+ LOG(DEBUG, "wait for the guest to suspend");
+
+ rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
+ suspend_common_wait_guest_watch,
+ "@releaseDomain");
+ if (rc) goto err;
+
+ rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+ suspend_common_wait_guest_timeout,
+ 60*1000);
+ if (rc) goto err;
+ return;
+
+ err:
+ domain_suspend_common_failed(egc, dss);
+}
+
+static void suspend_common_wait_guest_watch(libxl__egc *egc,
+ libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
+ suspend_common_wait_guest_check(egc, dss);
+}
+
+static void suspend_common_wait_guest_check(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ STATE_AO_GC(dss->ao);
+ xc_domaininfo_t info;
+ int ret;
+ int shutdown_reason;
+
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+
+ ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
+ if (ret < 0) {
+ LOGE(ERROR, "unable to check for status of guest %"PRId32"", domid);
+ goto err;
+ }
+
+ if (!(ret == 1 && info.domain == domid)) {
+ LOGE(ERROR, "guest %"PRId32" we were suspending has been destroyed",
+ domid);
+ goto err;
+ }
+
+ if (!(info.flags & XEN_DOMINF_shutdown))
+ /* keep waiting */
+ return;
+
+ shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
+ & XEN_DOMINF_shutdownmask;
+ if (shutdown_reason != SHUTDOWN_suspend) {
+ LOG(DEBUG, "guest %"PRId32" we were suspending has shut down"
+ " with unexpected reason code %d", domid, shutdown_reason);
+ goto err;
+ }
+
+ LOG(DEBUG, "guest has suspended");
+ domain_suspend_common_guest_suspended(egc, dss);
+ return;
+
+ err:
+ domain_suspend_common_failed(egc, dss);
+}
+
+static void suspend_common_wait_guest_timeout(libxl__egc *egc,
+ libxl__ev_time *ev, const struct timeval *requested_abs)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
+ STATE_AO_GC(dss->ao);
+ LOG(ERROR, "guest did not suspend, timed out");
+ domain_suspend_common_failed(egc, dss);
+}
+
+static void domain_suspend_common_guest_suspended(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ STATE_AO_GC(dss->ao);
+ int ret;
+
+ libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+ libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
+ libxl__ev_time_deregister(gc, &dss->guest_timeout);
+
+ if (dss->hvm) {
+ ret = libxl__domain_suspend_device_model(gc, dss);
+ if (ret) {
+ LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
+ domain_suspend_common_failed(egc, dss);
+ return;
+ }
+ }
+ domain_suspend_common_done(egc, dss, 1);
+}
+
+static void domain_suspend_common_failed(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ domain_suspend_common_done(egc, dss, 0);
+}
+
+static void domain_suspend_common_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ bool ok)
+{
+ EGC_GC;
+ assert(!libxl__xswait_inuse(&dss->pvcontrol));
+ libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+ libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
+ libxl__ev_time_deregister(gc, &dss->guest_timeout);
+ dss->callback_common_done(egc, dss, ok);
+}
+
+void libxl__domain_suspend_callback(void *data)
+{
+ libxl__save_helper_state *shs = data;
+ libxl__egc *egc = shs->egc;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+
+ dss->callback_common_done = domain_suspend_callback_common_done;
+ domain_suspend_callback_common(egc, dss);
+}
+
+static void domain_suspend_callback_common_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int ok)
+{
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 19ebaab..54c3cc1 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3318,6 +3318,12 @@ _hidden void libxl__domain_save_device_model(libxl__egc *egc,
_hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t domid);
+/* calls dss->callback_common_done when done */
+_hidden void libxl__domain_suspend(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+/* used by libxc to suspend the guest during migration */
+_hidden void libxl__domain_suspend_callback(void *data);
+
/*
* Convenience macros.
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* [PATCH v3 COLOPre 03/26] tools/libxl: move domain resume code into libxl_dom_suspend.c
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 01/26] tools/libxl: rename libxl__domain_suspend to libxl__domain_save Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 02/26] tools/libxl: move domain suspend code into libxl_dom_suspend.c Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 15:44 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 04/26] tools/libxl: move remus code into libxl_remus.c Yang Hongyang
` (22 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
move domain resume code into libxl_dom_suspend.c.
pure code motion except for making
libxl__domain_resume_device_model() static.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
---
tools/libxl/libxl.c | 33 -------------------------
tools/libxl/libxl_dom.c | 20 ---------------
tools/libxl/libxl_dom_suspend.c | 55 +++++++++++++++++++++++++++++++++++++++++
tools/libxl/libxl_internal.h | 1 -
4 files changed, 55 insertions(+), 54 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 5a70062..1f52bf0 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -510,39 +510,6 @@ int libxl_domain_rename(libxl_ctx *ctx, uint32_t domid,
return rc;
}
-int libxl__domain_resume(libxl__gc *gc, uint32_t domid, int suspend_cancel)
-{
- int rc = 0;
-
- if (xc_domain_resume(CTX->xch, domid, suspend_cancel)) {
- LOGE(ERROR, "xc_domain_resume failed for domain %u", domid);
- rc = ERROR_FAIL;
- goto out;
- }
-
- libxl_domain_type type = libxl__domain_type(gc, domid);
- if (type == LIBXL_DOMAIN_TYPE_INVALID) {
- rc = ERROR_FAIL;
- goto out;
- }
-
- if (type == LIBXL_DOMAIN_TYPE_HVM) {
- rc = libxl__domain_resume_device_model(gc, domid);
- if (rc) {
- LOG(ERROR, "failed to resume device model for domain %u:%d",
- domid, rc);
- goto out;
- }
- }
-
- if (!xs_resume_domain(CTX->xsh, domid)) {
- LOGE(ERROR, "xs_resume_domain failed for domain %u", domid);
- rc = ERROR_FAIL;
- }
-out:
- return rc;
-}
-
int libxl_domain_resume(libxl_ctx *ctx, uint32_t domid, int suspend_cancel,
const libxl_asyncop_how *ao_how)
{
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 3b02562..f457f72 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1326,26 +1326,6 @@ static void switch_logdirty_done(libxl__egc *egc,
libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
}
-int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
-{
-
- switch (libxl__device_model_version_running(gc, domid)) {
- case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
- libxl__qemu_traditional_cmd(gc, domid, "continue");
- libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
- break;
- }
- case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
- if (libxl__qmp_resume(gc, domid))
- return ERROR_FAIL;
- break;
- default:
- return ERROR_INVAL;
- }
-
- return 0;
-}
-
static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
uint32_t domid,
char *phys_offset, char *node)
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
index 4edf936..5f1f4fd 100644
--- a/tools/libxl/libxl_dom_suspend.c
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -372,6 +372,61 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
{
libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
}
+
+/*======================= Domain resume ========================*/
+
+static int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
+{
+
+ switch (libxl__device_model_version_running(gc, domid)) {
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
+ libxl__qemu_traditional_cmd(gc, domid, "continue");
+ libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
+ break;
+ }
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+ if (libxl__qmp_resume(gc, domid))
+ return ERROR_FAIL;
+ break;
+ default:
+ return ERROR_INVAL;
+ }
+
+ return 0;
+}
+
+int libxl__domain_resume(libxl__gc *gc, uint32_t domid, int suspend_cancel)
+{
+ int rc = 0;
+
+ if (xc_domain_resume(CTX->xch, domid, suspend_cancel)) {
+ LOGE(ERROR, "xc_domain_resume failed for domain %u", domid);
+ rc = ERROR_FAIL;
+ goto out;
+ }
+
+ libxl_domain_type type = libxl__domain_type(gc, domid);
+ if (type == LIBXL_DOMAIN_TYPE_INVALID) {
+ rc = ERROR_FAIL;
+ goto out;
+ }
+
+ if (type == LIBXL_DOMAIN_TYPE_HVM) {
+ rc = libxl__domain_resume_device_model(gc, domid);
+ if (rc) {
+ LOG(ERROR, "failed to resume device model for domain %u:%d",
+ domid, rc);
+ goto out;
+ }
+ }
+
+ if (!xs_resume_domain(CTX->xsh, domid)) {
+ LOGE(ERROR, "xs_resume_domain failed for domain %u", domid);
+ rc = ERROR_FAIL;
+ }
+out:
+ return rc;
+}
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 54c3cc1..28959fa 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1027,7 +1027,6 @@ _hidden int libxl__domain_rename(libxl__gc *gc, uint32_t domid,
_hidden int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
uint32_t size, void *data);
-_hidden int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid);
_hidden const char *libxl__userdata_path(libxl__gc *gc, uint32_t domid,
const char *userdata_userid,
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 03/26] tools/libxl: move domain resume code into libxl_dom_suspend.c
2015-06-25 6:25 ` [PATCH v3 COLOPre 03/26] tools/libxl: move domain resume " Yang Hongyang
@ 2015-06-29 15:44 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 15:44 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> move domain resume code into libxl_dom_suspend.c.
> pure code motion except for making
> libxl__domain_resume_device_model() static.
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 04/26] tools/libxl: move remus code into libxl_remus.c
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (2 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 03/26] tools/libxl: move domain resume " Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 15:48 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 05/26] tools/libxl: move save/restore code into libxl_dom_save.c Yang Hongyang
` (21 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
Do the following things:
- There are 2 checkpoint callbacks, rename them to:
libxl__remus_domain_{save/restore}_checkpoint_callback
- Moving remus callbacks into libxl_remus.c, export following
callbacks for internal use:
* libxl__remus_domain_suspend_callback
* libxl__remus_domain_resume_callback
* libxl__remus_domain_save_checkpoint_callback
* libxl__remus_domain_restore_checkpoint_callback
- Refactoring Remus setup/teardown, introduce 2 APIs to
setup/teardown Remus:
* libxl__remus_setup
* libxl__remus_teardown
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
---
tools/libxl/Makefile | 2 +-
tools/libxl/libxl.c | 55 +-------
tools/libxl/libxl_create.c | 24 +---
tools/libxl/libxl_dom.c | 204 +--------------------------
tools/libxl/libxl_internal.h | 12 ++
tools/libxl/libxl_remus.c | 325 +++++++++++++++++++++++++++++++++++++++++++
6 files changed, 342 insertions(+), 280 deletions(-)
create mode 100644 tools/libxl/libxl_remus.c
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index e98e26f..0c04ae7 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -56,7 +56,7 @@ else
LIBXL_OBJS-y += libxl_nonetbuffer.o
endif
-LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o
+LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 1f52bf0..b939f2f 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -792,10 +792,6 @@ out:
return ptr;
}
-static void libxl__remus_setup_done(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc);
-static void libxl__remus_setup_failed(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc);
static void remus_failover_cb(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc);
@@ -844,63 +840,14 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
assert(info);
- /* Convenience aliases */
- libxl__remus_devices_state *const rds = &dss->rds;
-
- if (libxl_defbool_val(info->netbuf)) {
- if (!libxl__netbuffer_enabled(gc)) {
- LOG(ERROR, "Remus: No support for network buffering");
- rc = ERROR_FAIL;
- goto out;
- }
- rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
- }
-
- if (libxl_defbool_val(info->diskbuf))
- rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
-
- rds->ao = ao;
- rds->domid = domid;
- rds->callback = libxl__remus_setup_done;
-
/* Point of no return */
- libxl__remus_devices_setup(egc, rds);
+ libxl__remus_setup(egc, dss);
return AO_INPROGRESS;
out:
return AO_ABORT(rc);
}
-static void libxl__remus_setup_done(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
- STATE_AO_GC(dss->ao);
-
- if (!rc) {
- libxl__domain_save(egc, dss);
- return;
- }
-
- LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
- dss->domid, rc);
- rds->callback = libxl__remus_setup_failed;
- libxl__remus_devices_teardown(egc, rds);
-}
-
-static void libxl__remus_setup_failed(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
- STATE_AO_GC(dss->ao);
-
- if (rc)
- LOG(ERROR, "Remus: failed to teardown device after setup failed"
- " for guest with domid %u, rc %d", dss->domid, rc);
-
- dss->callback(egc, dss, rc);
-}
-
static void remus_failover_cb(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc)
{
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index ac918bd..dfea992 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -747,27 +747,6 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
libxl_device_model_version_to_string(b_info->device_model_version));
}
-/*----- remus asynchronous checkpoint callback -----*/
-
-static void remus_checkpoint_stream_done(
- libxl__egc *egc, libxl__domain_create_state *dcs, int rc);
-
-static void libxl__remus_domain_checkpoint_callback(void *data)
-{
- libxl__save_helper_state *shs = data;
- libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
- libxl__egc *egc = dcs->shs.egc;
- STATE_AO_GC(dcs->ao);
-
- libxl__stream_read_start_checkpoint(egc, &dcs->srs);
-}
-
-static void remus_checkpoint_stream_done(
- libxl__egc *egc, libxl__domain_create_state *dcs, int rc)
-{
- libxl__xc_domain_saverestore_async_callback_done(egc, &dcs->shs, rc);
-}
-
/*----- main domain creation -----*/
/* We have a linear control flow; only one event callback is
@@ -1058,7 +1037,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
}
/* Restore */
- callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
+ callbacks->checkpoint = libxl__remus_domain_restore_checkpoint_callback;
rc = libxl__build_pre(gc, domid, d_config, state);
if (rc)
@@ -1068,7 +1047,6 @@ static void domcreate_bootloader_done(libxl__egc *egc,
dcs->srs.fd = restore_fd;
dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
dcs->srs.completion_callback = domcreate_stream_done;
- dcs->srs.checkpoint_callback = remus_checkpoint_stream_done;
libxl__stream_read_start(egc, &dcs->srs);
return;
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index f457f72..6f4cda8 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1416,184 +1416,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
return 0;
}
-/*----- remus callbacks -----*/
-static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok);
-static void remus_devices_postsuspend_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc);
-static void remus_devices_preresume_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc);
-
-static void libxl__remus_domain_suspend_callback(void *data)
-{
- libxl__save_helper_state *shs = data;
- libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-
- dss->callback_common_done = remus_domain_suspend_callback_common_done;
- libxl__domain_suspend(egc, dss);
-}
-
-static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok)
-{
- if (!ok)
- goto out;
-
- libxl__remus_devices_state *const rds = &dss->rds;
- rds->callback = remus_devices_postsuspend_cb;
- libxl__remus_devices_postsuspend(egc, rds);
- return;
-
-out:
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
-static void remus_devices_postsuspend_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc)
-{
- int ok = 0;
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-
- if (rc)
- goto out;
-
- ok = 1;
-
-out:
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
-static void libxl__remus_domain_resume_callback(void *data)
-{
- libxl__save_helper_state *shs = data;
- libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
- STATE_AO_GC(dss->ao);
-
- libxl__remus_devices_state *const rds = &dss->rds;
- rds->callback = remus_devices_preresume_cb;
- libxl__remus_devices_preresume(egc, rds);
-}
-
-static void remus_devices_preresume_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc)
-{
- int ok = 0;
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
- STATE_AO_GC(dss->ao);
-
- if (rc)
- goto out;
-
- /* Resumes the domain and the device model */
- rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
- if (rc)
- goto out;
-
- ok = 1;
-
-out:
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
-}
-
-/*----- remus asynchronous checkpoint callback -----*/
-
-static void remus_checkpoint_stream_written(
- libxl__egc *egc, libxl__domain_suspend_state *dss, int rc);
-static void remus_devices_commit_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc);
-static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
- const struct timeval *requested_abs);
-
-static void libxl__remus_domain_checkpoint_callback(void *data)
-{
- libxl__save_helper_state *shs = data;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
- libxl__egc *egc = dss->shs.egc;
- STATE_AO_GC(dss->ao);
-
- libxl__stream_write_start_checkpoint(egc, &dss->sws);
-}
-
-static void remus_checkpoint_stream_written(
- libxl__egc *egc, libxl__domain_suspend_state *dss, int rc)
-{
- /* Convenience aliases */
- libxl__remus_devices_state *const rds = &dss->rds;
-
- STATE_AO_GC(dss->ao);
-
- if (rc) {
- LOG(ERROR, "Failed to save device model. Terminating Remus..");
- goto out;
- }
-
- rds->callback = remus_devices_commit_cb;
- libxl__remus_devices_commit(egc, rds);
-
- return;
-
-out:
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
-}
-
-static void remus_devices_commit_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
-
- STATE_AO_GC(dss->ao);
-
- if (rc) {
- LOG(ERROR, "Failed to do device commit op."
- " Terminating Remus..");
- goto out;
- }
-
- /*
- * At this point, we have successfully checkpointed the guest and
- * committed it at the backup. We'll come back after the checkpoint
- * interval to checkpoint the guest again. Until then, let the guest
- * continue execution.
- */
-
- /* Set checkpoint interval timeout */
- rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
- remus_next_checkpoint,
- dss->interval);
-
- if (rc)
- goto out;
-
- return;
-
-out:
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
-}
-
-static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
- const struct timeval *requested_abs)
-{
- libxl__domain_suspend_state *dss =
- CONTAINER_OF(ev, *dss, checkpoint_timeout);
-
- STATE_AO_GC(dss->ao);
-
- /*
- * Time to checkpoint the guest again. We return 1 to libxc
- * (xc_domain_save.c). in order to continue executing the infinite loop
- * (suspend, checkpoint, resume) in xc_domain_save().
- */
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
-}
-
/*----- main code for suspending, in order of execution -----*/
void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
@@ -1666,8 +1488,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
if (r_info != NULL) {
callbacks->suspend = libxl__remus_domain_suspend_callback;
callbacks->postcopy = libxl__remus_domain_resume_callback;
- callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
- dss->sws.checkpoint_callback = remus_checkpoint_stream_written;
+ callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback;
} else
callbacks->suspend = libxl__domain_suspend_callback;
@@ -1782,10 +1603,6 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
dss->save_dm_callback(egc, dss, our_rc);
}
-static void remus_teardown_done(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc);
-
static void domain_save_done(libxl__egc *egc,
libxl__domain_suspend_state *dss, int rc)
{
@@ -1811,24 +1628,7 @@ static void domain_save_done(libxl__egc *egc,
* from sending checkpoints. Teardown the network buffers and
* release netlink resources. This is an async op.
*/
- LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
- " teardown Remus devices...", rc);
- dss->rds.callback = remus_teardown_done;
- libxl__remus_devices_teardown(egc, &dss->rds);
-}
-
-static void remus_teardown_done(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- int rc)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
- STATE_AO_GC(dss->ao);
-
- if (rc)
- LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
- " rc %d", dss->domid, rc);
-
- dss->callback(egc, dss, rc);
+ libxl__remus_teardown(egc, dss, rc);
}
/*==================== Miscellaneous ====================*/
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 28959fa..b342988 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3323,6 +3323,18 @@ _hidden void libxl__domain_suspend(libxl__egc *egc,
/* used by libxc to suspend the guest during migration */
_hidden void libxl__domain_suspend_callback(void *data);
+/* Remus callbacks for save */
+_hidden void libxl__remus_domain_suspend_callback(void *data);
+_hidden void libxl__remus_domain_resume_callback(void *data);
+_hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
+/* Remus callbacks for restore */
+_hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
+/* Remus setup and teardown*/
+_hidden void libxl__remus_setup(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+_hidden void libxl__remus_teardown(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc);
/*
* Convenience macros.
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
new file mode 100644
index 0000000..504cd44
--- /dev/null
+++ b/tools/libxl/libxl_remus.c
@@ -0,0 +1,325 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+/*----- Remus setup and teardown -----*/
+
+static void remus_setup_done(libxl__egc *egc,
+ libxl__remus_devices_state *rds, int rc);
+static void remus_setup_failed(libxl__egc *egc,
+ libxl__remus_devices_state *rds, int rc);
+
+void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
+{
+ /* Convenience aliases */
+ libxl__remus_devices_state *const rds = &dss->rds;
+ const libxl_domain_remus_info *const info = dss->remus;
+
+ STATE_AO_GC(dss->ao);
+
+ if (libxl_defbool_val(info->netbuf)) {
+ if (!libxl__netbuffer_enabled(gc)) {
+ LOG(ERROR, "Remus: No support for network buffering");
+ goto out;
+ }
+ rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
+ }
+
+ if (libxl_defbool_val(info->diskbuf))
+ rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
+
+ rds->ao = ao;
+ rds->domid = dss->domid;
+ rds->callback = remus_setup_done;
+
+ libxl__remus_devices_setup(egc, rds);
+ return;
+
+out:
+ dss->callback(egc, dss, ERROR_FAIL);
+}
+
+static void remus_setup_done(libxl__egc *egc,
+ libxl__remus_devices_state *rds, int rc)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ STATE_AO_GC(dss->ao);
+
+ if (!rc) {
+ libxl__domain_save(egc, dss);
+ return;
+ }
+
+ LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
+ dss->domid, rc);
+ rds->callback = remus_setup_failed;
+ libxl__remus_devices_teardown(egc, rds);
+}
+
+static void remus_setup_failed(libxl__egc *egc,
+ libxl__remus_devices_state *rds, int rc)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ STATE_AO_GC(dss->ao);
+
+ if (rc)
+ LOG(ERROR, "Remus: failed to teardown device after setup failed"
+ " for guest with domid %u, rc %d", dss->domid, rc);
+
+ dss->callback(egc, dss, rc);
+}
+
+static void remus_teardown_done(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc);
+void libxl__remus_teardown(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc)
+{
+ EGC_GC;
+
+ LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
+ " teardown Remus devices...", rc);
+ dss->rds.callback = remus_teardown_done;
+ libxl__remus_devices_teardown(egc, &dss->rds);
+}
+
+static void remus_teardown_done(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ STATE_AO_GC(dss->ao);
+
+ if (rc)
+ LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
+ " rc %d", dss->domid, rc);
+
+ dss->callback(egc, dss, rc);
+}
+
+/*---------------------- remus callbacks (save) -----------------------*/
+
+static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int ok);
+static void remus_devices_postsuspend_cb(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc);
+static void remus_devices_preresume_cb(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc);
+
+void libxl__remus_domain_suspend_callback(void *data)
+{
+ libxl__save_helper_state *shs = data;
+ libxl__egc *egc = shs->egc;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+
+ dss->callback_common_done = remus_domain_suspend_callback_common_done;
+ libxl__domain_suspend(egc, dss);
+}
+
+static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int ok)
+{
+ if (!ok)
+ goto out;
+
+ libxl__remus_devices_state *const rds = &dss->rds;
+ rds->callback = remus_devices_postsuspend_cb;
+ libxl__remus_devices_postsuspend(egc, rds);
+ return;
+
+out:
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+static void remus_devices_postsuspend_cb(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc)
+{
+ int ok = 0;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+
+ if (rc)
+ goto out;
+
+ ok = 1;
+
+out:
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+void libxl__remus_domain_resume_callback(void *data)
+{
+ libxl__save_helper_state *shs = data;
+ libxl__egc *egc = shs->egc;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ STATE_AO_GC(dss->ao);
+
+ libxl__remus_devices_state *const rds = &dss->rds;
+ rds->callback = remus_devices_preresume_cb;
+ libxl__remus_devices_preresume(egc, rds);
+}
+
+static void remus_devices_preresume_cb(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc)
+{
+ int ok = 0;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ STATE_AO_GC(dss->ao);
+
+ if (rc)
+ goto out;
+
+ /* Resumes the domain and the device model */
+ rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
+ if (rc)
+ goto out;
+
+ ok = 1;
+
+out:
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+/*----- remus asynchronous checkpoint callback -----*/
+
+static void remus_checkpoint_stream_written(
+ libxl__egc *egc, libxl__domain_suspend_state *dss, int rc);
+static void remus_devices_commit_cb(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc);
+static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
+ const struct timeval *requested_abs);
+
+void libxl__remus_domain_save_checkpoint_callback(void *data)
+{
+ libxl__save_helper_state *shs = data;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__egc *egc = dss->shs.egc;
+ STATE_AO_GC(dss->ao);
+
+ dss->sws.checkpoint_callback = remus_checkpoint_stream_written;
+ libxl__stream_write_start_checkpoint(egc, &dss->sws);
+}
+
+static void remus_checkpoint_stream_written(
+ libxl__egc *egc, libxl__domain_suspend_state *dss, int rc)
+{
+ /* Convenience aliases */
+ libxl__remus_devices_state *const rds = &dss->rds;
+
+ STATE_AO_GC(dss->ao);
+
+ if (rc) {
+ LOG(ERROR, "Failed to save device model. Terminating Remus..");
+ goto out;
+ }
+
+ rds->callback = remus_devices_commit_cb;
+ libxl__remus_devices_commit(egc, rds);
+
+ return;
+
+out:
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+}
+
+static void remus_devices_commit_cb(libxl__egc *egc,
+ libxl__remus_devices_state *rds,
+ int rc)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+
+ STATE_AO_GC(dss->ao);
+
+ if (rc) {
+ LOG(ERROR, "Failed to do device commit op."
+ " Terminating Remus..");
+ goto out;
+ }
+
+ /*
+ * At this point, we have successfully checkpointed the guest and
+ * committed it at the backup. We'll come back after the checkpoint
+ * interval to checkpoint the guest again. Until then, let the guest
+ * continue execution.
+ */
+
+ /* Set checkpoint interval timeout */
+ rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
+ remus_next_checkpoint,
+ dss->interval);
+
+ if (rc)
+ goto out;
+
+ return;
+
+out:
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+}
+
+static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
+ const struct timeval *requested_abs)
+{
+ libxl__domain_suspend_state *dss =
+ CONTAINER_OF(ev, *dss, checkpoint_timeout);
+
+ STATE_AO_GC(dss->ao);
+
+ /*
+ * Time to checkpoint the guest again. We return 1 to libxc
+ * (xc_domain_save.c). in order to continue executing the infinite loop
+ * (suspend, checkpoint, resume) in xc_domain_save().
+ */
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
+}
+
+/*---------------------- remus callbacks (restore) -----------------------*/
+
+/*----- remus asynchronous checkpoint callback -----*/
+
+static void remus_checkpoint_stream_done(
+ libxl__egc *egc, libxl__domain_create_state *dcs, int rc);
+
+void libxl__remus_domain_restore_checkpoint_callback(void *data)
+{
+ libxl__save_helper_state *shs = data;
+ libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
+ libxl__egc *egc = dcs->shs.egc;
+ STATE_AO_GC(dcs->ao);
+
+ dcs->srs.checkpoint_callback = remus_checkpoint_stream_done;
+ libxl__stream_read_start_checkpoint(egc, &dcs->srs);
+}
+
+static void remus_checkpoint_stream_done(
+ libxl__egc *egc, libxl__domain_create_state *dcs, int rc)
+{
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dcs->shs, rc);
+}
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 04/26] tools/libxl: move remus code into libxl_remus.c
2015-06-25 6:25 ` [PATCH v3 COLOPre 04/26] tools/libxl: move remus code into libxl_remus.c Yang Hongyang
@ 2015-06-29 15:48 ` Ian Campbell
2015-06-30 9:36 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 15:48 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Do the following things:
> - There are 2 checkpoint callbacks, rename them to:
> libxl__remus_domain_{save/restore}_checkpoint_callback
> - Moving remus callbacks into libxl_remus.c, export following
> callbacks for internal use:
> * libxl__remus_domain_suspend_callback
> * libxl__remus_domain_resume_callback
> * libxl__remus_domain_save_checkpoint_callback
> * libxl__remus_domain_restore_checkpoint_callback
> - Refactoring Remus setup/teardown, introduce 2 APIs to
> setup/teardown Remus:
> * libxl__remus_setup
> * libxl__remus_teardown
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> [...]
> +/* Remus callbacks for save */
> +_hidden void libxl__remus_domain_suspend_callback(void *data);
> +_hidden void libxl__remus_domain_resume_callback(void *data);
At first glance having resume under save rather than under restore
seemed odd, but I suppose it is just that you resume after a failed
save, but on restore you just start the VM, is that the right logic?
> +_hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
> +/* Remus callbacks for restore */
> +_hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
> +/* Remus setup and teardown*/
> +_hidden void libxl__remus_setup(libxl__egc *egc,
> + libxl__domain_suspend_state *dss);
> +_hidden void libxl__remus_teardown(libxl__egc *egc,
> + libxl__domain_suspend_state *dss,
> + int rc);
>
> /*
> * Convenience macros.
> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
> new file mode 100644
> index 0000000..504cd44
> --- /dev/null
> +++ b/tools/libxl/libxl_remus.c
> @@ -0,0 +1,325 @@
> +/*
> + * Copyright (c) 2015 FUJITSU LIMITED
> + * Author Yang Hongyang <yanghy@cn.fujitsu.com>
Given the amount of this which is code motion I think you should also be
copying the existing copyright and licenses from the files where the
code originated.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 04/26] tools/libxl: move remus code into libxl_remus.c
2015-06-29 15:48 ` Ian Campbell
@ 2015-06-30 9:36 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 9:36 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/29/2015 11:48 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Do the following things:
>> - There are 2 checkpoint callbacks, rename them to:
>> libxl__remus_domain_{save/restore}_checkpoint_callback
>> - Moving remus callbacks into libxl_remus.c, export following
>> callbacks for internal use:
>> * libxl__remus_domain_suspend_callback
>> * libxl__remus_domain_resume_callback
>> * libxl__remus_domain_save_checkpoint_callback
>> * libxl__remus_domain_restore_checkpoint_callback
>> - Refactoring Remus setup/teardown, introduce 2 APIs to
>> setup/teardown Remus:
>> * libxl__remus_setup
>> * libxl__remus_teardown
>>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>
>> [...]
>> +/* Remus callbacks for save */
>> +_hidden void libxl__remus_domain_suspend_callback(void *data);
>> +_hidden void libxl__remus_domain_resume_callback(void *data);
>
> At first glance having resume under save rather than under restore
> seemed odd, but I suppose it is just that you resume after a failed
> save, but on restore you just start the VM, is that the right logic?
With Remus, save side should continue to work after migration and
after every checkpoint, while we need to suspend guest to transfer dirty
pages, we must resume it after.
>
>> +_hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
>> +/* Remus callbacks for restore */
>> +_hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
>> +/* Remus setup and teardown*/
>> +_hidden void libxl__remus_setup(libxl__egc *egc,
>> + libxl__domain_suspend_state *dss);
>> +_hidden void libxl__remus_teardown(libxl__egc *egc,
>> + libxl__domain_suspend_state *dss,
>> + int rc);
>>
>> /*
>> * Convenience macros.
>> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
>> new file mode 100644
>> index 0000000..504cd44
>> --- /dev/null
>> +++ b/tools/libxl/libxl_remus.c
>> @@ -0,0 +1,325 @@
>> +/*
>> + * Copyright (c) 2015 FUJITSU LIMITED
>> + * Author Yang Hongyang <yanghy@cn.fujitsu.com>
>
> Given the amount of this which is code motion I think you should also be
> copying the existing copyright and licenses from the files where the
> code originated.
Sure, will add it in the next version.
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 05/26] tools/libxl: move save/restore code into libxl_dom_save.c
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (3 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 04/26] tools/libxl: move remus code into libxl_remus.c Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 15:49 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state Yang Hongyang
` (20 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
This is purely code motion.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
---
tools/libxl/Makefile | 2 +-
tools/libxl/libxl_dom.c | 607 -----------------------------------------
tools/libxl/libxl_dom_save.c | 634 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 635 insertions(+), 608 deletions(-)
create mode 100644 tools/libxl/libxl_dom_save.c
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 0c04ae7..d61c191 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -96,7 +96,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \
libxl_stream_read.o libxl_stream_write.o \
libxl_save_callout.o _libxl_save_msgs_callout.o \
- libxl_convert_callout.o libxl_dom_suspend.o \
+ libxl_convert_callout.o libxl_dom_suspend.o libxl_dom_save.o \
libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
LIBXL_OBJS += libxl_genid.o
LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 6f4cda8..ccab2f3 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1024,613 +1024,6 @@ int libxl__qemu_traditional_cmd(libxl__gc *gc, uint32_t domid,
return libxl__xs_write(gc, XBT_NULL, path, "%s", cmd);
}
-struct libxl__physmap_info {
- uint64_t phys_offset;
- uint64_t start_addr;
- uint64_t size;
- uint32_t namelen;
- char name[];
-};
-
-#define TOOLSTACK_SAVE_VERSION 1
-
-static inline char *restore_helper(libxl__gc *gc, uint32_t dm_domid,
- uint32_t domid,
- uint64_t phys_offset, char *node)
-{
- return libxl__device_model_xs_path(gc, dm_domid, domid,
- "/physmap/%"PRIx64"/%s",
- phys_offset, node);
-}
-
-int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
- uint32_t size, void *user)
-{
- libxl__save_helper_state *shs = user;
- libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
- STATE_AO_GC(dcs->ao);
- int i, ret;
- const uint8_t *ptr = buf;
- uint32_t count = 0, version = 0;
- struct libxl__physmap_info* pi;
- char *xs_path;
- uint32_t dm_domid;
-
- LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, size);
-
- if (size < sizeof(version) + sizeof(count)) {
- LOG(ERROR, "wrong size");
- return -1;
- }
-
- memcpy(&version, ptr, sizeof(version));
- ptr += sizeof(version);
-
- if (version != TOOLSTACK_SAVE_VERSION) {
- LOG(ERROR, "wrong version");
- return -1;
- }
-
- memcpy(&count, ptr, sizeof(count));
- ptr += sizeof(count);
-
- if (size < sizeof(version) + sizeof(count) +
- count * (sizeof(struct libxl__physmap_info))) {
- LOG(ERROR, "wrong size");
- return -1;
- }
-
- dm_domid = libxl_get_stubdom_id(CTX, domid);
- for (i = 0; i < count; i++) {
- pi = (struct libxl__physmap_info*) ptr;
- ptr += sizeof(struct libxl__physmap_info) + pi->namelen;
-
- xs_path = restore_helper(gc, dm_domid, domid,
- pi->phys_offset, "start_addr");
- ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->start_addr);
- if (ret)
- return -1;
- xs_path = restore_helper(gc, dm_domid, domid, pi->phys_offset, "size");
- ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->size);
- if (ret)
- return -1;
- if (pi->namelen > 0) {
- xs_path = restore_helper(gc, dm_domid, domid,
- pi->phys_offset, "name");
- ret = libxl__xs_write(gc, 0, xs_path, "%s", pi->name);
- if (ret)
- return -1;
- }
- }
- return 0;
-}
-
-/*==================== Domain suspend (save) ====================*/
-
-static void stream_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
-static void domain_save_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
-
-/*----- complicated callback, called by xc_domain_save -----*/
-
-/*
- * We implement the other end of protocol for controlling qemu-dm's
- * logdirty. There is no documentation for this protocol, but our
- * counterparty's implementation is in
- * qemu-xen-traditional.git:xenstore.c in the function
- * xenstore_process_logdirty_event
- */
-
-static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
- const struct timeval *requested_abs);
-static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
- const char *watch_path, const char *event_path);
-static void switch_logdirty_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok);
-
-static void logdirty_init(libxl__logdirty_switch *lds)
-{
- lds->cmd_path = 0;
- libxl__ev_xswatch_init(&lds->watch);
- libxl__ev_time_init(&lds->timeout);
-}
-
-static void domain_suspend_switch_qemu_xen_traditional_logdirty
- (int domid, unsigned enable,
- libxl__save_helper_state *shs)
-{
- libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
- libxl__logdirty_switch *lds = &dss->logdirty;
- STATE_AO_GC(dss->ao);
- int rc;
- xs_transaction_t t = 0;
- const char *got;
-
- if (!lds->cmd_path) {
- uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
- lds->cmd_path = libxl__device_model_xs_path(gc, dm_domid, domid,
- "/logdirty/cmd");
- lds->ret_path = libxl__device_model_xs_path(gc, dm_domid, domid,
- "/logdirty/ret");
- }
- lds->cmd = enable ? "enable" : "disable";
-
- rc = libxl__ev_xswatch_register(gc, &lds->watch,
- switch_logdirty_xswatch, lds->ret_path);
- if (rc) goto out;
-
- rc = libxl__ev_time_register_rel(gc, &lds->timeout,
- switch_logdirty_timeout, 10*1000);
- if (rc) goto out;
-
- for (;;) {
- rc = libxl__xs_transaction_start(gc, &t);
- if (rc) goto out;
-
- rc = libxl__xs_read_checked(gc, t, lds->cmd_path, &got);
- if (rc) goto out;
-
- if (got) {
- const char *got_ret;
- rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got_ret);
- if (rc) goto out;
-
- if (!got_ret || strcmp(got, got_ret)) {
- LOG(ERROR,"controlling logdirty: qemu was already sent"
- " command `%s' (xenstore path `%s') but result is `%s'",
- got, lds->cmd_path, got_ret ? got_ret : "<none>");
- rc = ERROR_FAIL;
- goto out;
- }
- rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
- if (rc) goto out;
- }
-
- rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
- if (rc) goto out;
-
- rc = libxl__xs_write_checked(gc, t, lds->cmd_path, lds->cmd);
- if (rc) goto out;
-
- rc = libxl__xs_transaction_commit(gc, &t);
- if (!rc) break;
- if (rc<0) goto out;
- }
-
- /* OK, wait for some callback */
- return;
-
- out:
- LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
- libxl__xs_transaction_abort(gc, &t);
- switch_logdirty_done(egc,dss,-1);
-}
-
-static void domain_suspend_switch_qemu_xen_logdirty
- (int domid, unsigned enable,
- libxl__save_helper_state *shs)
-{
- libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
- STATE_AO_GC(dss->ao);
- int rc;
-
- rc = libxl__qmp_set_global_dirty_log(gc, domid, enable);
- if (!rc) {
- libxl__xc_domain_saverestore_async_callback_done(egc, shs, 0);
- } else {
- LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
- libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
- }
-}
-
-void libxl__domain_suspend_common_switch_qemu_logdirty
- (int domid, unsigned enable, void *user)
-{
- libxl__save_helper_state *shs = user;
- libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
- STATE_AO_GC(dss->ao);
-
- switch (libxl__device_model_version_running(gc, domid)) {
- case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
- domain_suspend_switch_qemu_xen_traditional_logdirty(domid, enable, shs);
- break;
- case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
- domain_suspend_switch_qemu_xen_logdirty(domid, enable, shs);
- break;
- default:
- LOG(ERROR,"logdirty switch failed"
- ", no valid device model version found, aborting suspend");
- libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
- }
-}
-static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
- const struct timeval *requested_abs)
-{
- libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
- STATE_AO_GC(dss->ao);
- LOG(ERROR,"logdirty switch: wait for device model timed out");
- switch_logdirty_done(egc,dss,-1);
-}
-
-static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
- const char *watch_path, const char *event_path)
-{
- libxl__domain_suspend_state *dss =
- CONTAINER_OF(watch, *dss, logdirty.watch);
- libxl__logdirty_switch *lds = &dss->logdirty;
- STATE_AO_GC(dss->ao);
- const char *got;
- xs_transaction_t t = 0;
- int rc;
-
- for (;;) {
- rc = libxl__xs_transaction_start(gc, &t);
- if (rc) goto out;
-
- rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got);
- if (rc) goto out;
-
- if (!got) {
- rc = +1;
- goto out;
- }
-
- if (strcmp(got, lds->cmd)) {
- LOG(ERROR,"logdirty switch: sent command `%s' but got reply `%s'"
- " (xenstore paths `%s' / `%s')", lds->cmd, got,
- lds->cmd_path, lds->ret_path);
- rc = ERROR_FAIL;
- goto out;
- }
-
- rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
- if (rc) goto out;
-
- rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
- if (rc) goto out;
-
- rc = libxl__xs_transaction_commit(gc, &t);
- if (!rc) break;
- if (rc<0) goto out;
- }
-
- out:
- /* rc < 0: error
- * rc == 0: ok, we are done
- * rc == +1: need to keep waiting
- */
- libxl__xs_transaction_abort(gc, &t);
-
- if (!rc) {
- switch_logdirty_done(egc,dss,0);
- } else if (rc < 0) {
- LOG(ERROR,"logdirty switch: failed (rc=%d)",rc);
- switch_logdirty_done(egc,dss,-1);
- }
-}
-
-static void switch_logdirty_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
- int broke)
-{
- STATE_AO_GC(dss->ao);
- libxl__logdirty_switch *lds = &dss->logdirty;
-
- libxl__ev_xswatch_deregister(gc, &lds->watch);
- libxl__ev_time_deregister(gc, &lds->timeout);
-
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
-}
-
-static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
- uint32_t domid,
- char *phys_offset, char *node)
-{
- return libxl__device_model_xs_path(gc, dm_domid, domid,
- "/physmap/%s/%s",
- phys_offset, node);
-}
-
-int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
- uint32_t *len, void *dss_void)
-{
- libxl__domain_suspend_state *dss = dss_void;
- STATE_AO_GC(dss->ao);
- int i = 0;
- char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
- unsigned int num = 0;
- uint32_t count = 0, version = TOOLSTACK_SAVE_VERSION, namelen = 0;
- uint8_t *ptr = NULL;
- char **entries = NULL;
- struct libxl__physmap_info *pi;
- uint32_t dm_domid;
-
- dm_domid = libxl_get_stubdom_id(CTX, domid);
-
- entries = libxl__xs_directory(gc, 0,
- libxl__device_model_xs_path(gc, dm_domid, domid, "/physmap"),
- &num);
- count = num;
-
- *len = sizeof(version) + sizeof(count);
- *buf = calloc(1, *len);
- ptr = *buf;
- if (*buf == NULL)
- return -1;
-
- memcpy(ptr, &version, sizeof(version));
- ptr += sizeof(version);
- memcpy(ptr, &count, sizeof(count));
- ptr += sizeof(count);
-
- for (i = 0; i < count; i++) {
- unsigned long offset;
- char *xs_path;
- phys_offset = entries[i];
- if (phys_offset == NULL) {
- LOG(ERROR, "phys_offset %d is NULL", i);
- return -1;
- }
-
- xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "start_addr");
- start_addr = libxl__xs_read(gc, 0, xs_path);
- if (start_addr == NULL) {
- LOG(ERROR, "%s is NULL", xs_path);
- return -1;
- }
-
- xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "size");
- size = libxl__xs_read(gc, 0, xs_path);
- if (size == NULL) {
- LOG(ERROR, "%s is NULL", xs_path);
- return -1;
- }
-
- xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "name");
- name = libxl__xs_read(gc, 0, xs_path);
- if (name == NULL)
- namelen = 0;
- else
- namelen = strlen(name) + 1;
- *len += namelen + sizeof(struct libxl__physmap_info);
- offset = ptr - (*buf);
- *buf = realloc(*buf, *len);
- if (*buf == NULL)
- return -1;
- ptr = (*buf) + offset;
- pi = (struct libxl__physmap_info *) ptr;
- pi->phys_offset = strtoll(phys_offset, NULL, 16);
- pi->start_addr = strtoll(start_addr, NULL, 16);
- pi->size = strtoll(size, NULL, 16);
- pi->namelen = namelen;
- memcpy(pi->name, name, namelen);
- ptr += sizeof(struct libxl__physmap_info) + namelen;
- }
-
- LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, *len);
-
- return 0;
-}
-
-/*----- main code for suspending, in order of execution -----*/
-
-void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
-{
- STATE_AO_GC(dss->ao);
- int port;
- int rc = ERROR_FAIL;
-
- /* Convenience aliases */
- const uint32_t domid = dss->domid;
- const libxl_domain_type type = dss->type;
- const int live = dss->live;
- const int debug = dss->debug;
- const libxl_domain_remus_info *const r_info = dss->remus;
- libxl__srm_save_autogen_callbacks *const callbacks =
- &dss->shs.callbacks.save.a;
-
- logdirty_init(&dss->logdirty);
- libxl__xswait_init(&dss->pvcontrol);
- libxl__ev_evtchn_init(&dss->guest_evtchn);
- libxl__ev_xswatch_init(&dss->guest_watch);
- libxl__ev_time_init(&dss->guest_timeout);
-
- switch (type) {
- case LIBXL_DOMAIN_TYPE_HVM: {
- dss->hvm = 1;
- break;
- }
- case LIBXL_DOMAIN_TYPE_PV:
- dss->hvm = 0;
- break;
- default:
- abort();
- }
-
- dss->xcflags = (live ? XCFLAGS_LIVE : 0)
- | (debug ? XCFLAGS_DEBUG : 0)
- | (dss->hvm ? XCFLAGS_HVM : 0);
-
- dss->guest_evtchn.port = -1;
- dss->guest_evtchn_lockfd = -1;
- dss->guest_responded = 0;
- dss->dm_savefile = libxl__device_model_savefile(gc, domid);
-
- if (r_info != NULL) {
- dss->interval = r_info->interval;
- dss->xcflags |= XCFLAGS_CHECKPOINTED;
- if (libxl_defbool_val(r_info->compression))
- dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
- }
-
- port = xs_suspend_evtchn_port(dss->domid);
-
- if (port >= 0) {
- rc = libxl__ctx_evtchn_init(gc);
- if (rc) goto out;
-
- dss->guest_evtchn.port =
- xc_suspend_evtchn_init_exclusive(CTX->xch, CTX->xce,
- dss->domid, port, &dss->guest_evtchn_lockfd);
-
- if (dss->guest_evtchn.port < 0) {
- LOG(WARN, "Suspend event channel initialization failed");
- rc = ERROR_FAIL;
- goto out;
- }
- }
-
- memset(callbacks, 0, sizeof(*callbacks));
- if (r_info != NULL) {
- callbacks->suspend = libxl__remus_domain_suspend_callback;
- callbacks->postcopy = libxl__remus_domain_resume_callback;
- callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback;
- } else
- callbacks->suspend = libxl__domain_suspend_callback;
-
- callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
-
- dss->sws.fd = dss->fd;
- dss->sws.ao = dss->ao;
- dss->sws.completion_callback = stream_done;
-
- libxl__stream_write_start(egc, &dss->sws);
- return;
-
- out:
- domain_save_done(egc, dss, rc);
-}
-
-static void stream_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
-{
- domain_save_done(egc, dss, rc);
-}
-
-static void save_device_model_datacopier_done(libxl__egc *egc,
- libxl__datacopier_state *dc, int onwrite, int errnoval);
-
-void libxl__domain_save_device_model(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
- libxl__save_device_model_cb *callback)
-{
- STATE_AO_GC(dss->ao);
- struct stat st;
- uint32_t qemu_state_len;
- int rc;
-
- dss->save_dm_callback = callback;
-
- /* Convenience aliases */
- const char *const filename = dss->dm_savefile;
- const int fd = dss->fd;
-
- libxl__datacopier_state *dc = &dss->save_dm_datacopier;
- memset(dc, 0, sizeof(*dc));
- dc->readwhat = GCSPRINTF("qemu save file %s", filename);
- dc->ao = ao;
- dc->readfd = -1;
- dc->writefd = fd;
- dc->maxsz = INT_MAX;
- dc->bytes_to_read = -1;
- dc->copywhat = GCSPRINTF("qemu save file for domain %"PRIu32, dss->domid);
- dc->writewhat = "save/migration stream";
- dc->callback = save_device_model_datacopier_done;
-
- dc->readfd = open(filename, O_RDONLY);
- if (dc->readfd < 0) {
- LOGE(ERROR, "unable to open %s", dc->readwhat);
- goto out;
- }
-
- if (fstat(dc->readfd, &st))
- {
- LOGE(ERROR, "unable to fstat %s", dc->readwhat);
- goto out;
- }
-
- if (!S_ISREG(st.st_mode)) {
- LOG(ERROR, "%s is not a plain file!", dc->readwhat);
- goto out;
- }
-
- qemu_state_len = st.st_size;
- LOG(DEBUG, "%s is %d bytes", dc->readwhat, qemu_state_len);
-
- rc = libxl__datacopier_start(dc);
- if (rc) goto out;
-
- libxl__datacopier_prefixdata(egc, dc,
- QEMU_SIGNATURE, strlen(QEMU_SIGNATURE));
-
- libxl__datacopier_prefixdata(egc, dc,
- &qemu_state_len, sizeof(qemu_state_len));
- return;
-
- out:
- save_device_model_datacopier_done(egc, dc, -1, 0);
-}
-
-static void save_device_model_datacopier_done(libxl__egc *egc,
- libxl__datacopier_state *dc, int onwrite, int errnoval)
-{
- libxl__domain_suspend_state *dss =
- CONTAINER_OF(dc, *dss, save_dm_datacopier);
- STATE_AO_GC(dss->ao);
-
- /* Convenience aliases */
- const char *const filename = dss->dm_savefile;
- int our_rc = 0;
- int rc;
-
- libxl__datacopier_kill(dc);
-
- if (onwrite || errnoval)
- our_rc = ERROR_FAIL;
-
- if (dc->readfd >= 0) {
- close(dc->readfd);
- dc->readfd = -1;
- }
-
- rc = libxl__remove_file(gc, filename);
- if (!our_rc) our_rc = rc;
-
- dss->save_dm_callback(egc, dss, our_rc);
-}
-
-static void domain_save_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
-{
- STATE_AO_GC(dss->ao);
-
- /* Convenience aliases */
- const uint32_t domid = dss->domid;
-
- libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
-
- if (dss->guest_evtchn.port > 0)
- xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
- dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
-
- if (!dss->remus) {
- dss->callback(egc, dss, rc);
- return;
- }
-
- /*
- * With Remus, if we reach this point, it means either
- * backup died or some network error occurred preventing us
- * from sending checkpoints. Teardown the network buffers and
- * release netlink resources. This is an async op.
- */
- libxl__remus_teardown(egc, dss, rc);
-}
-
/*==================== Miscellaneous ====================*/
char *libxl__uuid2string(libxl__gc *gc, const libxl_uuid uuid)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
new file mode 100644
index 0000000..9ee74ba
--- /dev/null
+++ b/tools/libxl/libxl_dom_save.c
@@ -0,0 +1,634 @@
+/*
+ * Copyright (C) 2009 Citrix Ltd.
+ * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+struct libxl__physmap_info {
+ uint64_t phys_offset;
+ uint64_t start_addr;
+ uint64_t size;
+ uint32_t namelen;
+ char name[];
+};
+
+#define TOOLSTACK_SAVE_VERSION 1
+
+/*========================= Domain save ============================*/
+
+static void stream_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc);
+static void domain_save_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc);
+
+/*----- complicated callback, called by xc_domain_save -----*/
+
+/*
+ * We implement the other end of protocol for controlling qemu-dm's
+ * logdirty. There is no documentation for this protocol, but our
+ * counterparty's implementation is in
+ * qemu-xen-traditional.git:xenstore.c in the function
+ * xenstore_process_logdirty_event
+ */
+
+static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
+ const struct timeval *requested_abs);
+static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
+ const char *watch_path, const char *event_path);
+static void switch_logdirty_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int ok);
+
+static void logdirty_init(libxl__logdirty_switch *lds)
+{
+ lds->cmd_path = 0;
+ libxl__ev_xswatch_init(&lds->watch);
+ libxl__ev_time_init(&lds->timeout);
+}
+
+static void domain_suspend_switch_qemu_xen_traditional_logdirty
+ (int domid, unsigned enable,
+ libxl__save_helper_state *shs)
+{
+ libxl__egc *egc = shs->egc;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__logdirty_switch *lds = &dss->logdirty;
+ STATE_AO_GC(dss->ao);
+ int rc;
+ xs_transaction_t t = 0;
+ const char *got;
+
+ if (!lds->cmd_path) {
+ uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
+ lds->cmd_path = libxl__device_model_xs_path(gc, dm_domid, domid,
+ "/logdirty/cmd");
+ lds->ret_path = libxl__device_model_xs_path(gc, dm_domid, domid,
+ "/logdirty/ret");
+ }
+ lds->cmd = enable ? "enable" : "disable";
+
+ rc = libxl__ev_xswatch_register(gc, &lds->watch,
+ switch_logdirty_xswatch, lds->ret_path);
+ if (rc) goto out;
+
+ rc = libxl__ev_time_register_rel(gc, &lds->timeout,
+ switch_logdirty_timeout, 10*1000);
+ if (rc) goto out;
+
+ for (;;) {
+ rc = libxl__xs_transaction_start(gc, &t);
+ if (rc) goto out;
+
+ rc = libxl__xs_read_checked(gc, t, lds->cmd_path, &got);
+ if (rc) goto out;
+
+ if (got) {
+ const char *got_ret;
+ rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got_ret);
+ if (rc) goto out;
+
+ if (!got_ret || strcmp(got, got_ret)) {
+ LOG(ERROR,"controlling logdirty: qemu was already sent"
+ " command `%s' (xenstore path `%s') but result is `%s'",
+ got, lds->cmd_path, got_ret ? got_ret : "<none>");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
+ if (rc) goto out;
+ }
+
+ rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
+ if (rc) goto out;
+
+ rc = libxl__xs_write_checked(gc, t, lds->cmd_path, lds->cmd);
+ if (rc) goto out;
+
+ rc = libxl__xs_transaction_commit(gc, &t);
+ if (!rc) break;
+ if (rc<0) goto out;
+ }
+
+ /* OK, wait for some callback */
+ return;
+
+ out:
+ LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
+ libxl__xs_transaction_abort(gc, &t);
+ switch_logdirty_done(egc,dss,-1);
+}
+
+static void domain_suspend_switch_qemu_xen_logdirty
+ (int domid, unsigned enable,
+ libxl__save_helper_state *shs)
+{
+ libxl__egc *egc = shs->egc;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ STATE_AO_GC(dss->ao);
+ int rc;
+
+ rc = libxl__qmp_set_global_dirty_log(gc, domid, enable);
+ if (!rc) {
+ libxl__xc_domain_saverestore_async_callback_done(egc, shs, 0);
+ } else {
+ LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
+ libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
+ }
+}
+
+void libxl__domain_suspend_common_switch_qemu_logdirty
+ (int domid, unsigned enable, void *user)
+{
+ libxl__save_helper_state *shs = user;
+ libxl__egc *egc = shs->egc;
+ libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ STATE_AO_GC(dss->ao);
+
+ switch (libxl__device_model_version_running(gc, domid)) {
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
+ domain_suspend_switch_qemu_xen_traditional_logdirty(domid, enable, shs);
+ break;
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+ domain_suspend_switch_qemu_xen_logdirty(domid, enable, shs);
+ break;
+ default:
+ LOG(ERROR,"logdirty switch failed"
+ ", no valid device model version found, aborting suspend");
+ libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
+ }
+}
+static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
+ const struct timeval *requested_abs)
+{
+ libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
+ STATE_AO_GC(dss->ao);
+ LOG(ERROR,"logdirty switch: wait for device model timed out");
+ switch_logdirty_done(egc,dss,-1);
+}
+
+static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
+ const char *watch_path, const char *event_path)
+{
+ libxl__domain_suspend_state *dss =
+ CONTAINER_OF(watch, *dss, logdirty.watch);
+ libxl__logdirty_switch *lds = &dss->logdirty;
+ STATE_AO_GC(dss->ao);
+ const char *got;
+ xs_transaction_t t = 0;
+ int rc;
+
+ for (;;) {
+ rc = libxl__xs_transaction_start(gc, &t);
+ if (rc) goto out;
+
+ rc = libxl__xs_read_checked(gc, t, lds->ret_path, &got);
+ if (rc) goto out;
+
+ if (!got) {
+ rc = +1;
+ goto out;
+ }
+
+ if (strcmp(got, lds->cmd)) {
+ LOG(ERROR,"logdirty switch: sent command `%s' but got reply `%s'"
+ " (xenstore paths `%s' / `%s')", lds->cmd, got,
+ lds->cmd_path, lds->ret_path);
+ rc = ERROR_FAIL;
+ goto out;
+ }
+
+ rc = libxl__xs_rm_checked(gc, t, lds->cmd_path);
+ if (rc) goto out;
+
+ rc = libxl__xs_rm_checked(gc, t, lds->ret_path);
+ if (rc) goto out;
+
+ rc = libxl__xs_transaction_commit(gc, &t);
+ if (!rc) break;
+ if (rc<0) goto out;
+ }
+
+ out:
+ /* rc < 0: error
+ * rc == 0: ok, we are done
+ * rc == +1: need to keep waiting
+ */
+ libxl__xs_transaction_abort(gc, &t);
+
+ if (!rc) {
+ switch_logdirty_done(egc,dss,0);
+ } else if (rc < 0) {
+ LOG(ERROR,"logdirty switch: failed (rc=%d)",rc);
+ switch_logdirty_done(egc,dss,-1);
+ }
+}
+
+static void switch_logdirty_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int broke)
+{
+ STATE_AO_GC(dss->ao);
+ libxl__logdirty_switch *lds = &dss->logdirty;
+
+ libxl__ev_xswatch_deregister(gc, &lds->watch);
+ libxl__ev_time_deregister(gc, &lds->timeout);
+
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
+}
+
+static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
+ uint32_t domid,
+ char *phys_offset, char *node)
+{
+ return libxl__device_model_xs_path(gc, dm_domid, domid,
+ "/physmap/%s/%s",
+ phys_offset, node);
+}
+
+int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
+ uint32_t *len, void *dss_void)
+{
+ libxl__domain_suspend_state *dss = dss_void;
+ STATE_AO_GC(dss->ao);
+ int i = 0;
+ char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
+ unsigned int num = 0;
+ uint32_t count = 0, version = TOOLSTACK_SAVE_VERSION, namelen = 0;
+ uint8_t *ptr = NULL;
+ char **entries = NULL;
+ struct libxl__physmap_info *pi;
+ uint32_t dm_domid;
+
+ dm_domid = libxl_get_stubdom_id(CTX, domid);
+
+ entries = libxl__xs_directory(gc, 0,
+ libxl__device_model_xs_path(gc, dm_domid, domid, "/physmap"),
+ &num);
+ count = num;
+
+ *len = sizeof(version) + sizeof(count);
+ *buf = calloc(1, *len);
+ ptr = *buf;
+ if (*buf == NULL)
+ return -1;
+
+ memcpy(ptr, &version, sizeof(version));
+ ptr += sizeof(version);
+ memcpy(ptr, &count, sizeof(count));
+ ptr += sizeof(count);
+
+ for (i = 0; i < count; i++) {
+ unsigned long offset;
+ char *xs_path;
+ phys_offset = entries[i];
+ if (phys_offset == NULL) {
+ LOG(ERROR, "phys_offset %d is NULL", i);
+ return -1;
+ }
+
+ xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "start_addr");
+ start_addr = libxl__xs_read(gc, 0, xs_path);
+ if (start_addr == NULL) {
+ LOG(ERROR, "%s is NULL", xs_path);
+ return -1;
+ }
+
+ xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "size");
+ size = libxl__xs_read(gc, 0, xs_path);
+ if (size == NULL) {
+ LOG(ERROR, "%s is NULL", xs_path);
+ return -1;
+ }
+
+ xs_path = physmap_path(gc, dm_domid, domid, phys_offset, "name");
+ name = libxl__xs_read(gc, 0, xs_path);
+ if (name == NULL)
+ namelen = 0;
+ else
+ namelen = strlen(name) + 1;
+ *len += namelen + sizeof(struct libxl__physmap_info);
+ offset = ptr - (*buf);
+ *buf = realloc(*buf, *len);
+ if (*buf == NULL)
+ return -1;
+ ptr = (*buf) + offset;
+ pi = (struct libxl__physmap_info *) ptr;
+ pi->phys_offset = strtoll(phys_offset, NULL, 16);
+ pi->start_addr = strtoll(start_addr, NULL, 16);
+ pi->size = strtoll(size, NULL, 16);
+ pi->namelen = namelen;
+ memcpy(pi->name, name, namelen);
+ ptr += sizeof(struct libxl__physmap_info) + namelen;
+ }
+
+ LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, *len);
+
+ return 0;
+}
+
+/*----- main code for suspending, in order of execution -----*/
+
+void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
+{
+ STATE_AO_GC(dss->ao);
+ int port;
+ int rc = ERROR_FAIL;
+
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+ const libxl_domain_type type = dss->type;
+ const int live = dss->live;
+ const int debug = dss->debug;
+ const libxl_domain_remus_info *const r_info = dss->remus;
+ libxl__srm_save_autogen_callbacks *const callbacks =
+ &dss->shs.callbacks.save.a;
+
+ logdirty_init(&dss->logdirty);
+ libxl__xswait_init(&dss->pvcontrol);
+ libxl__ev_evtchn_init(&dss->guest_evtchn);
+ libxl__ev_xswatch_init(&dss->guest_watch);
+ libxl__ev_time_init(&dss->guest_timeout);
+
+ switch (type) {
+ case LIBXL_DOMAIN_TYPE_HVM: {
+ dss->hvm = 1;
+ break;
+ }
+ case LIBXL_DOMAIN_TYPE_PV:
+ dss->hvm = 0;
+ break;
+ default:
+ abort();
+ }
+
+ dss->xcflags = (live ? XCFLAGS_LIVE : 0)
+ | (debug ? XCFLAGS_DEBUG : 0)
+ | (dss->hvm ? XCFLAGS_HVM : 0);
+
+ dss->guest_evtchn.port = -1;
+ dss->guest_evtchn_lockfd = -1;
+ dss->guest_responded = 0;
+ dss->dm_savefile = libxl__device_model_savefile(gc, domid);
+
+ if (r_info != NULL) {
+ dss->interval = r_info->interval;
+ dss->xcflags |= XCFLAGS_CHECKPOINTED;
+ if (libxl_defbool_val(r_info->compression))
+ dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
+ }
+
+ port = xs_suspend_evtchn_port(dss->domid);
+
+ if (port >= 0) {
+ rc = libxl__ctx_evtchn_init(gc);
+ if (rc) goto out;
+
+ dss->guest_evtchn.port =
+ xc_suspend_evtchn_init_exclusive(CTX->xch, CTX->xce,
+ dss->domid, port, &dss->guest_evtchn_lockfd);
+
+ if (dss->guest_evtchn.port < 0) {
+ LOG(WARN, "Suspend event channel initialization failed");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ }
+
+ memset(callbacks, 0, sizeof(*callbacks));
+ if (r_info != NULL) {
+ callbacks->suspend = libxl__remus_domain_suspend_callback;
+ callbacks->postcopy = libxl__remus_domain_resume_callback;
+ callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback;
+ } else
+ callbacks->suspend = libxl__domain_suspend_callback;
+
+ callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
+
+ dss->sws.fd = dss->fd;
+ dss->sws.ao = dss->ao;
+ dss->sws.completion_callback = stream_done;
+
+ libxl__stream_write_start(egc, &dss->sws);
+ return;
+
+ out:
+ domain_save_done(egc, dss, rc);
+}
+
+static void stream_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc)
+{
+ domain_save_done(egc, dss, rc);
+}
+
+static void save_device_model_datacopier_done(libxl__egc *egc,
+ libxl__datacopier_state *dc, int onwrite, int errnoval);
+
+void libxl__domain_save_device_model(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ libxl__save_device_model_cb *callback)
+{
+ STATE_AO_GC(dss->ao);
+ struct stat st;
+ uint32_t qemu_state_len;
+ int rc;
+
+ dss->save_dm_callback = callback;
+
+ /* Convenience aliases */
+ const char *const filename = dss->dm_savefile;
+ const int fd = dss->fd;
+
+ libxl__datacopier_state *dc = &dss->save_dm_datacopier;
+ memset(dc, 0, sizeof(*dc));
+ dc->readwhat = GCSPRINTF("qemu save file %s", filename);
+ dc->ao = ao;
+ dc->readfd = -1;
+ dc->writefd = fd;
+ dc->maxsz = INT_MAX;
+ dc->bytes_to_read = -1;
+ dc->copywhat = GCSPRINTF("qemu save file for domain %"PRIu32, dss->domid);
+ dc->writewhat = "save/migration stream";
+ dc->callback = save_device_model_datacopier_done;
+
+ dc->readfd = open(filename, O_RDONLY);
+ if (dc->readfd < 0) {
+ LOGE(ERROR, "unable to open %s", dc->readwhat);
+ goto out;
+ }
+
+ if (fstat(dc->readfd, &st))
+ {
+ LOGE(ERROR, "unable to fstat %s", dc->readwhat);
+ goto out;
+ }
+
+ if (!S_ISREG(st.st_mode)) {
+ LOG(ERROR, "%s is not a plain file!", dc->readwhat);
+ goto out;
+ }
+
+ qemu_state_len = st.st_size;
+ LOG(DEBUG, "%s is %d bytes", dc->readwhat, qemu_state_len);
+
+ rc = libxl__datacopier_start(dc);
+ if (rc) goto out;
+
+ libxl__datacopier_prefixdata(egc, dc,
+ QEMU_SIGNATURE, strlen(QEMU_SIGNATURE));
+
+ libxl__datacopier_prefixdata(egc, dc,
+ &qemu_state_len, sizeof(qemu_state_len));
+ return;
+
+ out:
+ save_device_model_datacopier_done(egc, dc, -1, 0);
+}
+
+static void save_device_model_datacopier_done(libxl__egc *egc,
+ libxl__datacopier_state *dc, int onwrite, int errnoval)
+{
+ libxl__domain_suspend_state *dss =
+ CONTAINER_OF(dc, *dss, save_dm_datacopier);
+ STATE_AO_GC(dss->ao);
+
+ /* Convenience aliases */
+ const char *const filename = dss->dm_savefile;
+ int our_rc = 0;
+ int rc;
+
+ libxl__datacopier_kill(dc);
+
+ if (onwrite || errnoval)
+ our_rc = ERROR_FAIL;
+
+ if (dc->readfd >= 0) {
+ close(dc->readfd);
+ dc->readfd = -1;
+ }
+
+ rc = libxl__remove_file(gc, filename);
+ if (!our_rc) our_rc = rc;
+
+ dss->save_dm_callback(egc, dss, our_rc);
+}
+
+static void domain_save_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc)
+{
+ STATE_AO_GC(dss->ao);
+
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+
+ libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+
+ if (dss->guest_evtchn.port > 0)
+ xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
+ dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
+
+ if (!dss->remus) {
+ dss->callback(egc, dss, rc);
+ return;
+ }
+
+ /*
+ * With Remus, if we reach this point, it means either
+ * backup died or some network error occurred preventing us
+ * from sending checkpoints. Teardown the network buffers and
+ * release netlink resources. This is an async op.
+ */
+ libxl__remus_teardown(egc, dss, rc);
+}
+
+/*========================= Domain restore ============================*/
+
+static inline char *restore_helper(libxl__gc *gc, uint32_t dm_domid,
+ uint32_t domid,
+ uint64_t phys_offset, char *node)
+{
+ return libxl__device_model_xs_path(gc, dm_domid, domid,
+ "/physmap/%"PRIx64"/%s",
+ phys_offset, node);
+}
+
+int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
+ uint32_t size, void *user)
+{
+ libxl__save_helper_state *shs = user;
+ libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
+ STATE_AO_GC(dcs->ao);
+ int i, ret;
+ const uint8_t *ptr = buf;
+ uint32_t count = 0, version = 0;
+ struct libxl__physmap_info* pi;
+ char *xs_path;
+ uint32_t dm_domid;
+
+ LOG(DEBUG,"domain=%"PRIu32" toolstack data size=%"PRIu32, domid, size);
+
+ if (size < sizeof(version) + sizeof(count)) {
+ LOG(ERROR, "wrong size");
+ return -1;
+ }
+
+ memcpy(&version, ptr, sizeof(version));
+ ptr += sizeof(version);
+
+ if (version != TOOLSTACK_SAVE_VERSION) {
+ LOG(ERROR, "wrong version");
+ return -1;
+ }
+
+ memcpy(&count, ptr, sizeof(count));
+ ptr += sizeof(count);
+
+ if (size < sizeof(version) + sizeof(count) +
+ count * (sizeof(struct libxl__physmap_info))) {
+ LOG(ERROR, "wrong size");
+ return -1;
+ }
+
+ dm_domid = libxl_get_stubdom_id(CTX, domid);
+ for (i = 0; i < count; i++) {
+ pi = (struct libxl__physmap_info*) ptr;
+ ptr += sizeof(struct libxl__physmap_info) + pi->namelen;
+
+ xs_path = restore_helper(gc, dm_domid, domid,
+ pi->phys_offset, "start_addr");
+ ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->start_addr);
+ if (ret)
+ return -1;
+ xs_path = restore_helper(gc, dm_domid, domid, pi->phys_offset, "size");
+ ret = libxl__xs_write(gc, 0, xs_path, "%"PRIx64, pi->size);
+ if (ret)
+ return -1;
+ if (pi->namelen > 0) {
+ xs_path = restore_helper(gc, dm_domid, domid,
+ pi->phys_offset, "name");
+ ret = libxl__xs_write(gc, 0, xs_path, "%s", pi->name);
+ if (ret)
+ return -1;
+ }
+ }
+ return 0;
+}
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 05/26] tools/libxl: move save/restore code into libxl_dom_save.c
2015-06-25 6:25 ` [PATCH v3 COLOPre 05/26] tools/libxl: move save/restore code into libxl_dom_save.c Yang Hongyang
@ 2015-06-29 15:49 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 15:49 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> This is purely code motion.
On that basis:
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (4 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 05/26] tools/libxl: move save/restore code into libxl_dom_save.c Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 16:01 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record Yang Hongyang
` (19 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
Currently struct libxl__domain_suspend_state contains 2 type of states,
one is save state, another is suspend state. This patch separates those
two out.
The motivation of this is that COLO will need to do suspend/resume
continuously, we need a more common suspend state.
After this change, dss stands for libxl__domain_save_state,
dsps stands for libxl__domain_suspend_state.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/libxl/libxl.c | 10 +--
tools/libxl/libxl_dom_save.c | 74 +++++++++--------
tools/libxl/libxl_dom_suspend.c | 167 ++++++++++++++++++++-------------------
tools/libxl/libxl_internal.h | 55 +++++++------
tools/libxl/libxl_netbuffer.c | 2 +-
tools/libxl/libxl_remus.c | 39 ++++-----
tools/libxl/libxl_save_callout.c | 2 +-
tools/libxl/libxl_stream_write.c | 26 +++---
8 files changed, 199 insertions(+), 176 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index b939f2f..8b63eff 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -793,7 +793,7 @@ out:
}
static void remus_failover_cb(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
+ libxl__domain_save_state *dss, int rc);
/* TODO: Explicit Checkpoint acknowledgements via recv_fd. */
int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
@@ -801,7 +801,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
const libxl_asyncop_how *ao_how)
{
AO_CREATE(ctx, domid, ao_how);
- libxl__domain_suspend_state *dss;
+ libxl__domain_save_state *dss;
int rc;
libxl_domain_type type = libxl__domain_type(gc, domid);
@@ -849,7 +849,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
}
static void remus_failover_cb(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
+ libxl__domain_save_state *dss, int rc)
{
STATE_AO_GC(dss->ao);
/*
@@ -861,7 +861,7 @@ static void remus_failover_cb(libxl__egc *egc,
}
static void domain_suspend_cb(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
+ libxl__domain_save_state *dss, int rc)
{
STATE_AO_GC(dss->ao);
libxl__ao_complete(egc,ao,rc);
@@ -880,7 +880,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
goto out_err;
}
- libxl__domain_suspend_state *dss;
+ libxl__domain_save_state *dss;
GCNEW(dss);
dss->ao = ao;
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 9ee74ba..93061c7 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -30,9 +30,10 @@ struct libxl__physmap_info {
/*========================= Domain save ============================*/
static void stream_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
+ libxl__domain_save_state *dss, int rc);
static void domain_save_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
+ libxl__domain_save_state *dss, int rc);
+
/*----- complicated callback, called by xc_domain_save -----*/
@@ -49,7 +50,7 @@ static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
const char *watch_path, const char *event_path);
static void switch_logdirty_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok);
+ libxl__domain_save_state *dss, int ok);
static void logdirty_init(libxl__logdirty_switch *lds)
{
@@ -63,7 +64,7 @@ static void domain_suspend_switch_qemu_xen_traditional_logdirty
libxl__save_helper_state *shs)
{
libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
libxl__logdirty_switch *lds = &dss->logdirty;
STATE_AO_GC(dss->ao);
int rc;
@@ -135,7 +136,7 @@ static void domain_suspend_switch_qemu_xen_logdirty
libxl__save_helper_state *shs)
{
libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
STATE_AO_GC(dss->ao);
int rc;
@@ -153,7 +154,7 @@ void libxl__domain_suspend_common_switch_qemu_logdirty
{
libxl__save_helper_state *shs = user;
libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
STATE_AO_GC(dss->ao);
switch (libxl__device_model_version_running(gc, domid)) {
@@ -172,7 +173,7 @@ void libxl__domain_suspend_common_switch_qemu_logdirty
static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
const struct timeval *requested_abs)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
+ libxl__domain_save_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
STATE_AO_GC(dss->ao);
LOG(ERROR,"logdirty switch: wait for device model timed out");
switch_logdirty_done(egc,dss,-1);
@@ -181,7 +182,7 @@ static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
const char *watch_path, const char *event_path)
{
- libxl__domain_suspend_state *dss =
+ libxl__domain_save_state *dss =
CONTAINER_OF(watch, *dss, logdirty.watch);
libxl__logdirty_switch *lds = &dss->logdirty;
STATE_AO_GC(dss->ao);
@@ -236,7 +237,7 @@ static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
}
static void switch_logdirty_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
int broke)
{
STATE_AO_GC(dss->ao);
@@ -260,7 +261,7 @@ static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
uint32_t *len, void *dss_void)
{
- libxl__domain_suspend_state *dss = dss_void;
+ libxl__domain_save_state *dss = dss_void;
STATE_AO_GC(dss->ao);
int i = 0;
char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
@@ -338,9 +339,9 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
return 0;
}
-/*----- main code for suspending, in order of execution -----*/
+/*----- main code for saving, in order of execution -----*/
-void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
{
STATE_AO_GC(dss->ao);
int port;
@@ -354,20 +355,23 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
const libxl_domain_remus_info *const r_info = dss->remus;
libxl__srm_save_autogen_callbacks *const callbacks =
&dss->shs.callbacks.save.a;
+ libxl__domain_suspend_state *dsps = &dss->dsps;
logdirty_init(&dss->logdirty);
- libxl__xswait_init(&dss->pvcontrol);
- libxl__ev_evtchn_init(&dss->guest_evtchn);
- libxl__ev_xswatch_init(&dss->guest_watch);
- libxl__ev_time_init(&dss->guest_timeout);
+ libxl__xswait_init(&dsps->pvcontrol);
+ libxl__ev_evtchn_init(&dsps->guest_evtchn);
+ libxl__ev_xswatch_init(&dsps->guest_watch);
+ libxl__ev_time_init(&dsps->guest_timeout);
switch (type) {
case LIBXL_DOMAIN_TYPE_HVM: {
dss->hvm = 1;
+ dsps->hvm = 1;
break;
}
case LIBXL_DOMAIN_TYPE_PV:
dss->hvm = 0;
+ dsps->hvm = 0;
break;
default:
abort();
@@ -377,10 +381,12 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
| (debug ? XCFLAGS_DEBUG : 0)
| (dss->hvm ? XCFLAGS_HVM : 0);
- dss->guest_evtchn.port = -1;
- dss->guest_evtchn_lockfd = -1;
- dss->guest_responded = 0;
- dss->dm_savefile = libxl__device_model_savefile(gc, domid);
+ dsps->ao = ao;
+ dsps->domid = domid;
+ dsps->guest_evtchn.port = -1;
+ dsps->guest_evtchn_lockfd = -1;
+ dsps->guest_responded = 0;
+ dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
if (r_info != NULL) {
dss->interval = r_info->interval;
@@ -389,17 +395,18 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
}
- port = xs_suspend_evtchn_port(dss->domid);
+ port = xs_suspend_evtchn_port(dsps->domid);
if (port >= 0) {
rc = libxl__ctx_evtchn_init(gc);
if (rc) goto out;
- dss->guest_evtchn.port =
+ dsps->guest_evtchn.port =
xc_suspend_evtchn_init_exclusive(CTX->xch, CTX->xce,
- dss->domid, port, &dss->guest_evtchn_lockfd);
+ dsps->domid, port,
+ &dsps->guest_evtchn_lockfd);
- if (dss->guest_evtchn.port < 0) {
+ if (dsps->guest_evtchn.port < 0) {
LOG(WARN, "Suspend event channel initialization failed");
rc = ERROR_FAIL;
goto out;
@@ -428,7 +435,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
}
static void stream_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
+ libxl__domain_save_state *dss, int rc)
{
domain_save_done(egc, dss, rc);
}
@@ -437,7 +444,7 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
libxl__datacopier_state *dc, int onwrite, int errnoval);
void libxl__domain_save_device_model(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
libxl__save_device_model_cb *callback)
{
STATE_AO_GC(dss->ao);
@@ -448,7 +455,7 @@ void libxl__domain_save_device_model(libxl__egc *egc,
dss->save_dm_callback = callback;
/* Convenience aliases */
- const char *const filename = dss->dm_savefile;
+ const char *const filename = dss->dsps.dm_savefile;
const int fd = dss->fd;
libxl__datacopier_state *dc = &dss->save_dm_datacopier;
@@ -500,12 +507,12 @@ void libxl__domain_save_device_model(libxl__egc *egc,
static void save_device_model_datacopier_done(libxl__egc *egc,
libxl__datacopier_state *dc, int onwrite, int errnoval)
{
- libxl__domain_suspend_state *dss =
+ libxl__domain_save_state *dss =
CONTAINER_OF(dc, *dss, save_dm_datacopier);
STATE_AO_GC(dss->ao);
/* Convenience aliases */
- const char *const filename = dss->dm_savefile;
+ const char *const filename = dss->dsps.dm_savefile;
int our_rc = 0;
int rc;
@@ -526,18 +533,19 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
}
static void domain_save_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
+ libxl__domain_save_state *dss, int rc)
{
STATE_AO_GC(dss->ao);
/* Convenience aliases */
const uint32_t domid = dss->domid;
+ libxl__domain_suspend_state *dsps = &dss->dsps;
- libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
+ libxl__ev_evtchn_cancel(gc, &dsps->guest_evtchn);
- if (dss->guest_evtchn.port > 0)
+ if (dsps->guest_evtchn.port > 0)
xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
- dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
+ dsps->guest_evtchn.port, &dsps->guest_evtchn_lockfd);
if (!dss->remus) {
dss->callback(egc, dss, rc);
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
index 5f1f4fd..1c486c4 100644
--- a/tools/libxl/libxl_dom_suspend.c
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -22,11 +22,11 @@
/*----- callbacks, called by xc_domain_save -----*/
int libxl__domain_suspend_device_model(libxl__gc *gc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
int ret = 0;
- uint32_t const domid = dss->domid;
- const char *const filename = dss->dm_savefile;
+ uint32_t const domid = dsps->domid;
+ const char *const filename = dsps->dm_savefile;
switch (libxl__device_model_version_running(gc, domid)) {
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
@@ -51,9 +51,9 @@ int libxl__domain_suspend_device_model(libxl__gc *gc,
}
static void domain_suspend_common_wait_guest(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
static void domain_suspend_common_guest_suspended(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
libxl__xswait_state *xswa, int rc, const char *state);
@@ -62,64 +62,63 @@ static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
static void suspend_common_wait_guest_watch(libxl__egc *egc,
libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path);
static void suspend_common_wait_guest_check(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
static void suspend_common_wait_guest_timeout(libxl__egc *egc,
libxl__ev_time *ev, const struct timeval *requested_abs);
static void domain_suspend_common_failed(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
static void domain_suspend_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_suspend_state *dsps,
bool ok);
static void domain_suspend_callback_common(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
static void domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok);
+ libxl__domain_suspend_state *dsps, int ok);
-/* calls dss->callback_common_done when done */
+/* calls dsps->callback_common_done when done */
void libxl__domain_suspend(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
- domain_suspend_callback_common(egc, dss);
+ domain_suspend_callback_common(egc, dsps);
}
-
static bool domain_suspend_pvcontrol_acked(const char *state) {
/* any value other than "suspend", including ENOENT (i.e. !state), is OK */
if (!state) return 1;
return strcmp(state,"suspend");
}
-/* calls dss->callback_common_done when done */
+/* calls dsps->callback_common_done when done */
static void domain_suspend_callback_common(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
- STATE_AO_GC(dss->ao);
+ STATE_AO_GC(dsps->ao);
uint64_t hvm_s_state = 0, hvm_pvdrv = 0;
int ret, rc;
/* Convenience aliases */
- const uint32_t domid = dss->domid;
+ const uint32_t domid = dsps->domid;
- if (dss->hvm) {
+ if (dsps->hvm) {
xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ, &hvm_pvdrv);
xc_hvm_param_get(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE, &hvm_s_state);
}
- if ((hvm_s_state == 0) && (dss->guest_evtchn.port >= 0)) {
+ if ((hvm_s_state == 0) && (dsps->guest_evtchn.port >= 0)) {
LOG(DEBUG, "issuing %s suspend request via event channel",
- dss->hvm ? "PVHVM" : "PV");
- ret = xc_evtchn_notify(CTX->xce, dss->guest_evtchn.port);
+ dsps->hvm ? "PVHVM" : "PV");
+ ret = xc_evtchn_notify(CTX->xce, dsps->guest_evtchn.port);
if (ret < 0) {
LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
goto err;
}
- dss->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
- rc = libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
+ dsps->guest_evtchn.callback = domain_suspend_common_wait_guest_evtchn;
+ rc = libxl__ev_evtchn_wait(gc, &dsps->guest_evtchn);
if (rc) goto err;
- rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+ rc = libxl__ev_time_register_rel(gc, &dsps->guest_timeout,
suspend_common_wait_guest_timeout,
60*1000);
if (rc) goto err;
@@ -127,7 +126,7 @@ static void domain_suspend_callback_common(libxl__egc *egc,
return;
}
- if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
+ if (dsps->hvm && (!hvm_pvdrv || hvm_s_state)) {
LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
ret = xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
if (ret < 0) {
@@ -135,55 +134,55 @@ static void domain_suspend_callback_common(libxl__egc *egc,
goto err;
}
/* The guest does not (need to) respond to this sort of request. */
- dss->guest_responded = 1;
- domain_suspend_common_wait_guest(egc, dss);
+ dsps->guest_responded = 1;
+ domain_suspend_common_wait_guest(egc, dsps);
return;
}
LOG(DEBUG, "issuing %s suspend request via XenBus control node",
- dss->hvm ? "PVHVM" : "PV");
+ dsps->hvm ? "PVHVM" : "PV");
libxl__domain_pvcontrol_write(gc, XBT_NULL, domid, "suspend");
- dss->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
- if (!dss->pvcontrol.path) goto err;
+ dsps->pvcontrol.path = libxl__domain_pvcontrol_xspath(gc, domid);
+ if (!dsps->pvcontrol.path) goto err;
- dss->pvcontrol.ao = ao;
- dss->pvcontrol.what = "guest acknowledgement of suspend request";
- dss->pvcontrol.timeout_ms = 60 * 1000;
- dss->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
- libxl__xswait_start(gc, &dss->pvcontrol);
+ dsps->pvcontrol.ao = ao;
+ dsps->pvcontrol.what = "guest acknowledgement of suspend request";
+ dsps->pvcontrol.timeout_ms = 60 * 1000;
+ dsps->pvcontrol.callback = domain_suspend_common_pvcontrol_suspending;
+ libxl__xswait_start(gc, &dsps->pvcontrol);
return;
err:
- domain_suspend_common_failed(egc, dss);
+ domain_suspend_common_failed(egc, dsps);
}
static void domain_suspend_common_wait_guest_evtchn(libxl__egc *egc,
libxl__ev_evtchn *evev)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(evev, *dss, guest_evtchn);
- STATE_AO_GC(dss->ao);
+ libxl__domain_suspend_state *dsps = CONTAINER_OF(evev, *dsps, guest_evtchn);
+ STATE_AO_GC(dsps->ao);
/* If we should be done waiting, suspend_common_wait_guest_check
* will end up calling domain_suspend_common_guest_suspended or
* domain_suspend_common_failed, both of which cancel the evtchn
* wait. So re-enable it now. */
- libxl__ev_evtchn_wait(gc, &dss->guest_evtchn);
- suspend_common_wait_guest_check(egc, dss);
+ libxl__ev_evtchn_wait(gc, &dsps->guest_evtchn);
+ suspend_common_wait_guest_check(egc, dsps);
}
static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
libxl__xswait_state *xswa, int rc, const char *state)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(xswa, *dss, pvcontrol);
- STATE_AO_GC(dss->ao);
+ libxl__domain_suspend_state *dsps = CONTAINER_OF(xswa, *dsps, pvcontrol);
+ STATE_AO_GC(dsps->ao);
xs_transaction_t t = 0;
if (!rc && !domain_suspend_pvcontrol_acked(state))
/* keep waiting */
return;
- libxl__xswait_stop(gc, &dss->pvcontrol);
+ libxl__xswait_stop(gc, &dsps->pvcontrol);
if (rc == ERROR_TIMEDOUT) {
/*
@@ -226,56 +225,56 @@ static void domain_suspend_common_pvcontrol_suspending(libxl__egc *egc,
LOG(DEBUG, "guest acknowledged suspend request");
libxl__xs_transaction_abort(gc, &t);
- dss->guest_responded = 1;
- domain_suspend_common_wait_guest(egc,dss);
+ dsps->guest_responded = 1;
+ domain_suspend_common_wait_guest(egc,dsps);
return;
err:
libxl__xs_transaction_abort(gc, &t);
- domain_suspend_common_failed(egc, dss);
+ domain_suspend_common_failed(egc, dsps);
return;
}
static void domain_suspend_common_wait_guest(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
- STATE_AO_GC(dss->ao);
+ STATE_AO_GC(dsps->ao);
int rc;
LOG(DEBUG, "wait for the guest to suspend");
- rc = libxl__ev_xswatch_register(gc, &dss->guest_watch,
+ rc = libxl__ev_xswatch_register(gc, &dsps->guest_watch,
suspend_common_wait_guest_watch,
"@releaseDomain");
if (rc) goto err;
- rc = libxl__ev_time_register_rel(gc, &dss->guest_timeout,
+ rc = libxl__ev_time_register_rel(gc, &dsps->guest_timeout,
suspend_common_wait_guest_timeout,
60*1000);
if (rc) goto err;
return;
err:
- domain_suspend_common_failed(egc, dss);
+ domain_suspend_common_failed(egc, dsps);
}
static void suspend_common_wait_guest_watch(libxl__egc *egc,
libxl__ev_xswatch *xsw, const char *watch_path, const char *event_path)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(xsw, *dss, guest_watch);
- suspend_common_wait_guest_check(egc, dss);
+ libxl__domain_suspend_state *dsps = CONTAINER_OF(xsw, *dsps, guest_watch);
+ suspend_common_wait_guest_check(egc, dsps);
}
static void suspend_common_wait_guest_check(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
- STATE_AO_GC(dss->ao);
+ STATE_AO_GC(dsps->ao);
xc_domaininfo_t info;
int ret;
int shutdown_reason;
/* Convenience aliases */
- const uint32_t domid = dss->domid;
+ const uint32_t domid = dsps->domid;
ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
if (ret < 0) {
@@ -302,74 +301,76 @@ static void suspend_common_wait_guest_check(libxl__egc *egc,
}
LOG(DEBUG, "guest has suspended");
- domain_suspend_common_guest_suspended(egc, dss);
+ domain_suspend_common_guest_suspended(egc, dsps);
return;
err:
- domain_suspend_common_failed(egc, dss);
+ domain_suspend_common_failed(egc, dsps);
}
static void suspend_common_wait_guest_timeout(libxl__egc *egc,
libxl__ev_time *ev, const struct timeval *requested_abs)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(ev, *dss, guest_timeout);
- STATE_AO_GC(dss->ao);
+ libxl__domain_suspend_state *dsps = CONTAINER_OF(ev, *dsps, guest_timeout);
+ STATE_AO_GC(dsps->ao);
LOG(ERROR, "guest did not suspend, timed out");
- domain_suspend_common_failed(egc, dss);
+ domain_suspend_common_failed(egc, dsps);
}
static void domain_suspend_common_guest_suspended(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
- STATE_AO_GC(dss->ao);
+ STATE_AO_GC(dsps->ao);
int ret;
- libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
- libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
- libxl__ev_time_deregister(gc, &dss->guest_timeout);
+ libxl__ev_evtchn_cancel(gc, &dsps->guest_evtchn);
+ libxl__ev_xswatch_deregister(gc, &dsps->guest_watch);
+ libxl__ev_time_deregister(gc, &dsps->guest_timeout);
- if (dss->hvm) {
- ret = libxl__domain_suspend_device_model(gc, dss);
+ if (dsps->hvm) {
+ ret = libxl__domain_suspend_device_model(gc, dsps);
if (ret) {
LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d", ret);
- domain_suspend_common_failed(egc, dss);
+ domain_suspend_common_failed(egc, dsps);
return;
}
}
- domain_suspend_common_done(egc, dss, 1);
+ domain_suspend_common_done(egc, dsps, 1);
}
static void domain_suspend_common_failed(libxl__egc *egc,
- libxl__domain_suspend_state *dss)
+ libxl__domain_suspend_state *dsps)
{
- domain_suspend_common_done(egc, dss, 0);
+ domain_suspend_common_done(egc, dsps, 0);
}
static void domain_suspend_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_suspend_state *dsps,
bool ok)
{
EGC_GC;
- assert(!libxl__xswait_inuse(&dss->pvcontrol));
- libxl__ev_evtchn_cancel(gc, &dss->guest_evtchn);
- libxl__ev_xswatch_deregister(gc, &dss->guest_watch);
- libxl__ev_time_deregister(gc, &dss->guest_timeout);
- dss->callback_common_done(egc, dss, ok);
+ assert(!libxl__xswait_inuse(&dsps->pvcontrol));
+ libxl__ev_evtchn_cancel(gc, &dsps->guest_evtchn);
+ libxl__ev_xswatch_deregister(gc, &dsps->guest_watch);
+ libxl__ev_time_deregister(gc, &dsps->guest_timeout);
+ dsps->callback_common_done(egc, dsps, ok);
}
void libxl__domain_suspend_callback(void *data)
{
libxl__save_helper_state *shs = data;
libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_suspend_state *dsps = &dss->dsps;
- dss->callback_common_done = domain_suspend_callback_common_done;
- domain_suspend_callback_common(egc, dss);
+ dsps->callback_common_done = domain_suspend_callback_common_done;
+ domain_suspend_callback_common(egc, dsps);
}
static void domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok)
+ libxl__domain_suspend_state *dsps, int ok)
{
+ libxl__domain_save_state *dss = CONTAINER_OF(dsps, *dss, dsps);
libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
}
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index b342988..4fdf9d8 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2859,11 +2859,12 @@ static inline bool libxl__convert_legacy_stream_inuse(
/*----- Domain suspend (save) state structure -----*/
typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
+typedef struct libxl__domain_save_state libxl__domain_save_state;
-typedef void libxl__domain_suspend_cb(libxl__egc*,
- libxl__domain_suspend_state*, int rc);
+typedef void libxl__domain_save_cb(libxl__egc*,
+ libxl__domain_save_state*, int rc);
typedef void libxl__save_device_model_cb(libxl__egc*,
- libxl__domain_suspend_state*, int rc);
+ libxl__domain_save_state*, int rc);
/* State for writing a libxl migration v2 stream */
typedef struct libxl__stream_write_state libxl__stream_write_state;
@@ -2874,10 +2875,10 @@ struct libxl__stream_write_state {
int fd;
uint32_t domid;
void (*completion_callback)(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
int rc);
void (*checkpoint_callback)(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
int rc);
/* Private */
int rc;
@@ -2913,9 +2914,27 @@ typedef struct libxl__logdirty_switch {
} libxl__logdirty_switch;
struct libxl__domain_suspend_state {
+ /* set by caller of domain_suspend_callback_common */
+ libxl__ao *ao;
+
+ uint32_t domid;
+ int hvm;
+ /* private */
+ libxl__ev_evtchn guest_evtchn;
+ int guest_evtchn_lockfd;
+ int guest_responded;
+ libxl__xswait_state pvcontrol;
+ libxl__ev_xswatch guest_watch;
+ libxl__ev_time guest_timeout;
+ const char *dm_savefile;
+ void (*callback_common_done)(libxl__egc*,
+ struct libxl__domain_suspend_state*, int ok);
+};
+
+struct libxl__domain_save_state {
/* set by caller of libxl__domain_suspend */
libxl__ao *ao;
- libxl__domain_suspend_cb *callback;
+ libxl__domain_save_cb *callback;
uint32_t domid;
int fd;
@@ -2924,22 +2943,14 @@ struct libxl__domain_suspend_state {
int debug;
const libxl_domain_remus_info *remus;
/* private */
- libxl__ev_evtchn guest_evtchn;
- int guest_evtchn_lockfd;
+ libxl__domain_suspend_state dsps;
int hvm;
int xcflags;
- int guest_responded;
- libxl__xswait_state pvcontrol;
- libxl__ev_xswatch guest_watch;
- libxl__ev_time guest_timeout;
- const char *dm_savefile;
libxl__remus_devices_state rds;
libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
int interval; /* checkpoint interval (for Remus) */
libxl__save_helper_state shs;
libxl__logdirty_switch logdirty;
- void (*callback_common_done)(libxl__egc*,
- struct libxl__domain_suspend_state*, int ok);
/* private for libxl__domain_save_device_model */
libxl__save_device_model_cb *save_dm_callback;
libxl__datacopier_state save_dm_datacopier;
@@ -3264,11 +3275,11 @@ struct libxl__domain_create_state {
/* calls dss->callback when done */
_hidden void libxl__domain_save(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_save_state *dss);
/* calls libxl__xc_domain_suspend_done when done */
-_hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_suspend_state*);
+_hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_save_state*);
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
@@ -3310,16 +3321,16 @@ static inline bool libxl__save_helper_inuse(const libxl__save_helper_state *shs)
/* Each time the dm needs to be saved, we must call suspend and then save */
_hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
_hidden void libxl__domain_save_device_model(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
libxl__save_device_model_cb *callback);
_hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t domid);
/* calls dss->callback_common_done when done */
_hidden void libxl__domain_suspend(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_suspend_state *dsps);
/* used by libxc to suspend the guest during migration */
_hidden void libxl__domain_suspend_callback(void *data);
@@ -3331,9 +3342,9 @@ _hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
_hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
/* Remus setup and teardown*/
_hidden void libxl__remus_setup(libxl__egc *egc,
- libxl__domain_suspend_state *dss);
+ libxl__domain_save_state *dss);
_hidden void libxl__remus_teardown(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
int rc);
/*
diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
index edc6843..71c6531 100644
--- a/tools/libxl/libxl_netbuffer.c
+++ b/tools/libxl/libxl_netbuffer.c
@@ -41,7 +41,7 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
int init_subkind_nic(libxl__remus_devices_state *rds)
{
int rc, ret;
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
STATE_AO_GC(rds->ao);
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index 504cd44..01cc9e7 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -24,7 +24,7 @@ static void remus_setup_done(libxl__egc *egc,
static void remus_setup_failed(libxl__egc *egc,
libxl__remus_devices_state *rds, int rc);
-void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
{
/* Convenience aliases */
libxl__remus_devices_state *const rds = &dss->rds;
@@ -57,7 +57,7 @@ out:
static void remus_setup_done(libxl__egc *egc,
libxl__remus_devices_state *rds, int rc)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
STATE_AO_GC(dss->ao);
if (!rc) {
@@ -74,7 +74,7 @@ static void remus_setup_done(libxl__egc *egc,
static void remus_setup_failed(libxl__egc *egc,
libxl__remus_devices_state *rds, int rc)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -88,7 +88,7 @@ static void remus_teardown_done(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc);
void libxl__remus_teardown(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
int rc)
{
EGC_GC;
@@ -103,7 +103,7 @@ static void remus_teardown_done(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -116,7 +116,7 @@ static void remus_teardown_done(libxl__egc *egc,
/*---------------------- remus callbacks (save) -----------------------*/
static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok);
+ libxl__domain_suspend_state *dsps, int ok);
static void remus_devices_postsuspend_cb(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc);
@@ -128,15 +128,18 @@ void libxl__remus_domain_suspend_callback(void *data)
{
libxl__save_helper_state *shs = data;
libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_suspend_state *dsps = &dss->dsps;
- dss->callback_common_done = remus_domain_suspend_callback_common_done;
- libxl__domain_suspend(egc, dss);
+ dsps->callback_common_done = remus_domain_suspend_callback_common_done;
+ libxl__domain_suspend(egc, dsps);
}
static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int ok)
+ libxl__domain_suspend_state *dsps, int ok)
{
+ libxl__domain_save_state *dss = CONTAINER_OF(dsps, *dss, dsps);
+
if (!ok)
goto out;
@@ -154,7 +157,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
int rc)
{
int ok = 0;
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
if (rc)
goto out;
@@ -169,7 +172,7 @@ void libxl__remus_domain_resume_callback(void *data)
{
libxl__save_helper_state *shs = data;
libxl__egc *egc = shs->egc;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
STATE_AO_GC(dss->ao);
libxl__remus_devices_state *const rds = &dss->rds;
@@ -182,7 +185,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
int rc)
{
int ok = 0;
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -202,7 +205,7 @@ out:
/*----- remus asynchronous checkpoint callback -----*/
static void remus_checkpoint_stream_written(
- libxl__egc *egc, libxl__domain_suspend_state *dss, int rc);
+ libxl__egc *egc, libxl__domain_save_state *dss, int rc);
static void remus_devices_commit_cb(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc);
@@ -212,7 +215,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
void libxl__remus_domain_save_checkpoint_callback(void *data)
{
libxl__save_helper_state *shs = data;
- libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+ libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
libxl__egc *egc = dss->shs.egc;
STATE_AO_GC(dss->ao);
@@ -221,7 +224,7 @@ void libxl__remus_domain_save_checkpoint_callback(void *data)
}
static void remus_checkpoint_stream_written(
- libxl__egc *egc, libxl__domain_suspend_state *dss, int rc)
+ libxl__egc *egc, libxl__domain_save_state *dss, int rc)
{
/* Convenience aliases */
libxl__remus_devices_state *const rds = &dss->rds;
@@ -246,7 +249,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
STATE_AO_GC(dss->ao);
@@ -280,7 +283,7 @@ out:
static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
const struct timeval *requested_abs)
{
- libxl__domain_suspend_state *dss =
+ libxl__domain_save_state *dss =
CONTAINER_OF(ev, *dss, checkpoint_timeout);
STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 02e0190..a5039e6 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -74,7 +74,7 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
argnums, ARRAY_SIZE(argnums));
}
-void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss)
+void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
{
STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index 40f2cb7..1ebe306 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -60,7 +60,7 @@ static void stream_done(libxl__egc *egc,
libxl__stream_write_state *stream);
static void check_stream_finished(libxl__egc *egc,
- libxl__domain_suspend_state *dcs,
+ libxl__domain_save_state *dcs,
int rc, const char *what);
/* Event callbacks for plain VM. */
@@ -187,7 +187,7 @@ static void stream_failed(libxl__egc *egc,
static void stream_done(libxl__egc *egc,
libxl__stream_write_state *stream)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
assert(!stream->running);
assert(!stream->in_checkpoint);
@@ -196,7 +196,7 @@ static void stream_done(libxl__egc *egc,
}
static void check_stream_finished(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+ libxl__domain_save_state *dss,
int rc, const char *what)
{
libxl__stream_write_state *stream = &dss->sws;
@@ -270,7 +270,7 @@ static void libxc_header_done(libxl__egc *egc,
int onwrite, int errnoval)
{
libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
- libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
STATE_AO_GC(stream->ao);
int ret = 0;
@@ -290,7 +290,7 @@ static void libxc_header_done(libxl__egc *egc,
void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
int rc, int retval, int errnoval)
{
- libxl__domain_suspend_state *dss = dss_void;
+ libxl__domain_save_state *dss = dss_void;
libxl__stream_write_state *stream = &dss->sws;
STATE_AO_GC(dss->ao);
@@ -299,10 +299,10 @@ void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
if (retval) {
LOGEV(ERROR, errnoval, "saving domain: %s",
- dss->guest_responded ?
+ dss->dsps.guest_responded ?
"domain responded to suspend request" :
"domain did not respond to suspend request");
- if ( !dss->guest_responded )
+ if ( !dss->dsps.guest_responded )
rc = ERROR_GUEST_TIMEDOUT;
else
rc = ERROR_FAIL;
@@ -320,7 +320,7 @@ void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
static void write_toolstack_record(libxl__egc *egc,
libxl__stream_write_state *stream)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
libxl__datacopier_state *dc = &stream->dc;
STATE_AO_GC(stream->ao);
struct libxl_sr_rec_hdr rec = { REC_TYPE_XENSTORE_DATA, 0 };
@@ -364,7 +364,7 @@ static void toolstack_record_done(libxl__egc *egc,
int onwrite, int errnoval)
{
libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
- libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
STATE_AO_GC(stream->ao);
int ret = 0;
@@ -392,7 +392,7 @@ static void toolstack_record_done(libxl__egc *egc,
static void write_emulator_record(libxl__egc *egc,
libxl__stream_write_state *stream)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
libxl__datacopier_state *dc = &stream->dc;
STATE_AO_GC(stream->ao);
struct libxl_sr_rec_hdr rec = { REC_TYPE_EMULATOR_CONTEXT, 0 };
@@ -404,7 +404,7 @@ static void write_emulator_record(libxl__egc *egc,
assert(dss->type == LIBXL_DOMAIN_TYPE_HVM);
/* Convenience aliases */
- const char *const filename = dss->dm_savefile;
+ const char *const filename = dss->dsps.dm_savefile;
const uint32_t domid = dss->domid;
switch(libxl__device_model_version_running(gc, domid)) {
@@ -421,7 +421,7 @@ static void write_emulator_record(libxl__egc *egc,
goto err;
}
- ret = libxl__domain_suspend_device_model(gc, dss);
+ ret = libxl__domain_suspend_device_model(gc, &dss->dsps);
if (ret)
goto err;
@@ -577,7 +577,7 @@ static void checkpoint_done(libxl__egc *egc,
libxl__stream_write_state *stream,
int rc)
{
- libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
assert(stream->in_checkpoint);
stream->in_checkpoint = false;
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state
2015-06-25 6:25 ` [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state Yang Hongyang
@ 2015-06-29 16:01 ` Ian Campbell
2015-06-30 9:43 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 16:01 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
[...]
> switch (type) {
> case LIBXL_DOMAIN_TYPE_HVM: {
> dss->hvm = 1;
> + dsps->hvm = 1;
> break;
> }
> case LIBXL_DOMAIN_TYPE_PV:
> dss->hvm = 0;
> + dsps->hvm = 0;
> break;
[...]
> @@ -2913,9 +2914,27 @@ typedef struct libxl__logdirty_switch {
> } libxl__logdirty_switch;
>
> struct libxl__domain_suspend_state {
> + /* set by caller of domain_suspend_callback_common */
> + libxl__ao *ao;
> +
> + uint32_t domid;
> + int hvm;
> + /* private */
> + libxl__ev_evtchn guest_evtchn;
> + int guest_evtchn_lockfd;
> + int guest_responded;
> + libxl__xswait_state pvcontrol;
> + libxl__ev_xswatch guest_watch;
> + libxl__ev_time guest_timeout;
> + const char *dm_savefile;
> + void (*callback_common_done)(libxl__egc*,
> + struct libxl__domain_suspend_state*, int ok);
> +};
> +
> +struct libxl__domain_save_state {
> /* set by caller of libxl__domain_suspend */
> libxl__ao *ao;
> - libxl__domain_suspend_cb *callback;
> + libxl__domain_save_cb *callback;
>
> uint32_t domid;
> int fd;
> @@ -2924,22 +2943,14 @@ struct libxl__domain_suspend_state {
> int debug;
> const libxl_domain_remus_info *remus;
> /* private */
> - libxl__ev_evtchn guest_evtchn;
> - int guest_evtchn_lockfd;
> + libxl__domain_suspend_state dsps;
> int hvm;
I wonder if, given that any domain suspend must necessarily be contained
within a domain save it would be preferable to have "suspend" code
upcast the suspend_state to the containing save_state in order to look
at ->hvm, rather than duplicating it. Likewise domid.
For ao I can imagine that the suspend and save might actually have
separate ao lifetimes, in which case these do of course need to remain
different.
Would that make any sense at all?
FYI it was the dual initialisation of hvm quoted above which lead me to
think along these lines.
Alternatively perhaps suspend_state should a separate init function to
logically separate it from the save_state initialiser. Perhaps taking
the latter as an argument? Or possibly just the relevant fields.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state
2015-06-29 16:01 ` Ian Campbell
@ 2015-06-30 9:43 ` Yang Hongyang
2015-06-30 9:50 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 9:43 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/30/2015 12:01 AM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> [...]
>> switch (type) {
>> case LIBXL_DOMAIN_TYPE_HVM: {
>> dss->hvm = 1;
>> + dsps->hvm = 1;
>> break;
>> }
>> case LIBXL_DOMAIN_TYPE_PV:
>> dss->hvm = 0;
>> + dsps->hvm = 0;
>> break;
> [...]
>> @@ -2913,9 +2914,27 @@ typedef struct libxl__logdirty_switch {
>> } libxl__logdirty_switch;
>>
>> struct libxl__domain_suspend_state {
>> + /* set by caller of domain_suspend_callback_common */
>> + libxl__ao *ao;
>> +
>> + uint32_t domid;
>> + int hvm;
>> + /* private */
>> + libxl__ev_evtchn guest_evtchn;
>> + int guest_evtchn_lockfd;
>> + int guest_responded;
>> + libxl__xswait_state pvcontrol;
>> + libxl__ev_xswatch guest_watch;
>> + libxl__ev_time guest_timeout;
>> + const char *dm_savefile;
>> + void (*callback_common_done)(libxl__egc*,
>> + struct libxl__domain_suspend_state*, int ok);
>> +};
>> +
>> +struct libxl__domain_save_state {
>> /* set by caller of libxl__domain_suspend */
>> libxl__ao *ao;
>> - libxl__domain_suspend_cb *callback;
>> + libxl__domain_save_cb *callback;
>>
>> uint32_t domid;
>> int fd;
>> @@ -2924,22 +2943,14 @@ struct libxl__domain_suspend_state {
>> int debug;
>> const libxl_domain_remus_info *remus;
>> /* private */
>> - libxl__ev_evtchn guest_evtchn;
>> - int guest_evtchn_lockfd;
>> + libxl__domain_suspend_state dsps;
>> int hvm;
>
> I wonder if, given that any domain suspend must necessarily be contained
> within a domain save it would be preferable to have "suspend" code
> upcast the suspend_state to the containing save_state in order to look
> at ->hvm, rather than duplicating it. Likewise domid.
The reason we include hvm in suspend state is that we need a more common
suspend function we will use on restore side (with COLO). It should not touch
the upper struct, because it will be used on both save/restore side.
>
> For ao I can imagine that the suspend and save might actually have
> separate ao lifetimes, in which case these do of course need to remain
> different.
>
> Would that make any sense at all?
>
> FYI it was the dual initialisation of hvm quoted above which lead me to
> think along these lines.
>
> Alternatively perhaps suspend_state should a separate init function to
> logically separate it from the save_state initialiser. Perhaps taking
> the latter as an argument? Or possibly just the relevant fields.
Yes, the latter should be better. I will separate the init of the
suspend state.
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state
2015-06-30 9:43 ` Yang Hongyang
@ 2015-06-30 9:50 ` Ian Campbell
2015-06-30 10:05 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 9:50 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Tue, 2015-06-30 at 17:43 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 12:01 AM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> > [...]
> >> switch (type) {
> >> case LIBXL_DOMAIN_TYPE_HVM: {
> >> dss->hvm = 1;
> >> + dsps->hvm = 1;
> >> break;
> >> }
> >> case LIBXL_DOMAIN_TYPE_PV:
> >> dss->hvm = 0;
> >> + dsps->hvm = 0;
> >> break;
> > [...]
> >> @@ -2913,9 +2914,27 @@ typedef struct libxl__logdirty_switch {
> >> } libxl__logdirty_switch;
> >>
> >> struct libxl__domain_suspend_state {
> >> + /* set by caller of domain_suspend_callback_common */
> >> + libxl__ao *ao;
> >> +
> >> + uint32_t domid;
> >> + int hvm;
> >> + /* private */
> >> + libxl__ev_evtchn guest_evtchn;
> >> + int guest_evtchn_lockfd;
> >> + int guest_responded;
> >> + libxl__xswait_state pvcontrol;
> >> + libxl__ev_xswatch guest_watch;
> >> + libxl__ev_time guest_timeout;
> >> + const char *dm_savefile;
> >> + void (*callback_common_done)(libxl__egc*,
> >> + struct libxl__domain_suspend_state*, int ok);
> >> +};
> >> +
> >> +struct libxl__domain_save_state {
> >> /* set by caller of libxl__domain_suspend */
> >> libxl__ao *ao;
> >> - libxl__domain_suspend_cb *callback;
> >> + libxl__domain_save_cb *callback;
> >>
> >> uint32_t domid;
> >> int fd;
> >> @@ -2924,22 +2943,14 @@ struct libxl__domain_suspend_state {
> >> int debug;
> >> const libxl_domain_remus_info *remus;
> >> /* private */
> >> - libxl__ev_evtchn guest_evtchn;
> >> - int guest_evtchn_lockfd;
> >> + libxl__domain_suspend_state dsps;
> >> int hvm;
> >
> > I wonder if, given that any domain suspend must necessarily be contained
> > within a domain save it would be preferable to have "suspend" code
> > upcast the suspend_state to the containing save_state in order to look
> > at ->hvm, rather than duplicating it. Likewise domid.
>
> The reason we include hvm in suspend state is that we need a more common
> suspend function we will use on restore side (with COLO). It should not touch
> the upper struct, because it will be used on both save/restore side.
OK makes sense, thanks. You may as well mention this in the commit
message so I don't forget next time ;-)
> > For ao I can imagine that the suspend and save might actually have
> > separate ao lifetimes, in which case these do of course need to remain
> > different.
> >
> > Would that make any sense at all?
> >
> > FYI it was the dual initialisation of hvm quoted above which lead me to
> > think along these lines.
> >
> > Alternatively perhaps suspend_state should a separate init function to
> > logically separate it from the save_state initialiser. Perhaps taking
> > the latter as an argument? Or possibly just the relevant fields.
>
> Yes, the latter should be better. I will separate the init of the
> suspend state.
Thanks.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state
2015-06-30 9:50 ` Ian Campbell
@ 2015-06-30 10:05 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 10:05 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/30/2015 05:50 PM, Ian Campbell wrote:
> On Tue, 2015-06-30 at 17:43 +0800, Yang Hongyang wrote:
>>
>> On 06/30/2015 12:01 AM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>> [...]
>>>> switch (type) {
>>>> case LIBXL_DOMAIN_TYPE_HVM: {
>>>> dss->hvm = 1;
>>>> + dsps->hvm = 1;
>>>> break;
>>>> }
>>>> case LIBXL_DOMAIN_TYPE_PV:
>>>> dss->hvm = 0;
>>>> + dsps->hvm = 0;
>>>> break;
>>> [...]
>>>> @@ -2913,9 +2914,27 @@ typedef struct libxl__logdirty_switch {
>>>> } libxl__logdirty_switch;
>>>>
>>>> struct libxl__domain_suspend_state {
>>>> + /* set by caller of domain_suspend_callback_common */
>>>> + libxl__ao *ao;
>>>> +
>>>> + uint32_t domid;
>>>> + int hvm;
>>>> + /* private */
>>>> + libxl__ev_evtchn guest_evtchn;
>>>> + int guest_evtchn_lockfd;
>>>> + int guest_responded;
>>>> + libxl__xswait_state pvcontrol;
>>>> + libxl__ev_xswatch guest_watch;
>>>> + libxl__ev_time guest_timeout;
>>>> + const char *dm_savefile;
>>>> + void (*callback_common_done)(libxl__egc*,
>>>> + struct libxl__domain_suspend_state*, int ok);
>>>> +};
>>>> +
>>>> +struct libxl__domain_save_state {
>>>> /* set by caller of libxl__domain_suspend */
>>>> libxl__ao *ao;
>>>> - libxl__domain_suspend_cb *callback;
>>>> + libxl__domain_save_cb *callback;
>>>>
>>>> uint32_t domid;
>>>> int fd;
>>>> @@ -2924,22 +2943,14 @@ struct libxl__domain_suspend_state {
>>>> int debug;
>>>> const libxl_domain_remus_info *remus;
>>>> /* private */
>>>> - libxl__ev_evtchn guest_evtchn;
>>>> - int guest_evtchn_lockfd;
>>>> + libxl__domain_suspend_state dsps;
>>>> int hvm;
>>>
>>> I wonder if, given that any domain suspend must necessarily be contained
>>> within a domain save it would be preferable to have "suspend" code
>>> upcast the suspend_state to the containing save_state in order to look
>>> at ->hvm, rather than duplicating it. Likewise domid.
>>
>> The reason we include hvm in suspend state is that we need a more common
>> suspend function we will use on restore side (with COLO). It should not touch
>> the upper struct, because it will be used on both save/restore side.
>
> OK makes sense, thanks. You may as well mention this in the commit
> message so I don't forget next time ;-)
Ok, will do :)
>
>>> For ao I can imagine that the suspend and save might actually have
>>> separate ao lifetimes, in which case these do of course need to remain
>>> different.
>>>
>>> Would that make any sense at all?
>>>
>>> FYI it was the dual initialisation of hvm quoted above which lead me to
>>> think along these lines.
>>>
>>> Alternatively perhaps suspend_state should a separate init function to
>>> logically separate it from the save_state initialiser. Perhaps taking
>>> the latter as an argument? Or possibly just the relevant fields.
>>
>> Yes, the latter should be better. I will separate the init of the
>> suspend state.
>
> Thanks.
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (5 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 06/26] libxl/save: Refactor libxl__domain_suspend_state Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 16:07 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests Yang Hongyang
` (18 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
If the err is RECORD_NOT_PROCESSED, and it is an optional record,
restore will still fail. There're two options to fix this,
a, setting rc to 0 when it is an optional record.
b, do the error handling in the caller.
We choose b because:
There will be another error type in COLO, which indicates a failover,
that needs to be handled in restore(), so moving the error handling out
to make the logic clearer...Otherwise, in process_record,
RECORD_NOT_PROCESSED is handled, and in restore another error type
returned from process_record is handled.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/libxc/xc_sr_restore.c | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
index fd45775..d5645e0 100644
--- a/tools/libxc/xc_sr_restore.c
+++ b/tools/libxc/xc_sr_restore.c
@@ -569,19 +569,6 @@ static int process_record(struct xc_sr_context *ctx, struct xc_sr_record *rec)
free(rec->data);
rec->data = NULL;
- if ( rc == RECORD_NOT_PROCESSED )
- {
- if ( rec->type & REC_TYPE_OPTIONAL )
- DPRINTF("Ignoring optional record %#x (%s)",
- rec->type, rec_type_to_str(rec->type));
- else
- {
- ERROR("Mandatory record %#x (%s) not handled",
- rec->type, rec_type_to_str(rec->type));
- rc = -1;
- }
- }
-
return rc;
}
@@ -669,7 +656,20 @@ static int restore(struct xc_sr_context *ctx)
else
{
rc = process_record(ctx, &rec);
- if ( rc )
+ if ( rc == RECORD_NOT_PROCESSED )
+ {
+ if ( rec.type & REC_TYPE_OPTIONAL )
+ DPRINTF("Ignoring optional record %#x (%s)",
+ rec.type, rec_type_to_str(rec.type));
+ else
+ {
+ ERROR("Mandatory record %#x (%s) not handled",
+ rec.type, rec_type_to_str(rec.type));
+ rc = -1;
+ goto err;
+ }
+ }
+ else if ( rc )
goto err;
}
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record
2015-06-25 6:25 ` [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record Yang Hongyang
@ 2015-06-29 16:07 ` Ian Campbell
2015-06-30 9:45 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 16:07 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
ITYM "handling" in the subject. And perhaps "change" rather than "fix"?
> If the err is RECORD_NOT_PROCESSED, and it is an optional record,
> restore will still fail. There're two options to fix this,
> a, setting rc to 0 when it is an optional record.
> b, do the error handling in the caller.
> We choose b because:
> There will be another error type in COLO, which indicates a failover,
> that needs to be handled in restore(), so moving the error handling out
> to make the logic clearer...Otherwise, in process_record,
> RECORD_NOT_PROCESSED is handled, and in restore another error type
> returned from process_record is handled.
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> CC: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> tools/libxc/xc_sr_restore.c | 28 ++++++++++++++--------------
> 1 file changed, 14 insertions(+), 14 deletions(-)
>
> diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
> index fd45775..d5645e0 100644
> --- a/tools/libxc/xc_sr_restore.c
> +++ b/tools/libxc/xc_sr_restore.c
> @@ -569,19 +569,6 @@ static int process_record(struct xc_sr_context *ctx, struct xc_sr_record *rec)
> free(rec->data);
> rec->data = NULL;
>
> - if ( rc == RECORD_NOT_PROCESSED )
> - {
> - if ( rec->type & REC_TYPE_OPTIONAL )
> - DPRINTF("Ignoring optional record %#x (%s)",
> - rec->type, rec_type_to_str(rec->type));
> - else
> - {
> - ERROR("Mandatory record %#x (%s) not handled",
> - rec->type, rec_type_to_str(rec->type));
> - rc = -1;
> - }
> - }
> -
> return rc;
> }
>
> @@ -669,7 +656,20 @@ static int restore(struct xc_sr_context *ctx)
There is a second caller of process_record in handle_checkpoint, so this
change will result in a change of behaviour for that case, I think.
If that is intentional then it ought to be mentioned in the commit log,
since otherwise it looks a lot like this change is supposed to result in
no overall difference.
Ian.
> else
> {
> rc = process_record(ctx, &rec);
> - if ( rc )
> + if ( rc == RECORD_NOT_PROCESSED )
> + {
> + if ( rec.type & REC_TYPE_OPTIONAL )
> + DPRINTF("Ignoring optional record %#x (%s)",
> + rec.type, rec_type_to_str(rec.type));
> + else
> + {
> + ERROR("Mandatory record %#x (%s) not handled",
> + rec.type, rec_type_to_str(rec.type));
> + rc = -1;
> + goto err;
> + }
> + }
> + else if ( rc )
> goto err;
> }
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record
2015-06-29 16:07 ` Ian Campbell
@ 2015-06-30 9:45 ` Yang Hongyang
2015-07-03 3:12 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 9:45 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/30/2015 12:07 AM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>
> ITYM "handling" in the subject. And perhaps "change" rather than "fix"?
>
>> If the err is RECORD_NOT_PROCESSED, and it is an optional record,
>> restore will still fail. There're two options to fix this,
>> a, setting rc to 0 when it is an optional record.
>> b, do the error handling in the caller.
>> We choose b because:
>> There will be another error type in COLO, which indicates a failover,
>> that needs to be handled in restore(), so moving the error handling out
>> to make the logic clearer...Otherwise, in process_record,
>> RECORD_NOT_PROCESSED is handled, and in restore another error type
>> returned from process_record is handled.
>>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
>> CC: Wei Liu <wei.liu2@citrix.com>
>> CC: Andrew Cooper <andrew.cooper3@citrix.com>
>> ---
>> tools/libxc/xc_sr_restore.c | 28 ++++++++++++++--------------
>> 1 file changed, 14 insertions(+), 14 deletions(-)
>>
>> diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
>> index fd45775..d5645e0 100644
>> --- a/tools/libxc/xc_sr_restore.c
>> +++ b/tools/libxc/xc_sr_restore.c
>> @@ -569,19 +569,6 @@ static int process_record(struct xc_sr_context *ctx, struct xc_sr_record *rec)
>> free(rec->data);
>> rec->data = NULL;
>>
>> - if ( rc == RECORD_NOT_PROCESSED )
>> - {
>> - if ( rec->type & REC_TYPE_OPTIONAL )
>> - DPRINTF("Ignoring optional record %#x (%s)",
>> - rec->type, rec_type_to_str(rec->type));
>> - else
>> - {
>> - ERROR("Mandatory record %#x (%s) not handled",
>> - rec->type, rec_type_to_str(rec->type));
>> - rc = -1;
>> - }
>> - }
>> -
>> return rc;
>> }
>>
>> @@ -669,7 +656,20 @@ static int restore(struct xc_sr_context *ctx)
>
> There is a second caller of process_record in handle_checkpoint, so this
> change will result in a change of behaviour for that case, I think.
Ah, seems I missed the other caller, good catch, and I will fix it in the
next version.
>
> If that is intentional then it ought to be mentioned in the commit log,
> since otherwise it looks a lot like this change is supposed to result in
> no overall difference.
>
> Ian.
>
>
>> else
>> {
>> rc = process_record(ctx, &rec);
>> - if ( rc )
>> + if ( rc == RECORD_NOT_PROCESSED )
>> + {
>> + if ( rec.type & REC_TYPE_OPTIONAL )
>> + DPRINTF("Ignoring optional record %#x (%s)",
>> + rec.type, rec_type_to_str(rec.type));
>> + else
>> + {
>> + ERROR("Mandatory record %#x (%s) not handled",
>> + rec.type, rec_type_to_str(rec.type));
>> + rc = -1;
>> + goto err;
>> + }
>> + }
>> + else if ( rc )
>> goto err;
>> }
>>
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record
2015-06-30 9:45 ` Yang Hongyang
@ 2015-07-03 3:12 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-03 3:12 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/30/2015 05:45 PM, Yang Hongyang wrote:
>
>
> On 06/30/2015 12:07 AM, Ian Campbell wrote:
>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>
>> ITYM "handling" in the subject. And perhaps "change" rather than "fix"?
>>
>>> If the err is RECORD_NOT_PROCESSED, and it is an optional record,
>>> restore will still fail. There're two options to fix this,
>>> a, setting rc to 0 when it is an optional record.
>>> b, do the error handling in the caller.
>>> We choose b because:
>>> There will be another error type in COLO, which indicates a failover,
>>> that needs to be handled in restore(), so moving the error handling out
>>> to make the logic clearer...Otherwise, in process_record,
>>> RECORD_NOT_PROCESSED is handled, and in restore another error type
>>> returned from process_record is handled.
>>>
>>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>>> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
>>> CC: Wei Liu <wei.liu2@citrix.com>
>>> CC: Andrew Cooper <andrew.cooper3@citrix.com>
>>> ---
>>> tools/libxc/xc_sr_restore.c | 28 ++++++++++++++--------------
>>> 1 file changed, 14 insertions(+), 14 deletions(-)
>>>
>>> diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
>>> index fd45775..d5645e0 100644
>>> --- a/tools/libxc/xc_sr_restore.c
>>> +++ b/tools/libxc/xc_sr_restore.c
>>> @@ -569,19 +569,6 @@ static int process_record(struct xc_sr_context *ctx,
>>> struct xc_sr_record *rec)
>>> free(rec->data);
>>> rec->data = NULL;
>>>
>>> - if ( rc == RECORD_NOT_PROCESSED )
>>> - {
>>> - if ( rec->type & REC_TYPE_OPTIONAL )
>>> - DPRINTF("Ignoring optional record %#x (%s)",
>>> - rec->type, rec_type_to_str(rec->type));
>>> - else
>>> - {
>>> - ERROR("Mandatory record %#x (%s) not handled",
>>> - rec->type, rec_type_to_str(rec->type));
>>> - rc = -1;
>>> - }
>>> - }
>>> -
>>> return rc;
>>> }
>>>
>>> @@ -669,7 +656,20 @@ static int restore(struct xc_sr_context *ctx)
>>
>> There is a second caller of process_record in handle_checkpoint, so this
>> change will result in a change of behaviour for that case, I think.
>
> Ah, seems I missed the other caller, good catch, and I will fix it in the
> next version.
>
>>
>> If that is intentional then it ought to be mentioned in the commit log,
I rechecked the process_record in handle_checkpoint, it just return the error
code, finally, the error will be handled by the upper caller in restore(),
so it is intentional, I wll mention this in the commit log.
>> since otherwise it looks a lot like this change is supposed to result in
>> no overall difference.
>>
>> Ian.
>>
>>
>>> else
>>> {
>>> rc = process_record(ctx, &rec);
>>> - if ( rc )
>>> + if ( rc == RECORD_NOT_PROCESSED )
>>> + {
>>> + if ( rec.type & REC_TYPE_OPTIONAL )
>>> + DPRINTF("Ignoring optional record %#x (%s)",
>>> + rec.type, rec_type_to_str(rec.type));
>>> + else
>>> + {
>>> + ERROR("Mandatory record %#x (%s) not handled",
>>> + rec.type, rec_type_to_str(rec.type));
>>> + rc = -1;
>>> + goto err;
>>> + }
>>> + }
>>> + else if ( rc )
>>> goto err;
>>> }
>>>
>>
>>
>> .
>>
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (6 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 07/26] libxc/restore: fix error handle of process_record Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 16:27 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream Yang Hongyang
` (17 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
From: Wen Congyang <wency@cn.fujitsu.com>
1. suspend
a. PVHVM and PV: we use the same way to suspend the guest(send the suspend
request to the guest). If the guest doesn't support evtchn, the xenstore
variant will be used, suspending the guest via XenBus control node.
b. pure HVM: we call xc_domain_shutdown(..., SHUTDOWN_suspend) to suspend
the guest
2. Resume:
a. fast path
In this case, we don't change the guest's state.
PV: modify the return code to 1, and than call the domctl:
XEN_DOMCTL_resumedomain
PVHVM: same with PV
HVM: do nothing in modify_returncode, and than call the domctl:
XEN_DOMCTL_resumedomain
b. slow
In this case, we have changed the guest's state.
PV: update start info, and reset all secondary CPU states. Than call the
domctl: XEN_DOMCTL_resumedomain
PVHVM and HVM can not be resumed.
For PVHVM, in my test, only call the domctl: XEN_DOMCTL_resumedomain
can work. I am not sure if we should update start info and reset all
secondary CPU states.
For pure HVM guest, in my test, only call the domctl:
XEN_DOMCTL_resumedomain can work.
So we can call libxl__domain_resume(..., 1) if we don't change the guest
state, otherwise call libxl__domain_resume(..., 0).
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxc/xc_resume.c | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
diff --git a/tools/libxc/xc_resume.c b/tools/libxc/xc_resume.c
index e67bebd..bd82334 100644
--- a/tools/libxc/xc_resume.c
+++ b/tools/libxc/xc_resume.c
@@ -109,6 +109,23 @@ static int xc_domain_resume_cooperative(xc_interface *xch, uint32_t domid)
return do_domctl(xch, &domctl);
}
+static int xc_domain_resume_hvm(xc_interface *xch, uint32_t domid)
+{
+ DECLARE_DOMCTL;
+
+ /*
+ * If it is PVHVM, the hypercall return code is 0, because this
+ * is not a fast path resume, we do not modify_returncode as in
+ * xc_domain_resume_cooperative.
+ * (resuming it in a new domain context)
+ *
+ * If it is a HVM, the hypercall is a NOP.
+ */
+ domctl.cmd = XEN_DOMCTL_resumedomain;
+ domctl.domain = domid;
+ return do_domctl(xch, &domctl);
+}
+
static int xc_domain_resume_any(xc_interface *xch, uint32_t domid)
{
DECLARE_DOMCTL;
@@ -138,10 +155,7 @@ static int xc_domain_resume_any(xc_interface *xch, uint32_t domid)
*/
#if defined(__i386__) || defined(__x86_64__)
if ( info.hvm )
- {
- ERROR("Cannot resume uncooperative HVM guests");
- return rc;
- }
+ return xc_domain_resume_hvm(xch, domid);
if ( xc_domain_get_guest_width(xch, domid, &dinfo->guest_width) != 0 )
{
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests
2015-06-25 6:25 ` [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests Yang Hongyang
@ 2015-06-29 16:27 ` Ian Campbell
2015-06-30 10:08 ` Wen Congyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 16:27 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> From: Wen Congyang <wency@cn.fujitsu.com>
>
> 1. suspend
> a. PVHVM and PV: we use the same way to suspend the guest(send the suspend
space between "guest" and the open parenthesis please.
> request to the guest). If the guest doesn't support evtchn, the xenstore
> variant will be used, suspending the guest via XenBus control node.
> b. pure HVM: we call xc_domain_shutdown(..., SHUTDOWN_suspend) to suspend
> the guest
>
> 2. Resume:
> a. fast path
> In this case, we don't change the guest's state.
> PV: modify the return code to 1, and than call the domctl:
> XEN_DOMCTL_resumedomain
> PVHVM: same with PV
> HVM: do nothing in modify_returncode, and than call the domctl:
> XEN_DOMCTL_resumedomain
> b. slow
> In this case, we have changed the guest's state.
"have" or "will"? AIUI the latter would be more accurate.
> PV: update start info, and reset all secondary CPU states. Than call the
> domctl: XEN_DOMCTL_resumedomain
> PVHVM and HVM can not be resumed.
I'm confused -- isn't the purpose of this patch to make PVHM support
resume?
> For PVHVM, in my test, only call the domctl: XEN_DOMCTL_resumedomain
> can work. I am not sure if we should update start info and reset all
> secondary CPU states.
>
> For pure HVM guest, in my test, only call the domctl:
> XEN_DOMCTL_resumedomain can work.
>
> So we can call libxl__domain_resume(..., 1) if we don't change the guest
> state, otherwise call libxl__domain_resume(..., 0).
Hrm, so it sounds here like the correctness of this new functionality
requires the caller to have not messed with the domain's state? What
sort of changes are to the guest state are we talking about here?
Isn't that a new requirement for this call? If so then it should be
documented somewhere, specifically what sorts of changes are and are not
allowed and the types of guests which are affected.
>
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
> tools/libxc/xc_resume.c | 22 ++++++++++++++++++----
> 1 file changed, 18 insertions(+), 4 deletions(-)
>
> diff --git a/tools/libxc/xc_resume.c b/tools/libxc/xc_resume.c
> index e67bebd..bd82334 100644
> --- a/tools/libxc/xc_resume.c
> +++ b/tools/libxc/xc_resume.c
> @@ -109,6 +109,23 @@ static int xc_domain_resume_cooperative(xc_interface *xch, uint32_t domid)
> return do_domctl(xch, &domctl);
> }
>
> +static int xc_domain_resume_hvm(xc_interface *xch, uint32_t domid)
> +{
> + DECLARE_DOMCTL;
> +
> + /*
> + * If it is PVHVM, the hypercall return code is 0, because this
> + * is not a fast path resume, we do not modify_returncode as in
> + * xc_domain_resume_cooperative.
> + * (resuming it in a new domain context)
> + *
> + * If it is a HVM, the hypercall is a NOP.
> + */
> + domctl.cmd = XEN_DOMCTL_resumedomain;
> + domctl.domain = domid;
> + return do_domctl(xch, &domctl);
There are already several open coded instances of this
XEN_DOMCTL_resumedomain, and I think putting this particular one into a
helper is actually more confusing than just inlining this at the caller.
In particular when reading this function my first question was "how do
we know this is not a fast path resume", the answer being that the only
caller is the slow path resume case, but that's not evident from the
context (what if someone adds a second call?)
So I think at least the comment ought to go at the callsite, at which
point this function doesn't add much.
(Ideally all the open coded do_domctl would go into a single
do_domainresume or something, but I don't think you need to do that
unless you really want to).
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests
2015-06-29 16:27 ` Ian Campbell
@ 2015-06-30 10:08 ` Wen Congyang
2015-06-30 10:59 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Wen Congyang @ 2015-06-30 10:08 UTC (permalink / raw)
To: Ian Campbell, Yang Hongyang
Cc: wei.liu2, andrew.cooper3, yunhong.jiang, eddie.dong, xen-devel,
guijianfeng, rshriram, ian.jackson
On 06/30/2015 12:27 AM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> From: Wen Congyang <wency@cn.fujitsu.com>
>>
>> 1. suspend
>> a. PVHVM and PV: we use the same way to suspend the guest(send the suspend
>
> space between "guest" and the open parenthesis please.
>
>> request to the guest). If the guest doesn't support evtchn, the xenstore
>> variant will be used, suspending the guest via XenBus control node.
>> b. pure HVM: we call xc_domain_shutdown(..., SHUTDOWN_suspend) to suspend
>> the guest
>>
>> 2. Resume:
>> a. fast path
>> In this case, we don't change the guest's state.
>> PV: modify the return code to 1, and than call the domctl:
>> XEN_DOMCTL_resumedomain
>> PVHVM: same with PV
>> HVM: do nothing in modify_returncode, and than call the domctl:
>> XEN_DOMCTL_resumedomain
>> b. slow
>> In this case, we have changed the guest's state.
>
> "have" or "will"? AIUI the latter would be more accurate.
>
>> PV: update start info, and reset all secondary CPU states. Than call the
>> domctl: XEN_DOMCTL_resumedomain
>> PVHVM and HVM can not be resumed.
>
> I'm confused -- isn't the purpose of this patch to make PVHM support
> resume?
Without this patch, HVM(both PVHVM and pure HVM) can not be resumed in slow
path if its state is changed.
>
>> For PVHVM, in my test, only call the domctl: XEN_DOMCTL_resumedomain
>> can work. I am not sure if we should update start info and reset all
>> secondary CPU states.
>>
>> For pure HVM guest, in my test, only call the domctl:
>> XEN_DOMCTL_resumedomain can work.
>>
>> So we can call libxl__domain_resume(..., 1) if we don't change the guest
>> state, otherwise call libxl__domain_resume(..., 0).
>
> Hrm, so it sounds here like the correctness of this new functionality
> requires the caller to have not messed with the domain's state? What
> sort of changes are to the guest state are we talking about here?
Any state can be changed: memory, device state. If the caller messes
with the domain's state, the guest may be crashed.
>
> Isn't that a new requirement for this call? If so then it should be
> documented somewhere, specifically what sorts of changes are and are not
> allowed and the types of guests which are affected.
The state should be the one saved by xen.
>
>>
>> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> ---
>> tools/libxc/xc_resume.c | 22 ++++++++++++++++++----
>> 1 file changed, 18 insertions(+), 4 deletions(-)
>>
>> diff --git a/tools/libxc/xc_resume.c b/tools/libxc/xc_resume.c
>> index e67bebd..bd82334 100644
>> --- a/tools/libxc/xc_resume.c
>> +++ b/tools/libxc/xc_resume.c
>> @@ -109,6 +109,23 @@ static int xc_domain_resume_cooperative(xc_interface *xch, uint32_t domid)
>> return do_domctl(xch, &domctl);
>> }
>>
>> +static int xc_domain_resume_hvm(xc_interface *xch, uint32_t domid)
>> +{
>> + DECLARE_DOMCTL;
>> +
>> + /*
>> + * If it is PVHVM, the hypercall return code is 0, because this
>> + * is not a fast path resume, we do not modify_returncode as in
>> + * xc_domain_resume_cooperative.
>> + * (resuming it in a new domain context)
>> + *
>> + * If it is a HVM, the hypercall is a NOP.
>> + */
>> + domctl.cmd = XEN_DOMCTL_resumedomain;
>> + domctl.domain = domid;
>> + return do_domctl(xch, &domctl);
>
> There are already several open coded instances of this
> XEN_DOMCTL_resumedomain, and I think putting this particular one into a
> helper is actually more confusing than just inlining this at the caller.
>
> In particular when reading this function my first question was "how do
> we know this is not a fast path resume", the answer being that the only
> caller is the slow path resume case, but that's not evident from the
> context (what if someone adds a second call?)
Only the caller know it. See xc_domain_resume()'s third parameter.
Thanks
Wen Congyang
>
> So I think at least the comment ought to go at the callsite, at which
> point this function doesn't add much.
>
> (Ideally all the open coded do_domctl would go into a single
> do_domainresume or something, but I don't think you need to do that
> unless you really want to).
>
>
>
> .
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests
2015-06-30 10:08 ` Wen Congyang
@ 2015-06-30 10:59 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:59 UTC (permalink / raw)
To: Wen Congyang
Cc: wei.liu2, andrew.cooper3, yunhong.jiang, eddie.dong, xen-devel,
guijianfeng, rshriram, Yang Hongyang, ian.jackson
On Tue, 2015-06-30 at 18:08 +0800, Wen Congyang wrote:
> On 06/30/2015 12:27 AM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> From: Wen Congyang <wency@cn.fujitsu.com>
> >>
> >> 1. suspend
> >> a. PVHVM and PV: we use the same way to suspend the guest(send the suspend
> >
> > space between "guest" and the open parenthesis please.
> >
> >> request to the guest). If the guest doesn't support evtchn, the xenstore
> >> variant will be used, suspending the guest via XenBus control node.
> >> b. pure HVM: we call xc_domain_shutdown(..., SHUTDOWN_suspend) to suspend
> >> the guest
> >>
> >> 2. Resume:
> >> a. fast path
> >> In this case, we don't change the guest's state.
> >> PV: modify the return code to 1, and than call the domctl:
> >> XEN_DOMCTL_resumedomain
> >> PVHVM: same with PV
> >> HVM: do nothing in modify_returncode, and than call the domctl:
> >> XEN_DOMCTL_resumedomain
> >> b. slow
> >> In this case, we have changed the guest's state.
> >
> > "have" or "will"? AIUI the latter would be more accurate.
> >
> >> PV: update start info, and reset all secondary CPU states. Than call the
> >> domctl: XEN_DOMCTL_resumedomain
> >> PVHVM and HVM can not be resumed.
> >
> > I'm confused -- isn't the purpose of this patch to make PVHM support
> > resume?
>
> Without this patch, HVM(both PVHVM and pure HVM) can not be resumed in slow
> path if its state is changed.
So your commit message is confusingly and without saying so talking
about what the previous case was, not what this patch itself makes work.
>
> >
> >> For PVHVM, in my test, only call the domctl: XEN_DOMCTL_resumedomain
> >> can work. I am not sure if we should update start info and reset all
> >> secondary CPU states.
> >>
> >> For pure HVM guest, in my test, only call the domctl:
> >> XEN_DOMCTL_resumedomain can work.
> >>
> >> So we can call libxl__domain_resume(..., 1) if we don't change the guest
> >> state, otherwise call libxl__domain_resume(..., 0).
> >
> > Hrm, so it sounds here like the correctness of this new functionality
> > requires the caller to have not messed with the domain's state? What
> > sort of changes are to the guest state are we talking about here?
>
> Any state can be changed: memory, device state. If the caller messes
> with the domain's state, the guest may be crashed.
>
> >
> > Isn't that a new requirement for this call? If so then it should be
> > documented somewhere, specifically what sorts of changes are and are not
> > allowed and the types of guests which are affected.
>
> The state should be the one saved by xen.
I don't know what that refers to.
I'm afraid I think the commit message for this patch (and the associated
doc comments) need revisiting almost from scratch, to clearly explain
what this patch is doing and why and what the constraints on the new
functionality will be.
At the moment it mostly talks in a confusing way about the old behaviour
and adds very specific assumptions to the new function which are not
made clear.
>
> >
> >>
> >> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> >> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> >> ---
> >> tools/libxc/xc_resume.c | 22 ++++++++++++++++++----
> >> 1 file changed, 18 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/tools/libxc/xc_resume.c b/tools/libxc/xc_resume.c
> >> index e67bebd..bd82334 100644
> >> --- a/tools/libxc/xc_resume.c
> >> +++ b/tools/libxc/xc_resume.c
> >> @@ -109,6 +109,23 @@ static int xc_domain_resume_cooperative(xc_interface *xch, uint32_t domid)
> >> return do_domctl(xch, &domctl);
> >> }
> >>
> >> +static int xc_domain_resume_hvm(xc_interface *xch, uint32_t domid)
> >> +{
> >> + DECLARE_DOMCTL;
> >> +
> >> + /*
> >> + * If it is PVHVM, the hypercall return code is 0, because this
> >> + * is not a fast path resume, we do not modify_returncode as in
> >> + * xc_domain_resume_cooperative.
> >> + * (resuming it in a new domain context)
> >> + *
> >> + * If it is a HVM, the hypercall is a NOP.
> >> + */
> >> + domctl.cmd = XEN_DOMCTL_resumedomain;
> >> + domctl.domain = domid;
> >> + return do_domctl(xch, &domctl);
> >
> > There are already several open coded instances of this
> > XEN_DOMCTL_resumedomain, and I think putting this particular one into a
> > helper is actually more confusing than just inlining this at the caller.
> >
> > In particular when reading this function my first question was "how do
> > we know this is not a fast path resume", the answer being that the only
> > caller is the slow path resume case, but that's not evident from the
> > context (what if someone adds a second call?)
>
> Only the caller know it. See xc_domain_resume()'s third parameter.
Right, which is the problem I'm describing, this function is not in
itself self contained, it makes assumptions about its caller. Hence I
suggested:
> > So I think at least the comment ought to go at the callsite, at which
> > point this function doesn't add much.
The implication being the function should be removed and the actions it
takes just inlined at the caller
> > (Ideally all the open coded do_domctl would go into a single
> > do_domainresume or something, but I don't think you need to do that
> > unless you really want to).
With this as a potential cleanup to the whole file.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (7 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 08/26] tools/libxc: support to resume uncooperative HVM guests Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 16:30 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 10/26] migration/save: pass checkpointed_stream from libxl to libxc Yang Hongyang
` (16 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
introduce enum type libxl_checkpointed_stream in IDL.
rename the last argument of migrate_receive from "remus" to
"checkpointed" since the semantics of this parameter has
changed.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxl/libxl_types.idl | 5 +++++
tools/libxl/xl_cmdimpl.c | 13 +++++++------
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 7418d92..7c82f33 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -198,6 +198,11 @@ libxl_viridian_enlightenment = Enumeration("viridian_enlightenment", [
(3, "reference_tsc"),
])
+libxl_checkpointed_stream = Enumeration("checkpointed_stream", [
+ (0, "NONE"),
+ (1, "REMUS"),
+ ])
+
#
# Complex libxl types
#
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 35bc26d..c965ef5 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -4249,7 +4249,7 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
}
static void migrate_receive(int debug, int daemonize, int monitor,
- int send_fd, int recv_fd, int remus)
+ int send_fd, int recv_fd, int checkpointed)
{
uint32_t domid;
int rc, rc2;
@@ -4274,7 +4274,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
dom_info.paused = 1;
dom_info.migrate_fd = recv_fd;
dom_info.migration_domname_r = &migration_domname;
- dom_info.checkpointed_stream = remus;
+ dom_info.checkpointed_stream = checkpointed;
rc = create_domain(&dom_info);
if (rc < 0) {
@@ -4285,7 +4285,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
domid = rc;
- if (remus) {
+ if (checkpointed) {
/* If we are here, it means that the sender (primary) has crashed.
* TODO: Split-Brain Check.
*/
@@ -4456,7 +4456,8 @@ int main_restore(int argc, char **argv)
int main_migrate_receive(int argc, char **argv)
{
- int debug = 0, daemonize = 1, monitor = 1, remus = 0;
+ int debug = 0, daemonize = 1, monitor = 1;
+ int checkpointed = LIBXL_CHECKPOINTED_STREAM_NONE;
int opt;
SWITCH_FOREACH_OPT(opt, "Fedr", NULL, "migrate-receive", 0) {
@@ -4471,7 +4472,7 @@ int main_migrate_receive(int argc, char **argv)
debug = 1;
break;
case 'r':
- remus = 1;
+ checkpointed = LIBXL_CHECKPOINTED_STREAM_REMUS;
break;
}
@@ -4481,7 +4482,7 @@ int main_migrate_receive(int argc, char **argv)
}
migrate_receive(debug, daemonize, monitor,
STDOUT_FILENO, STDIN_FILENO,
- remus);
+ checkpointed);
return 0;
}
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-06-25 6:25 ` [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream Yang Hongyang
@ 2015-06-29 16:30 ` Ian Campbell
2015-06-30 9:53 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 16:30 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> introduce enum type libxl_checkpointed_stream in IDL.
> rename the last argument of migrate_receive from "remus" to
> "checkpointed" since the semantics of this parameter has
> changed.
I don't see anything here which is hitting the libxl API layer, it all
looks to be internal to xl. Is that going to change in a future patch
i.e. is something in libxl.h going to now accept a
libxl_checkpointed_stream?
If this is completely internal to xl you should just make an enum there
I think (you could add xl_types.idl, but that seems like overkill)
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-06-29 16:30 ` Ian Campbell
@ 2015-06-30 9:53 ` Yang Hongyang
2015-06-30 10:52 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 9:53 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 12:30 AM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> introduce enum type libxl_checkpointed_stream in IDL.
>> rename the last argument of migrate_receive from "remus" to
>> "checkpointed" since the semantics of this parameter has
>> changed.
>
> I don't see anything here which is hitting the libxl API layer, it all
> looks to be internal to xl. Is that going to change in a future patch
> i.e. is something in libxl.h going to now accept a
> libxl_checkpointed_stream?
It is stored in restore_params.checkpointed_stream, does this a libxl API layer
structure?
>
> If this is completely internal to xl you should just make an enum there
> I think (you could add xl_types.idl, but that seems like overkill)
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-06-30 9:53 ` Yang Hongyang
@ 2015-06-30 10:52 ` Ian Campbell
2015-07-01 2:05 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:52 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Tue, 2015-06-30 at 17:53 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 12:30 AM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> introduce enum type libxl_checkpointed_stream in IDL.
> >> rename the last argument of migrate_receive from "remus" to
> >> "checkpointed" since the semantics of this parameter has
> >> changed.
> >
> > I don't see anything here which is hitting the libxl API layer, it all
> > looks to be internal to xl. Is that going to change in a future patch
> > i.e. is something in libxl.h going to now accept a
> > libxl_checkpointed_stream?
>
> It is stored in restore_params.checkpointed_stream, does this a libxl API layer
> structure?
Is that in a later patch? I didn't spot it if so.
Ian
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-06-30 10:52 ` Ian Campbell
@ 2015-07-01 2:05 ` Yang Hongyang
2015-07-01 10:36 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:05 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:52 PM, Ian Campbell wrote:
> On Tue, 2015-06-30 at 17:53 +0800, Yang Hongyang wrote:
>>
>> On 06/30/2015 12:30 AM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>> introduce enum type libxl_checkpointed_stream in IDL.
>>>> rename the last argument of migrate_receive from "remus" to
>>>> "checkpointed" since the semantics of this parameter has
>>>> changed.
>>>
>>> I don't see anything here which is hitting the libxl API layer, it all
>>> looks to be internal to xl. Is that going to change in a future patch
>>> i.e. is something in libxl.h going to now accept a
>>> libxl_checkpointed_stream?
>>
>> It is stored in restore_params.checkpointed_stream, does this a libxl API layer
>> structure?
>
> Is that in a later patch? I didn't spot it if so.
It's in the existing code:
xl_cmdimpl.c create_domain()
2720 if ( restoring ) {
2721 libxl_domain_restore_params params;
2722
2723 libxl_domain_restore_params_init(¶ms);
2724
2725 params.checkpointed_stream = dom_info->checkpointed_stream;
2726 ret = libxl_domain_create_restore(ctx, &d_config,
...
>
> Ian
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-07-01 2:05 ` Yang Hongyang
@ 2015-07-01 10:36 ` Ian Campbell
2015-07-01 13:43 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:36 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Wed, 2015-07-01 at 10:05 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 06:52 PM, Ian Campbell wrote:
> > On Tue, 2015-06-30 at 17:53 +0800, Yang Hongyang wrote:
> >>
> >> On 06/30/2015 12:30 AM, Ian Campbell wrote:
> >>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >>>> introduce enum type libxl_checkpointed_stream in IDL.
> >>>> rename the last argument of migrate_receive from "remus" to
> >>>> "checkpointed" since the semantics of this parameter has
> >>>> changed.
> >>>
> >>> I don't see anything here which is hitting the libxl API layer, it all
> >>> looks to be internal to xl. Is that going to change in a future patch
> >>> i.e. is something in libxl.h going to now accept a
> >>> libxl_checkpointed_stream?
> >>
> >> It is stored in restore_params.checkpointed_stream, does this a libxl API layer
> >> structure?
> >
> > Is that in a later patch? I didn't spot it if so.
>
> It's in the existing code:
>
> xl_cmdimpl.c create_domain()
Ah sorry, since it didn't change it wasn't in the patch and my grep
seems to have missed it.
But wait, libxl_domain_restore_params isn't changed here, shouldn't it
be? Otherwise params.checkpointed_stream is still an int. I suspect some
part of a later patch should appear here instead.
Ian.
>
> 2720 if ( restoring ) {
> 2721 libxl_domain_restore_params params;
> 2722
> 2723 libxl_domain_restore_params_init(¶ms);
> 2724
> 2725 params.checkpointed_stream = dom_info->checkpointed_stream;
> 2726 ret = libxl_domain_create_restore(ctx, &d_config,
> ...
>
>
> >
> > Ian
> >
> >
> > .
> >
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-07-01 10:36 ` Ian Campbell
@ 2015-07-01 13:43 ` Yang Hongyang
2015-07-01 14:09 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 13:43 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 07/01/2015 06:36 PM, Ian Campbell wrote:
> On Wed, 2015-07-01 at 10:05 +0800, Yang Hongyang wrote:
>>
>> On 06/30/2015 06:52 PM, Ian Campbell wrote:
>>> On Tue, 2015-06-30 at 17:53 +0800, Yang Hongyang wrote:
>>>>
>>>> On 06/30/2015 12:30 AM, Ian Campbell wrote:
>>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>>>> introduce enum type libxl_checkpointed_stream in IDL.
>>>>>> rename the last argument of migrate_receive from "remus" to
>>>>>> "checkpointed" since the semantics of this parameter has
>>>>>> changed.
>>>>>
>>>>> I don't see anything here which is hitting the libxl API layer, it all
>>>>> looks to be internal to xl. Is that going to change in a future patch
>>>>> i.e. is something in libxl.h going to now accept a
>>>>> libxl_checkpointed_stream?
>>>>
>>>> It is stored in restore_params.checkpointed_stream, does this a libxl API layer
>>>> structure?
>>>
>>> Is that in a later patch? I didn't spot it if so.
>>
>> It's in the existing code:
>>
>> xl_cmdimpl.c create_domain()
>
> Ah sorry, since it didn't change it wasn't in the patch and my grep
> seems to have missed it.
>
> But wait, libxl_domain_restore_params isn't changed here, shouldn't it
> be? Otherwise params.checkpointed_stream is still an int. I suspect some
> part of a later patch should appear here instead.
It better to be changed, but I'm not sure if this change will impact the
existing users, if we do not change, there won't be problem at least.
>
> Ian.
>
>>
>> 2720 if ( restoring ) {
>> 2721 libxl_domain_restore_params params;
>> 2722
>> 2723 libxl_domain_restore_params_init(¶ms);
>> 2724
>> 2725 params.checkpointed_stream = dom_info->checkpointed_stream;
>> 2726 ret = libxl_domain_create_restore(ctx, &d_config,
>> ...
>>
>>
>>>
>>> Ian
>>>
>>>
>>> .
>>>
>>
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream
2015-07-01 13:43 ` Yang Hongyang
@ 2015-07-01 14:09 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 14:09 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Wed, 2015-07-01 at 21:43 +0800, Yang Hongyang wrote:
>
> On 07/01/2015 06:36 PM, Ian Campbell wrote:
> > On Wed, 2015-07-01 at 10:05 +0800, Yang Hongyang wrote:
> >>
> >> On 06/30/2015 06:52 PM, Ian Campbell wrote:
> >>> On Tue, 2015-06-30 at 17:53 +0800, Yang Hongyang wrote:
> >>>>
> >>>> On 06/30/2015 12:30 AM, Ian Campbell wrote:
> >>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >>>>>> introduce enum type libxl_checkpointed_stream in IDL.
> >>>>>> rename the last argument of migrate_receive from "remus" to
> >>>>>> "checkpointed" since the semantics of this parameter has
> >>>>>> changed.
> >>>>>
> >>>>> I don't see anything here which is hitting the libxl API layer, it all
> >>>>> looks to be internal to xl. Is that going to change in a future patch
> >>>>> i.e. is something in libxl.h going to now accept a
> >>>>> libxl_checkpointed_stream?
> >>>>
> >>>> It is stored in restore_params.checkpointed_stream, does this a libxl API layer
> >>>> structure?
> >>>
> >>> Is that in a later patch? I didn't spot it if so.
> >>
> >> It's in the existing code:
> >>
> >> xl_cmdimpl.c create_domain()
> >
> > Ah sorry, since it didn't change it wasn't in the patch and my grep
> > seems to have missed it.
> >
> > But wait, libxl_domain_restore_params isn't changed here, shouldn't it
> > be? Otherwise params.checkpointed_stream is still an int. I suspect some
> > part of a later patch should appear here instead.
>
> It better to be changed, but I'm not sure if this change will impact the
> existing users, if we do not change, there won't be problem at least.
It surely has to change eventually and other callers will have to be
updated to cope (and there should be LIBXL_HAVE_...).
If it's going to change it may as well be here.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 10/26] migration/save: pass checkpointed_stream from libxl to libxc
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (8 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 09/26] tools/libxl: introduce enum type libxl_checkpointed_stream Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 16:33 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state Yang Hongyang
` (15 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
Pass checkpointed_stream from libxl to libxc.
It won't affact legacy migration because legacy migration
won't use this param.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/libxc/include/xenguest.h | 9 ++++++---
tools/libxc/xc_domain_save.c | 6 ++++--
tools/libxc/xc_nomigrate.c | 3 ++-
tools/libxc/xc_sr_common.h | 2 +-
tools/libxc/xc_sr_save.c | 5 +++--
tools/libxl/libxl.c | 2 ++
tools/libxl/libxl_dom_save.c | 12 +++++++++---
tools/libxl/libxl_internal.h | 1 +
tools/libxl/libxl_save_callout.c | 2 +-
tools/libxl/libxl_save_helper.c | 3 ++-
10 files changed, 31 insertions(+), 14 deletions(-)
diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
index b0d27ed..d7c8fe4 100644
--- a/tools/libxc/include/xenguest.h
+++ b/tools/libxc/include/xenguest.h
@@ -30,7 +30,6 @@
#define XCFLAGS_HVM (1 << 2)
#define XCFLAGS_STDVGA (1 << 3)
#define XCFLAGS_CHECKPOINT_COMPRESS (1 << 4)
-#define XCFLAGS_CHECKPOINTED (1 << 5)
#define X86_64_B_SIZE 64
#define X86_32_B_SIZE 32
@@ -85,16 +84,20 @@ struct save_callbacks {
* @parm xch a handle to an open hypervisor interface
* @parm fd the file descriptor to save a domain to
* @parm dom the id of the domain
+ * @parm checkpointed_stream non-zero if the far end of the stream is using
+ * checkpointing
* @return 0 on success, -1 on failure
*/
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */,
- struct save_callbacks* callbacks, int hvm);
+ struct save_callbacks* callbacks, int hvm,
+ int checkpointed_stream);
/* Domain Save v2 */
int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags,
- struct save_callbacks* callbacks, int hvm);
+ struct save_callbacks* callbacks, int hvm,
+ int checkpointed_stream);
/* callbacks provided by xc_domain_restore */
struct restore_callbacks {
diff --git a/tools/libxc/xc_domain_save.c b/tools/libxc/xc_domain_save.c
index 301e770..c76980e 100644
--- a/tools/libxc/xc_domain_save.c
+++ b/tools/libxc/xc_domain_save.c
@@ -802,7 +802,8 @@ static int save_tsc_info(xc_interface *xch, uint32_t dom, int io_fd)
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags,
- struct save_callbacks* callbacks, int hvm)
+ struct save_callbacks* callbacks, int hvm,
+ int checkpointed_stream)
{
xc_dominfo_t info;
DECLARE_DOMCTL;
@@ -897,7 +898,8 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iter
if ( getenv("XG_MIGRATION_V2") )
{
return xc_domain_save2(xch, io_fd, dom, max_iters,
- max_factor, flags, callbacks, hvm);
+ max_factor, flags, callbacks, hvm,
+ checkpointed_stream);
}
DPRINTF("%s: starting save of domid %u", __func__, dom);
diff --git a/tools/libxc/xc_nomigrate.c b/tools/libxc/xc_nomigrate.c
index 76978a0..374d5bf 100644
--- a/tools/libxc/xc_nomigrate.c
+++ b/tools/libxc/xc_nomigrate.c
@@ -23,7 +23,8 @@
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags,
- struct save_callbacks* callbacks, int hvm)
+ struct save_callbacks* callbacks, int hvm,
+ int checkpointed_stream)
{
errno = ENOSYS;
return -1;
diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
index 42af55b..04f32f5 100644
--- a/tools/libxc/xc_sr_common.h
+++ b/tools/libxc/xc_sr_common.h
@@ -175,7 +175,7 @@ struct xc_sr_context
bool live;
/* Plain VM, or checkpoints over time. */
- bool checkpointed;
+ int checkpointed;
/* Further debugging information in the stream. */
bool debug;
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index d63b783..6102b66 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -820,7 +820,8 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
uint32_t max_iters, uint32_t max_factor, uint32_t flags,
- struct save_callbacks* callbacks, int hvm)
+ struct save_callbacks* callbacks, int hvm,
+ int checkpointed_stream)
{
xen_pfn_t nr_pfns;
struct xc_sr_context ctx =
@@ -833,7 +834,7 @@ int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
ctx.save.callbacks = callbacks;
ctx.save.live = !!(flags & XCFLAGS_LIVE);
ctx.save.debug = !!(flags & XCFLAGS_DEBUG);
- ctx.save.checkpointed = !!(flags & XCFLAGS_CHECKPOINTED);
+ ctx.save.checkpointed = checkpointed_stream;
/*
* TODO: Find some time to better tweak the live migration algorithm.
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 8b63eff..2ca59ea 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -837,6 +837,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
dss->live = 1;
dss->debug = 0;
dss->remus = info;
+ dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_REMUS;
assert(info);
@@ -891,6 +892,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
dss->type = type;
dss->live = flags & LIBXL_SUSPEND_LIVE;
dss->debug = flags & LIBXL_SUSPEND_DEBUG;
+ dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
libxl__domain_save(egc, dss);
return AO_INPROGRESS;
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 93061c7..8fe1625 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -357,6 +357,13 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
&dss->shs.callbacks.save.a;
libxl__domain_suspend_state *dsps = &dss->dsps;
+ if (dss->checkpointed_stream && !r_info) {
+ LOG(ERROR, "Migration stream is checkpointed, but there's no "
+ "checkpoint info!");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+
logdirty_init(&dss->logdirty);
libxl__xswait_init(&dsps->pvcontrol);
libxl__ev_evtchn_init(&dsps->guest_evtchn);
@@ -388,9 +395,8 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
dsps->guest_responded = 0;
dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
- if (r_info != NULL) {
+ if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
dss->interval = r_info->interval;
- dss->xcflags |= XCFLAGS_CHECKPOINTED;
if (libxl_defbool_val(r_info->compression))
dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
}
@@ -414,7 +420,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
}
memset(callbacks, 0, sizeof(*callbacks));
- if (r_info != NULL) {
+ if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
callbacks->suspend = libxl__remus_domain_suspend_callback;
callbacks->postcopy = libxl__remus_domain_resume_callback;
callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 4fdf9d8..573c844 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2941,6 +2941,7 @@ struct libxl__domain_save_state {
libxl_domain_type type;
int live;
int debug;
+ int checkpointed_stream;
const libxl_domain_remus_info *remus;
/* private */
libxl__domain_suspend_state dsps;
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index a5039e6..12cc57e 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -83,7 +83,7 @@ void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
const unsigned long argnums[] = {
dss->domid, 0, 0, dss->xcflags, dss->hvm,
- cbflags,
+ cbflags, dss->checkpointed_stream,
};
dss->shs.ao = ao;
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
index b8f0390..9d2cbf6 100644
--- a/tools/libxl/libxl_save_helper.c
+++ b/tools/libxl/libxl_save_helper.c
@@ -248,6 +248,7 @@ int main(int argc, char **argv)
uint32_t flags = strtoul(NEXTARG,0,10);
int hvm = atoi(NEXTARG);
unsigned cbflags = strtoul(NEXTARG,0,10);
+ int checkpointed_stream = strtoul(NEXTARG,0,10);
assert(!*++argv);
helper_setcallbacks_save(&helper_save_callbacks, cbflags);
@@ -256,7 +257,7 @@ int main(int argc, char **argv)
setup_signals(save_signal_handler);
r = xc_domain_save2(xch, io_fd, dom, max_iters, max_factor, flags,
- &helper_save_callbacks, hvm);
+ &helper_save_callbacks, hvm, checkpointed_stream);
complete(r);
} else if (!strcmp(mode,"--restore-domain")) {
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 10/26] migration/save: pass checkpointed_stream from libxl to libxc
2015-06-25 6:25 ` [PATCH v3 COLOPre 10/26] migration/save: pass checkpointed_stream from libxl to libxc Yang Hongyang
@ 2015-06-29 16:33 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 16:33 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Pass checkpointed_stream from libxl to libxc.
Hrm, so now we seem to have a purely libxl internal usage of this new
enum, but still nothing crossing the libxl.h boundary. I suspect I've
missed something!
> It won't affact legacy migration because legacy migration
> won't use this param.
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> CC: Ian Campbell <Ian.Campbell@citrix.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> CC: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> tools/libxc/include/xenguest.h | 9 ++++++---
> tools/libxc/xc_domain_save.c | 6 ++++--
> tools/libxc/xc_nomigrate.c | 3 ++-
> tools/libxc/xc_sr_common.h | 2 +-
> tools/libxc/xc_sr_save.c | 5 +++--
> tools/libxl/libxl.c | 2 ++
> tools/libxl/libxl_dom_save.c | 12 +++++++++---
> tools/libxl/libxl_internal.h | 1 +
> tools/libxl/libxl_save_callout.c | 2 +-
> tools/libxl/libxl_save_helper.c | 3 ++-
> 10 files changed, 31 insertions(+), 14 deletions(-)
>
> diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
> index b0d27ed..d7c8fe4 100644
> --- a/tools/libxc/include/xenguest.h
> +++ b/tools/libxc/include/xenguest.h
> @@ -30,7 +30,6 @@
> #define XCFLAGS_HVM (1 << 2)
> #define XCFLAGS_STDVGA (1 << 3)
> #define XCFLAGS_CHECKPOINT_COMPRESS (1 << 4)
> -#define XCFLAGS_CHECKPOINTED (1 << 5)
>
> #define X86_64_B_SIZE 64
> #define X86_32_B_SIZE 32
> @@ -85,16 +84,20 @@ struct save_callbacks {
> * @parm xch a handle to an open hypervisor interface
> * @parm fd the file descriptor to save a domain to
> * @parm dom the id of the domain
> + * @parm checkpointed_stream non-zero if the far end of the stream is using
> + * checkpointing
> * @return 0 on success, -1 on failure
> */
> int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
> uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */,
> - struct save_callbacks* callbacks, int hvm);
> + struct save_callbacks* callbacks, int hvm,
> + int checkpointed_stream);
>
> /* Domain Save v2 */
> int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
> uint32_t max_factor, uint32_t flags,
> - struct save_callbacks* callbacks, int hvm);
> + struct save_callbacks* callbacks, int hvm,
> + int checkpointed_stream);
>
> /* callbacks provided by xc_domain_restore */
> struct restore_callbacks {
> diff --git a/tools/libxc/xc_domain_save.c b/tools/libxc/xc_domain_save.c
> index 301e770..c76980e 100644
> --- a/tools/libxc/xc_domain_save.c
> +++ b/tools/libxc/xc_domain_save.c
> @@ -802,7 +802,8 @@ static int save_tsc_info(xc_interface *xch, uint32_t dom, int io_fd)
>
> int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
> uint32_t max_factor, uint32_t flags,
> - struct save_callbacks* callbacks, int hvm)
> + struct save_callbacks* callbacks, int hvm,
> + int checkpointed_stream)
> {
> xc_dominfo_t info;
> DECLARE_DOMCTL;
> @@ -897,7 +898,8 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iter
> if ( getenv("XG_MIGRATION_V2") )
> {
> return xc_domain_save2(xch, io_fd, dom, max_iters,
> - max_factor, flags, callbacks, hvm);
> + max_factor, flags, callbacks, hvm,
> + checkpointed_stream);
> }
>
> DPRINTF("%s: starting save of domid %u", __func__, dom);
> diff --git a/tools/libxc/xc_nomigrate.c b/tools/libxc/xc_nomigrate.c
> index 76978a0..374d5bf 100644
> --- a/tools/libxc/xc_nomigrate.c
> +++ b/tools/libxc/xc_nomigrate.c
> @@ -23,7 +23,8 @@
>
> int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
> uint32_t max_factor, uint32_t flags,
> - struct save_callbacks* callbacks, int hvm)
> + struct save_callbacks* callbacks, int hvm,
> + int checkpointed_stream)
> {
> errno = ENOSYS;
> return -1;
> diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
> index 42af55b..04f32f5 100644
> --- a/tools/libxc/xc_sr_common.h
> +++ b/tools/libxc/xc_sr_common.h
> @@ -175,7 +175,7 @@ struct xc_sr_context
> bool live;
>
> /* Plain VM, or checkpoints over time. */
> - bool checkpointed;
> + int checkpointed;
>
> /* Further debugging information in the stream. */
> bool debug;
> diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
> index d63b783..6102b66 100644
> --- a/tools/libxc/xc_sr_save.c
> +++ b/tools/libxc/xc_sr_save.c
> @@ -820,7 +820,8 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
>
> int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
> uint32_t max_iters, uint32_t max_factor, uint32_t flags,
> - struct save_callbacks* callbacks, int hvm)
> + struct save_callbacks* callbacks, int hvm,
> + int checkpointed_stream)
> {
> xen_pfn_t nr_pfns;
> struct xc_sr_context ctx =
> @@ -833,7 +834,7 @@ int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
> ctx.save.callbacks = callbacks;
> ctx.save.live = !!(flags & XCFLAGS_LIVE);
> ctx.save.debug = !!(flags & XCFLAGS_DEBUG);
> - ctx.save.checkpointed = !!(flags & XCFLAGS_CHECKPOINTED);
> + ctx.save.checkpointed = checkpointed_stream;
>
> /*
> * TODO: Find some time to better tweak the live migration algorithm.
> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> index 8b63eff..2ca59ea 100644
> --- a/tools/libxl/libxl.c
> +++ b/tools/libxl/libxl.c
> @@ -837,6 +837,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
> dss->live = 1;
> dss->debug = 0;
> dss->remus = info;
> + dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_REMUS;
>
> assert(info);
>
> @@ -891,6 +892,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
> dss->type = type;
> dss->live = flags & LIBXL_SUSPEND_LIVE;
> dss->debug = flags & LIBXL_SUSPEND_DEBUG;
> + dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
>
> libxl__domain_save(egc, dss);
> return AO_INPROGRESS;
> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
> index 93061c7..8fe1625 100644
> --- a/tools/libxl/libxl_dom_save.c
> +++ b/tools/libxl/libxl_dom_save.c
> @@ -357,6 +357,13 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
> &dss->shs.callbacks.save.a;
> libxl__domain_suspend_state *dsps = &dss->dsps;
>
> + if (dss->checkpointed_stream && !r_info) {
> + LOG(ERROR, "Migration stream is checkpointed, but there's no "
> + "checkpoint info!");
> + rc = ERROR_FAIL;
> + goto out;
> + }
> +
> logdirty_init(&dss->logdirty);
> libxl__xswait_init(&dsps->pvcontrol);
> libxl__ev_evtchn_init(&dsps->guest_evtchn);
> @@ -388,9 +395,8 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
> dsps->guest_responded = 0;
> dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
>
> - if (r_info != NULL) {
> + if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
> dss->interval = r_info->interval;
> - dss->xcflags |= XCFLAGS_CHECKPOINTED;
> if (libxl_defbool_val(r_info->compression))
> dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
> }
> @@ -414,7 +420,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
> }
>
> memset(callbacks, 0, sizeof(*callbacks));
> - if (r_info != NULL) {
> + if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
> callbacks->suspend = libxl__remus_domain_suspend_callback;
> callbacks->postcopy = libxl__remus_domain_resume_callback;
> callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback;
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 4fdf9d8..573c844 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -2941,6 +2941,7 @@ struct libxl__domain_save_state {
> libxl_domain_type type;
> int live;
> int debug;
> + int checkpointed_stream;
> const libxl_domain_remus_info *remus;
> /* private */
> libxl__domain_suspend_state dsps;
> diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
> index a5039e6..12cc57e 100644
> --- a/tools/libxl/libxl_save_callout.c
> +++ b/tools/libxl/libxl_save_callout.c
> @@ -83,7 +83,7 @@ void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>
> const unsigned long argnums[] = {
> dss->domid, 0, 0, dss->xcflags, dss->hvm,
> - cbflags,
> + cbflags, dss->checkpointed_stream,
> };
>
> dss->shs.ao = ao;
> diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
> index b8f0390..9d2cbf6 100644
> --- a/tools/libxl/libxl_save_helper.c
> +++ b/tools/libxl/libxl_save_helper.c
> @@ -248,6 +248,7 @@ int main(int argc, char **argv)
> uint32_t flags = strtoul(NEXTARG,0,10);
> int hvm = atoi(NEXTARG);
> unsigned cbflags = strtoul(NEXTARG,0,10);
> + int checkpointed_stream = strtoul(NEXTARG,0,10);
> assert(!*++argv);
>
> helper_setcallbacks_save(&helper_save_callbacks, cbflags);
> @@ -256,7 +257,7 @@ int main(int argc, char **argv)
> setup_signals(save_signal_handler);
>
> r = xc_domain_save2(xch, io_fd, dom, max_iters, max_factor, flags,
> - &helper_save_callbacks, hvm);
> + &helper_save_callbacks, hvm, checkpointed_stream);
> complete(r);
>
> } else if (!strcmp(mode,"--restore-domain")) {
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (9 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 10/26] migration/save: pass checkpointed_stream from libxl to libxc Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-29 16:38 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen Yang Hongyang
` (14 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, Anthony Perard, guijianfeng, rshriram, ian.jackson
Secondary vm is running in colo mode. So we will do
the following things again and again:
1. suspend both primay vm and secondary vm
2. sync the state
3. resume both primary vm and secondary vm
We will send qemu's state each time in step2, and
slave's qemu should read it each time before resuming
secondary vm. Introduce a new API libxl__domain_restore()
to do it. This API should be called before resuming
secondary vm.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Cc: Anthony Perard <anthony.perard@citrix.com>
---
tools/libxl/libxl_dom_save.c | 49 ++++++++++++++++++++++++++++++++++++++++++++
tools/libxl/libxl_internal.h | 3 +++
tools/libxl/libxl_qmp.c | 10 +++++++++
3 files changed, 62 insertions(+)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 8fe1625..0ad2894 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -639,6 +639,55 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
}
return 0;
}
+
+static int libxl__domain_restore_device_model(libxl__gc *gc, uint32_t domid);
+
+int libxl__domain_restore(libxl__gc *gc, uint32_t domid)
+{
+ int rc = 0;
+
+ libxl_domain_type type = libxl__domain_type(gc, domid);
+ if (type != LIBXL_DOMAIN_TYPE_HVM) {
+ rc = ERROR_FAIL;
+ goto out;
+ }
+
+ rc = libxl__domain_restore_device_model(gc, domid);
+ if (rc)
+ LOG(ERROR, "failed to restore device mode for domain %u:%d",
+ domid, rc);
+out:
+ return rc;
+}
+
+static int libxl__domain_restore_device_model(libxl__gc *gc, uint32_t domid)
+{
+ char *state_file;
+ int rc;
+
+ switch (libxl__device_model_version_running(gc, domid)) {
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
+ /* not supported now */
+ rc = ERROR_INVAL;
+ break;
+ case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+ /*
+ * This function may be called too many times for the same gc,
+ * so we use NOGC, and free the memory before return to avoid
+ * OOM.
+ */
+ state_file = libxl__sprintf(NOGC,
+ XC_DEVICE_MODEL_RESTORE_FILE".%d",
+ domid);
+ rc = libxl__qmp_restore(gc, domid, state_file);
+ free(state_file);
+ break;
+ default:
+ rc = ERROR_INVAL;
+ }
+
+ return rc;
+}
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 573c844..0adc9b4 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1043,6 +1043,7 @@ _hidden int libxl__userdata_store(libxl__gc *gc, uint32_t domid,
const char *userdata_userid,
const uint8_t *data, int datalen);
+_hidden int libxl__domain_restore(libxl__gc *gc, uint32_t domid);
_hidden int libxl__domain_resume(libxl__gc *gc, uint32_t domid,
int suspend_cancel);
@@ -1654,6 +1655,8 @@ _hidden int libxl__qmp_stop(libxl__gc *gc, int domid);
_hidden int libxl__qmp_resume(libxl__gc *gc, int domid);
/* Save current QEMU state into fd. */
_hidden int libxl__qmp_save(libxl__gc *gc, int domid, const char *filename);
+/* Load current QEMU state from fd. */
+_hidden int libxl__qmp_restore(libxl__gc *gc, int domid, const char *filename);
/* Set dirty bitmap logging status */
_hidden int libxl__qmp_set_global_dirty_log(libxl__gc *gc, int domid, bool enable);
_hidden int libxl__qmp_insert_cdrom(libxl__gc *gc, int domid, const libxl_device_disk *disk);
diff --git a/tools/libxl/libxl_qmp.c b/tools/libxl/libxl_qmp.c
index 9aa7e2e..a6f1a21 100644
--- a/tools/libxl/libxl_qmp.c
+++ b/tools/libxl/libxl_qmp.c
@@ -892,6 +892,16 @@ int libxl__qmp_save(libxl__gc *gc, int domid, const char *filename)
NULL, NULL);
}
+int libxl__qmp_restore(libxl__gc *gc, int domid, const char *state_file)
+{
+ libxl__json_object *args = NULL;
+
+ qmp_parameters_add_string(gc, &args, "filename", state_file);
+
+ return qmp_run_command(gc, domid, "xen-load-devices-state", args,
+ NULL, NULL);
+}
+
static int qmp_change(libxl__gc *gc, libxl__qmp_handler *qmp,
char *device, char *target, char *arg)
{
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state
2015-06-25 6:25 ` [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state Yang Hongyang
@ 2015-06-29 16:38 ` Ian Campbell
2015-06-30 10:04 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-29 16:38 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, rshriram, guijianfeng, Anthony Perard, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Secondary vm is running in colo mode. So we will do
> the following things again and again:
> 1. suspend both primay vm and secondary vm
> 2. sync the state
> 3. resume both primary vm and secondary vm
> We will send qemu's state each time in step2, and
> slave's qemu should read it each time before resuming
> secondary vm. Introduce a new API libxl__domain_restore()
> to do it. This API should be called before resuming
> secondary vm.
I think before this patch the state was passed to qemu as a parameter
when it was launched, is that correct? If so then that would be worth
mentioning for completeness.
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> Cc: Anthony Perard <anthony.perard@citrix.com>
> ---
> tools/libxl/libxl_dom_save.c | 49 ++++++++++++++++++++++++++++++++++++++++++++
> tools/libxl/libxl_internal.h | 3 +++
> tools/libxl/libxl_qmp.c | 10 +++++++++
> 3 files changed, 62 insertions(+)
>
> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
> index 8fe1625..0ad2894 100644
> --- a/tools/libxl/libxl_dom_save.c
> +++ b/tools/libxl/libxl_dom_save.c
> @@ -639,6 +639,55 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
> }
> return 0;
> }
> +
> +static int libxl__domain_restore_device_model(libxl__gc *gc, uint32_t domid);
> +
> +int libxl__domain_restore(libxl__gc *gc, uint32_t domid)
We don't have any libxl__domain_save counterpart, but we do have
libxl__domain_save_device_model, so I wonder if the upcoming callers
ought to just call that direct? Especially given that this function
isn't any kind of generic domain restore, but has rather specific
functionality (in particular it fails for PV guests).
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state
2015-06-29 16:38 ` Ian Campbell
@ 2015-06-30 10:04 ` Yang Hongyang
2015-06-30 10:54 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-30 10:04 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, rshriram, guijianfeng, Anthony Perard, ian.jackson
On 06/30/2015 12:38 AM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Secondary vm is running in colo mode. So we will do
>> the following things again and again:
>> 1. suspend both primay vm and secondary vm
>> 2. sync the state
>> 3. resume both primary vm and secondary vm
>> We will send qemu's state each time in step2, and
>> slave's qemu should read it each time before resuming
>> secondary vm. Introduce a new API libxl__domain_restore()
>> to do it. This API should be called before resuming
>> secondary vm.
>
> I think before this patch the state was passed to qemu as a parameter
> when it was launched, is that correct? If so then that would be worth
> mentioning for completeness.
Inaccurate I think. What you said before is the normal migration, in that
case, yes, the state was passed to qemu as a parameter. With COLO, the
first step is live migration, so the state is still passed to qemu as a
parameter when the live migration ended. The new introduced API only used
when we need to restore the DM state after a checkpoint, at this point,
guest QEMU already started, we can not pass the state as a parameter like
we do on first boot, so we introduce this API to restore the state after
QEMU has started.
>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
>> Cc: Anthony Perard <anthony.perard@citrix.com>
>> ---
>> tools/libxl/libxl_dom_save.c | 49 ++++++++++++++++++++++++++++++++++++++++++++
>> tools/libxl/libxl_internal.h | 3 +++
>> tools/libxl/libxl_qmp.c | 10 +++++++++
>> 3 files changed, 62 insertions(+)
>>
>> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
>> index 8fe1625..0ad2894 100644
>> --- a/tools/libxl/libxl_dom_save.c
>> +++ b/tools/libxl/libxl_dom_save.c
>> @@ -639,6 +639,55 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
>> }
>> return 0;
>> }
>> +
>> +static int libxl__domain_restore_device_model(libxl__gc *gc, uint32_t domid);
>> +
>> +int libxl__domain_restore(libxl__gc *gc, uint32_t domid)
>
> We don't have any libxl__domain_save counterpart, but we do have
> libxl__domain_save_device_model, so I wonder if the upcoming callers
> ought to just call that direct? Especially given that this function
> isn't any kind of generic domain restore, but has rather specific
> functionality (in particular it fails for PV guests).
Maybe we just introduce libxl__domain_restore_device_model() and call
this when needed, discard the new libxl__domain_restore() API, what do
you think?
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state
2015-06-30 10:04 ` Yang Hongyang
@ 2015-06-30 10:54 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:54 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, Anthony Perard, guijianfeng, rshriram, ian.jackson
On Tue, 2015-06-30 at 18:04 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 12:38 AM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> Secondary vm is running in colo mode. So we will do
> >> the following things again and again:
> >> 1. suspend both primay vm and secondary vm
> >> 2. sync the state
> >> 3. resume both primary vm and secondary vm
> >> We will send qemu's state each time in step2, and
> >> slave's qemu should read it each time before resuming
> >> secondary vm. Introduce a new API libxl__domain_restore()
> >> to do it. This API should be called before resuming
> >> secondary vm.
> >
> > I think before this patch the state was passed to qemu as a parameter
> > when it was launched, is that correct? If so then that would be worth
> > mentioning for completeness.
>
> Inaccurate I think. What you said before is the normal migration, in that
> case, yes, the state was passed to qemu as a parameter.
Which is what I think I said.
> With COLO, the
> first step is live migration, so the state is still passed to qemu as a
> parameter when the live migration ended. The new introduced API only used
> when we need to restore the DM state after a checkpoint, at this point,
> guest QEMU already started, we can not pass the state as a parameter like
> we do on first boot, so we introduce this API to restore the state after
> QEMU has started.
This sort of explanation would be good to have in the commit message,
especially due to the subtlety around first and subsequent checkpoints.
> >> +int libxl__domain_restore(libxl__gc *gc, uint32_t domid)
> >
> > We don't have any libxl__domain_save counterpart, but we do have
> > libxl__domain_save_device_model, so I wonder if the upcoming callers
> > ought to just call that direct? Especially given that this function
> > isn't any kind of generic domain restore, but has rather specific
> > functionality (in particular it fails for PV guests).
>
> Maybe we just introduce libxl__domain_restore_device_model() and call
> this when needed, discard the new libxl__domain_restore() API, what do
> you think?
Yes, that's what I think I meant to say (but didn't).
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (10 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 11/26] tools/libxl: introduce a new API libxl__domain_restore() to load qemu state Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:00 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 13/26] tools/libxl: introduce libxl__domain_common_switch_qemu_logdirty() Yang Hongyang
` (13 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
Currently, libxl__domain_unpause() only supports
qemu-xen-traditional. Update it to support qemu-xen.
We use libxl__domain_resume_device_model to unpause guest dm.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxl/libxl.c | 15 +++++----------
tools/libxl/libxl_dom_suspend.c | 15 ++++++++++++---
tools/libxl/libxl_internal.h | 1 +
3 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 2ca59ea..59e2dfe 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -938,8 +938,6 @@ out:
int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
{
GC_INIT(ctx);
- char *path;
- char *state;
int ret, rc = 0;
libxl_domain_type type = libxl__domain_type(gc, domid);
@@ -949,14 +947,11 @@ int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
}
if (type == LIBXL_DOMAIN_TYPE_HVM) {
- uint32_t dm_domid = libxl_get_stubdom_id(ctx, domid);
-
- path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
- state = libxl__xs_read(gc, XBT_NULL, path);
- if (state != NULL && !strcmp(state, "paused")) {
- libxl__qemu_traditional_cmd(gc, domid, "continue");
- libxl__wait_for_device_model_deprecated(gc, domid, "running",
- NULL, NULL, NULL);
+ rc = libxl__domain_resume_device_model(gc, domid);
+ if (rc < 0) {
+ LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to unpause device model "
+ "for domain %u:%d", domid, rc);
+ goto out;
}
}
ret = xc_domain_unpause(ctx->xch, domid);
diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
index 1c486c4..3edbd2e 100644
--- a/tools/libxl/libxl_dom_suspend.c
+++ b/tools/libxl/libxl_dom_suspend.c
@@ -376,13 +376,22 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
/*======================= Domain resume ========================*/
-static int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
+int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
{
+ char *path;
+ char *state;
switch (libxl__device_model_version_running(gc, domid)) {
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
- libxl__qemu_traditional_cmd(gc, domid, "continue");
- libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
+ uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
+
+ path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
+ state = libxl__xs_read(gc, XBT_NULL, path);
+ if (state != NULL && !strcmp(state, "paused")) {
+ libxl__qemu_traditional_cmd(gc, domid, "continue");
+ libxl__wait_for_device_model_deprecated(gc, domid, "running",
+ NULL, NULL, NULL);
+ }
break;
}
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 0adc9b4..6960280 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3326,6 +3326,7 @@ static inline bool libxl__save_helper_inuse(const libxl__save_helper_state *shs)
/* Each time the dm needs to be saved, we must call suspend and then save */
_hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
libxl__domain_suspend_state *dsps);
+_hidden int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid);
_hidden void libxl__domain_save_device_model(libxl__egc *egc,
libxl__domain_save_state *dss,
libxl__save_device_model_cb *callback);
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen
2015-06-25 6:25 ` [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen Yang Hongyang
@ 2015-06-30 10:00 ` Ian Campbell
2015-07-01 2:10 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:00 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Currently, libxl__domain_unpause() only supports
> qemu-xen-traditional. Update it to support qemu-xen.
> We use libxl__domain_resume_device_model to unpause guest dm.
There appears to be at least two significant semantic changes to the
libxl__domain_resume_device_model function (namely it now deals
internally with stubdom and checks the current status before doing
anything). What is the impact on the existing callers of those changes?
I think those changes should be two separate preparatory patches.
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
> tools/libxl/libxl.c | 15 +++++----------
> tools/libxl/libxl_dom_suspend.c | 15 ++++++++++++---
> tools/libxl/libxl_internal.h | 1 +
> 3 files changed, 18 insertions(+), 13 deletions(-)
>
> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> index 2ca59ea..59e2dfe 100644
> --- a/tools/libxl/libxl.c
> +++ b/tools/libxl/libxl.c
> @@ -938,8 +938,6 @@ out:
> int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
> {
> GC_INIT(ctx);
> - char *path;
> - char *state;
> int ret, rc = 0;
>
> libxl_domain_type type = libxl__domain_type(gc, domid);
> @@ -949,14 +947,11 @@ int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
> }
>
> if (type == LIBXL_DOMAIN_TYPE_HVM) {
> - uint32_t dm_domid = libxl_get_stubdom_id(ctx, domid);
> -
> - path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
> - state = libxl__xs_read(gc, XBT_NULL, path);
> - if (state != NULL && !strcmp(state, "paused")) {
> - libxl__qemu_traditional_cmd(gc, domid, "continue");
> - libxl__wait_for_device_model_deprecated(gc, domid, "running",
> - NULL, NULL, NULL);
> + rc = libxl__domain_resume_device_model(gc, domid);
> + if (rc < 0) {
> + LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to unpause device model "
> + "for domain %u:%d", domid, rc);
> + goto out;
> }
> }
> ret = xc_domain_unpause(ctx->xch, domid);
> diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
> index 1c486c4..3edbd2e 100644
> --- a/tools/libxl/libxl_dom_suspend.c
> +++ b/tools/libxl/libxl_dom_suspend.c
> @@ -376,13 +376,22 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
>
> /*======================= Domain resume ========================*/
>
> -static int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
> +int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
> {
> + char *path;
> + char *state;
>
> switch (libxl__device_model_version_running(gc, domid)) {
> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
> - libxl__qemu_traditional_cmd(gc, domid, "continue");
> - libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
> + uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
> +
> + path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
> + state = libxl__xs_read(gc, XBT_NULL, path);
> + if (state != NULL && !strcmp(state, "paused")) {
> + libxl__qemu_traditional_cmd(gc, domid, "continue");
> + libxl__wait_for_device_model_deprecated(gc, domid, "running",
> + NULL, NULL, NULL);
> + }
> break;
> }
> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 0adc9b4..6960280 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -3326,6 +3326,7 @@ static inline bool libxl__save_helper_inuse(const libxl__save_helper_state *shs)
> /* Each time the dm needs to be saved, we must call suspend and then save */
> _hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
> libxl__domain_suspend_state *dsps);
> +_hidden int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid);
> _hidden void libxl__domain_save_device_model(libxl__egc *egc,
> libxl__domain_save_state *dss,
> libxl__save_device_model_cb *callback);
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen
2015-06-30 10:00 ` Ian Campbell
@ 2015-07-01 2:10 ` Yang Hongyang
2015-07-01 10:38 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:10 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:00 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Currently, libxl__domain_unpause() only supports
>> qemu-xen-traditional. Update it to support qemu-xen.
>> We use libxl__domain_resume_device_model to unpause guest dm.
>
> There appears to be at least two significant semantic changes to the
> libxl__domain_resume_device_model function (namely it now deals
> internally with stubdom and checks the current status before doing
> anything). What is the impact on the existing callers of those changes?
This change was suggested by Wei on last round(v2), he thought it is an
enhancement to the existing callers.
>
> I think those changes should be two separate preparatory patches.
>
>
>>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> ---
>> tools/libxl/libxl.c | 15 +++++----------
>> tools/libxl/libxl_dom_suspend.c | 15 ++++++++++++---
>> tools/libxl/libxl_internal.h | 1 +
>> 3 files changed, 18 insertions(+), 13 deletions(-)
>>
>> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
>> index 2ca59ea..59e2dfe 100644
>> --- a/tools/libxl/libxl.c
>> +++ b/tools/libxl/libxl.c
>> @@ -938,8 +938,6 @@ out:
>> int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
>> {
>> GC_INIT(ctx);
>> - char *path;
>> - char *state;
>> int ret, rc = 0;
>>
>> libxl_domain_type type = libxl__domain_type(gc, domid);
>> @@ -949,14 +947,11 @@ int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
>> }
>>
>> if (type == LIBXL_DOMAIN_TYPE_HVM) {
>> - uint32_t dm_domid = libxl_get_stubdom_id(ctx, domid);
>> -
>> - path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
>> - state = libxl__xs_read(gc, XBT_NULL, path);
>> - if (state != NULL && !strcmp(state, "paused")) {
>> - libxl__qemu_traditional_cmd(gc, domid, "continue");
>> - libxl__wait_for_device_model_deprecated(gc, domid, "running",
>> - NULL, NULL, NULL);
>> + rc = libxl__domain_resume_device_model(gc, domid);
>> + if (rc < 0) {
>> + LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to unpause device model "
>> + "for domain %u:%d", domid, rc);
>> + goto out;
>> }
>> }
>> ret = xc_domain_unpause(ctx->xch, domid);
>> diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
>> index 1c486c4..3edbd2e 100644
>> --- a/tools/libxl/libxl_dom_suspend.c
>> +++ b/tools/libxl/libxl_dom_suspend.c
>> @@ -376,13 +376,22 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
>>
>> /*======================= Domain resume ========================*/
>>
>> -static int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>> +int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>> {
>> + char *path;
>> + char *state;
>>
>> switch (libxl__device_model_version_running(gc, domid)) {
>> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
>> - libxl__qemu_traditional_cmd(gc, domid, "continue");
>> - libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
>> + uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
>> +
>> + path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
>> + state = libxl__xs_read(gc, XBT_NULL, path);
>> + if (state != NULL && !strcmp(state, "paused")) {
>> + libxl__qemu_traditional_cmd(gc, domid, "continue");
>> + libxl__wait_for_device_model_deprecated(gc, domid, "running",
>> + NULL, NULL, NULL);
>> + }
>> break;
>> }
>> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index 0adc9b4..6960280 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
>> @@ -3326,6 +3326,7 @@ static inline bool libxl__save_helper_inuse(const libxl__save_helper_state *shs)
>> /* Each time the dm needs to be saved, we must call suspend and then save */
>> _hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
>> libxl__domain_suspend_state *dsps);
>> +_hidden int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid);
>> _hidden void libxl__domain_save_device_model(libxl__egc *egc,
>> libxl__domain_save_state *dss,
>> libxl__save_device_model_cb *callback);
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen
2015-07-01 2:10 ` Yang Hongyang
@ 2015-07-01 10:38 ` Ian Campbell
2015-07-01 13:38 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:38 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Wed, 2015-07-01 at 10:10 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 06:00 PM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> Currently, libxl__domain_unpause() only supports
> >> qemu-xen-traditional. Update it to support qemu-xen.
> >> We use libxl__domain_resume_device_model to unpause guest dm.
> >
> > There appears to be at least two significant semantic changes to the
> > libxl__domain_resume_device_model function (namely it now deals
> > internally with stubdom and checks the current status before doing
> > anything). What is the impact on the existing callers of those changes?
>
> This change was suggested by Wei on last round(v2), he thought it is an
> enhancement to the existing callers.
I don't think that contradicts what I've said. If multiple improvements
are needed to make libxl__domain_resume_device_model suitable for your
uses then there should be several patches. This will also give you an
opportunity to explain in each commit message why there is no impact (or
a positive impact) on the existing callers.
>
> >
> > I think those changes should be two separate preparatory patches.
> >
> >
> >>
> >> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> >> ---
> >> tools/libxl/libxl.c | 15 +++++----------
> >> tools/libxl/libxl_dom_suspend.c | 15 ++++++++++++---
> >> tools/libxl/libxl_internal.h | 1 +
> >> 3 files changed, 18 insertions(+), 13 deletions(-)
> >>
> >> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> >> index 2ca59ea..59e2dfe 100644
> >> --- a/tools/libxl/libxl.c
> >> +++ b/tools/libxl/libxl.c
> >> @@ -938,8 +938,6 @@ out:
> >> int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
> >> {
> >> GC_INIT(ctx);
> >> - char *path;
> >> - char *state;
> >> int ret, rc = 0;
> >>
> >> libxl_domain_type type = libxl__domain_type(gc, domid);
> >> @@ -949,14 +947,11 @@ int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
> >> }
> >>
> >> if (type == LIBXL_DOMAIN_TYPE_HVM) {
> >> - uint32_t dm_domid = libxl_get_stubdom_id(ctx, domid);
> >> -
> >> - path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
> >> - state = libxl__xs_read(gc, XBT_NULL, path);
> >> - if (state != NULL && !strcmp(state, "paused")) {
> >> - libxl__qemu_traditional_cmd(gc, domid, "continue");
> >> - libxl__wait_for_device_model_deprecated(gc, domid, "running",
> >> - NULL, NULL, NULL);
> >> + rc = libxl__domain_resume_device_model(gc, domid);
> >> + if (rc < 0) {
> >> + LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to unpause device model "
> >> + "for domain %u:%d", domid, rc);
> >> + goto out;
> >> }
> >> }
> >> ret = xc_domain_unpause(ctx->xch, domid);
> >> diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
> >> index 1c486c4..3edbd2e 100644
> >> --- a/tools/libxl/libxl_dom_suspend.c
> >> +++ b/tools/libxl/libxl_dom_suspend.c
> >> @@ -376,13 +376,22 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
> >>
> >> /*======================= Domain resume ========================*/
> >>
> >> -static int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
> >> +int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
> >> {
> >> + char *path;
> >> + char *state;
> >>
> >> switch (libxl__device_model_version_running(gc, domid)) {
> >> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
> >> - libxl__qemu_traditional_cmd(gc, domid, "continue");
> >> - libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
> >> + uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
> >> +
> >> + path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
> >> + state = libxl__xs_read(gc, XBT_NULL, path);
> >> + if (state != NULL && !strcmp(state, "paused")) {
> >> + libxl__qemu_traditional_cmd(gc, domid, "continue");
> >> + libxl__wait_for_device_model_deprecated(gc, domid, "running",
> >> + NULL, NULL, NULL);
> >> + }
> >> break;
> >> }
> >> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
> >> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> >> index 0adc9b4..6960280 100644
> >> --- a/tools/libxl/libxl_internal.h
> >> +++ b/tools/libxl/libxl_internal.h
> >> @@ -3326,6 +3326,7 @@ static inline bool libxl__save_helper_inuse(const libxl__save_helper_state *shs)
> >> /* Each time the dm needs to be saved, we must call suspend and then save */
> >> _hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
> >> libxl__domain_suspend_state *dsps);
> >> +_hidden int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid);
> >> _hidden void libxl__domain_save_device_model(libxl__egc *egc,
> >> libxl__domain_save_state *dss,
> >> libxl__save_device_model_cb *callback);
> >
> >
> > .
> >
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen
2015-07-01 10:38 ` Ian Campbell
@ 2015-07-01 13:38 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 13:38 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 07/01/2015 06:38 PM, Ian Campbell wrote:
> On Wed, 2015-07-01 at 10:10 +0800, Yang Hongyang wrote:
>>
>> On 06/30/2015 06:00 PM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>> Currently, libxl__domain_unpause() only supports
>>>> qemu-xen-traditional. Update it to support qemu-xen.
>>>> We use libxl__domain_resume_device_model to unpause guest dm.
>>>
>>> There appears to be at least two significant semantic changes to the
>>> libxl__domain_resume_device_model function (namely it now deals
>>> internally with stubdom and checks the current status before doing
>>> anything). What is the impact on the existing callers of those changes?
>>
>> This change was suggested by Wei on last round(v2), he thought it is an
>> enhancement to the existing callers.
>
> I don't think that contradicts what I've said. If multiple improvements
> are needed to make libxl__domain_resume_device_model suitable for your
> uses then there should be several patches. This will also give you an
> opportunity to explain in each commit message why there is no impact (or
> a positive impact) on the existing callers.
Yes, for this part, I agree with you, will separate it in the next version!
thank you!
>
>>
>>>
>>> I think those changes should be two separate preparatory patches.
>>>
>>>
>>>>
>>>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>>>> ---
>>>> tools/libxl/libxl.c | 15 +++++----------
>>>> tools/libxl/libxl_dom_suspend.c | 15 ++++++++++++---
>>>> tools/libxl/libxl_internal.h | 1 +
>>>> 3 files changed, 18 insertions(+), 13 deletions(-)
>>>>
>>>> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
>>>> index 2ca59ea..59e2dfe 100644
>>>> --- a/tools/libxl/libxl.c
>>>> +++ b/tools/libxl/libxl.c
>>>> @@ -938,8 +938,6 @@ out:
>>>> int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
>>>> {
>>>> GC_INIT(ctx);
>>>> - char *path;
>>>> - char *state;
>>>> int ret, rc = 0;
>>>>
>>>> libxl_domain_type type = libxl__domain_type(gc, domid);
>>>> @@ -949,14 +947,11 @@ int libxl_domain_unpause(libxl_ctx *ctx, uint32_t domid)
>>>> }
>>>>
>>>> if (type == LIBXL_DOMAIN_TYPE_HVM) {
>>>> - uint32_t dm_domid = libxl_get_stubdom_id(ctx, domid);
>>>> -
>>>> - path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
>>>> - state = libxl__xs_read(gc, XBT_NULL, path);
>>>> - if (state != NULL && !strcmp(state, "paused")) {
>>>> - libxl__qemu_traditional_cmd(gc, domid, "continue");
>>>> - libxl__wait_for_device_model_deprecated(gc, domid, "running",
>>>> - NULL, NULL, NULL);
>>>> + rc = libxl__domain_resume_device_model(gc, domid);
>>>> + if (rc < 0) {
>>>> + LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to unpause device model "
>>>> + "for domain %u:%d", domid, rc);
>>>> + goto out;
>>>> }
>>>> }
>>>> ret = xc_domain_unpause(ctx->xch, domid);
>>>> diff --git a/tools/libxl/libxl_dom_suspend.c b/tools/libxl/libxl_dom_suspend.c
>>>> index 1c486c4..3edbd2e 100644
>>>> --- a/tools/libxl/libxl_dom_suspend.c
>>>> +++ b/tools/libxl/libxl_dom_suspend.c
>>>> @@ -376,13 +376,22 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
>>>>
>>>> /*======================= Domain resume ========================*/
>>>>
>>>> -static int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>>>> +int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid)
>>>> {
>>>> + char *path;
>>>> + char *state;
>>>>
>>>> switch (libxl__device_model_version_running(gc, domid)) {
>>>> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
>>>> - libxl__qemu_traditional_cmd(gc, domid, "continue");
>>>> - libxl__wait_for_device_model_deprecated(gc, domid, "running", NULL, NULL, NULL);
>>>> + uint32_t dm_domid = libxl_get_stubdom_id(CTX, domid);
>>>> +
>>>> + path = libxl__device_model_xs_path(gc, dm_domid, domid, "/state");
>>>> + state = libxl__xs_read(gc, XBT_NULL, path);
>>>> + if (state != NULL && !strcmp(state, "paused")) {
>>>> + libxl__qemu_traditional_cmd(gc, domid, "continue");
>>>> + libxl__wait_for_device_model_deprecated(gc, domid, "running",
>>>> + NULL, NULL, NULL);
>>>> + }
>>>> break;
>>>> }
>>>> case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
>>>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>>>> index 0adc9b4..6960280 100644
>>>> --- a/tools/libxl/libxl_internal.h
>>>> +++ b/tools/libxl/libxl_internal.h
>>>> @@ -3326,6 +3326,7 @@ static inline bool libxl__save_helper_inuse(const libxl__save_helper_state *shs)
>>>> /* Each time the dm needs to be saved, we must call suspend and then save */
>>>> _hidden int libxl__domain_suspend_device_model(libxl__gc *gc,
>>>> libxl__domain_suspend_state *dsps);
>>>> +_hidden int libxl__domain_resume_device_model(libxl__gc *gc, uint32_t domid);
>>>> _hidden void libxl__domain_save_device_model(libxl__egc *egc,
>>>> libxl__domain_save_state *dss,
>>>> libxl__save_device_model_cb *callback);
>>>
>>>
>>> .
>>>
>>
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 13/26] tools/libxl: introduce libxl__domain_common_switch_qemu_logdirty()
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (11 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 12/26] tools/libxl: Update libxl_domain_unpause() to support qemu-xen Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 14/26] tools/libxl: export logdirty_init Yang Hongyang
` (12 subsequent siblings)
25 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
Secondary vm is running in colo mode, we need to send
secondary vm's dirty page information to master at checkpoint,
so we have to enable qemu logdirty on secondary.
libxl__domain_suspend_common_switch_qemu_logdirty() is to enable
qemu logdirty. But it uses domain_save_state, and calls
libxl__xc_domain_saverestore_async_callback_done()
before exits. This can not be used for secondary vm.
Update libxl__domain_suspend_common_switch_qemu_logdirty() to
introduce a new API libxl__domain_common_switch_qemu_logdirty().
This API only uses libxl__logdirty_switch, and calls
lds->callback before exits.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
tools/libxl/libxl_dom_save.c | 78 ++++++++++++++++++++++++++------------------
tools/libxl/libxl_internal.h | 8 +++++
2 files changed, 54 insertions(+), 32 deletions(-)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 0ad2894..5becc68 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -50,7 +50,7 @@ static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
const char *watch_path, const char *event_path);
static void switch_logdirty_done(libxl__egc *egc,
- libxl__domain_save_state *dss, int ok);
+ libxl__logdirty_switch *lds, int ok);
static void logdirty_init(libxl__logdirty_switch *lds)
{
@@ -60,13 +60,10 @@ static void logdirty_init(libxl__logdirty_switch *lds)
}
static void domain_suspend_switch_qemu_xen_traditional_logdirty
- (int domid, unsigned enable,
- libxl__save_helper_state *shs)
+ (libxl__egc *egc, int domid, unsigned enable,
+ libxl__logdirty_switch *lds)
{
- libxl__egc *egc = shs->egc;
- libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
- libxl__logdirty_switch *lds = &dss->logdirty;
- STATE_AO_GC(dss->ao);
+ STATE_AO_GC(lds->ao);
int rc;
xs_transaction_t t = 0;
const char *got;
@@ -128,64 +125,81 @@ static void domain_suspend_switch_qemu_xen_traditional_logdirty
out:
LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
libxl__xs_transaction_abort(gc, &t);
- switch_logdirty_done(egc,dss,-1);
+ switch_logdirty_done(egc,lds,-1);
}
static void domain_suspend_switch_qemu_xen_logdirty
- (int domid, unsigned enable,
- libxl__save_helper_state *shs)
+ (libxl__egc *egc, int domid, unsigned enable,
+ libxl__logdirty_switch *lds)
{
- libxl__egc *egc = shs->egc;
- libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
- STATE_AO_GC(dss->ao);
+ STATE_AO_GC(lds->ao);
int rc;
rc = libxl__qmp_set_global_dirty_log(gc, domid, enable);
if (!rc) {
- libxl__xc_domain_saverestore_async_callback_done(egc, shs, 0);
+ lds->callback(egc, lds, 0);
} else {
LOG(ERROR,"logdirty switch failed (rc=%d), aborting suspend",rc);
- libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
+ lds->callback(egc, lds, -1);
}
}
+static void libxl__domain_suspend_switch_qemu_logdirty_done
+ (libxl__egc *egc, libxl__logdirty_switch *lds, int rc)
+{
+ libxl__domain_save_state *dss = CONTAINER_OF(lds, *dss, logdirty);
+
+ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, rc);
+}
+
void libxl__domain_suspend_common_switch_qemu_logdirty
(int domid, unsigned enable, void *user)
{
libxl__save_helper_state *shs = user;
libxl__egc *egc = shs->egc;
libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
- STATE_AO_GC(dss->ao);
+
+ /* convenience aliases */
+ libxl__logdirty_switch *const lds = &dss->logdirty;
+
+ lds->callback = libxl__domain_suspend_switch_qemu_logdirty_done;
+ libxl__domain_common_switch_qemu_logdirty(egc, domid, enable, lds);
+}
+
+void libxl__domain_common_switch_qemu_logdirty(libxl__egc *egc,
+ int domid, unsigned enable,
+ libxl__logdirty_switch *lds)
+{
+ STATE_AO_GC(lds->ao);
switch (libxl__device_model_version_running(gc, domid)) {
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
- domain_suspend_switch_qemu_xen_traditional_logdirty(domid, enable, shs);
+ domain_suspend_switch_qemu_xen_traditional_logdirty(egc, domid, enable,
+ lds);
break;
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
- domain_suspend_switch_qemu_xen_logdirty(domid, enable, shs);
+ domain_suspend_switch_qemu_xen_logdirty(egc, domid, enable, lds);
break;
default:
LOG(ERROR,"logdirty switch failed"
", no valid device model version found, aborting suspend");
- libxl__xc_domain_saverestore_async_callback_done(egc, shs, -1);
+ lds->callback(egc, lds, -1);
}
}
static void switch_logdirty_timeout(libxl__egc *egc, libxl__ev_time *ev,
const struct timeval *requested_abs)
{
- libxl__domain_save_state *dss = CONTAINER_OF(ev, *dss, logdirty.timeout);
- STATE_AO_GC(dss->ao);
+ libxl__logdirty_switch *lds = CONTAINER_OF(ev, *lds, timeout);
+ STATE_AO_GC(lds->ao);
LOG(ERROR,"logdirty switch: wait for device model timed out");
- switch_logdirty_done(egc,dss,-1);
+ switch_logdirty_done(egc,lds,-1);
}
static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
const char *watch_path, const char *event_path)
{
- libxl__domain_save_state *dss =
- CONTAINER_OF(watch, *dss, logdirty.watch);
- libxl__logdirty_switch *lds = &dss->logdirty;
- STATE_AO_GC(dss->ao);
+ libxl__logdirty_switch *lds = CONTAINER_OF(watch, *lds, watch);
+ STATE_AO_GC(lds->ao);
const char *got;
xs_transaction_t t = 0;
int rc;
@@ -229,24 +243,23 @@ static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch *watch,
libxl__xs_transaction_abort(gc, &t);
if (!rc) {
- switch_logdirty_done(egc,dss,0);
+ switch_logdirty_done(egc,lds,0);
} else if (rc < 0) {
LOG(ERROR,"logdirty switch: failed (rc=%d)",rc);
- switch_logdirty_done(egc,dss,-1);
+ switch_logdirty_done(egc,lds,-1);
}
}
static void switch_logdirty_done(libxl__egc *egc,
- libxl__domain_save_state *dss,
+ libxl__logdirty_switch *lds,
int broke)
{
- STATE_AO_GC(dss->ao);
- libxl__logdirty_switch *lds = &dss->logdirty;
+ STATE_AO_GC(lds->ao);
libxl__ev_xswatch_deregister(gc, &lds->watch);
libxl__ev_time_deregister(gc, &lds->timeout);
- libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, broke);
+ lds->callback(egc, lds, broke);
}
static inline char *physmap_path(libxl__gc *gc, uint32_t dm_domid,
@@ -365,6 +378,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
}
logdirty_init(&dss->logdirty);
+ dss->logdirty.ao = ao;
libxl__xswait_init(&dsps->pvcontrol);
libxl__ev_evtchn_init(&dsps->guest_evtchn);
libxl__ev_xswatch_init(&dsps->guest_watch);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 6960280..b15c24a 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2909,6 +2909,11 @@ static inline bool libxl__stream_write_inuse(
}
typedef struct libxl__logdirty_switch {
+ /* set by caller of libxl__domain_common_switch_qemu_logdirty */
+ libxl__ao *ao;
+ void (*callback)(libxl__egc *egc, struct libxl__logdirty_switch *lds,
+ int rc);
+
const char *cmd;
const char *cmd_path;
const char *ret_path;
@@ -3300,6 +3305,9 @@ void libxl__xc_domain_saverestore_async_callback_done(libxl__egc *egc,
_hidden void libxl__domain_suspend_common_switch_qemu_logdirty
(int domid, unsigned int enable, void *data);
+_hidden void libxl__domain_common_switch_qemu_logdirty(libxl__egc *egc,
+ int domid, unsigned enable,
+ libxl__logdirty_switch *lds);
_hidden int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
uint32_t *len, void *data);
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* [PATCH v3 COLOPre 14/26] tools/libxl: export logdirty_init
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (12 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 13/26] tools/libxl: introduce libxl__domain_common_switch_qemu_logdirty() Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:01 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back Yang Hongyang
` (11 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
We need to enable logdirty on secondary, so we export logdirty_init
for internal use. Rename it to libxl__logdirty_init.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxl/libxl_dom_save.c | 4 ++--
tools/libxl/libxl_internal.h | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 5becc68..5797148 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -52,7 +52,7 @@ static void switch_logdirty_xswatch(libxl__egc *egc, libxl__ev_xswatch*,
static void switch_logdirty_done(libxl__egc *egc,
libxl__logdirty_switch *lds, int ok);
-static void logdirty_init(libxl__logdirty_switch *lds)
+void libxl__logdirty_init(libxl__logdirty_switch *lds)
{
lds->cmd_path = 0;
libxl__ev_xswatch_init(&lds->watch);
@@ -377,7 +377,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
goto out;
}
- logdirty_init(&dss->logdirty);
+ libxl__logdirty_init(&dss->logdirty);
dss->logdirty.ao = ao;
libxl__xswait_init(&dsps->pvcontrol);
libxl__ev_evtchn_init(&dsps->guest_evtchn);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index b15c24a..5f875ee 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2921,6 +2921,8 @@ typedef struct libxl__logdirty_switch {
libxl__ev_time timeout;
} libxl__logdirty_switch;
+_hidden void libxl__logdirty_init(libxl__logdirty_switch *lds);
+
struct libxl__domain_suspend_state {
/* set by caller of domain_suspend_callback_common */
libxl__ao *ao;
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 14/26] tools/libxl: export logdirty_init
2015-06-25 6:25 ` [PATCH v3 COLOPre 14/26] tools/libxl: export logdirty_init Yang Hongyang
@ 2015-06-30 10:01 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:01 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> We need to enable logdirty on secondary, so we export logdirty_init
> for internal use. Rename it to libxl__logdirty_init.
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (13 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 14/26] tools/libxl: export logdirty_init Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:07 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc Yang Hongyang
` (10 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
From: Wen Congyang <wency@cn.fujitsu.com>
In colo mode, slave needs to send data to master, but the io_fd
only can be written in master, and only can be read in slave.
Save recv_fd in domain_suspend_state, and send_fd in
domain_create_state.
Extend libxl_domain_create_restore API, add a send_fd param to
it.
Add LIBXL_HAVE_CREATE_RESTORE_SEND_FD to indicate the API change.
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxl/libxl.c | 2 +-
tools/libxl/libxl.h | 29 +++++++++++++++++++++++++++--
tools/libxl/libxl_create.c | 9 +++++----
tools/libxl/libxl_internal.h | 2 ++
tools/libxl/libxl_types.idl | 1 +
tools/libxl/xl_cmdimpl.c | 8 +++++++-
6 files changed, 43 insertions(+), 8 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 59e2dfe..a837ff2 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -832,7 +832,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
dss->callback = remus_failover_cb;
dss->domid = domid;
dss->fd = send_fd;
- /* TODO do something with recv_fd */
+ dss->recv_fd = recv_fd;
dss->type = type;
dss->live = 1;
dss->debug = 0;
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 0a7913b..cd595cf 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -604,6 +604,14 @@ typedef struct libxl__ctx libxl_ctx;
#define LIBXL_HAVE_DOMAIN_CREATE_RESTORE_PARAMS 1
/*
+ * LIBXL_HAVE_DOMAIN_CREATE_RESTORE_SEND_FD 1
+ *
+ * If this is defined, libxl_domain_create_restore()'s API has changed to
+ * include a send_fd param which used for libxl migration back channel.
+ */
+#define LIBXL_HAVE_DOMAIN_CREATE_RESTORE_SEND_FD 1
+
+/*
* LIBXL_HAVE_CREATEINFO_PVH
* If this is defined, then libxl supports creation of a PVH guest.
*/
@@ -981,7 +989,7 @@ int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
const libxl_asyncprogress_how *aop_console_how)
LIBXL_EXTERNAL_CALLERS_ONLY;
int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
- uint32_t *domid, int restore_fd,
+ uint32_t *domid, int restore_fd, int send_fd,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1002,7 +1010,7 @@ int static inline libxl_domain_create_restore_0x040200(
libxl_domain_restore_params_init(¶ms);
ret = libxl_domain_create_restore(
- ctx, d_config, domid, restore_fd, ¶ms, ao_how, aop_console_how);
+ ctx, d_config, domid, restore_fd, -1, ¶ms, ao_how, aop_console_how);
libxl_domain_restore_params_dispose(¶ms);
return ret;
@@ -1010,6 +1018,23 @@ int static inline libxl_domain_create_restore_0x040200(
#define libxl_domain_create_restore libxl_domain_create_restore_0x040200
+#elif defined(LIBXL_API_VERSION) && LIBXL_API_VERSION >= 0x040400 \
+ && LIBXL_API_VERSION < 0x040600
+
+int static inline libxl_domain_create_restore_0x040400(
+ libxl_ctx *ctx, libxl_domain_config *d_config,
+ uint32_t *domid, int restore_fd,
+ const libxl_domain_restore_params *params,
+ const libxl_asyncop_how *ao_how,
+ const libxl_asyncprogress_how *aop_console_how)
+ LIBXL_EXTERNAL_CALLERS_ONLY
+{
+ return libxl_domain_create_restore(ctx, d_config, domid, restore_fd,
+ -1, params, ao_how, aop_console_how);
+}
+
+#define libxl_domain_create_restore libxl_domain_create_restore_0x040400
+
#endif
/* A progress report will be made via ao_console_how, of type
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index dfea992..6fa5ab7 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1561,7 +1561,7 @@ static void domain_create_cb(libxl__egc *egc,
int rc, uint32_t domid);
static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
- uint32_t *domid, int restore_fd,
+ uint32_t *domid, int restore_fd, int send_fd,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1576,6 +1576,7 @@ static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
libxl_domain_config_copy(ctx, &cdcs->dcs.guest_config_saved, d_config);
cdcs->dcs.restore_fd = restore_fd;
if (params) cdcs->dcs.restore_params = *params;
+ cdcs->dcs.send_fd = send_fd;
cdcs->dcs.callback = domain_create_cb;
libxl__ao_progress_gethow(&cdcs->dcs.aop_console_how, aop_console_how);
cdcs->domid_out = domid;
@@ -1603,17 +1604,17 @@ int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
{
- return do_domain_create(ctx, d_config, domid, -1, NULL,
+ return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
ao_how, aop_console_how);
}
int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
- uint32_t *domid, int restore_fd,
+ uint32_t *domid, int restore_fd, int send_fd,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
{
- return do_domain_create(ctx, d_config, domid, restore_fd, params,
+ return do_domain_create(ctx, d_config, domid, restore_fd, send_fd, params,
ao_how, aop_console_how);
}
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 5f875ee..13d8271 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2948,6 +2948,7 @@ struct libxl__domain_save_state {
uint32_t domid;
int fd;
+ int recv_fd;
libxl_domain_type type;
int live;
int debug;
@@ -3264,6 +3265,7 @@ struct libxl__domain_create_state {
libxl_domain_config *guest_config;
libxl_domain_config guest_config_saved; /* vanilla config */
int restore_fd;
+ int send_fd;
libxl_domain_restore_params restore_params;
libxl__domain_create_cb *callback;
libxl_asyncprogress_how aop_console_how;
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 7c82f33..5c7b82d 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -201,6 +201,7 @@ libxl_viridian_enlightenment = Enumeration("viridian_enlightenment", [
libxl_checkpointed_stream = Enumeration("checkpointed_stream", [
(0, "NONE"),
(1, "REMUS"),
+ (2, "COLO"),
])
#
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index c965ef5..83164bc 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -157,6 +157,7 @@ struct domain_create {
const char *extra_config; /* extra config string */
const char *restore_file;
int migrate_fd; /* -1 means none */
+ int send_fd; /* -1 means none */
char **migration_domname_r; /* from malloc */
};
@@ -2536,6 +2537,7 @@ static uint32_t create_domain(struct domain_create *dom_info)
void *config_data = 0;
int config_len = 0;
int restore_fd = -1;
+ int send_fd = -1;
const libxl_asyncprogress_how *autoconnect_console_how;
struct save_file_header hdr;
@@ -2552,6 +2554,7 @@ static uint32_t create_domain(struct domain_create *dom_info)
if (migrate_fd >= 0) {
restore_source = "<incoming migration stream>";
restore_fd = migrate_fd;
+ send_fd = dom_info->send_fd;
} else {
restore_source = restore_file;
restore_fd = open(restore_file, O_RDONLY);
@@ -2731,7 +2734,7 @@ start:
ret = libxl_domain_create_restore(ctx, &d_config,
&domid, restore_fd,
- ¶ms,
+ send_fd, ¶ms,
0, autoconnect_console_how);
libxl_domain_restore_params_dispose(¶ms);
@@ -4273,6 +4276,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
dom_info.monitor = monitor;
dom_info.paused = 1;
dom_info.migrate_fd = recv_fd;
+ dom_info.send_fd = send_fd;
dom_info.migration_domname_r = &migration_domname;
dom_info.checkpointed_stream = checkpointed;
@@ -4443,6 +4447,7 @@ int main_restore(int argc, char **argv)
dom_info.config_file = config_file;
dom_info.restore_file = checkpoint_file;
dom_info.migrate_fd = -1;
+ dom_info.send_fd = -1;
dom_info.vnc = vnc;
dom_info.vncautopass = vncautopass;
dom_info.console_autoconnect = console_autoconnect;
@@ -4894,6 +4899,7 @@ int main_create(int argc, char **argv)
dom_info.config_file = filename;
dom_info.extra_config = extra_config;
dom_info.migrate_fd = -1;
+ dom_info.send_fd = -1;
dom_info.vnc = vnc;
dom_info.vncautopass = vncautopass;
dom_info.console_autoconnect = console_autoconnect;
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back
2015-06-25 6:25 ` [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back Yang Hongyang
@ 2015-06-30 10:07 ` Ian Campbell
2015-07-01 2:28 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:07 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> + * LIBXL_HAVE_DOMAIN_CREATE_RESTORE_SEND_FD 1
> + *
> + * If this is defined, libxl_domain_create_restore()'s API has changed to
> + * include a send_fd param which used for libxl migration back channel.
^is
Perhaps also end the sentence with "during $foo operations"? Since it is
not used for standard restore.
> int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
> - uint32_t *domid, int restore_fd,
> + uint32_t *domid, int restore_fd, int send_fd,
This is probably not a good idea, but what about "int fd[2]", i.e.
mimicking pipe(2). I suspect this will just make things harder on
callers, especially those who wish to work with both versions. Thoughts?
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back
2015-06-30 10:07 ` Ian Campbell
@ 2015-07-01 2:28 ` Yang Hongyang
2015-07-01 10:40 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:28 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:07 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> + * LIBXL_HAVE_DOMAIN_CREATE_RESTORE_SEND_FD 1
>> + *
>> + * If this is defined, libxl_domain_create_restore()'s API has changed to
>> + * include a send_fd param which used for libxl migration back channel.
>
> ^is
>
> Perhaps also end the sentence with "during $foo operations"? Since it is
> not used for standard restore.
IIRC, at hackthon, someone from alibaba also mentioned that they need a back
channel on normal migration...For the standard restore, just pass -1 as
send_fd.
>
>> int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
>> - uint32_t *domid, int restore_fd,
>> + uint32_t *domid, int restore_fd, int send_fd,
>
> This is probably not a good idea, but what about "int fd[2]", i.e.
> mimicking pipe(2). I suspect this will just make things harder on
I suspect so...
> callers, especially those who wish to work with both versions. Thoughts?
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back
2015-07-01 2:28 ` Yang Hongyang
@ 2015-07-01 10:40 ` Ian Campbell
2015-07-01 13:46 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:40 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Wed, 2015-07-01 at 10:28 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 06:07 PM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> + * LIBXL_HAVE_DOMAIN_CREATE_RESTORE_SEND_FD 1
> >> + *
> >> + * If this is defined, libxl_domain_create_restore()'s API has changed to
> >> + * include a send_fd param which used for libxl migration back channel.
> >
> > ^is
> >
> > Perhaps also end the sentence with "during $foo operations"? Since it is
> > not used for standard restore.
>
> IIRC, at hackthon, someone from alibaba also mentioned that they need a back
> channel on normal migration
There is no code like that in tree, if such is added then comments may
need to be adjusted.
> ...For the standard restore, just pass -1 as
> send_fd.
Correct.
I'm asking for a clarification to thew comment here, not a code change.
>
> >
> >> int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
> >> - uint32_t *domid, int restore_fd,
> >> + uint32_t *domid, int restore_fd, int send_fd,
> >
> > This is probably not a good idea, but what about "int fd[2]", i.e.
> > mimicking pipe(2). I suspect this will just make things harder on
>
> I suspect so...
>
> > callers, especially those who wish to work with both versions. Thoughts?
> >
> > Ian.
> >
> > .
> >
>
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back
2015-07-01 10:40 ` Ian Campbell
@ 2015-07-01 13:46 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 13:46 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 07/01/2015 06:40 PM, Ian Campbell wrote:
> On Wed, 2015-07-01 at 10:28 +0800, Yang Hongyang wrote:
>>
>> On 06/30/2015 06:07 PM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>> + * LIBXL_HAVE_DOMAIN_CREATE_RESTORE_SEND_FD 1
>>>> + *
>>>> + * If this is defined, libxl_domain_create_restore()'s API has changed to
>>>> + * include a send_fd param which used for libxl migration back channel.
>>>
>>> ^is
>>>
>>> Perhaps also end the sentence with "during $foo operations"? Since it is
>>> not used for standard restore.
>>
>> IIRC, at hackthon, someone from alibaba also mentioned that they need a back
>> channel on normal migration
>
> There is no code like that in tree, if such is added then comments may
> need to be adjusted.
>
>> ...For the standard restore, just pass -1 as
>> send_fd.
>
> Correct.
>
> I'm asking for a clarification to thew comment here, not a code change.
Sure and thanks!
>
>>
>>>
>>>> int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
>>>> - uint32_t *domid, int restore_fd,
>>>> + uint32_t *domid, int restore_fd, int send_fd,
>>>
>>> This is probably not a good idea, but what about "int fd[2]", i.e.
>>> mimicking pipe(2). I suspect this will just make things harder on
>>
>> I suspect so...
>>
>>> callers, especially those who wish to work with both versions. Thoughts?
>>>
>>> Ian.
>>>
>>> .
>>>
>>
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (14 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 15/26] tools/libxl: Add back channel to allow migration target send data back Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:10 ` Ian Campbell
2015-06-30 10:17 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback Yang Hongyang
` (9 subsequent siblings)
25 siblings, 2 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, Ian Campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, Ian Jackson
We need to send secondary's dirty page pfns back to primary.
Primary will then send pages that are both dirtied on
primary/secondary to secondary. in this way the secondary's
memory will be consistent with primary.
We do this on libxc side, so we need to introduce back
channel to libxc.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/libxc/include/xenguest.h | 8 ++++----
tools/libxc/xc_domain_restore.c | 4 ++--
tools/libxc/xc_domain_save.c | 4 ++--
tools/libxc/xc_sr_restore.c | 2 +-
tools/libxc/xc_sr_save.c | 2 +-
tools/libxl/libxl_save_callout.c | 23 +++++++++++++++++------
tools/libxl/libxl_save_helper.c | 8 ++++++--
7 files changed, 33 insertions(+), 18 deletions(-)
diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
index d7c8fe4..3e48a97 100644
--- a/tools/libxc/include/xenguest.h
+++ b/tools/libxc/include/xenguest.h
@@ -91,13 +91,13 @@ struct save_callbacks {
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */,
struct save_callbacks* callbacks, int hvm,
- int checkpointed_stream);
+ int checkpointed_stream, int back_fd);
/* Domain Save v2 */
int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags,
struct save_callbacks* callbacks, int hvm,
- int checkpointed_stream);
+ int checkpointed_stream, int back_fd);
/* callbacks provided by xc_domain_restore */
struct restore_callbacks {
@@ -136,7 +136,7 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_mfn, domid_t console_domid,
unsigned int hvm, unsigned int pae, int superpages,
int checkpointed_stream,
- struct restore_callbacks *callbacks);
+ struct restore_callbacks *callbacks, int back_fd);
/* Domain Restore v2 */
int xc_domain_restore2(xc_interface *xch, int io_fd, uint32_t dom,
@@ -145,7 +145,7 @@ int xc_domain_restore2(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_mfn, domid_t console_domid,
unsigned int hvm, unsigned int pae, int superpages,
int checkpointed_stream,
- struct restore_callbacks *callbacks);
+ struct restore_callbacks *callbacks, int back_fd);
/**
* xc_domain_restore writes a file to disk that contains the device
* model saved state.
diff --git a/tools/libxc/xc_domain_restore.c b/tools/libxc/xc_domain_restore.c
index 3cd3483..63d1e6b 100644
--- a/tools/libxc/xc_domain_restore.c
+++ b/tools/libxc/xc_domain_restore.c
@@ -1515,7 +1515,7 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_mfn, domid_t console_domid,
unsigned int hvm, unsigned int pae, int superpages,
int checkpointed_stream,
- struct restore_callbacks *callbacks)
+ struct restore_callbacks *callbacks, int back_fd)
{
DECLARE_DOMCTL;
xc_dominfo_t info;
@@ -1578,7 +1578,7 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
return xc_domain_restore2(
xch, io_fd, dom, store_evtchn, store_mfn,
store_domid, console_evtchn, console_mfn, console_domid,
- hvm, pae, superpages, checkpointed_stream, callbacks);
+ hvm, pae, superpages, checkpointed_stream, callbacks, back_fd);
}
DPRINTF("%s: starting restore of new domid %u", __func__, dom);
diff --git a/tools/libxc/xc_domain_save.c b/tools/libxc/xc_domain_save.c
index c76980e..de2f852 100644
--- a/tools/libxc/xc_domain_save.c
+++ b/tools/libxc/xc_domain_save.c
@@ -803,7 +803,7 @@ static int save_tsc_info(xc_interface *xch, uint32_t dom, int io_fd)
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags,
struct save_callbacks* callbacks, int hvm,
- int checkpointed_stream)
+ int checkpointed_stream, int back_fd)
{
xc_dominfo_t info;
DECLARE_DOMCTL;
@@ -899,7 +899,7 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iter
{
return xc_domain_save2(xch, io_fd, dom, max_iters,
max_factor, flags, callbacks, hvm,
- checkpointed_stream);
+ checkpointed_stream, back_fd);
}
DPRINTF("%s: starting save of domid %u", __func__, dom);
diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
index d5645e0..f8b6f2f 100644
--- a/tools/libxc/xc_sr_restore.c
+++ b/tools/libxc/xc_sr_restore.c
@@ -710,7 +710,7 @@ int xc_domain_restore2(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_gfn, domid_t console_domid,
unsigned int hvm, unsigned int pae, int superpages,
int checkpointed_stream,
- struct restore_callbacks *callbacks)
+ struct restore_callbacks *callbacks, int back_fd)
{
struct xc_sr_context ctx =
{
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index 6102b66..d12e5b1 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -821,7 +821,7 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
uint32_t max_iters, uint32_t max_factor, uint32_t flags,
struct save_callbacks* callbacks, int hvm,
- int checkpointed_stream)
+ int checkpointed_stream, int back_fd)
{
xen_pfn_t nr_pfns;
struct xc_sr_context ctx =
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 12cc57e..25817d6 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -27,7 +27,7 @@
*/
static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
const char *mode_arg,
- int stream_fd,
+ int stream_fd, int back_fd,
const int *preserve_fds, int num_preserve_fds,
const unsigned long *argnums, int num_argnums);
@@ -70,8 +70,8 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
dcs->shs.need_results = 1;
dcs->shs.toolstack_data_file = 0;
- run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, 0,0,
- argnums, ARRAY_SIZE(argnums));
+ run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, dcs->send_fd,
+ 0,0, argnums, ARRAY_SIZE(argnums));
}
void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
@@ -93,7 +93,7 @@ void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
dss->shs.caller_state = dss;
dss->shs.need_results = 0;
- run_helper(egc, &dss->shs, "--save-domain", dss->fd,
+ run_helper(egc, &dss->shs, "--save-domain", dss->fd, dss->recv_fd,
NULL, 0,
argnums, ARRAY_SIZE(argnums));
return;
@@ -111,12 +111,12 @@ void libxl__xc_domain_saverestore_async_callback_done(libxl__egc *egc,
/*----- helper execution -----*/
static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
- const char *mode_arg, int stream_fd,
+ const char *mode_arg, int stream_fd, int back_fd,
const int *preserve_fds, int num_preserve_fds,
const unsigned long *argnums, int num_argnums)
{
STATE_AO_GC(shs->ao);
- const char *args[4 + num_argnums];
+ const char *args[5 + num_argnums];
const char **arg = args;
int i, rc;
@@ -140,6 +140,7 @@ static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
*arg++ = getenv("LIBXL_SAVE_HELPER") ?: LIBEXEC_BIN "/" "libxl-save-helper";
*arg++ = mode_arg;
const char **stream_fd_arg = arg++;
+ const char **back_fd_arg = arg++;
for (i=0; i<num_argnums; i++)
*arg++ = GCSPRINTF("%lu", argnums[i]);
*arg++ = 0;
@@ -174,6 +175,16 @@ static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
libxl_fd_set_cloexec(CTX, stream_fd, 0);
*stream_fd_arg = GCSPRINTF("%d", stream_fd);
+ if (back_fd <= 2) {
+ back_fd = dup(back_fd);
+ if (back_fd < 0) {
+ LOGE(ERROR,"dup migration back channel");
+ exit(-1);
+ }
+ }
+ libxl_fd_set_cloexec(CTX, back_fd, 0);
+ *back_fd_arg = GCSPRINTF("%d", back_fd);
+
for (i=0; i<num_preserve_fds; i++)
if (preserve_fds[i] >= 0) {
assert(preserve_fds[i] > 2);
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
index 9d2cbf6..81498bd 100644
--- a/tools/libxl/libxl_save_helper.c
+++ b/tools/libxl/libxl_save_helper.c
@@ -233,6 +233,7 @@ static struct restore_callbacks helper_restore_callbacks;
int main(int argc, char **argv)
{
int r;
+ int back_fd;
#define NEXTARG (++argv, assert(*argv), *argv)
@@ -242,6 +243,7 @@ int main(int argc, char **argv)
if (!strcmp(mode,"--save-domain")) {
io_fd = atoi(NEXTARG);
+ back_fd = atoi(NEXTARG);
uint32_t dom = strtoul(NEXTARG,0,10);
uint32_t max_iters = strtoul(NEXTARG,0,10);
uint32_t max_factor = strtoul(NEXTARG,0,10);
@@ -257,12 +259,14 @@ int main(int argc, char **argv)
setup_signals(save_signal_handler);
r = xc_domain_save2(xch, io_fd, dom, max_iters, max_factor, flags,
- &helper_save_callbacks, hvm, checkpointed_stream);
+ &helper_save_callbacks, hvm, checkpointed_stream,
+ back_fd);
complete(r);
} else if (!strcmp(mode,"--restore-domain")) {
io_fd = atoi(NEXTARG);
+ back_fd = atoi(NEXTARG);
uint32_t dom = strtoul(NEXTARG,0,10);
unsigned store_evtchn = strtoul(NEXTARG,0,10);
domid_t store_domid = strtoul(NEXTARG,0,10);
@@ -287,7 +291,7 @@ int main(int argc, char **argv)
store_domid, console_evtchn, &console_mfn,
console_domid, hvm, pae, superpages,
checkpointed,
- &helper_restore_callbacks);
+ &helper_restore_callbacks, back_fd);
helper_stub_restore_results(store_mfn,console_mfn,0);
complete(r);
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-06-25 6:25 ` [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc Yang Hongyang
@ 2015-06-30 10:10 ` Ian Campbell
2015-07-01 2:38 ` Yang Hongyang
2015-06-30 10:17 ` Ian Campbell
1 sibling, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:10 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> We need to send secondary's dirty page pfns back to primary.
In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
In the pdf
http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
linked from the wiki page
http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
it says that the secondary keeps a copy of the original contents of
its dirty pages. So I don't understand why you need to send the dirty
bitmap to the primary.
Which I don't see an answer for in my archive. Have I missed (or
misplaced) the answer?
Ian
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-06-30 10:10 ` Ian Campbell
@ 2015-07-01 2:38 ` Yang Hongyang
2015-07-01 10:42 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:38 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/30/2015 06:10 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> We need to send secondary's dirty page pfns back to primary.
>
> In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
>
> In the pdf
> http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
> linked from the wiki page
> http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
> it says that the secondary keeps a copy of the original contents of
> its dirty pages. So I don't understand why you need to send the dirty
> bitmap to the primary.
>
> Which I don't see an answer for in my archive. Have I missed (or
> misplaced) the answer?
Sorry, seems that I misplaced the answer to:
[PATCH v2 COLOPre 09/13] tools/libxl: Update libxl_save_msgs_gen.pl to support
return data from xl to xc
> Thanks for this. I would have some comments on the details, but first
> I want to properly understand your use case. So while I'm the author
> and maintainer of this save helper, I won't review this in detail just
> yet. I'm following the thread about what this is for...
We need to send secondary's dirty page pfn back to primary. Primary will
then send pages that are both dirtied on primary/secondary to secondary.
in this way the secondary's memory will be consistent with primary.
As we disscussed in [PATCH v2 COLOPre 04/13] tools/libxc: export xc_bitops.h
If we move this operation to libxc layer, this patch could be dropped.
>
> Ian
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 2:38 ` Yang Hongyang
@ 2015-07-01 10:42 ` Ian Campbell
2015-07-01 11:01 ` Andrew Cooper
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:42 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Wed, 2015-07-01 at 10:38 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 06:10 PM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> We need to send secondary's dirty page pfns back to primary.
> >
> > In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
> >
> > In the pdf
> > http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
> > linked from the wiki page
> > http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
> > it says that the secondary keeps a copy of the original contents of
> > its dirty pages. So I don't understand why you need to send the dirty
> > bitmap to the primary.
> >
> > Which I don't see an answer for in my archive. Have I missed (or
> > misplaced) the answer?
>
> Sorry, seems that I misplaced the answer to:
> [PATCH v2 COLOPre 09/13] tools/libxl: Update libxl_save_msgs_gen.pl to support
> return data from xl to xc
>
> > Thanks for this. I would have some comments on the details, but first
> > I want to properly understand your use case. So while I'm the author
> > and maintainer of this save helper, I won't review this in detail just
> > yet. I'm following the thread about what this is for...
>
> We need to send secondary's dirty page pfn back to primary. Primary will
> then send pages that are both dirtied on primary/secondary to secondary.
> in this way the secondary's memory will be consistent with primary.
>
> As we disscussed in [PATCH v2 COLOPre 04/13] tools/libxc: export xc_bitops.h
> If we move this operation to libxc layer, this patch could be dropped.
This doesn't seem to be a response to Ian's question which I quoted
above.
The crux of the question is that the design contained in those links
does not appear to require a back channel, because it does not require a
dirty bitmap to go from secondary to primary. Asserting a need to do so
does not answer the question.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 10:42 ` Ian Campbell
@ 2015-07-01 11:01 ` Andrew Cooper
2015-07-01 11:21 ` Ian Campbell
2015-07-01 13:54 ` Yang Hongyang
0 siblings, 2 replies; 103+ messages in thread
From: Andrew Cooper @ 2015-07-01 11:01 UTC (permalink / raw)
To: Ian Campbell, Yang Hongyang
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Ian Jackson
On 01/07/15 11:42, Ian Campbell wrote:
> On Wed, 2015-07-01 at 10:38 +0800, Yang Hongyang wrote:
>> On 06/30/2015 06:10 PM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>> We need to send secondary's dirty page pfns back to primary.
>>> In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
>>>
>>> In the pdf
>>> http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
>>> linked from the wiki page
>>> http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
>>> it says that the secondary keeps a copy of the original contents of
>>> its dirty pages. So I don't understand why you need to send the dirty
>>> bitmap to the primary.
>>>
>>> Which I don't see an answer for in my archive. Have I missed (or
>>> misplaced) the answer?
>> Sorry, seems that I misplaced the answer to:
>> [PATCH v2 COLOPre 09/13] tools/libxl: Update libxl_save_msgs_gen.pl to support
>> return data from xl to xc
>>
>> > Thanks for this. I would have some comments on the details, but first
>> > I want to properly understand your use case. So while I'm the author
>> > and maintainer of this save helper, I won't review this in detail just
>> > yet. I'm following the thread about what this is for...
>>
>> We need to send secondary's dirty page pfn back to primary. Primary will
>> then send pages that are both dirtied on primary/secondary to secondary.
>> in this way the secondary's memory will be consistent with primary.
>>
>> As we disscussed in [PATCH v2 COLOPre 04/13] tools/libxc: export xc_bitops.h
>> If we move this operation to libxc layer, this patch could be dropped.
> This doesn't seem to be a response to Ian's question which I quoted
> above.
>
> The crux of the question is that the design contained in those links
> does not appear to require a back channel, because it does not require a
> dirty bitmap to go from secondary to primary. Asserting a need to do so
> does not answer the question.
It very definitely does require a dirty bitmap moving from the secondary
to the primary.
Lets see whether I can try explaining it in a different way.
In COLO mode, both VMs are running, and are considered in sync if the
visible network traffic is identical. After some time, they fall out of
sync.
At this point, the two VMs have definitely diverged. Lets call the
primary dirty bitmap set A, while the secondary dirty bitmap set B.
Sets A and B are different.
Under normal migration, the page data for set A will be sent form the
primary to the secondary.
However, the set difference B - A (lets call this C) is out-of-date on
the secondary (with respect to the primary) and will not be sent by the
primary, as it was not memory dirtied by the primary. The secondary
needs the page data for C to reconstruct an exact copy of the primary at
the checkpoint.
The secondary cannot calculate C as it doesn't know A. Instead, the
secondary must send B to the primary, at which point the primary
calculates the union of A and B (lets call this D) which is all the
pages dirtied by both the primary and the secondary, and sends all page
data covered by D.
In the general case, D is a superset of both A and B. Without the
backchannel dirty bitmap, a COLO checkpoint can't reconstruct a valid
copy of the primary.
~Andrew
P.S. I have suggested an investigation of the CoW support in Xen as a
potential optimisation, as this could be used to prevent the secondary
losing C, but this is very definitely future work and not appropriate at
this point in COLO.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 11:01 ` Andrew Cooper
@ 2015-07-01 11:21 ` Ian Campbell
2015-07-01 12:07 ` Ian Jackson
2015-07-01 13:54 ` Yang Hongyang
1 sibling, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 11:21 UTC (permalink / raw)
To: Andrew Cooper
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Yang Hongyang, Ian Jackson
On Wed, 2015-07-01 at 12:01 +0100, Andrew Cooper wrote:
> On 01/07/15 11:42, Ian Campbell wrote:
> > On Wed, 2015-07-01 at 10:38 +0800, Yang Hongyang wrote:
> >> On 06/30/2015 06:10 PM, Ian Campbell wrote:
> >>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >>>> We need to send secondary's dirty page pfns back to primary.
> >>> In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
> >>>
> >>> In the pdf
> >>> http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
> >>> linked from the wiki page
> >>> http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
> >>> it says that the secondary keeps a copy of the original contents of
> >>> its dirty pages. So I don't understand why you need to send the dirty
> >>> bitmap to the primary.
> >>>
> >>> Which I don't see an answer for in my archive. Have I missed (or
> >>> misplaced) the answer?
> >> Sorry, seems that I misplaced the answer to:
> >> [PATCH v2 COLOPre 09/13] tools/libxl: Update libxl_save_msgs_gen.pl to support
> >> return data from xl to xc
> >>
> >> > Thanks for this. I would have some comments on the details, but first
> >> > I want to properly understand your use case. So while I'm the author
> >> > and maintainer of this save helper, I won't review this in detail just
> >> > yet. I'm following the thread about what this is for...
> >>
> >> We need to send secondary's dirty page pfn back to primary. Primary will
> >> then send pages that are both dirtied on primary/secondary to secondary.
> >> in this way the secondary's memory will be consistent with primary.
> >>
> >> As we disscussed in [PATCH v2 COLOPre 04/13] tools/libxc: export xc_bitops.h
> >> If we move this operation to libxc layer, this patch could be dropped.
> > This doesn't seem to be a response to Ian's question which I quoted
> > above.
> >
> > The crux of the question is that the design contained in those links
> > does not appear to require a back channel, because it does not require a
> > dirty bitmap to go from secondary to primary. Asserting a need to do so
> > does not answer the question.
>
> It very definitely does require a dirty bitmap moving from the secondary
> to the primary.
The current implementation might work as you describe, but the
design/paper which Ian references suggests that the secondary keeps
copies of all the clean pages sent by the primary, which it can then use
on checkpoint to reestablish consistency. See the last paragraph of
section 4.2 which states:
COLO solves the memory checkpointing issue by
keeping a local copy of the previous checkpoint’s mem-
ory contents, and reverting locally modified memory
pages to the previous checkpoint before applying the
delta memory pages from the PVM. Therefore, only
Dp is transmitted, saving CPU and network resources.
For device state, COLO uses the device suspend/resume
process that was introduced by live migration [16] to
gracefully bring both the PVM and SVM to the ini-
tial state, and rebuilds the machine state using active-
checkpointing.
Now perhaps Ian and I have both misinterpreted that part of the paper or
perhaps there is some reason why the current implementation deviates
from the design described there but as it stands your contention that a
back channel is a fundamental requirement is unfounded as far as I can
tell.
So to restate the question: Why does the current design deviate from the
design in the paper, or does the paper not say what we think it says.
> However, the set difference B - A (lets call this C) is out-of-date on
> the secondary (with respect to the primary) and will not be sent by the
> primary, as it was not memory dirtied by the primary.
This is where we deviate. According to the paper there is no need to
resend because the secondary already has a non-dirty copy of any memory
which is dirty in B but not A.
Ian.
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 11:21 ` Ian Campbell
@ 2015-07-01 12:07 ` Ian Jackson
2015-07-01 13:56 ` Yang Hongyang
2015-07-01 14:21 ` Ian Campbell
0 siblings, 2 replies; 103+ messages in thread
From: Ian Jackson @ 2015-07-01 12:07 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, Andrew Cooper, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Yang Hongyang
Ian Campbell writes ("Re: [Xen-devel] [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc"):
> So to restate the question: Why does the current design deviate from the
> design in the paper, or does the paper not say what we think it says.
To be clear, I have no problem if the design has changed since the
paper was written. I just want:
* A clear high-level explanation of the actually-implemented
arrangements to exist somewhere
* The commit messages, or code, to refer to that explanation
A description and explanation of the difference from some other
somewhat different previously-published document is IMO necessary in
this case because the primary design reference is that
previously-published document, which does not correspond to the actual
code.
Having a design document which disagrees with the implementation is
dangerous because future programmers will look to the design to
understand what is going on.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 12:07 ` Ian Jackson
@ 2015-07-01 13:56 ` Yang Hongyang
2015-07-01 13:58 ` Ian Jackson
2015-07-01 14:21 ` Ian Campbell
1 sibling, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 13:56 UTC (permalink / raw)
To: Ian Jackson, Ian Campbell
Cc: wei.liu2, wency, Andrew Cooper, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram
On 07/01/2015 08:07 PM, Ian Jackson wrote:
> Ian Campbell writes ("Re: [Xen-devel] [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc"):
>> So to restate the question: Why does the current design deviate from the
>> design in the paper, or does the paper not say what we think it says.
>
> To be clear, I have no problem if the design has changed since the
> paper was written. I just want:
>
> * A clear high-level explanation of the actually-implemented
> arrangements to exist somewhere
>
> * The commit messages, or code, to refer to that explanation
>
> A description and explanation of the difference from some other
> somewhat different previously-published document is IMO necessary in
> this case because the primary design reference is that
> previously-published document, which does not correspond to the actual
> code.
>
> Having a design document which disagrees with the implementation is
> dangerous because future programmers will look to the design to
> understand what is going on.
I will add the explanation to code comment and commit message, thank
you!
>
> Ian.
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 13:56 ` Yang Hongyang
@ 2015-07-01 13:58 ` Ian Jackson
0 siblings, 0 replies; 103+ messages in thread
From: Ian Jackson @ 2015-07-01 13:58 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, Ian Campbell, wency, Andrew Cooper, yunhong.jiang,
eddie.dong, xen-devel, guijianfeng, rshriram
Yang Hongyang writes ("Re: [Xen-devel] [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc"):
> I will add the explanation to code comment and commit message, thank
> you!
Right, thanks.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 12:07 ` Ian Jackson
2015-07-01 13:56 ` Yang Hongyang
@ 2015-07-01 14:21 ` Ian Campbell
1 sibling, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 14:21 UTC (permalink / raw)
To: Ian Jackson
Cc: wei.liu2, wency, Andrew Cooper, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Yang Hongyang
On Wed, 2015-07-01 at 13:07 +0100, Ian Jackson wrote:
> Ian Campbell writes ("Re: [Xen-devel] [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc"):
> > So to restate the question: Why does the current design deviate from the
> > design in the paper, or does the paper not say what we think it says.
>
> To be clear, I have no problem if the design has changed since the
> paper was written. I just want:
>
> * A clear high-level explanation of the actually-implemented
> arrangements to exist somewhere
>
> * The commit messages, or code, to refer to that explanation
>
> A description and explanation of the difference from some other
> somewhat different previously-published document is IMO necessary in
> this case because the primary design reference is that
> previously-published document, which does not correspond to the actual
> code.
Also in this case the implementation requires a significant new bit of
functionality (the back channel) which the previously-published design
did not, so knowing what about that design was
wrong/inadequate/impractical is useful so we can see that the new
functionality is justified.
>
> Having a design document which disagrees with the implementation is
> dangerous because future programmers will look to the design to
> understand what is going on.
>
> Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 11:01 ` Andrew Cooper
2015-07-01 11:21 ` Ian Campbell
@ 2015-07-01 13:54 ` Yang Hongyang
2015-07-01 14:03 ` Andrew Cooper
1 sibling, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 13:54 UTC (permalink / raw)
To: Andrew Cooper, Ian Campbell
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Ian Jackson
On 07/01/2015 07:01 PM, Andrew Cooper wrote:
> On 01/07/15 11:42, Ian Campbell wrote:
>> On Wed, 2015-07-01 at 10:38 +0800, Yang Hongyang wrote:
>>> On 06/30/2015 06:10 PM, Ian Campbell wrote:
>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>>> We need to send secondary's dirty page pfns back to primary.
>>>> In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
>>>>
>>>> In the pdf
>>>> http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
>>>> linked from the wiki page
>>>> http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
>>>> it says that the secondary keeps a copy of the original contents of
>>>> its dirty pages. So I don't understand why you need to send the dirty
>>>> bitmap to the primary.
>>>>
>>>> Which I don't see an answer for in my archive. Have I missed (or
>>>> misplaced) the answer?
>>> Sorry, seems that I misplaced the answer to:
>>> [PATCH v2 COLOPre 09/13] tools/libxl: Update libxl_save_msgs_gen.pl to support
>>> return data from xl to xc
>>>
>>> > Thanks for this. I would have some comments on the details, but first
>>> > I want to properly understand your use case. So while I'm the author
>>> > and maintainer of this save helper, I won't review this in detail just
>>> > yet. I'm following the thread about what this is for...
>>>
>>> We need to send secondary's dirty page pfn back to primary. Primary will
>>> then send pages that are both dirtied on primary/secondary to secondary.
>>> in this way the secondary's memory will be consistent with primary.
>>>
>>> As we disscussed in [PATCH v2 COLOPre 04/13] tools/libxc: export xc_bitops.h
>>> If we move this operation to libxc layer, this patch could be dropped.
>> This doesn't seem to be a response to Ian's question which I quoted
>> above.
>>
>> The crux of the question is that the design contained in those links
>> does not appear to require a back channel, because it does not require a
>> dirty bitmap to go from secondary to primary. Asserting a need to do so
>> does not answer the question.
>
> It very definitely does require a dirty bitmap moving from the secondary
> to the primary.
>
> Lets see whether I can try explaining it in a different way.
>
> In COLO mode, both VMs are running, and are considered in sync if the
> visible network traffic is identical. After some time, they fall out of
> sync.
>
> At this point, the two VMs have definitely diverged. Lets call the
> primary dirty bitmap set A, while the secondary dirty bitmap set B.
>
> Sets A and B are different.
>
> Under normal migration, the page data for set A will be sent form the
> primary to the secondary.
>
> However, the set difference B - A (lets call this C) is out-of-date on
> the secondary (with respect to the primary) and will not be sent by the
> primary, as it was not memory dirtied by the primary. The secondary
> needs the page data for C to reconstruct an exact copy of the primary at
> the checkpoint.
>
> The secondary cannot calculate C as it doesn't know A. Instead, the
> secondary must send B to the primary, at which point the primary
> calculates the union of A and B (lets call this D) which is all the
> pages dirtied by both the primary and the secondary, and sends all page
> data covered by D.
>
> In the general case, D is a superset of both A and B. Without the
> backchannel dirty bitmap, a COLO checkpoint can't reconstruct a valid
> copy of the primary.
Thank you Andy! The explaination is clear enough, do you mind if I copy your
comments into the code comment or commit message and with your sob?
>
> ~Andrew
>
> P.S. I have suggested an investigation of the CoW support in Xen as a
> potential optimisation, as this could be used to prevent the secondary
> losing C, but this is very definitely future work and not appropriate at
> this point in COLO.
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-07-01 13:54 ` Yang Hongyang
@ 2015-07-01 14:03 ` Andrew Cooper
0 siblings, 0 replies; 103+ messages in thread
From: Andrew Cooper @ 2015-07-01 14:03 UTC (permalink / raw)
To: Yang Hongyang, Ian Campbell
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Ian Jackson
On 01/07/15 14:54, Yang Hongyang wrote:
>
>
> On 07/01/2015 07:01 PM, Andrew Cooper wrote:
>> On 01/07/15 11:42, Ian Campbell wrote:
>>> On Wed, 2015-07-01 at 10:38 +0800, Yang Hongyang wrote:
>>>> On 06/30/2015 06:10 PM, Ian Campbell wrote:
>>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>>>> We need to send secondary's dirty page pfns back to primary.
>>>>> In v2 Ian asked (<21888.2988.774072.32946@mariner.uk.xensource.com>):
>>>>>
>>>>> In the pdf
>>>>>
>>>>> http://www.socc2013.org/home/program/a3-dong.pdf?attredirects=0
>>>>> linked from the wiki page
>>>>>
>>>>> http://wiki.xen.org/wiki/COLO_-_Coarse_Grain_Lock_Stepping
>>>>> it says that the secondary keeps a copy of the original
>>>>> contents of
>>>>> its dirty pages. So I don't understand why you need to
>>>>> send the dirty
>>>>> bitmap to the primary.
>>>>>
>>>>> Which I don't see an answer for in my archive. Have I missed (or
>>>>> misplaced) the answer?
>>>> Sorry, seems that I misplaced the answer to:
>>>> [PATCH v2 COLOPre 09/13] tools/libxl: Update libxl_save_msgs_gen.pl
>>>> to support
>>>> return data from xl to xc
>>>>
>>>> > Thanks for this. I would have some comments on the details,
>>>> but first
>>>> > I want to properly understand your use case. So while I'm
>>>> the author
>>>> > and maintainer of this save helper, I won't review this in
>>>> detail just
>>>> > yet. I'm following the thread about what this is for...
>>>>
>>>> We need to send secondary's dirty page pfn back to primary.
>>>> Primary will
>>>> then send pages that are both dirtied on primary/secondary to
>>>> secondary.
>>>> in this way the secondary's memory will be consistent with
>>>> primary.
>>>>
>>>> As we disscussed in [PATCH v2 COLOPre 04/13] tools/libxc:
>>>> export xc_bitops.h
>>>> If we move this operation to libxc layer, this patch could be
>>>> dropped.
>>> This doesn't seem to be a response to Ian's question which I quoted
>>> above.
>>>
>>> The crux of the question is that the design contained in those links
>>> does not appear to require a back channel, because it does not
>>> require a
>>> dirty bitmap to go from secondary to primary. Asserting a need to do so
>>> does not answer the question.
>>
>> It very definitely does require a dirty bitmap moving from the secondary
>> to the primary.
>>
>> Lets see whether I can try explaining it in a different way.
>>
>> In COLO mode, both VMs are running, and are considered in sync if the
>> visible network traffic is identical. After some time, they fall out of
>> sync.
>>
>> At this point, the two VMs have definitely diverged. Lets call the
>> primary dirty bitmap set A, while the secondary dirty bitmap set B.
>>
>> Sets A and B are different.
>>
>> Under normal migration, the page data for set A will be sent form the
>> primary to the secondary.
>>
>> However, the set difference B - A (lets call this C) is out-of-date on
>> the secondary (with respect to the primary) and will not be sent by the
>> primary, as it was not memory dirtied by the primary. The secondary
>> needs the page data for C to reconstruct an exact copy of the primary at
>> the checkpoint.
>>
>> The secondary cannot calculate C as it doesn't know A. Instead, the
>> secondary must send B to the primary, at which point the primary
>> calculates the union of A and B (lets call this D) which is all the
>> pages dirtied by both the primary and the secondary, and sends all page
>> data covered by D.
>>
>> In the general case, D is a superset of both A and B. Without the
>> backchannel dirty bitmap, a COLO checkpoint can't reconstruct a valid
>> copy of the primary.
>
> Thank you Andy! The explaination is clear enough, do you mind if I
> copy your
> comments into the code comment or commit message and with your sob?
Feel free to borrow any/all of the description which you would feel
would be useful, although you probably don't want to take it all
verbatim for a commit message.
~Andrew
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-06-25 6:25 ` [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc Yang Hongyang
2015-06-30 10:10 ` Ian Campbell
@ 2015-06-30 10:17 ` Ian Campbell
2015-07-01 2:40 ` Yang Hongyang
1 sibling, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:17 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
> index 12cc57e..25817d6 100644
> --- a/tools/libxl/libxl_save_callout.c
> +++ b/tools/libxl/libxl_save_callout.c
> @@ -27,7 +27,7 @@
> */
> static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
> const char *mode_arg,
> - int stream_fd,
> + int stream_fd, int back_fd,
> const int *preserve_fds, int num_preserve_fds,
> const unsigned long *argnums, int num_argnums);
>
> @@ -70,8 +70,8 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
> dcs->shs.need_results = 1;
> dcs->shs.toolstack_data_file = 0;
>
> - run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, 0,0,
> - argnums, ARRAY_SIZE(argnums));
> + run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, dcs->send_fd,
restore_fd comes from convenience aliases, it would be in keeping to do
the same for send_fd too (and elsewhere)
> @@ -174,6 +175,16 @@ static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
> libxl_fd_set_cloexec(CTX, stream_fd, 0);
> *stream_fd_arg = GCSPRINTF("%d", stream_fd);
>
> + if (back_fd <= 2) {
> + back_fd = dup(back_fd);
> + if (back_fd < 0) {
> + LOGE(ERROR,"dup migration back channel");
> + exit(-1);
> + }
> + }
> + libxl_fd_set_cloexec(CTX, back_fd, 0);
> + *back_fd_arg = GCSPRINTF("%d", back_fd);
This is an exact duplicate of the stream_fd setup, I think you should
make it a helper which takes a gc, an fd and a const char *what
(=="stream" or "backchannel" for logging) and returns a string
representing a safe (i.e. possibly dupd) version of the fd.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc
2015-06-30 10:17 ` Ian Campbell
@ 2015-07-01 2:40 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:40 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, Ian Jackson
On 06/30/2015 06:17 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>
>> diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
>> index 12cc57e..25817d6 100644
>> --- a/tools/libxl/libxl_save_callout.c
>> +++ b/tools/libxl/libxl_save_callout.c
>> @@ -27,7 +27,7 @@
>> */
>> static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
>> const char *mode_arg,
>> - int stream_fd,
>> + int stream_fd, int back_fd,
>> const int *preserve_fds, int num_preserve_fds,
>> const unsigned long *argnums, int num_argnums);
>>
>> @@ -70,8 +70,8 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
>> dcs->shs.need_results = 1;
>> dcs->shs.toolstack_data_file = 0;
>>
>> - run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, 0,0,
>> - argnums, ARRAY_SIZE(argnums));
>> + run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, dcs->send_fd,
>
> restore_fd comes from convenience aliases, it would be in keeping to do
> the same for send_fd too (and elsewhere)
Ok
>
>> @@ -174,6 +175,16 @@ static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
>> libxl_fd_set_cloexec(CTX, stream_fd, 0);
>> *stream_fd_arg = GCSPRINTF("%d", stream_fd);
>>
>> + if (back_fd <= 2) {
>> + back_fd = dup(back_fd);
>> + if (back_fd < 0) {
>> + LOGE(ERROR,"dup migration back channel");
>> + exit(-1);
>> + }
>> + }
>> + libxl_fd_set_cloexec(CTX, back_fd, 0);
>> + *back_fd_arg = GCSPRINTF("%d", back_fd);
>
> This is an exact duplicate of the stream_fd setup, I think you should
> make it a helper which takes a gc, an fd and a const char *what
> (=="stream" or "backchannel" for logging) and returns a string
> representing a safe (i.e. possibly dupd) version of the fd.
ok, will do in the next version.
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (15 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 16/26] tools/libx{l, c}: add back channel to libxc Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:19 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 18/26] tools/libx{l, c}: add postcopy/suspend callback to restore side Yang Hongyang
` (8 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
Under COLO, we are doing checkpoint on demand, if this
callback returns 1, we will take another checkpoint.
0 indicates unexpected error.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxc/include/xenguest.h | 18 ++++++++++++++++++
tools/libxl/libxl_save_msgs_gen.pl | 7 ++++---
2 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
index 3e48a97..e804a1d 100644
--- a/tools/libxc/include/xenguest.h
+++ b/tools/libxc/include/xenguest.h
@@ -63,6 +63,15 @@ struct save_callbacks {
* 1: take another checkpoint */
int (*checkpoint)(void* data);
+ /*
+ * Called after the checkpoint callback.
+ *
+ * returns:
+ * 0: terminate checkpointing gracefully
+ * 1: take another checkpoint
+ */
+ int (*should_checkpoint)(void* data);
+
/* Enable qemu-dm logging dirty pages to xen */
int (*switch_qemu_logdirty)(int domid, unsigned enable, void *data); /* HVM only */
@@ -108,6 +117,15 @@ struct restore_callbacks {
/* A checkpoint record has been found in the stream */
int (*checkpoint)(void* data);
+ /*
+ * Called after the checkpoint callback.
+ *
+ * returns:
+ * 0: terminate checkpointing gracefully
+ * 1: take another checkpoint
+ */
+ int (*should_checkpoint)(void* data);
+
/* to be provided as the last argument to each callback function */
void* data;
};
diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl
index dc17c6b..7284975 100755
--- a/tools/libxl/libxl_save_msgs_gen.pl
+++ b/tools/libxl/libxl_save_msgs_gen.pl
@@ -26,11 +26,12 @@ our @msgs = (
[ 3, 'scxA', "suspend", [] ],
[ 4, 'scxA', "postcopy", [] ],
[ 5, 'srcxA', "checkpoint", [] ],
- [ 6, 'scxA', "switch_qemu_logdirty", [qw(int domid
+ [ 6, 'srcxA', "should_checkpoint", [] ],
+ [ 7, 'scxA', "switch_qemu_logdirty", [qw(int domid
unsigned enable)] ],
- [ 7, 'r', "restore_results", ['unsigned long', 'store_mfn',
+ [ 8, 'r', "restore_results", ['unsigned long', 'store_mfn',
'unsigned long', 'console_mfn'] ],
- [ 8, 'srW', "complete", [qw(int retval
+ [ 9, 'srW', "complete", [qw(int retval
int errnoval)] ],
);
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback
2015-06-25 6:25 ` [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback Yang Hongyang
@ 2015-06-30 10:19 ` Ian Campbell
2015-07-01 2:43 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:19 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Under COLO, we are doing checkpoint on demand, if this
> callback returns 1, we will take another checkpoint.
> 0 indicates unexpected error.
Is this checkpoint therefore expected to be blocking until another
checkpoint is desired (or an error occurs)?
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
> tools/libxc/include/xenguest.h | 18 ++++++++++++++++++
> tools/libxl/libxl_save_msgs_gen.pl | 7 ++++---
> 2 files changed, 22 insertions(+), 3 deletions(-)
>
> diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
> index 3e48a97..e804a1d 100644
> --- a/tools/libxc/include/xenguest.h
> +++ b/tools/libxc/include/xenguest.h
> @@ -63,6 +63,15 @@ struct save_callbacks {
> * 1: take another checkpoint */
> int (*checkpoint)(void* data);
>
> + /*
> + * Called after the checkpoint callback.
> + *
> + * returns:
> + * 0: terminate checkpointing gracefully
The commit message describes this as an "unexpected error" which this
seems more like the desired response to such an error.
And what would gracefully involve in this context, resuming on sender or
receiver?
> + * 1: take another checkpoint
> + */
> + int (*should_checkpoint)(void* data);
> +
> /* Enable qemu-dm logging dirty pages to xen */
> int (*switch_qemu_logdirty)(int domid, unsigned enable, void *data); /* HVM only */
>
> @@ -108,6 +117,15 @@ struct restore_callbacks {
> /* A checkpoint record has been found in the stream */
> int (*checkpoint)(void* data);
>
> + /*
> + * Called after the checkpoint callback.
> + *
> + * returns:
> + * 0: terminate checkpointing gracefully
Again.
> + * 1: take another checkpoint
> + */
> + int (*should_checkpoint)(void* data);
> +
> /* to be provided as the last argument to each callback function */
> void* data;
> };
> diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl
> index dc17c6b..7284975 100755
> --- a/tools/libxl/libxl_save_msgs_gen.pl
> +++ b/tools/libxl/libxl_save_msgs_gen.pl
> @@ -26,11 +26,12 @@ our @msgs = (
> [ 3, 'scxA', "suspend", [] ],
> [ 4, 'scxA', "postcopy", [] ],
> [ 5, 'srcxA', "checkpoint", [] ],
> - [ 6, 'scxA', "switch_qemu_logdirty", [qw(int domid
> + [ 6, 'srcxA', "should_checkpoint", [] ],
> + [ 7, 'scxA', "switch_qemu_logdirty", [qw(int domid
> unsigned enable)] ],
> - [ 7, 'r', "restore_results", ['unsigned long', 'store_mfn',
> + [ 8, 'r', "restore_results", ['unsigned long', 'store_mfn',
> 'unsigned long', 'console_mfn'] ],
> - [ 8, 'srW', "complete", [qw(int retval
> + [ 9, 'srW', "complete", [qw(int retval
> int errnoval)] ],
> );
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback
2015-06-30 10:19 ` Ian Campbell
@ 2015-07-01 2:43 ` Yang Hongyang
2015-07-01 10:43 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:43 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:19 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Under COLO, we are doing checkpoint on demand, if this
>> callback returns 1, we will take another checkpoint.
>> 0 indicates unexpected error.
>
> Is this checkpoint therefore expected to be blocking until another
> checkpoint is desired (or an error occurs)?
Right.
>
>>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> ---
>> tools/libxc/include/xenguest.h | 18 ++++++++++++++++++
>> tools/libxl/libxl_save_msgs_gen.pl | 7 ++++---
>> 2 files changed, 22 insertions(+), 3 deletions(-)
>>
>> diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
>> index 3e48a97..e804a1d 100644
>> --- a/tools/libxc/include/xenguest.h
>> +++ b/tools/libxc/include/xenguest.h
>> @@ -63,6 +63,15 @@ struct save_callbacks {
>> * 1: take another checkpoint */
>> int (*checkpoint)(void* data);
>>
>> + /*
>> + * Called after the checkpoint callback.
>> + *
>> + * returns:
>> + * 0: terminate checkpointing gracefully
>
> The commit message describes this as an "unexpected error" which this
> seems more like the desired response to such an error.
>
> And what would gracefully involve in this context, resuming on sender or
> receiver?
It depends on the error. If it indicates a failover on restore side, we will
resuming the receiver. I will make the comment clearer next version.
>
>> + * 1: take another checkpoint
>> + */
>> + int (*should_checkpoint)(void* data);
>> +
>> /* Enable qemu-dm logging dirty pages to xen */
>> int (*switch_qemu_logdirty)(int domid, unsigned enable, void *data); /* HVM only */
>>
>> @@ -108,6 +117,15 @@ struct restore_callbacks {
>> /* A checkpoint record has been found in the stream */
>> int (*checkpoint)(void* data);
>>
>> + /*
>> + * Called after the checkpoint callback.
>> + *
>> + * returns:
>> + * 0: terminate checkpointing gracefully
>
> Again.
>
>> + * 1: take another checkpoint
>> + */
>> + int (*should_checkpoint)(void* data);
>> +
>> /* to be provided as the last argument to each callback function */
>> void* data;
>> };
>> diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl
>> index dc17c6b..7284975 100755
>> --- a/tools/libxl/libxl_save_msgs_gen.pl
>> +++ b/tools/libxl/libxl_save_msgs_gen.pl
>> @@ -26,11 +26,12 @@ our @msgs = (
>> [ 3, 'scxA', "suspend", [] ],
>> [ 4, 'scxA', "postcopy", [] ],
>> [ 5, 'srcxA', "checkpoint", [] ],
>> - [ 6, 'scxA', "switch_qemu_logdirty", [qw(int domid
>> + [ 6, 'srcxA', "should_checkpoint", [] ],
>> + [ 7, 'scxA', "switch_qemu_logdirty", [qw(int domid
>> unsigned enable)] ],
>> - [ 7, 'r', "restore_results", ['unsigned long', 'store_mfn',
>> + [ 8, 'r', "restore_results", ['unsigned long', 'store_mfn',
>> 'unsigned long', 'console_mfn'] ],
>> - [ 8, 'srW', "complete", [qw(int retval
>> + [ 9, 'srW', "complete", [qw(int retval
>> int errnoval)] ],
>> );
>>
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback
2015-07-01 2:43 ` Yang Hongyang
@ 2015-07-01 10:43 ` Ian Campbell
2015-07-01 13:58 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:43 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Wed, 2015-07-01 at 10:43 +0800, Yang Hongyang wrote:
>
> On 06/30/2015 06:19 PM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> Under COLO, we are doing checkpoint on demand, if this
> >> callback returns 1, we will take another checkpoint.
> >> 0 indicates unexpected error.
> >
> > Is this checkpoint therefore expected to be blocking until another
> > checkpoint is desired (or an error occurs)?
>
> Right.
IMHO this should therefore be documented. Some care will need to be
taken to avoid blocking any non-async part of the main libxl process.
The means by which this is achieved (which I suppose is that the call
back is in the context of the helper process) should be mentioned in the
commit message.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback
2015-07-01 10:43 ` Ian Campbell
@ 2015-07-01 13:58 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 13:58 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 07/01/2015 06:43 PM, Ian Campbell wrote:
> On Wed, 2015-07-01 at 10:43 +0800, Yang Hongyang wrote:
>>
>> On 06/30/2015 06:19 PM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>> Under COLO, we are doing checkpoint on demand, if this
>>>> callback returns 1, we will take another checkpoint.
>>>> 0 indicates unexpected error.
>>>
>>> Is this checkpoint therefore expected to be blocking until another
>>> checkpoint is desired (or an error occurs)?
>>
>> Right.
>
> IMHO this should therefore be documented. Some care will need to be
> taken to avoid blocking any non-async part of the main libxl process.
> The means by which this is achieved (which I suppose is that the call
> back is in the context of the helper process) should be mentioned in the
> commit message.
Ok, thank you very much!
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 18/26] tools/libx{l, c}: add postcopy/suspend callback to restore side
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (16 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 17/26] tools/libx{l, c}: introduce should_checkpoint callback Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:21 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records Yang Hongyang
` (7 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
Secondary(restore side) is running under COLO, we also need
postcopy/suspend callbacks.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxc/include/xenguest.h | 10 ++++++++++
tools/libxl/libxl_save_msgs_gen.pl | 4 ++--
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
index e804a1d..dcc441a 100644
--- a/tools/libxc/include/xenguest.h
+++ b/tools/libxc/include/xenguest.h
@@ -114,6 +114,16 @@ struct restore_callbacks {
int (*toolstack_restore)(uint32_t domid, const uint8_t *buf,
uint32_t size, void* data);
+ /* Called after a new checkpoint to suspend the guest.
+ */
+ int (*suspend)(void* data);
+
+ /* Called after the secondary vm is ready to resume.
+ * Callback function resumes the guest & the device model,
+ * returns to xc_domain_restore.
+ */
+ int (*postcopy)(void* data);
+
/* A checkpoint record has been found in the stream */
int (*checkpoint)(void* data);
diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl
index 7284975..86cd395 100755
--- a/tools/libxl/libxl_save_msgs_gen.pl
+++ b/tools/libxl/libxl_save_msgs_gen.pl
@@ -23,8 +23,8 @@ our @msgs = (
STRING doing_what),
'unsigned long', 'done',
'unsigned long', 'total'] ],
- [ 3, 'scxA', "suspend", [] ],
- [ 4, 'scxA', "postcopy", [] ],
+ [ 3, 'srcxA', "suspend", [] ],
+ [ 4, 'srcxA', "postcopy", [] ],
[ 5, 'srcxA', "checkpoint", [] ],
[ 6, 'srcxA', "should_checkpoint", [] ],
[ 7, 'scxA', "switch_qemu_logdirty", [qw(int domid
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 18/26] tools/libx{l, c}: add postcopy/suspend callback to restore side
2015-06-25 6:25 ` [PATCH v3 COLOPre 18/26] tools/libx{l, c}: add postcopy/suspend callback to restore side Yang Hongyang
@ 2015-06-30 10:21 ` Ian Campbell
2015-07-01 2:48 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:21 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Secondary(restore side) is running under COLO, we also need
> postcopy/suspend callbacks.
I wonder if this and the previous patch ought to be folded into the
patches which introduce the actual use of the callback?
>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
> tools/libxc/include/xenguest.h | 10 ++++++++++
> tools/libxl/libxl_save_msgs_gen.pl | 4 ++--
> 2 files changed, 12 insertions(+), 2 deletions(-)
>
> diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
> index e804a1d..dcc441a 100644
> --- a/tools/libxc/include/xenguest.h
> +++ b/tools/libxc/include/xenguest.h
> @@ -114,6 +114,16 @@ struct restore_callbacks {
> int (*toolstack_restore)(uint32_t domid, const uint8_t *buf,
> uint32_t size, void* data);
>
> + /* Called after a new checkpoint to suspend the guest.
> + */
> + int (*suspend)(void* data);
> +
> + /* Called after the secondary vm is ready to resume.
> + * Callback function resumes the guest & the device model,
> + * returns to xc_domain_restore.
> + */
> + int (*postcopy)(void* data);
> +
> /* A checkpoint record has been found in the stream */
> int (*checkpoint)(void* data);
>
> diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl
> index 7284975..86cd395 100755
> --- a/tools/libxl/libxl_save_msgs_gen.pl
> +++ b/tools/libxl/libxl_save_msgs_gen.pl
> @@ -23,8 +23,8 @@ our @msgs = (
> STRING doing_what),
> 'unsigned long', 'done',
> 'unsigned long', 'total'] ],
> - [ 3, 'scxA', "suspend", [] ],
> - [ 4, 'scxA', "postcopy", [] ],
> + [ 3, 'srcxA', "suspend", [] ],
> + [ 4, 'srcxA', "postcopy", [] ],
> [ 5, 'srcxA', "checkpoint", [] ],
> [ 6, 'srcxA', "should_checkpoint", [] ],
> [ 7, 'scxA', "switch_qemu_logdirty", [qw(int domid
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 18/26] tools/libx{l, c}: add postcopy/suspend callback to restore side
2015-06-30 10:21 ` Ian Campbell
@ 2015-07-01 2:48 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 2:48 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:21 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Secondary(restore side) is running under COLO, we also need
>> postcopy/suspend callbacks.
>
> I wonder if this and the previous patch ought to be folded into the
> patches which introduce the actual use of the callback?
No problem, will squash these 2 patches.
>>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> ---
>> tools/libxc/include/xenguest.h | 10 ++++++++++
>> tools/libxl/libxl_save_msgs_gen.pl | 4 ++--
>> 2 files changed, 12 insertions(+), 2 deletions(-)
>>
>> diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
>> index e804a1d..dcc441a 100644
>> --- a/tools/libxc/include/xenguest.h
>> +++ b/tools/libxc/include/xenguest.h
>> @@ -114,6 +114,16 @@ struct restore_callbacks {
>> int (*toolstack_restore)(uint32_t domid, const uint8_t *buf,
>> uint32_t size, void* data);
>>
>> + /* Called after a new checkpoint to suspend the guest.
>> + */
>> + int (*suspend)(void* data);
>> +
>> + /* Called after the secondary vm is ready to resume.
>> + * Callback function resumes the guest & the device model,
>> + * returns to xc_domain_restore.
>> + */
>> + int (*postcopy)(void* data);
>> +
>> /* A checkpoint record has been found in the stream */
>> int (*checkpoint)(void* data);
>>
>> diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl
>> index 7284975..86cd395 100755
>> --- a/tools/libxl/libxl_save_msgs_gen.pl
>> +++ b/tools/libxl/libxl_save_msgs_gen.pl
>> @@ -23,8 +23,8 @@ our @msgs = (
>> STRING doing_what),
>> 'unsigned long', 'done',
>> 'unsigned long', 'total'] ],
>> - [ 3, 'scxA', "suspend", [] ],
>> - [ 4, 'scxA', "postcopy", [] ],
>> + [ 3, 'srcxA', "suspend", [] ],
>> + [ 4, 'srcxA', "postcopy", [] ],
>> [ 5, 'srcxA', "checkpoint", [] ],
>> [ 6, 'srcxA', "should_checkpoint", [] ],
>> [ 7, 'scxA', "switch_qemu_logdirty", [qw(int domid
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (17 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 18/26] tools/libx{l, c}: add postcopy/suspend callback to restore side Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:24 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 20/26] libxc/migration: export read_record for common use Yang Hongyang
` (6 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
Used by secondary to send it's dirty bitmap to primary under COLO.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
docs/specs/libxc-migration-stream.pandoc | 23 ++++++++++++++++++++++-
tools/libxc/xc_sr_common.c | 1 +
tools/libxc/xc_sr_stream_format.h | 1 +
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/docs/specs/libxc-migration-stream.pandoc b/docs/specs/libxc-migration-stream.pandoc
index 68fa513..718b4aa 100644
--- a/docs/specs/libxc-migration-stream.pandoc
+++ b/docs/specs/libxc-migration-stream.pandoc
@@ -227,7 +227,9 @@ type 0x00000000: END
0x0000000E: CHECKPOINT
- 0x0000000F - 0x7FFFFFFF: Reserved for future _mandatory_
+ 0x0000000F: DIRTY_BITMAP
+
+ 0x00000010 - 0x7FFFFFFF: Reserved for future _mandatory_
records.
0x80000000 - 0xFFFFFFFF: Reserved for future _optional_
@@ -601,6 +603,25 @@ CHECKPOINT record or an END record.
\clearpage
+DIRTY_BITMAP
+------------
+
+A dirty_bitmap record is used for secondary to send it's dirty bitmap
+to primary while doing a checkpoint under COLO.
+
+ 0 1 2 3 4 5 6 7 octet
+ +-------------------------------------------------+
+ | pfn[0] |
+ +-------------------------------------------------+
+ ...
+ +-------------------------------------------------+
+ | pfn[C-1] |
+ +-------------------------------------------------+
+
+The count of the pfn is: record->length/sizeof(uint64_t).
+
+\clearpage
+
Layout
======
diff --git a/tools/libxc/xc_sr_common.c b/tools/libxc/xc_sr_common.c
index 945cfa6..becc0f4 100644
--- a/tools/libxc/xc_sr_common.c
+++ b/tools/libxc/xc_sr_common.c
@@ -35,6 +35,7 @@ static const char *mandatory_rec_types[] =
[REC_TYPE_X86_PV_VCPU_MSRS] = "x86 PV vcpu msrs",
[REC_TYPE_VERIFY] = "Verify",
[REC_TYPE_CHECKPOINT] = "Checkpoint",
+ [REC_TYPE_DIRTY_BITMAP] = "Dirty bitmap",
};
const char *rec_type_to_str(uint32_t type)
diff --git a/tools/libxc/xc_sr_stream_format.h b/tools/libxc/xc_sr_stream_format.h
index 6d0f8fd..43a0209 100644
--- a/tools/libxc/xc_sr_stream_format.h
+++ b/tools/libxc/xc_sr_stream_format.h
@@ -75,6 +75,7 @@ struct xc_sr_rhdr
#define REC_TYPE_X86_PV_VCPU_MSRS 0x0000000cU
#define REC_TYPE_VERIFY 0x0000000dU
#define REC_TYPE_CHECKPOINT 0x0000000eU
+#define REC_TYPE_DIRTY_BITMAP 0x0000000fU
#define REC_TYPE_OPTIONAL 0x80000000U
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-06-25 6:25 ` [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records Yang Hongyang
@ 2015-06-30 10:24 ` Ian Campbell
2015-07-01 3:07 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:24 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Used by secondary to send it's dirty bitmap to primary under COLO.
This is the backchannel, right?
It seems to me that this ought to be described more clearly as a
separate stream in the opposite direction, rather than looking like just
another record in the forward channel.
Does the back channel not also need some sort of negotiation phase where
we check both ends are compatible (i.e. like the forward channel's
header). This might be easier than with the forward channel since you
might assert that the versions must match exactly for COLO to be
possible, that might not be true of some potential future user of the
backchannel though.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-06-30 10:24 ` Ian Campbell
@ 2015-07-01 3:07 ` Yang Hongyang
2015-07-01 10:16 ` Andrew Cooper
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 3:07 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:24 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Used by secondary to send it's dirty bitmap to primary under COLO.
>
> This is the backchannel, right?
Right.
>
> It seems to me that this ought to be described more clearly as a
> separate stream in the opposite direction, rather than looking like just
> another record in the forward channel.
Agreed, I'm not sure if having this back channel record is eligible,
Andy, thoughts?
>
> Does the back channel not also need some sort of negotiation phase where
> we check both ends are compatible (i.e. like the forward channel's
> header). This might be easier than with the forward channel since you
> might assert that the versions must match exactly for COLO to be
> possible, that might not be true of some potential future user of the
> backchannel though.
The negotiation record for COLO is introduced in the following patch
on libxl side. But that might be diffrent form what you said here, we
don't have a version check currently, if the 2 side doesn't match, for
example one has colo feature enabled and the other end do not, the
migration will simply fail.
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-07-01 3:07 ` Yang Hongyang
@ 2015-07-01 10:16 ` Andrew Cooper
2015-07-01 10:27 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Andrew Cooper @ 2015-07-01 10:16 UTC (permalink / raw)
To: Yang Hongyang, Ian Campbell
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, ian.jackson
On 01/07/15 04:07, Yang Hongyang wrote:
>
>
> On 06/30/2015 06:24 PM, Ian Campbell wrote:
>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>> Used by secondary to send it's dirty bitmap to primary under COLO.
>>
>> This is the backchannel, right?
>
> Right.
>
>>
>> It seems to me that this ought to be described more clearly as a
>> separate stream in the opposite direction, rather than looking like just
>> another record in the forward channel.
>
> Agreed, I'm not sure if having this back channel record is eligible,
> Andy, thoughts?
>
>>
>> Does the back channel not also need some sort of negotiation phase where
>> we check both ends are compatible (i.e. like the forward channel's
>> header). This might be easier than with the forward channel since you
>> might assert that the versions must match exactly for COLO to be
>> possible, that might not be true of some potential future user of the
>> backchannel though.
>
> The negotiation record for COLO is introduced in the following patch
> on libxl side. But that might be diffrent form what you said here, we
> don't have a version check currently, if the 2 side doesn't match, for
> example one has colo feature enabled and the other end do not, the
> migration will simply fail.
I do think that each backchannel level needs some kind of initial
negotiation to confirm everything is set up and working, but I think the
backchannel should also match the spec for its level, and all contained
in the single spec document.
So for both the libxc and libxl backchannels, I would have thought
something along these lines to be sensible:
Forwards sends a LIBX{C,L}_BACKCHANNEL_INIT, and waits to find a
LIBX{C,L}_BACKCHANNEL_REPLY on the backchannel. After that, processing
continues as normal, with records arriving on the backchannel when
appropriate.
~Andrew
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-07-01 10:16 ` Andrew Cooper
@ 2015-07-01 10:27 ` Ian Campbell
2015-07-01 10:39 ` Andrew Cooper
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:27 UTC (permalink / raw)
To: Andrew Cooper
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Yang Hongyang, ian.jackson
On Wed, 2015-07-01 at 11:16 +0100, Andrew Cooper wrote:
> On 01/07/15 04:07, Yang Hongyang wrote:
> >
> >
> > On 06/30/2015 06:24 PM, Ian Campbell wrote:
> >> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >>> Used by secondary to send it's dirty bitmap to primary under COLO.
> >>
> >> This is the backchannel, right?
> >
> > Right.
> >
> >>
> >> It seems to me that this ought to be described more clearly as a
> >> separate stream in the opposite direction, rather than looking like just
> >> another record in the forward channel.
> >
> > Agreed, I'm not sure if having this back channel record is eligible,
> > Andy, thoughts?
> >
> >>
> >> Does the back channel not also need some sort of negotiation phase where
> >> we check both ends are compatible (i.e. like the forward channel's
> >> header). This might be easier than with the forward channel since you
> >> might assert that the versions must match exactly for COLO to be
> >> possible, that might not be true of some potential future user of the
> >> backchannel though.
> >
> > The negotiation record for COLO is introduced in the following patch
> > on libxl side. But that might be diffrent form what you said here, we
> > don't have a version check currently, if the 2 side doesn't match, for
> > example one has colo feature enabled and the other end do not, the
> > migration will simply fail.
>
> I do think that each backchannel level needs some kind of initial
> negotiation to confirm everything is set up and working, but I think the
> backchannel should also match the spec for its level, and all contained
> in the single spec document.
In the same spec, sure. It's the presenting it as just another record
mixed in with all the others which I think is a problem.
At the very least every record should be tagged as either forward,
backward or bidirectional to indicate who can produce and who should
consume it.
Even better would br if we can convince ourselves there should be no
bidirectional fields, in which case I'd be further inclined to say that
the record space should be explicitly separate. i.e. the backchannel
should be a separate chapter in the doc and the records.
> So for both the libxc and libxl backchannels, I would have thought
> something along these lines to be sensible:
>
> Forwards sends a LIBX{C,L}_BACKCHANNEL_INIT, and waits to find a
> LIBX{C,L}_BACKCHANNEL_REPLY on the backchannel. After that, processing
> continues as normal, with records arriving on the backchannel when
> appropriate.
Yes, for init time something like this sounds sensible.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-07-01 10:27 ` Ian Campbell
@ 2015-07-01 10:39 ` Andrew Cooper
2015-07-01 11:00 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Andrew Cooper @ 2015-07-01 10:39 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Yang Hongyang, ian.jackson
On 01/07/15 11:27, Ian Campbell wrote:
> On Wed, 2015-07-01 at 11:16 +0100, Andrew Cooper wrote:
>> On 01/07/15 04:07, Yang Hongyang wrote:
>>>
>>> On 06/30/2015 06:24 PM, Ian Campbell wrote:
>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>>> Used by secondary to send it's dirty bitmap to primary under COLO.
>>>> This is the backchannel, right?
>>> Right.
>>>
>>>> It seems to me that this ought to be described more clearly as a
>>>> separate stream in the opposite direction, rather than looking like just
>>>> another record in the forward channel.
>>> Agreed, I'm not sure if having this back channel record is eligible,
>>> Andy, thoughts?
>>>
>>>> Does the back channel not also need some sort of negotiation phase where
>>>> we check both ends are compatible (i.e. like the forward channel's
>>>> header). This might be easier than with the forward channel since you
>>>> might assert that the versions must match exactly for COLO to be
>>>> possible, that might not be true of some potential future user of the
>>>> backchannel though.
>>> The negotiation record for COLO is introduced in the following patch
>>> on libxl side. But that might be diffrent form what you said here, we
>>> don't have a version check currently, if the 2 side doesn't match, for
>>> example one has colo feature enabled and the other end do not, the
>>> migration will simply fail.
>> I do think that each backchannel level needs some kind of initial
>> negotiation to confirm everything is set up and working, but I think the
>> backchannel should also match the spec for its level, and all contained
>> in the single spec document.
> In the same spec, sure. It's the presenting it as just another record
> mixed in with all the others which I think is a problem.
Ah I see. Yes - this would better be avoided.
>
> At the very least every record should be tagged as either forward,
> backward or bidirectional to indicate who can produce and who should
> consume it.
>
> Even better would br if we can convince ourselves there should be no
> bidirectional fields, in which case I'd be further inclined to say that
> the record space should be explicitly separate. i.e. the backchannel
> should be a separate chapter in the doc and the records.
I think it would be unwise to rule out the possibility of bidirectional
records. In the case that we get to a position of wanting/needing them,
we absolutely don't want a bidirectional record to have different id in
the forwards and backwards direction.
~Andrew
>
>> So for both the libxc and libxl backchannels, I would have thought
>> something along these lines to be sensible:
>>
>> Forwards sends a LIBX{C,L}_BACKCHANNEL_INIT, and waits to find a
>> LIBX{C,L}_BACKCHANNEL_REPLY on the backchannel. After that, processing
>> continues as normal, with records arriving on the backchannel when
>> appropriate.
> Yes, for init time something like this sounds sensible.
>
> Ian.
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-07-01 10:39 ` Andrew Cooper
@ 2015-07-01 11:00 ` Ian Campbell
2015-07-03 14:25 ` Andrew Cooper
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 11:00 UTC (permalink / raw)
To: Andrew Cooper
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Yang Hongyang, ian.jackson
On Wed, 2015-07-01 at 11:39 +0100, Andrew Cooper wrote:
> On 01/07/15 11:27, Ian Campbell wrote:
> > On Wed, 2015-07-01 at 11:16 +0100, Andrew Cooper wrote:
> >> On 01/07/15 04:07, Yang Hongyang wrote:
> >>>
> >>> On 06/30/2015 06:24 PM, Ian Campbell wrote:
> >>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >>>>> Used by secondary to send it's dirty bitmap to primary under COLO.
> >>>> This is the backchannel, right?
> >>> Right.
> >>>
> >>>> It seems to me that this ought to be described more clearly as a
> >>>> separate stream in the opposite direction, rather than looking like just
> >>>> another record in the forward channel.
> >>> Agreed, I'm not sure if having this back channel record is eligible,
> >>> Andy, thoughts?
> >>>
> >>>> Does the back channel not also need some sort of negotiation phase where
> >>>> we check both ends are compatible (i.e. like the forward channel's
> >>>> header). This might be easier than with the forward channel since you
> >>>> might assert that the versions must match exactly for COLO to be
> >>>> possible, that might not be true of some potential future user of the
> >>>> backchannel though.
> >>> The negotiation record for COLO is introduced in the following patch
> >>> on libxl side. But that might be diffrent form what you said here, we
> >>> don't have a version check currently, if the 2 side doesn't match, for
> >>> example one has colo feature enabled and the other end do not, the
> >>> migration will simply fail.
> >> I do think that each backchannel level needs some kind of initial
> >> negotiation to confirm everything is set up and working, but I think the
> >> backchannel should also match the spec for its level, and all contained
> >> in the single spec document.
> > In the same spec, sure. It's the presenting it as just another record
> > mixed in with all the others which I think is a problem.
>
> Ah I see. Yes - this would better be avoided.
>
> >
> > At the very least every record should be tagged as either forward,
> > backward or bidirectional to indicate who can produce and who should
> > consume it.
> >
> > Even better would br if we can convince ourselves there should be no
> > bidirectional fields, in which case I'd be further inclined to say that
> > the record space should be explicitly separate. i.e. the backchannel
> > should be a separate chapter in the doc and the records.
>
> I think it would be unwise to rule out the possibility of bidirectional
> records. In the case that we get to a position of wanting/needing them,
> we absolutely don't want a bidirectional record to have different id in
> the forwards and backwards direction.
Agreed.
Perhaps we should reserve some space for forward/backward/bidir records
in the record id space? Bit 31 is already the optional flag so e.g.
perhaps bit 30 = Backwards and bit 29 = !Forwards.
So a forward mandatory id would be 0x0......., backward would be
0x4....... and bidir would be 0x6......., optional bidir would be
0xe....... etc?
The weird inversion of Forward is in order to retain the existing record
ids.
>
> ~Andrew
>
> >
> >> So for both the libxc and libxl backchannels, I would have thought
> >> something along these lines to be sensible:
> >>
> >> Forwards sends a LIBX{C,L}_BACKCHANNEL_INIT, and waits to find a
> >> LIBX{C,L}_BACKCHANNEL_REPLY on the backchannel. After that, processing
> >> continues as normal, with records arriving on the backchannel when
> >> appropriate.
> > Yes, for init time something like this sounds sensible.
> >
> > Ian.
> >
>
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-07-01 11:00 ` Ian Campbell
@ 2015-07-03 14:25 ` Andrew Cooper
2015-07-03 14:41 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Andrew Cooper @ 2015-07-03 14:25 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Yang Hongyang, ian.jackson
On 01/07/15 12:00, Ian Campbell wrote:
> On Wed, 2015-07-01 at 11:39 +0100, Andrew Cooper wrote:
>> On 01/07/15 11:27, Ian Campbell wrote:
>>> On Wed, 2015-07-01 at 11:16 +0100, Andrew Cooper wrote:
>>>> On 01/07/15 04:07, Yang Hongyang wrote:
>>>>> On 06/30/2015 06:24 PM, Ian Campbell wrote:
>>>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>>>>> Used by secondary to send it's dirty bitmap to primary under COLO.
>>>>>> This is the backchannel, right?
>>>>> Right.
>>>>>
>>>>>> It seems to me that this ought to be described more clearly as a
>>>>>> separate stream in the opposite direction, rather than looking like just
>>>>>> another record in the forward channel.
>>>>> Agreed, I'm not sure if having this back channel record is eligible,
>>>>> Andy, thoughts?
>>>>>
>>>>>> Does the back channel not also need some sort of negotiation phase where
>>>>>> we check both ends are compatible (i.e. like the forward channel's
>>>>>> header). This might be easier than with the forward channel since you
>>>>>> might assert that the versions must match exactly for COLO to be
>>>>>> possible, that might not be true of some potential future user of the
>>>>>> backchannel though.
>>>>> The negotiation record for COLO is introduced in the following patch
>>>>> on libxl side. But that might be diffrent form what you said here, we
>>>>> don't have a version check currently, if the 2 side doesn't match, for
>>>>> example one has colo feature enabled and the other end do not, the
>>>>> migration will simply fail.
>>>> I do think that each backchannel level needs some kind of initial
>>>> negotiation to confirm everything is set up and working, but I think the
>>>> backchannel should also match the spec for its level, and all contained
>>>> in the single spec document.
>>> In the same spec, sure. It's the presenting it as just another record
>>> mixed in with all the others which I think is a problem.
>> Ah I see. Yes - this would better be avoided.
>>
>>> At the very least every record should be tagged as either forward,
>>> backward or bidirectional to indicate who can produce and who should
>>> consume it.
>>>
>>> Even better would br if we can convince ourselves there should be no
>>> bidirectional fields, in which case I'd be further inclined to say that
>>> the record space should be explicitly separate. i.e. the backchannel
>>> should be a separate chapter in the doc and the records.
>> I think it would be unwise to rule out the possibility of bidirectional
>> records. In the case that we get to a position of wanting/needing them,
>> we absolutely don't want a bidirectional record to have different id in
>> the forwards and backwards direction.
> Agreed.
>
> Perhaps we should reserve some space for forward/backward/bidir records
> in the record id space? Bit 31 is already the optional flag so e.g.
> perhaps bit 30 = Backwards and bit 29 = !Forwards.
>
> So a forward mandatory id would be 0x0......., backward would be
> 0x4....... and bidir would be 0x6......., optional bidir would be
> 0xe....... etc?
>
> The weird inversion of Forward is in order to retain the existing record
> ids.
I don't see how this helps. It will break binary compatibility if an
existing forward record wants to be changed to being bidirectional.
I think that some clear identification in the spec, and perhaps a helper
or two in xc_sr_common.c is going to be the easiest solution to this
problem.
~Andrew
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records
2015-07-03 14:25 ` Andrew Cooper
@ 2015-07-03 14:41 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-07-03 14:41 UTC (permalink / raw)
To: Andrew Cooper
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, Yang Hongyang, ian.jackson
On Fri, 2015-07-03 at 15:25 +0100, Andrew Cooper wrote:
> On 01/07/15 12:00, Ian Campbell wrote:
> > On Wed, 2015-07-01 at 11:39 +0100, Andrew Cooper wrote:
> >> On 01/07/15 11:27, Ian Campbell wrote:
> >>> On Wed, 2015-07-01 at 11:16 +0100, Andrew Cooper wrote:
> >>>> On 01/07/15 04:07, Yang Hongyang wrote:
> >>>>> On 06/30/2015 06:24 PM, Ian Campbell wrote:
> >>>>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >>>>>>> Used by secondary to send it's dirty bitmap to primary under COLO.
> >>>>>> This is the backchannel, right?
> >>>>> Right.
> >>>>>
> >>>>>> It seems to me that this ought to be described more clearly as a
> >>>>>> separate stream in the opposite direction, rather than looking like just
> >>>>>> another record in the forward channel.
> >>>>> Agreed, I'm not sure if having this back channel record is eligible,
> >>>>> Andy, thoughts?
> >>>>>
> >>>>>> Does the back channel not also need some sort of negotiation phase where
> >>>>>> we check both ends are compatible (i.e. like the forward channel's
> >>>>>> header). This might be easier than with the forward channel since you
> >>>>>> might assert that the versions must match exactly for COLO to be
> >>>>>> possible, that might not be true of some potential future user of the
> >>>>>> backchannel though.
> >>>>> The negotiation record for COLO is introduced in the following patch
> >>>>> on libxl side. But that might be diffrent form what you said here, we
> >>>>> don't have a version check currently, if the 2 side doesn't match, for
> >>>>> example one has colo feature enabled and the other end do not, the
> >>>>> migration will simply fail.
> >>>> I do think that each backchannel level needs some kind of initial
> >>>> negotiation to confirm everything is set up and working, but I think the
> >>>> backchannel should also match the spec for its level, and all contained
> >>>> in the single spec document.
> >>> In the same spec, sure. It's the presenting it as just another record
> >>> mixed in with all the others which I think is a problem.
> >> Ah I see. Yes - this would better be avoided.
> >>
> >>> At the very least every record should be tagged as either forward,
> >>> backward or bidirectional to indicate who can produce and who should
> >>> consume it.
> >>>
> >>> Even better would br if we can convince ourselves there should be no
> >>> bidirectional fields, in which case I'd be further inclined to say that
> >>> the record space should be explicitly separate. i.e. the backchannel
> >>> should be a separate chapter in the doc and the records.
> >> I think it would be unwise to rule out the possibility of bidirectional
> >> records. In the case that we get to a position of wanting/needing them,
> >> we absolutely don't want a bidirectional record to have different id in
> >> the forwards and backwards direction.
> > Agreed.
> >
> > Perhaps we should reserve some space for forward/backward/bidir records
> > in the record id space? Bit 31 is already the optional flag so e.g.
> > perhaps bit 30 = Backwards and bit 29 = !Forwards.
> >
> > So a forward mandatory id would be 0x0......., backward would be
> > 0x4....... and bidir would be 0x6......., optional bidir would be
> > 0xe....... etc?
> >
> > The weird inversion of Forward is in order to retain the existing record
> > ids.
>
> I don't see how this helps. It will break binary compatibility if an
> existing forward record wants to be changed to being bidirectional.
I'd say that was a feature not a bug, if you are going to be getting
these things thrown back at you from some point onwards then you need to
be prepared to cope (e.g. negotiate).
> I think that some clear identification in the spec, and perhaps a helper
> or two in xc_sr_common.c is going to be the easiest solution to this
> problem.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 20/26] libxc/migration: export read_record for common use
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (18 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 19/26] libxc/migration: Specification update for DIRTY_BITMAP records Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel Yang Hongyang
` (5 subsequent siblings)
25 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
read_record() could be used by primary to read dirty bitmap
record sent by secondary under COLO.
When used by save side, we need to pass the backchannel fd
instead of ctx->fd to read_record(), so we added a fd param to
it.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/libxc/xc_sr_common.c | 49 +++++++++++++++++++++++++++++++++++
tools/libxc/xc_sr_common.h | 14 ++++++++++
tools/libxc/xc_sr_restore.c | 63 +--------------------------------------------
3 files changed, 64 insertions(+), 62 deletions(-)
diff --git a/tools/libxc/xc_sr_common.c b/tools/libxc/xc_sr_common.c
index becc0f4..0ee607c 100644
--- a/tools/libxc/xc_sr_common.c
+++ b/tools/libxc/xc_sr_common.c
@@ -89,6 +89,55 @@ int write_split_record(struct xc_sr_context *ctx, struct xc_sr_record *rec,
return -1;
}
+int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec)
+{
+ xc_interface *xch = ctx->xch;
+ struct xc_sr_rhdr rhdr;
+ size_t datasz;
+
+ if ( read_exact(fd, &rhdr, sizeof(rhdr)) )
+ {
+ PERROR("Failed to read Record Header from stream");
+ return -1;
+ }
+ else if ( rhdr.length > REC_LENGTH_MAX )
+ {
+ ERROR("Record (0x%08x, %s) length %#x exceeds max (%#x)", rhdr.type,
+ rec_type_to_str(rhdr.type), rhdr.length, REC_LENGTH_MAX);
+ return -1;
+ }
+
+ datasz = ROUNDUP(rhdr.length, REC_ALIGN_ORDER);
+
+ if ( datasz )
+ {
+ rec->data = malloc(datasz);
+
+ if ( !rec->data )
+ {
+ ERROR("Unable to allocate %zu bytes for record data (0x%08x, %s)",
+ datasz, rhdr.type, rec_type_to_str(rhdr.type));
+ return -1;
+ }
+
+ if ( read_exact(fd, rec->data, datasz) )
+ {
+ free(rec->data);
+ rec->data = NULL;
+ PERROR("Failed to read %zu bytes of data for record (0x%08x, %s)",
+ datasz, rhdr.type, rec_type_to_str(rhdr.type));
+ return -1;
+ }
+ }
+ else
+ rec->data = NULL;
+
+ rec->type = rhdr.type;
+ rec->length = rhdr.length;
+
+ return 0;
+};
+
static void __attribute__((unused)) build_assertions(void)
{
XC_BUILD_BUG_ON(sizeof(struct xc_sr_ihdr) != 24);
diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
index 04f32f5..88ef135 100644
--- a/tools/libxc/xc_sr_common.h
+++ b/tools/libxc/xc_sr_common.h
@@ -353,6 +353,20 @@ static inline int write_record(struct xc_sr_context *ctx,
}
/*
+ * Reads a record from the stream, and fills in the record structure.
+ *
+ * Returns 0 on success and non-0 on failure.
+ *
+ * On success, the records type and size shall be valid.
+ * - If size is 0, data shall be NULL.
+ * - If size is non-0, data shall be a buffer allocated by malloc() which must
+ * be passed to free() by the caller.
+ *
+ * On failure, the contents of the record structure are undefined.
+ */
+int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec);
+
+/*
* This would ideally be private in restore.c, but is needed by
* x86_pv_localise_page() if we receive pagetables frames ahead of the
* contents of the frames they point at.
diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
index f8b6f2f..e6f00db 100644
--- a/tools/libxc/xc_sr_restore.c
+++ b/tools/libxc/xc_sr_restore.c
@@ -69,67 +69,6 @@ static int read_headers(struct xc_sr_context *ctx)
}
/*
- * Reads a record from the stream, and fills in the record structure.
- *
- * Returns 0 on success and non-0 on failure.
- *
- * On success, the records type and size shall be valid.
- * - If size is 0, data shall be NULL.
- * - If size is non-0, data shall be a buffer allocated by malloc() which must
- * be passed to free() by the caller.
- *
- * On failure, the contents of the record structure are undefined.
- */
-static int read_record(struct xc_sr_context *ctx, struct xc_sr_record *rec)
-{
- xc_interface *xch = ctx->xch;
- struct xc_sr_rhdr rhdr;
- size_t datasz;
-
- if ( read_exact(ctx->fd, &rhdr, sizeof(rhdr)) )
- {
- PERROR("Failed to read Record Header from stream");
- return -1;
- }
- else if ( rhdr.length > REC_LENGTH_MAX )
- {
- ERROR("Record (0x%08x, %s) length %#x exceeds max (%#x)", rhdr.type,
- rec_type_to_str(rhdr.type), rhdr.length, REC_LENGTH_MAX);
- return -1;
- }
-
- datasz = ROUNDUP(rhdr.length, REC_ALIGN_ORDER);
-
- if ( datasz )
- {
- rec->data = malloc(datasz);
-
- if ( !rec->data )
- {
- ERROR("Unable to allocate %zu bytes for record data (0x%08x, %s)",
- datasz, rhdr.type, rec_type_to_str(rhdr.type));
- return -1;
- }
-
- if ( read_exact(ctx->fd, rec->data, datasz) )
- {
- free(rec->data);
- rec->data = NULL;
- PERROR("Failed to read %zu bytes of data for record (0x%08x, %s)",
- datasz, rhdr.type, rec_type_to_str(rhdr.type));
- return -1;
- }
- }
- else
- rec->data = NULL;
-
- rec->type = rhdr.type;
- rec->length = rhdr.length;
-
- return 0;
-};
-
-/*
* Is a pfn populated?
*/
static bool pfn_is_populated(const struct xc_sr_context *ctx, xen_pfn_t pfn)
@@ -636,7 +575,7 @@ static int restore(struct xc_sr_context *ctx)
do
{
- rc = read_record(ctx, &rec);
+ rc = read_record(ctx, ctx->fd, &rec);
if ( rc )
{
if ( ctx->restore.buffer_all_records )
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (19 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 20/26] libxc/migration: export read_record for common use Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:28 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 22/26] tools/libxl: refactor read " Yang Hongyang
` (4 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
From: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
---
tools/libxl/libxl_dom_save.c | 7 +++++--
tools/libxl/libxl_internal.h | 9 +++++----
tools/libxl/libxl_remus.c | 8 +++++---
tools/libxl/libxl_stream_write.c | 22 ++++++++++++++++++----
4 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 5797148..9a3b33d 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -30,7 +30,7 @@ struct libxl__physmap_info {
/*========================= Domain save ============================*/
static void stream_done(libxl__egc *egc,
- libxl__domain_save_state *dss, int rc);
+ libxl__stream_write_state *stream, int rc);
static void domain_save_done(libxl__egc *egc,
libxl__domain_save_state *dss, int rc);
@@ -445,6 +445,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
dss->sws.fd = dss->fd;
dss->sws.ao = dss->ao;
+ dss->sws.back_channel = false;
dss->sws.completion_callback = stream_done;
libxl__stream_write_start(egc, &dss->sws);
@@ -455,8 +456,10 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
}
static void stream_done(libxl__egc *egc,
- libxl__domain_save_state *dss, int rc)
+ libxl__stream_write_state *stream, int rc)
{
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
+
domain_save_done(egc, dss, rc);
}
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 13d8271..82a327c 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2877,12 +2877,13 @@ struct libxl__stream_write_state {
libxl__ao *ao;
int fd;
uint32_t domid;
+ bool back_channel;
void (*completion_callback)(libxl__egc *egc,
- libxl__domain_save_state *dss,
- int rc);
- void (*checkpoint_callback)(libxl__egc *egc,
- libxl__domain_save_state *dss,
+ libxl__stream_write_state *stream,
int rc);
+ void (*write_records_callback)(libxl__egc *egc,
+ libxl__stream_write_state *stream,
+ int rc);
/* Private */
int rc;
int joined_rc;
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index 01cc9e7..e143b21 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -205,7 +205,7 @@ out:
/*----- remus asynchronous checkpoint callback -----*/
static void remus_checkpoint_stream_written(
- libxl__egc *egc, libxl__domain_save_state *dss, int rc);
+ libxl__egc *egc, libxl__stream_write_state *stream, int rc);
static void remus_devices_commit_cb(libxl__egc *egc,
libxl__remus_devices_state *rds,
int rc);
@@ -219,13 +219,15 @@ void libxl__remus_domain_save_checkpoint_callback(void *data)
libxl__egc *egc = dss->shs.egc;
STATE_AO_GC(dss->ao);
- dss->sws.checkpoint_callback = remus_checkpoint_stream_written;
+ dss->sws.write_records_callback = remus_checkpoint_stream_written;
libxl__stream_write_start_checkpoint(egc, &dss->sws);
}
static void remus_checkpoint_stream_written(
- libxl__egc *egc, libxl__domain_save_state *dss, int rc)
+ libxl__egc *egc, libxl__stream_write_state *stream, int rc)
{
+ libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
+
/* Convenience aliases */
libxl__remus_devices_state *const rds = &dss->rds;
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index 1ebe306..3f981f0 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -48,6 +48,13 @@
* - Toolstack record
* - if (hvm), Qemu record
* - Checkpoint end record
+ *
+ * For back channel stream:
+ * - libxl__stream_write_start()
+ * - Set up the stream to running state
+ *
+ * - Add a new API to write the record. When the record is written
+ * out, call stream->write_records_callback() to return.
*/
static const uint8_t zero_padding[1U << REC_ALIGN_ORDER] = { 0 };
@@ -111,6 +118,9 @@ void libxl__stream_write_start(libxl__egc *egc,
assert(!stream->running);
stream->running = true;
+ if (stream->back_channel)
+ return;
+
memset(dc, 0, sizeof(*dc));
dc->readwhat = "";
dc->copywhat = "suspend header";
@@ -143,6 +153,7 @@ void libxl__stream_write_start_checkpoint(libxl__egc *egc,
{
assert(stream->running);
assert(!stream->in_checkpoint);
+ assert(!stream->back_channel);
stream->in_checkpoint = true;
write_toolstack_record(egc, stream);
@@ -178,6 +189,11 @@ static void stream_failed(libxl__egc *egc,
return;
}
+ if (stream->back_channel) {
+ stream->completion_callback(egc, stream, rc);
+ return;
+ }
+
if (stream->running) {
stream->running = false;
stream_done(egc, stream);
@@ -231,7 +247,7 @@ static void check_stream_finished(libxl__egc *egc,
LOG(DEBUG, "save/restore still in use");
else {
LOG(INFO, "Join complete: result %d", stream->joined_rc);
- stream->completion_callback(egc, dss, stream->joined_rc);
+ stream->completion_callback(egc, stream, stream->joined_rc);
}
}
@@ -577,11 +593,9 @@ static void checkpoint_done(libxl__egc *egc,
libxl__stream_write_state *stream,
int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
-
assert(stream->in_checkpoint);
stream->in_checkpoint = false;
- stream->checkpoint_callback(egc, dss, rc);
+ stream->write_records_callback(egc, stream, rc);
}
static void write_checkpoint_end_record(libxl__egc *egc,
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel
2015-06-25 6:25 ` [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel Yang Hongyang
@ 2015-06-30 10:28 ` Ian Campbell
2015-07-01 5:33 ` Wen Congyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:28 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> From: Wen Congyang <wency@cn.fujitsu.com>
Refactor how?
(Tip: In general no change can be sufficiently described in just the
subject line unless it is absolutely the most trivial possible one).
It looks like mostly checkpoint_callback is renamed to
write_records_callback and the type of its argument and that of
completion_callback changed. But why?
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel
2015-06-30 10:28 ` Ian Campbell
@ 2015-07-01 5:33 ` Wen Congyang
2015-07-01 10:45 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Wen Congyang @ 2015-07-01 5:33 UTC (permalink / raw)
To: Ian Campbell, Yang Hongyang
Cc: wei.liu2, andrew.cooper3, yunhong.jiang, eddie.dong, xen-devel,
guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:28 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> From: Wen Congyang <wency@cn.fujitsu.com>
>
> Refactor how?
>
> (Tip: In general no change can be sufficiently described in just the
> subject line unless it is absolutely the most trivial possible one).
>
> It looks like mostly checkpoint_callback is renamed to
> write_records_callback and the type of its argument and that of
> completion_callback changed. But why?
checkpoint callback just write a record and return to the caller.
For back channel, we also write a record from secondary to primary.
So I rename it to write_records_callback.
In secondary side, there is no libxl__domain_save_state, so I change
the argument: use stream instead of dss. The caller can use container_of()
to get dss.
The back channel only needs this feature: use the read/write stream to read
and write record.
Thanks
Wen Congyang
>
>
>
> .
>
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel
2015-07-01 5:33 ` Wen Congyang
@ 2015-07-01 10:45 ` Ian Campbell
2015-07-01 11:09 ` Wen Congyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:45 UTC (permalink / raw)
To: Wen Congyang
Cc: wei.liu2, andrew.cooper3, yunhong.jiang, eddie.dong, xen-devel,
guijianfeng, rshriram, Yang Hongyang, ian.jackson
On Wed, 2015-07-01 at 13:33 +0800, Wen Congyang wrote:
> On 06/30/2015 06:28 PM, Ian Campbell wrote:
> > On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> >> From: Wen Congyang <wency@cn.fujitsu.com>
> >
> > Refactor how?
> >
> > (Tip: In general no change can be sufficiently described in just the
> > subject line unless it is absolutely the most trivial possible one).
> >
> > It looks like mostly checkpoint_callback is renamed to
> > write_records_callback and the type of its argument and that of
> > completion_callback changed. But why?
>
> checkpoint callback just write a record and return to the caller.
> For back channel, we also write a record from secondary to primary.
> So I rename it to write_records_callback.
>
> In secondary side, there is no libxl__domain_save_state, so I change
> the argument: use stream instead of dss. The caller can use container_of()
> to get dss.
>
> The back channel only needs this feature: use the read/write stream to read
> and write record.
Please use this to improve the commit message.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel
2015-07-01 10:45 ` Ian Campbell
@ 2015-07-01 11:09 ` Wen Congyang
0 siblings, 0 replies; 103+ messages in thread
From: Wen Congyang @ 2015-07-01 11:09 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, andrew.cooper3, yunhong.jiang, eddie.dong, xen-devel,
guijianfeng, rshriram, Yang Hongyang, ian.jackson
On 07/01/2015 06:45 PM, Ian Campbell wrote:
> On Wed, 2015-07-01 at 13:33 +0800, Wen Congyang wrote:
>> On 06/30/2015 06:28 PM, Ian Campbell wrote:
>>> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>>>> From: Wen Congyang <wency@cn.fujitsu.com>
>>>
>>> Refactor how?
>>>
>>> (Tip: In general no change can be sufficiently described in just the
>>> subject line unless it is absolutely the most trivial possible one).
>>>
>>> It looks like mostly checkpoint_callback is renamed to
>>> write_records_callback and the type of its argument and that of
>>> completion_callback changed. But why?
>>
>> checkpoint callback just write a record and return to the caller.
>> For back channel, we also write a record from secondary to primary.
>> So I rename it to write_records_callback.
>>
>> In secondary side, there is no libxl__domain_save_state, so I change
>> the argument: use stream instead of dss. The caller can use container_of()
>> to get dss.
>>
>> The back channel only needs this feature: use the read/write stream to read
>> and write record.
>
> Please use this to improve the commit message.
OK, I will do it.
Thanks
Wen Congyang
>
>
> .
>
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 22/26] tools/libxl: refactor read stream to support back channel
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (20 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 21/26] tools/libxl: refactor write stream to support back channel Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:39 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams Yang Hongyang
` (3 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
From: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
---
tools/libxl/libxl_create.c | 8 +++++---
tools/libxl/libxl_internal.h | 9 +++++----
tools/libxl/libxl_remus.c | 8 +++++---
tools/libxl/libxl_stream_read.c | 33 +++++++++++++++++++++++++++++----
4 files changed, 44 insertions(+), 14 deletions(-)
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 6fa5ab7..342aa01 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -780,7 +780,7 @@ static void domcreate_console_available(libxl__egc *egc,
libxl__domain_create_state *dcs);
static void domcreate_stream_done(libxl__egc *egc,
- libxl__domain_create_state *dcs,
+ libxl__stream_read_state *stream,
int ret);
static void domcreate_rebuild_done(libxl__egc *egc,
@@ -1046,13 +1046,14 @@ static void domcreate_bootloader_done(libxl__egc *egc,
dcs->srs.ao = ao;
dcs->srs.fd = restore_fd;
dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
+ dcs->srs.back_channel = false;
dcs->srs.completion_callback = domcreate_stream_done;
libxl__stream_read_start(egc, &dcs->srs);
return;
out:
- domcreate_stream_done(egc, dcs, rc);
+ domcreate_stream_done(egc, &dcs->srs, rc);
}
void libxl__srm_callout_callback_restore_results(unsigned long store_mfn,
@@ -1069,9 +1070,10 @@ void libxl__srm_callout_callback_restore_results(unsigned long store_mfn,
}
static void domcreate_stream_done(libxl__egc *egc,
- libxl__domain_create_state *dcs,
+ libxl__stream_read_state *stream,
int ret)
{
+ libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
STATE_AO_GC(dcs->ao);
libxl_ctx *ctx = libxl__gc_owner(gc);
char **vments = NULL, **localents = NULL;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 82a327c..bd369f2 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3223,12 +3223,13 @@ struct libxl__stream_read_state {
libxl__ao *ao;
int fd;
bool legacy;
+ bool back_channel;
void (*completion_callback)(libxl__egc *egc,
- libxl__domain_create_state *dcs,
- int rc);
- void (*checkpoint_callback)(libxl__egc *egc,
- libxl__domain_create_state *dcs,
+ libxl__stream_read_state *stream,
int rc);
+ void (*read_records_callback)(libxl__egc *egc,
+ libxl__stream_read_state *stream,
+ int rc);
/* Private */
libxl__carefd *v2_carefd;
int rc;
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index e143b21..0c449ad 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -303,7 +303,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
/*----- remus asynchronous checkpoint callback -----*/
static void remus_checkpoint_stream_done(
- libxl__egc *egc, libxl__domain_create_state *dcs, int rc);
+ libxl__egc *egc, libxl__stream_read_state *stream, int rc);
void libxl__remus_domain_restore_checkpoint_callback(void *data)
{
@@ -312,13 +312,15 @@ void libxl__remus_domain_restore_checkpoint_callback(void *data)
libxl__egc *egc = dcs->shs.egc;
STATE_AO_GC(dcs->ao);
- dcs->srs.checkpoint_callback = remus_checkpoint_stream_done;
+ dcs->srs.read_records_callback = remus_checkpoint_stream_done;
libxl__stream_read_start_checkpoint(egc, &dcs->srs);
}
static void remus_checkpoint_stream_done(
- libxl__egc *egc, libxl__domain_create_state *dcs, int rc)
+ libxl__egc *egc, libxl__stream_read_state *stream, int rc)
{
+ libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
+
libxl__xc_domain_saverestore_async_callback_done(egc, &dcs->shs, rc);
}
/*
diff --git a/tools/libxl/libxl_stream_read.c b/tools/libxl/libxl_stream_read.c
index 09ef0aa..72a9972 100644
--- a/tools/libxl/libxl_stream_read.c
+++ b/tools/libxl/libxl_stream_read.c
@@ -42,6 +42,15 @@
* Depending on the contents of the stream, there are likely to be several
* parallel tasks being managed. check_stream_finished() is used to join all
* tasks in both success and error cases.
+ *
+ * For back channel stream:
+ * - libxl__stream_read_start()
+ * - Set up the stream to running state
+ *
+ * - libxl__stream_read_continue()
+ * - Set up reading the next record from a started stream.
+ * You need to add some codes to process_record() to handle
+ * the record. Then call stream->read_records_callback() to return.
*/
static void stream_success(libxl__egc *egc,
@@ -94,6 +103,12 @@ void libxl__stream_read_start(libxl__egc *egc,
/* State initialisation. */
assert(!stream->running);
+ assert(!stream->legacy || !stream->back_channel);
+
+ if (stream->back_channel) {
+ stream->running = true;
+ return;
+ }
if (stream->legacy) {
/* Convert a legacy stream, if needed. */
@@ -174,6 +189,7 @@ void libxl__stream_read_start_checkpoint(libxl__egc *egc,
assert(stream->running);
assert(!stream->in_checkpoint);
+ assert(!stream->back_channel);
stream->in_checkpoint = true;
/* Read a record header. */
@@ -224,6 +240,11 @@ static void stream_failed(libxl__egc *egc,
return;
}
+ if (stream->back_channel) {
+ stream->completion_callback(egc, stream, rc);
+ return;
+ }
+
if (stream->running) {
stream->running = false;
stream_done(egc, stream);
@@ -237,6 +258,7 @@ static void stream_done(libxl__egc *egc,
assert(!stream->running);
assert(!stream->in_checkpoint);
+ assert(!stream->back_channel);
if (stream->v2_carefd)
libxl__carefd_close(stream->v2_carefd);
@@ -287,7 +309,7 @@ static void check_stream_finished(libxl__egc *egc,
LOG(DEBUG, "save/restore still in use");
else {
LOG(INFO, "Join complete: result %d", stream->joined_rc);
- stream->completion_callback(egc, dcs, stream->joined_rc);
+ stream->completion_callback(egc, stream, stream->joined_rc);
}
}
@@ -471,14 +493,17 @@ static void process_record(libxl__egc *egc,
switch (rec_hdr->type) {
case REC_TYPE_END:
+ assert(!stream->back_channel);
/* Handled later, after cleanup. */
break;
case REC_TYPE_LIBXC_CONTEXT:
+ assert(!stream->back_channel);
libxl__xc_domain_restore(egc, dcs, stream->fd, 0, 0, 0);
break;
case REC_TYPE_XENSTORE_DATA:
+ assert(!stream->back_channel);
ret = libxl__toolstack_restore(dcs->guest_domid, stream->rec_body,
rec_hdr->length, &dcs->shs);
if (ret)
@@ -492,10 +517,12 @@ static void process_record(libxl__egc *egc,
break;
case REC_TYPE_EMULATOR_CONTEXT:
+ assert(!stream->back_channel);
read_emulator_body(egc, stream);
break;
case REC_TYPE_CHECKPOINT_END:
+ assert(!stream->back_channel);
if (!stream->in_checkpoint) {
LOG(ERROR, "Unexpected CHECKPOINT_END record in stream");
ret = ERROR_FAIL;
@@ -647,11 +674,9 @@ static void emulator_padding_done(libxl__egc *egc,
static void checkpoint_done(libxl__egc *egc,
libxl__stream_read_state *stream, int rc)
{
- libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
-
assert(stream->in_checkpoint);
stream->in_checkpoint = false;
- stream->checkpoint_callback(egc, dcs, rc);
+ stream->read_records_callback(egc, stream, rc);
}
/*
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 22/26] tools/libxl: refactor read stream to support back channel
2015-06-25 6:25 ` [PATCH v3 COLOPre 22/26] tools/libxl: refactor read " Yang Hongyang
@ 2015-06-30 10:39 ` Ian Campbell
0 siblings, 0 replies; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:39 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> From: Wen Congyang <wency@cn.fujitsu.com>
Same comments as previous patch..
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (21 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 22/26] tools/libxl: refactor read " Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:42 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 24/26] tools/libxl: rename remus device to checkpoint device Yang Hongyang
` (2 subsequent siblings)
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
From: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
---
docs/specs/libxl-migration-stream.pandoc | 21 ++++++++++++++++++++-
tools/libxl/libxl_sr_stream_format.h | 11 +++++++++++
tools/python/xen/migration/libxl.py | 9 +++++++++
3 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/docs/specs/libxl-migration-stream.pandoc b/docs/specs/libxl-migration-stream.pandoc
index d41932a..7e1edaa 100644
--- a/docs/specs/libxl-migration-stream.pandoc
+++ b/docs/specs/libxl-migration-stream.pandoc
@@ -121,7 +121,9 @@ type 0x00000000: END
0x00000004: CHECKPOINT_END
- 0x00000005 - 0x7FFFFFFF: Reserved for future _mandatory_
+ 0x00000005: COLO_CONTEXT
+
+ 0x00000006 - 0x7FFFFFFF: Reserved for future _mandatory_
records.
0x80000000 - 0xFFFFFFFF: Reserved for future _optional_
@@ -216,3 +218,20 @@ A checkpoint end record marks the end of a checkpoint in the image.
The end record contains no fields; its body_length is 0.
+COLO\_CONTEXT
+--------------
+
+A COLO context record contains the control information for COLO.
+
+ 0 1 2 3 4 5 6 7 octet
+ +------------------------+------------------------+
+ | control_id | padding |
+ +------------------------+------------------------+
+
+--------------------------------------------------------------------
+Field Description
+------------ ---------------------------------------------------
+control_id 0x00000000: New checkpoint
+ 0x00000001: Secondary VM is suspended
+ 0x00000002: Secondary VM is ready
+ 0x00000003: Secondary VM is resumed
diff --git a/tools/libxl/libxl_sr_stream_format.h b/tools/libxl/libxl_sr_stream_format.h
index 5dfa55f..75208af 100644
--- a/tools/libxl/libxl_sr_stream_format.h
+++ b/tools/libxl/libxl_sr_stream_format.h
@@ -36,6 +36,7 @@ typedef struct libxl_sr_rec_hdr
#define REC_TYPE_XENSTORE_DATA 0x00000002U
#define REC_TYPE_EMULATOR_CONTEXT 0x00000003U
#define REC_TYPE_CHECKPOINT_END 0x00000004U
+#define REC_TYPE_COLO_CONTEXT 0x00000005U
typedef struct libxl_sr_emulator_hdr
{
@@ -47,6 +48,16 @@ typedef struct libxl_sr_emulator_hdr
#define EMULATOR_QEMU_TRADITIONAL 0x00000001U
#define EMULATOR_QEMU_UPSTREAM 0x00000002U
+typedef struct libxl_sr_colo_context
+{
+ uint32_t id;
+} libxl_sr_colo_context;
+
+#define COLO_NEW_CHECKPOINT 0x00000000U
+#define COLO_SVM_SUSPENDED 0x00000001U
+#define COLO_SVM_READY 0x00000002U
+#define COLO_SVM_RESUMED 0x00000003U
+
#endif /* LIBXL_SR_STREAM_FORMAT_H */
/*
diff --git a/tools/python/xen/migration/libxl.py b/tools/python/xen/migration/libxl.py
index 415502e..57031c6 100644
--- a/tools/python/xen/migration/libxl.py
+++ b/tools/python/xen/migration/libxl.py
@@ -37,6 +37,7 @@ REC_TYPE_libxc_context = 0x00000001
REC_TYPE_xenstore_data = 0x00000002
REC_TYPE_emulator_context = 0x00000003
REC_TYPE_checkpoint_end = 0x00000004
+REC_TYPE_colo_context = 0x00000005
rec_type_to_str = {
REC_TYPE_end : "End",
@@ -44,6 +45,7 @@ rec_type_to_str = {
REC_TYPE_xenstore_data : "Xenstore data",
REC_TYPE_emulator_context : "Emulator context",
REC_TYPE_checkpoint_end : "Checkpoint end",
+ REC_TYPE_colo_context : "COLO context"
}
# emulator_context
@@ -184,6 +186,11 @@ class VerifyLibxl(VerifyBase):
if len(content) != 0:
raise RecordError("Checkpoint end record with non-zero length")
+ def verify_record_colo_context(self, content):
+ """ COLO context """
+ if len(content) == 0:
+ raise RecordError("COLO context record with zero length")
+
record_verifiers = {
REC_TYPE_end:
@@ -196,4 +203,6 @@ record_verifiers = {
VerifyLibxl.verify_record_emulator_context,
REC_TYPE_checkpoint_end:
VerifyLibxl.verify_record_checkpoint_end,
+ REC_TYPE_colo_context:
+ VerifyLibxl.verify_record_colo_context,
}
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams
2015-06-25 6:25 ` [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams Yang Hongyang
@ 2015-06-30 10:42 ` Ian Campbell
2015-07-01 3:10 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:42 UTC (permalink / raw)
To: Yang Hongyang, Andrew Cooper
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> From: Wen Congyang <wency@cn.fujitsu.com>
This seems to mix up forward and backward facing information in a single
record?
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> ---
> docs/specs/libxl-migration-stream.pandoc | 21 ++++++++++++++++++++-
> tools/libxl/libxl_sr_stream_format.h | 11 +++++++++++
> tools/python/xen/migration/libxl.py | 9 +++++++++
> 3 files changed, 40 insertions(+), 1 deletion(-)
>
> diff --git a/docs/specs/libxl-migration-stream.pandoc b/docs/specs/libxl-migration-stream.pandoc
> index d41932a..7e1edaa 100644
> --- a/docs/specs/libxl-migration-stream.pandoc
> +++ b/docs/specs/libxl-migration-stream.pandoc
> @@ -121,7 +121,9 @@ type 0x00000000: END
>
> 0x00000004: CHECKPOINT_END
>
> - 0x00000005 - 0x7FFFFFFF: Reserved for future _mandatory_
> + 0x00000005: COLO_CONTEXT
> +
> + 0x00000006 - 0x7FFFFFFF: Reserved for future _mandatory_
> records.
>
> 0x80000000 - 0xFFFFFFFF: Reserved for future _optional_
> @@ -216,3 +218,20 @@ A checkpoint end record marks the end of a checkpoint in the image.
>
> The end record contains no fields; its body_length is 0.
>
> +COLO\_CONTEXT
> +--------------
> +
> +A COLO context record contains the control information for COLO.
I don't know what Andy thinks, but this seems like a rather generic
catch-all record type to me. It would seem better to have one or more
more concrete records for different aspects (e.g. status of the
secondary VM, although that seems like a backchannel thing too and as I
mentioned before I'm not sure those should be interleaved in the spec in
this way).
> +
> + 0 1 2 3 4 5 6 7 octet
> + +------------------------+------------------------+
> + | control_id | padding |
> + +------------------------+------------------------+
> +
> +--------------------------------------------------------------------
> +Field Description
> +------------ ---------------------------------------------------
> +control_id 0x00000000: New checkpoint
I think we already have a checkpoint record type, don't we?
> + 0x00000001: Secondary VM is suspended
> + 0x00000002: Secondary VM is ready
> + 0x00000003: Secondary VM is resumed
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams
2015-06-30 10:42 ` Ian Campbell
@ 2015-07-01 3:10 ` Yang Hongyang
2015-07-01 10:44 ` Ian Campbell
0 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 3:10 UTC (permalink / raw)
To: Ian Campbell, Andrew Cooper
Cc: wei.liu2, wency, guijianfeng, yunhong.jiang, eddie.dong,
xen-devel, rshriram, ian.jackson
On 06/30/2015 06:42 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> From: Wen Congyang <wency@cn.fujitsu.com>
>
> This seems to mix up forward and backward facing information in a single
> record?
Yes, it is the negotiation record for COLO.
>
>> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
>> ---
>> docs/specs/libxl-migration-stream.pandoc | 21 ++++++++++++++++++++-
>> tools/libxl/libxl_sr_stream_format.h | 11 +++++++++++
>> tools/python/xen/migration/libxl.py | 9 +++++++++
>> 3 files changed, 40 insertions(+), 1 deletion(-)
>>
>> diff --git a/docs/specs/libxl-migration-stream.pandoc b/docs/specs/libxl-migration-stream.pandoc
>> index d41932a..7e1edaa 100644
>> --- a/docs/specs/libxl-migration-stream.pandoc
>> +++ b/docs/specs/libxl-migration-stream.pandoc
>> @@ -121,7 +121,9 @@ type 0x00000000: END
>>
>> 0x00000004: CHECKPOINT_END
>>
>> - 0x00000005 - 0x7FFFFFFF: Reserved for future _mandatory_
>> + 0x00000005: COLO_CONTEXT
>> +
>> + 0x00000006 - 0x7FFFFFFF: Reserved for future _mandatory_
>> records.
>>
>> 0x80000000 - 0xFFFFFFFF: Reserved for future _optional_
>> @@ -216,3 +218,20 @@ A checkpoint end record marks the end of a checkpoint in the image.
>>
>> The end record contains no fields; its body_length is 0.
>>
>> +COLO\_CONTEXT
>> +--------------
>> +
>> +A COLO context record contains the control information for COLO.
>
> I don't know what Andy thinks, but this seems like a rather generic
> catch-all record type to me. It would seem better to have one or more
> more concrete records for different aspects (e.g. status of the
> secondary VM, although that seems like a backchannel thing too and as I
> mentioned before I'm not sure those should be interleaved in the spec in
> this way).
I will leave this to Andy.
>
>> +
>> + 0 1 2 3 4 5 6 7 octet
>> + +------------------------+------------------------+
>> + | control_id | padding |
>> + +------------------------+------------------------+
>> +
>> +--------------------------------------------------------------------
>> +Field Description
>> +------------ ---------------------------------------------------
>> +control_id 0x00000000: New checkpoint
>
> I think we already have a checkpoint record type, don't we?
Yes, but the meaning is diffrent, the checkpoint record means the
end of a checkpoint, this means a new checkpoint is needed.
>
>
>> + 0x00000001: Secondary VM is suspended
>> + 0x00000002: Secondary VM is ready
>> + 0x00000003: Secondary VM is resumed
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams
2015-07-01 3:10 ` Yang Hongyang
@ 2015-07-01 10:44 ` Ian Campbell
2015-07-01 14:05 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-07-01 10:44 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, Andrew Cooper, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Wed, 2015-07-01 at 11:10 +0800, Yang Hongyang wrote:
> >> +
> >> + 0 1 2 3 4 5 6 7 octet
> >> + +------------------------+------------------------+
> >> + | control_id | padding |
> >> + +------------------------+------------------------+
> >> +
> >> +--------------------------------------------------------------------
> >> +Field Description
> >> +------------ ---------------------------------------------------
> >> +control_id 0x00000000: New checkpoint
> >
> > I think we already have a checkpoint record type, don't we?
>
> Yes, but the meaning is diffrent, the checkpoint record means the
> end of a checkpoint, this means a new checkpoint is needed.
The wording needs to be improved then.
Ian.
^ permalink raw reply [flat|nested] 103+ messages in thread
* Re: [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams
2015-07-01 10:44 ` Ian Campbell
@ 2015-07-01 14:05 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 14:05 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, Andrew Cooper, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 07/01/2015 06:44 PM, Ian Campbell wrote:
> On Wed, 2015-07-01 at 11:10 +0800, Yang Hongyang wrote:
>>>> +
>>>> + 0 1 2 3 4 5 6 7 octet
>>>> + +------------------------+------------------------+
>>>> + | control_id | padding |
>>>> + +------------------------+------------------------+
>>>> +
>>>> +--------------------------------------------------------------------
>>>> +Field Description
>>>> +------------ ---------------------------------------------------
>>>> +control_id 0x00000000: New checkpoint
>>>
>>> I think we already have a checkpoint record type, don't we?
>>
>> Yes, but the meaning is diffrent, the checkpoint record means the
>> end of a checkpoint, this means a new checkpoint is needed.
>
> The wording needs to be improved then.
Ok, thanks!
>
> Ian.
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 24/26] tools/libxl: rename remus device to checkpoint device
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (22 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 23/26] docs/libxl: Introduce COLO_CONTEXT to support migration v2 colo streams Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:43 ` Ian Campbell
2015-06-25 6:25 ` [PATCH v3 COLOPre 25/26] tools/libxl: adjust the indentation Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device Yang Hongyang
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
This patch is auto generated by the following commands:
1. git mv tools/libxl/libxl_remus_device.c tools/libxl/libxl_checkpoint_device.c
2. perl -pi -e 's/libxl_remus_device/libxl_checkpoint_device/g' tools/libxl/Makefile
3. perl -pi -e 's/\blibxl__remus_devices/libxl__checkpoint_devices/g' tools/libxl/*.[ch]
4. perl -pi -e 's/\blibxl__remus_device\b/libxl__checkpoint_device/g' tools/libxl/*.[ch]
5. perl -pi -e 's/\blibxl__remus_device_instance_ops\b/libxl__checkpoint_device_instance_ops/g' tools/libxl/*.[ch]
6. perl -pi -e 's/\blibxl__remus_callback\b/libxl__checkpoint_callback/g' tools/libxl/*.[ch]
7. perl -pi -e 's/\bremus_device_init\b/checkpoint_device_init/g' tools/libxl/*.[ch]
8. perl -pi -e 's/\bremus_devices_setup\b/checkpoint_devices_setup/g' tools/libxl/*.[ch]
9. perl -pi -e 's/\bdefine_remus_checkpoint_api\b/define_checkpoint_api/g' tools/libxl/*.[ch]
10. perl -pi -e 's/\brds\b/cds/g' tools/libxl/*.[ch]
11. perl -pi -e 's/REMUS_DEVICE/CHECKPOINT_DEVICE/g' tools/libxl/*.[ch] tools/libxl/*.idl
12. perl -pi -e 's/REMUS_DEVOPS/CHECKPOINT_DEVOPS/g' tools/libxl/*.[ch] tools/libxl/*.idl
13. perl -pi -e 's/\bremus\b/checkpoint/g' tools/libxl/libxl_checkpoint_device.[ch]
14. perl -pi -e 's/\bremus device/checkpoint device/g' tools/libxl/libxl_internal.h
15. perl -pi -e 's/\bRemus device/checkpoint device/g' tools/libxl/libxl_internal.h
16. perl -pi -e 's/\bremus abstract/checkpoint abstract/g' tools/libxl/libxl_internal.h
17. perl -pi -e 's/\bremus invocation/checkpoint invocation/g' tools/libxl/libxl_internal.h
18. perl -pi -e 's/\blibxl__remus_device_\(/libxl__checkpoint_device_(/g' tools/libxl/libxl_internal.h
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxl/Makefile | 2 +-
tools/libxl/libxl_checkpoint_device.c | 327 ++++++++++++++++++++++++++++++++++
tools/libxl/libxl_internal.h | 112 ++++++------
tools/libxl/libxl_netbuffer.c | 108 +++++------
tools/libxl/libxl_nonetbuffer.c | 10 +-
tools/libxl/libxl_remus.c | 76 ++++----
tools/libxl/libxl_remus_device.c | 327 ----------------------------------
tools/libxl/libxl_remus_disk_drbd.c | 52 +++---
tools/libxl/libxl_types.idl | 4 +-
9 files changed, 509 insertions(+), 509 deletions(-)
create mode 100644 tools/libxl/libxl_checkpoint_device.c
delete mode 100644 tools/libxl/libxl_remus_device.c
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index d61c191..2f4efd4 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -56,7 +56,7 @@ else
LIBXL_OBJS-y += libxl_nonetbuffer.o
endif
-LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
+LIBXL_OBJS-y += libxl_remus.o libxl_checkpoint_device.o libxl_remus_disk_drbd.o
LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
new file mode 100644
index 0000000..109cd23
--- /dev/null
+++ b/tools/libxl/libxl_checkpoint_device.c
@@ -0,0 +1,327 @@
+/*
+ * Copyright (C) 2014 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+extern const libxl__checkpoint_device_instance_ops remus_device_nic;
+extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
+static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
+ &remus_device_nic,
+ &remus_device_drbd_disk,
+ NULL,
+};
+
+/*----- helper functions -----*/
+
+static int init_device_subkind(libxl__checkpoint_devices_state *cds)
+{
+ /* init device subkind-specific state in the libxl ctx */
+ int rc;
+ STATE_AO_GC(cds->ao);
+
+ if (libxl__netbuffer_enabled(gc)) {
+ rc = init_subkind_nic(cds);
+ if (rc) goto out;
+ }
+
+ rc = init_subkind_drbd_disk(cds);
+ if (rc) goto out;
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
+{
+ /* cleanup device subkind-specific state in the libxl ctx */
+ STATE_AO_GC(cds->ao);
+
+ if (libxl__netbuffer_enabled(gc))
+ cleanup_subkind_nic(cds);
+
+ cleanup_subkind_drbd_disk(cds);
+}
+
+/*----- setup() and teardown() -----*/
+
+/* callbacks */
+
+static void all_devices_setup_cb(libxl__egc *egc,
+ libxl__multidev *multidev,
+ int rc);
+static void device_setup_iterate(libxl__egc *egc,
+ libxl__ao_device *aodev);
+static void devices_teardown_cb(libxl__egc *egc,
+ libxl__multidev *multidev,
+ int rc);
+
+/* checkpoint device setup and teardown */
+
+static libxl__checkpoint_device* checkpoint_device_init(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds,
+ libxl__device_kind kind,
+ void *libxl_dev)
+{
+ libxl__checkpoint_device *dev = NULL;
+
+ STATE_AO_GC(cds->ao);
+ GCNEW(dev);
+ dev->backend_dev = libxl_dev;
+ dev->kind = kind;
+ dev->cds = cds;
+
+ return dev;
+}
+
+static void checkpoint_devices_setup(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds);
+
+void libxl__checkpoint_devices_setup(libxl__egc *egc, libxl__checkpoint_devices_state *cds)
+{
+ int i, rc;
+
+ STATE_AO_GC(cds->ao);
+
+ rc = init_device_subkind(cds);
+ if (rc)
+ goto out;
+
+ cds->num_devices = 0;
+ cds->num_nics = 0;
+ cds->num_disks = 0;
+
+ if (cds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF))
+ cds->nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
+
+ if (cds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD))
+ cds->disks = libxl_device_disk_list(CTX, cds->domid, &cds->num_disks);
+
+ if (cds->num_nics == 0 && cds->num_disks == 0)
+ goto out;
+
+ GCNEW_ARRAY(cds->devs, cds->num_nics + cds->num_disks);
+
+ for (i = 0; i < cds->num_nics; i++) {
+ cds->devs[cds->num_devices++] = checkpoint_device_init(egc, cds,
+ LIBXL__DEVICE_KIND_VIF,
+ &cds->nics[i]);
+ }
+
+ for (i = 0; i < cds->num_disks; i++) {
+ cds->devs[cds->num_devices++] = checkpoint_device_init(egc, cds,
+ LIBXL__DEVICE_KIND_VBD,
+ &cds->disks[i]);
+ }
+
+ checkpoint_devices_setup(egc, cds);
+
+ return;
+
+out:
+ cds->callback(egc, cds, rc);
+}
+
+static void checkpoint_devices_setup(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds)
+{
+ int i, rc;
+
+ STATE_AO_GC(cds->ao);
+
+ libxl__multidev_begin(ao, &cds->multidev);
+ cds->multidev.callback = all_devices_setup_cb;
+ for (i = 0; i < cds->num_devices; i++) {
+ libxl__checkpoint_device *dev = cds->devs[i];
+ dev->ops_index = -1;
+ libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);
+
+ dev->aodev.rc = ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED;
+ dev->aodev.callback = device_setup_iterate;
+ device_setup_iterate(egc,&dev->aodev);
+ }
+
+ rc = 0;
+ libxl__multidev_prepared(egc, &cds->multidev, rc);
+}
+
+
+static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
+{
+ libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
+ EGC_GC;
+
+ if (aodev->rc != ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED &&
+ aodev->rc != ERROR_CHECKPOINT_DEVOPS_DOES_NOT_MATCH)
+ /* might be success or disaster */
+ goto out;
+
+ do {
+ dev->ops = remus_ops[++dev->ops_index];
+ if (!dev->ops) {
+ libxl_device_nic * nic = NULL;
+ libxl_device_disk * disk = NULL;
+ uint32_t domid;
+ int devid;
+ if (dev->kind == LIBXL__DEVICE_KIND_VIF) {
+ nic = (libxl_device_nic *)dev->backend_dev;
+ domid = nic->backend_domid;
+ devid = nic->devid;
+ } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) {
+ disk = (libxl_device_disk *)dev->backend_dev;
+ domid = disk->backend_domid;
+ devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL);
+ } else {
+ LOG(ERROR,"device kind not handled by checkpoint: %s",
+ libxl__device_kind_to_string(dev->kind));
+ aodev->rc = ERROR_FAIL;
+ goto out;
+ }
+ LOG(ERROR,"device not handled by checkpoint"
+ " (device=%s:%"PRId32"/%"PRId32")",
+ libxl__device_kind_to_string(dev->kind),
+ domid, devid);
+ aodev->rc = ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED;
+ goto out;
+ }
+ } while (dev->ops->kind != dev->kind);
+
+ /* found the next ops_index to try */
+ assert(dev->aodev.callback == device_setup_iterate);
+ dev->ops->setup(egc,dev);
+ return;
+
+ out:
+ libxl__multidev_one_callback(egc,aodev);
+}
+
+static void all_devices_setup_cb(libxl__egc *egc,
+ libxl__multidev *multidev,
+ int rc)
+{
+ STATE_AO_GC(multidev->ao);
+
+ /* Convenience aliases */
+ libxl__checkpoint_devices_state *const cds =
+ CONTAINER_OF(multidev, *cds, multidev);
+
+ cds->callback(egc, cds, rc);
+}
+
+void libxl__checkpoint_devices_teardown(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds)
+{
+ int i;
+ libxl__checkpoint_device *dev;
+
+ STATE_AO_GC(cds->ao);
+
+ libxl__multidev_begin(ao, &cds->multidev);
+ cds->multidev.callback = devices_teardown_cb;
+ for (i = 0; i < cds->num_devices; i++) {
+ dev = cds->devs[i];
+ if (!dev->ops || !dev->matched)
+ continue;
+
+ libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);
+ dev->ops->teardown(egc,dev);
+ }
+
+ libxl__multidev_prepared(egc, &cds->multidev, 0);
+}
+
+static void devices_teardown_cb(libxl__egc *egc,
+ libxl__multidev *multidev,
+ int rc)
+{
+ int i;
+
+ STATE_AO_GC(multidev->ao);
+
+ /* Convenience aliases */
+ libxl__checkpoint_devices_state *const cds =
+ CONTAINER_OF(multidev, *cds, multidev);
+
+ /* clean nic */
+ for (i = 0; i < cds->num_nics; i++)
+ libxl_device_nic_dispose(&cds->nics[i]);
+ free(cds->nics);
+ cds->nics = NULL;
+ cds->num_nics = 0;
+
+ /* clean disk */
+ for (i = 0; i < cds->num_disks; i++)
+ libxl_device_disk_dispose(&cds->disks[i]);
+ free(cds->disks);
+ cds->disks = NULL;
+ cds->num_disks = 0;
+
+ cleanup_device_subkind(cds);
+
+ cds->callback(egc, cds, rc);
+}
+
+/*----- checkpointing APIs -----*/
+
+/* callbacks */
+
+static void devices_checkpoint_cb(libxl__egc *egc,
+ libxl__multidev *multidev,
+ int rc);
+
+/* API implementations */
+
+#define define_checkpoint_api(api) \
+void libxl__checkpoint_devices_##api(libxl__egc *egc, \
+ libxl__checkpoint_devices_state *cds) \
+{ \
+ int i; \
+ libxl__checkpoint_device *dev; \
+ \
+ STATE_AO_GC(cds->ao); \
+ \
+ libxl__multidev_begin(ao, &cds->multidev); \
+ cds->multidev.callback = devices_checkpoint_cb; \
+ for (i = 0; i < cds->num_devices; i++) { \
+ dev = cds->devs[i]; \
+ if (!dev->matched || !dev->ops->api) \
+ continue; \
+ libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);\
+ dev->ops->api(egc,dev); \
+ } \
+ \
+ libxl__multidev_prepared(egc, &cds->multidev, 0); \
+}
+
+define_checkpoint_api(postsuspend);
+
+define_checkpoint_api(preresume);
+
+define_checkpoint_api(commit);
+
+static void devices_checkpoint_cb(libxl__egc *egc,
+ libxl__multidev *multidev,
+ int rc)
+{
+ STATE_AO_GC(multidev->ao);
+
+ /* Convenience aliases */
+ libxl__checkpoint_devices_state *const cds =
+ CONTAINER_OF(multidev, *cds, multidev);
+
+ cds->callback(egc, cds, rc);
+}
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index bd369f2..ae3b964 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2651,9 +2651,9 @@ typedef struct libxl__save_helper_state {
* marshalling and xc callback functions */
} libxl__save_helper_state;
-/*----- remus device related state structure -----*/
+/*----- checkpoint device related state structure -----*/
/*
- * The abstract Remus device layer exposes a common
+ * The abstract checkpoint device layer exposes a common
* set of API to [external] libxl for manipulating devices attached to
* a guest protected by Remus. The device layer also exposes a set of
* [internal] interfaces that every device type must implement.
@@ -2661,34 +2661,34 @@ typedef struct libxl__save_helper_state {
* The following API are exposed to libxl:
*
* One-time configuration operations:
- * +libxl__remus_devices_setup
+ * +libxl__checkpoint_devices_setup
* > Enable output buffering for NICs, setup disk replication, etc.
- * +libxl__remus_devices_teardown
+ * +libxl__checkpoint_devices_teardown
* > Disable output buffering and disk replication; teardown any
* associated external setups like qdiscs for NICs.
*
* Operations executed every checkpoint (in order of invocation):
- * +libxl__remus_devices_postsuspend
- * +libxl__remus_devices_preresume
- * +libxl__remus_devices_commit
+ * +libxl__checkpoint_devices_postsuspend
+ * +libxl__checkpoint_devices_preresume
+ * +libxl__checkpoint_devices_commit
*
* Each device type needs to implement the interfaces specified in
- * the libxl__remus_device_instance_ops if it wishes to support Remus.
+ * the libxl__checkpoint_device_instance_ops if it wishes to support Remus.
*
- * The high-level control flow through the Remus device layer is shown below:
+ * The high-level control flow through the checkpoint device layer is shown below:
*
* xl remus
* |-> libxl_domain_remus_start
- * |-> libxl__remus_devices_setup
- * |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit]
+ * |-> libxl__checkpoint_devices_setup
+ * |-> Per-checkpoint libxl__checkpoint_devices_[postsuspend,preresume,commit]
* ...
* |-> On backup failure, network error or other internal errors:
- * libxl__remus_devices_teardown
+ * libxl__checkpoint_devices_teardown
*/
-typedef struct libxl__remus_device libxl__remus_device;
-typedef struct libxl__remus_devices_state libxl__remus_devices_state;
-typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops;
+typedef struct libxl__checkpoint_device libxl__checkpoint_device;
+typedef struct libxl__checkpoint_devices_state libxl__checkpoint_devices_state;
+typedef struct libxl__checkpoint_device_instance_ops libxl__checkpoint_device_instance_ops;
/*
* Interfaces to be implemented by every device subkind that wishes to
@@ -2698,7 +2698,7 @@ typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops
* synchronous and call dev->aodev.callback directly (as the last
* thing they do).
*/
-struct libxl__remus_device_instance_ops {
+struct libxl__checkpoint_device_instance_ops {
/* the device kind this ops belongs to... */
libxl__device_kind kind;
@@ -2709,12 +2709,12 @@ struct libxl__remus_device_instance_ops {
* Asynchronous.
*/
- void (*postsuspend)(libxl__egc *egc, libxl__remus_device *dev);
- void (*preresume)(libxl__egc *egc, libxl__remus_device *dev);
- void (*commit)(libxl__egc *egc, libxl__remus_device *dev);
+ void (*postsuspend)(libxl__egc *egc, libxl__checkpoint_device *dev);
+ void (*preresume)(libxl__egc *egc, libxl__checkpoint_device *dev);
+ void (*commit)(libxl__egc *egc, libxl__checkpoint_device *dev);
/*
- * setup() and teardown() are refer to the actual remus device.
+ * setup() and teardown() are refer to the actual checkpoint device.
* Asynchronous.
* teardown is called even if setup fails.
*/
@@ -2723,45 +2723,45 @@ struct libxl__remus_device_instance_ops {
* device. If matched, the device will then be managed with this set of
* subkind operations.
* Yields 0 if the device successfully set up.
- * REMUS_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
+ * CHECKPOINT_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
* any other rc indicates failure.
*/
- void (*setup)(libxl__egc *egc, libxl__remus_device *dev);
- void (*teardown)(libxl__egc *egc, libxl__remus_device *dev);
+ void (*setup)(libxl__egc *egc, libxl__checkpoint_device *dev);
+ void (*teardown)(libxl__egc *egc, libxl__checkpoint_device *dev);
};
-int init_subkind_nic(libxl__remus_devices_state *rds);
-void cleanup_subkind_nic(libxl__remus_devices_state *rds);
-int init_subkind_drbd_disk(libxl__remus_devices_state *rds);
-void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds);
+int init_subkind_nic(libxl__checkpoint_devices_state *cds);
+void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds);
+int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
+void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
-typedef void libxl__remus_callback(libxl__egc *,
- libxl__remus_devices_state *, int rc);
+typedef void libxl__checkpoint_callback(libxl__egc *,
+ libxl__checkpoint_devices_state *, int rc);
/*
- * State associated with a remus invocation, including parameters
- * passed to the remus abstract device layer by the remus
+ * State associated with a checkpoint invocation, including parameters
+ * passed to the checkpoint abstract device layer by the remus
* save/restore machinery.
*/
-struct libxl__remus_devices_state {
- /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/
+struct libxl__checkpoint_devices_state {
+ /*---- must be set by caller of libxl__checkpoint_device_(setup|teardown) ----*/
libxl__ao *ao;
uint32_t domid;
- libxl__remus_callback *callback;
+ libxl__checkpoint_callback *callback;
int device_kind_flags;
/*----- private for abstract layer only -----*/
int num_devices;
/*
- * this array is allocated before setup the remus devices by the
- * remus abstract layer.
- * devs may be NULL, means there's no remus devices that has been set up.
+ * this array is allocated before setup the checkpoint devices by the
+ * checkpoint abstract layer.
+ * devs may be NULL, means there's no checkpoint devices that has been set up.
* the size of this array is 'num_devices', which is the total number
* of libxl nic devices and disk devices(num_nics + num_disks).
*/
- libxl__remus_device **devs;
+ libxl__checkpoint_device **devs;
libxl_device_nic *nics;
int num_nics;
@@ -2783,20 +2783,20 @@ struct libxl__remus_devices_state {
/*
* Information about a single device being handled by remus.
- * Allocated by the remus abstract layer.
+ * Allocated by the checkpoint abstract layer.
*/
-struct libxl__remus_device {
+struct libxl__checkpoint_device {
/*----- shared between abstract and concrete layers -----*/
/*
* if this is true, that means the subkind ops match the device
*/
bool matched;
- /*----- set by remus device abstruct layer -----*/
- /* libxl__device_* which this remus device related to */
+ /*----- set by checkpoint device abstruct layer -----*/
+ /* libxl__device_* which this checkpoint device related to */
const void *backend_dev;
libxl__device_kind kind;
- libxl__remus_devices_state *rds;
+ libxl__checkpoint_devices_state *cds;
libxl__ao_device aodev;
/*----- private for abstract layer only -----*/
@@ -2807,7 +2807,7 @@ struct libxl__remus_device {
* individual devices.
*/
int ops_index;
- const libxl__remus_device_instance_ops *ops;
+ const libxl__checkpoint_device_instance_ops *ops;
/*----- private for concrete (device-specific) layer -----*/
@@ -2815,17 +2815,17 @@ struct libxl__remus_device {
void *concrete_data;
};
-/* the following 5 APIs are async ops, call rds->callback when done */
-_hidden void libxl__remus_devices_setup(libxl__egc *egc,
- libxl__remus_devices_state *rds);
-_hidden void libxl__remus_devices_teardown(libxl__egc *egc,
- libxl__remus_devices_state *rds);
-_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc,
- libxl__remus_devices_state *rds);
-_hidden void libxl__remus_devices_preresume(libxl__egc *egc,
- libxl__remus_devices_state *rds);
-_hidden void libxl__remus_devices_commit(libxl__egc *egc,
- libxl__remus_devices_state *rds);
+/* the following 5 APIs are async ops, call cds->callback when done */
+_hidden void libxl__checkpoint_devices_setup(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds);
+_hidden void libxl__checkpoint_devices_teardown(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds);
+_hidden void libxl__checkpoint_devices_postsuspend(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds);
+_hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds);
+_hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds);
_hidden int libxl__netbuffer_enabled(libxl__gc *gc);
/*----- Legacy conversion helper -----*/
@@ -2959,7 +2959,7 @@ struct libxl__domain_save_state {
libxl__domain_suspend_state dsps;
int hvm;
int xcflags;
- libxl__remus_devices_state rds;
+ libxl__checkpoint_devices_state cds;
libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
int interval; /* checkpoint interval (for Remus) */
libxl__save_helper_state shs;
diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
index 71c6531..86afba6 100644
--- a/tools/libxl/libxl_netbuffer.c
+++ b/tools/libxl/libxl_netbuffer.c
@@ -38,21 +38,21 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
return 1;
}
-int init_subkind_nic(libxl__remus_devices_state *rds)
+int init_subkind_nic(libxl__checkpoint_devices_state *cds)
{
int rc, ret;
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
- rds->nlsock = nl_socket_alloc();
- if (!rds->nlsock) {
+ cds->nlsock = nl_socket_alloc();
+ if (!cds->nlsock) {
LOG(ERROR, "cannot allocate nl socket");
rc = ERROR_FAIL;
goto out;
}
- ret = nl_connect(rds->nlsock, NETLINK_ROUTE);
+ ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
if (ret) {
LOG(ERROR, "failed to open netlink socket: %s",
nl_geterror(ret));
@@ -61,7 +61,7 @@ int init_subkind_nic(libxl__remus_devices_state *rds)
}
/* get list of all qdiscs installed on network devs. */
- ret = rtnl_qdisc_alloc_cache(rds->nlsock, &rds->qdisc_cache);
+ ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
if (ret) {
LOG(ERROR, "failed to allocate qdisc cache: %s",
nl_geterror(ret));
@@ -70,9 +70,9 @@ int init_subkind_nic(libxl__remus_devices_state *rds)
}
if (dss->remus->netbufscript) {
- rds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
+ cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
} else {
- rds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
+ cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
libxl__xen_script_dir_path());
}
@@ -82,22 +82,22 @@ out:
return rc;
}
-void cleanup_subkind_nic(libxl__remus_devices_state *rds)
+void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
{
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
/* free qdisc cache */
- if (rds->qdisc_cache) {
- nl_cache_clear(rds->qdisc_cache);
- nl_cache_free(rds->qdisc_cache);
- rds->qdisc_cache = NULL;
+ if (cds->qdisc_cache) {
+ nl_cache_clear(cds->qdisc_cache);
+ nl_cache_free(cds->qdisc_cache);
+ cds->qdisc_cache = NULL;
}
/* close & free nlsock */
- if (rds->nlsock) {
- nl_close(rds->nlsock);
- nl_socket_free(rds->nlsock);
- rds->nlsock = NULL;
+ if (cds->nlsock) {
+ nl_close(cds->nlsock);
+ nl_socket_free(cds->nlsock);
+ cds->nlsock = NULL;
}
}
@@ -111,17 +111,17 @@ void cleanup_subkind_nic(libxl__remus_devices_state *rds)
* it must ONLY be used for remus because if driver domains
* were in use it would constitute a security vulnerability.
*/
-static const char *get_vifname(libxl__remus_device *dev,
+static const char *get_vifname(libxl__checkpoint_device *dev,
const libxl_device_nic *nic)
{
const char *vifname = NULL;
const char *path;
int rc;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
/* Convenience aliases */
- const uint32_t domid = dev->rds->domid;
+ const uint32_t domid = dev->cds->domid;
path = GCSPRINTF("%s/backend/vif/%d/%d/vifname",
libxl__xs_get_dompath(gc, 0), domid, nic->devid);
@@ -144,19 +144,19 @@ static void free_qdisc(libxl__remus_device_nic *remus_nic)
remus_nic->qdisc = NULL;
}
-static int init_qdisc(libxl__remus_devices_state *rds,
+static int init_qdisc(libxl__checkpoint_devices_state *cds,
libxl__remus_device_nic *remus_nic)
{
int rc, ret, ifindex;
struct rtnl_link *ifb = NULL;
struct rtnl_qdisc *qdisc = NULL;
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
/* Now that we have brought up REMUS_IFB device with plug qdisc for
* this vif, so we need to refill the qdisc cache.
*/
- ret = nl_cache_refill(rds->nlsock, rds->qdisc_cache);
+ ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
if (ret) {
LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
rc = ERROR_FAIL;
@@ -164,7 +164,7 @@ static int init_qdisc(libxl__remus_devices_state *rds,
}
/* get a handle to the REMUS_IFB interface */
- ret = rtnl_link_get_kernel(rds->nlsock, 0, remus_nic->ifb, &ifb);
+ ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
if (ret) {
LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
nl_geterror(ret));
@@ -187,7 +187,7 @@ static int init_qdisc(libxl__remus_devices_state *rds,
* There is no need to explicitly free this qdisc as its just a
* reference from the qdisc cache we allocated earlier.
*/
- qdisc = rtnl_qdisc_get_by_parent(rds->qdisc_cache, ifindex, TC_H_ROOT);
+ qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
if (qdisc) {
const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
/* Sanity check: Ensure that the root qdisc is a plug qdisc. */
@@ -231,19 +231,19 @@ static void netbuf_teardown_script_cb(libxl__egc *egc,
* $REMUS_IFB (for teardown)
* setup/teardown as command line arg.
*/
-static void setup_async_exec(libxl__remus_device *dev, char *op)
+static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
{
int arraysize, nr = 0;
char **env = NULL, **args = NULL;
libxl__remus_device_nic *remus_nic = dev->concrete_data;
- libxl__remus_devices_state *rds = dev->rds;
+ libxl__checkpoint_devices_state *cds = dev->cds;
libxl__async_exec_state *aes = &dev->aodev.aes;
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
/* Convenience aliases */
- char *const script = libxl__strdup(gc, rds->netbufscript);
- const uint32_t domid = rds->domid;
+ char *const script = libxl__strdup(gc, cds->netbufscript);
+ const uint32_t domid = cds->domid;
const int dev_id = remus_nic->devid;
const char *const vif = remus_nic->vif;
const char *const ifb = remus_nic->ifb;
@@ -269,7 +269,7 @@ static void setup_async_exec(libxl__remus_device *dev, char *op)
args[nr++] = NULL;
assert(nr == arraysize);
- aes->ao = dev->rds->ao;
+ aes->ao = dev->cds->ao;
aes->what = GCSPRINTF("%s %s", args[0], args[1]);
aes->env = env;
aes->args = args;
@@ -286,13 +286,13 @@ static void setup_async_exec(libxl__remus_device *dev, char *op)
/* setup() and teardown() */
-static void nic_setup(libxl__egc *egc, libxl__remus_device *dev)
+static void nic_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
{
int rc;
libxl__remus_device_nic *remus_nic;
const libxl_device_nic *nic = dev->backend_dev;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
/*
* thers's no subkind of nic devices, so nic ops is always matched
@@ -330,16 +330,16 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
int status)
{
libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
- libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
+ libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
libxl__remus_device_nic *remus_nic = dev->concrete_data;
- libxl__remus_devices_state *rds = dev->rds;
+ libxl__checkpoint_devices_state *cds = dev->cds;
const char *out_path_base, *hotplug_error = NULL;
int rc;
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
/* Convenience aliases */
- const uint32_t domid = rds->domid;
+ const uint32_t domid = cds->domid;
const int devid = remus_nic->devid;
const char *const vif = remus_nic->vif;
const char **const ifb = &remus_nic->ifb;
@@ -373,7 +373,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
if (hotplug_error) {
LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
- rds->netbufscript, vif, hotplug_error);
+ cds->netbufscript, vif, hotplug_error);
rc = ERROR_FAIL;
goto out;
}
@@ -384,17 +384,17 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
}
LOG(DEBUG, "%s will buffer packets from vif %s", *ifb, vif);
- rc = init_qdisc(rds, remus_nic);
+ rc = init_qdisc(cds, remus_nic);
out:
aodev->rc = rc;
aodev->callback(egc, aodev);
}
-static void nic_teardown(libxl__egc *egc, libxl__remus_device *dev)
+static void nic_teardown(libxl__egc *egc, libxl__checkpoint_device *dev)
{
int rc;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
setup_async_exec(dev, "teardown");
@@ -415,7 +415,7 @@ static void netbuf_teardown_script_cb(libxl__egc *egc,
{
int rc;
libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
- libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
+ libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
libxl__remus_device_nic *remus_nic = dev->concrete_data;
if (status)
@@ -440,12 +440,12 @@ enum {
/* API implementations */
static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int buffer_op)
{
int rc, ret;
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
if (buffer_op == tc_buffer_start)
ret = rtnl_qdisc_plug_buffer(remus_nic->qdisc);
@@ -457,7 +457,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
goto out;
}
- ret = rtnl_qdisc_add(rds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
+ ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
if (ret) {
rc = ERROR_FAIL;
goto out;
@@ -474,33 +474,33 @@ out:
return rc;
}
-static void nic_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
+static void nic_postsuspend(libxl__egc *egc, libxl__checkpoint_device *dev)
{
int rc;
libxl__remus_device_nic *remus_nic = dev->concrete_data;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
- rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_start);
+ rc = remus_netbuf_op(remus_nic, dev->cds, tc_buffer_start);
dev->aodev.rc = rc;
dev->aodev.callback(egc, &dev->aodev);
}
-static void nic_commit(libxl__egc *egc, libxl__remus_device *dev)
+static void nic_commit(libxl__egc *egc, libxl__checkpoint_device *dev)
{
int rc;
libxl__remus_device_nic *remus_nic = dev->concrete_data;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
- rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_release);
+ rc = remus_netbuf_op(remus_nic, dev->cds, tc_buffer_release);
dev->aodev.rc = rc;
dev->aodev.callback(egc, &dev->aodev);
}
-const libxl__remus_device_instance_ops remus_device_nic = {
+const libxl__checkpoint_device_instance_ops remus_device_nic = {
.kind = LIBXL__DEVICE_KIND_VIF,
.setup = nic_setup,
.teardown = nic_teardown,
diff --git a/tools/libxl/libxl_nonetbuffer.c b/tools/libxl/libxl_nonetbuffer.c
index 3c659c2..4b68152 100644
--- a/tools/libxl/libxl_nonetbuffer.c
+++ b/tools/libxl/libxl_nonetbuffer.c
@@ -22,25 +22,25 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
return 0;
}
-int init_subkind_nic(libxl__remus_devices_state *rds)
+int init_subkind_nic(libxl__checkpoint_devices_state *cds)
{
return 0;
}
-void cleanup_subkind_nic(libxl__remus_devices_state *rds)
+void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
{
return;
}
-static void nic_setup(libxl__egc *egc, libxl__remus_device *dev)
+static void nic_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
{
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
dev->aodev.rc = ERROR_FAIL;
dev->aodev.callback(egc, &dev->aodev);
}
-const libxl__remus_device_instance_ops remus_device_nic = {
+const libxl__checkpoint_device_instance_ops remus_device_nic = {
.kind = LIBXL__DEVICE_KIND_VIF,
.setup = nic_setup,
};
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index 0c449ad..fe4acc8 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -20,14 +20,14 @@
/*----- Remus setup and teardown -----*/
static void remus_setup_done(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc);
+ libxl__checkpoint_devices_state *cds, int rc);
static void remus_setup_failed(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc);
+ libxl__checkpoint_devices_state *cds, int rc);
void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
{
/* Convenience aliases */
- libxl__remus_devices_state *const rds = &dss->rds;
+ libxl__checkpoint_devices_state *const cds = &dss->cds;
const libxl_domain_remus_info *const info = dss->remus;
STATE_AO_GC(dss->ao);
@@ -37,17 +37,17 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
LOG(ERROR, "Remus: No support for network buffering");
goto out;
}
- rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
+ cds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
}
if (libxl_defbool_val(info->diskbuf))
- rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
+ cds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
- rds->ao = ao;
- rds->domid = dss->domid;
- rds->callback = remus_setup_done;
+ cds->ao = ao;
+ cds->domid = dss->domid;
+ cds->callback = remus_setup_done;
- libxl__remus_devices_setup(egc, rds);
+ libxl__checkpoint_devices_setup(egc, cds);
return;
out:
@@ -55,9 +55,9 @@ out:
}
static void remus_setup_done(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc)
+ libxl__checkpoint_devices_state *cds, int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
STATE_AO_GC(dss->ao);
if (!rc) {
@@ -67,14 +67,14 @@ static void remus_setup_done(libxl__egc *egc,
LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
dss->domid, rc);
- rds->callback = remus_setup_failed;
- libxl__remus_devices_teardown(egc, rds);
+ cds->callback = remus_setup_failed;
+ libxl__checkpoint_devices_teardown(egc, cds);
}
static void remus_setup_failed(libxl__egc *egc,
- libxl__remus_devices_state *rds, int rc)
+ libxl__checkpoint_devices_state *cds, int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -85,7 +85,7 @@ static void remus_setup_failed(libxl__egc *egc,
}
static void remus_teardown_done(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc);
void libxl__remus_teardown(libxl__egc *egc,
libxl__domain_save_state *dss,
@@ -95,15 +95,15 @@ void libxl__remus_teardown(libxl__egc *egc,
LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
" teardown Remus devices...", rc);
- dss->rds.callback = remus_teardown_done;
- libxl__remus_devices_teardown(egc, &dss->rds);
+ dss->cds.callback = remus_teardown_done;
+ libxl__checkpoint_devices_teardown(egc, &dss->cds);
}
static void remus_teardown_done(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -118,10 +118,10 @@ static void remus_teardown_done(libxl__egc *egc,
static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
libxl__domain_suspend_state *dsps, int ok);
static void remus_devices_postsuspend_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc);
static void remus_devices_preresume_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc);
void libxl__remus_domain_suspend_callback(void *data)
@@ -143,9 +143,9 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
if (!ok)
goto out;
- libxl__remus_devices_state *const rds = &dss->rds;
- rds->callback = remus_devices_postsuspend_cb;
- libxl__remus_devices_postsuspend(egc, rds);
+ libxl__checkpoint_devices_state *const cds = &dss->cds;
+ cds->callback = remus_devices_postsuspend_cb;
+ libxl__checkpoint_devices_postsuspend(egc, cds);
return;
out:
@@ -153,11 +153,11 @@ out:
}
static void remus_devices_postsuspend_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc)
{
int ok = 0;
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
if (rc)
goto out;
@@ -175,17 +175,17 @@ void libxl__remus_domain_resume_callback(void *data)
libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
STATE_AO_GC(dss->ao);
- libxl__remus_devices_state *const rds = &dss->rds;
- rds->callback = remus_devices_preresume_cb;
- libxl__remus_devices_preresume(egc, rds);
+ libxl__checkpoint_devices_state *const cds = &dss->cds;
+ cds->callback = remus_devices_preresume_cb;
+ libxl__checkpoint_devices_preresume(egc, cds);
}
static void remus_devices_preresume_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc)
{
int ok = 0;
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -207,7 +207,7 @@ out:
static void remus_checkpoint_stream_written(
libxl__egc *egc, libxl__stream_write_state *stream, int rc);
static void remus_devices_commit_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc);
static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
const struct timeval *requested_abs);
@@ -229,7 +229,7 @@ static void remus_checkpoint_stream_written(
libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
/* Convenience aliases */
- libxl__remus_devices_state *const rds = &dss->rds;
+ libxl__checkpoint_devices_state *const cds = &dss->cds;
STATE_AO_GC(dss->ao);
@@ -238,8 +238,8 @@ static void remus_checkpoint_stream_written(
goto out;
}
- rds->callback = remus_devices_commit_cb;
- libxl__remus_devices_commit(egc, rds);
+ cds->callback = remus_devices_commit_cb;
+ libxl__checkpoint_devices_commit(egc, cds);
return;
@@ -248,10 +248,10 @@ out:
}
static void remus_devices_commit_cb(libxl__egc *egc,
- libxl__remus_devices_state *rds,
+ libxl__checkpoint_devices_state *cds,
int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c
deleted file mode 100644
index a6cb7f6..0000000
--- a/tools/libxl/libxl_remus_device.c
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Copyright (C) 2014 FUJITSU LIMITED
- * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published
- * by the Free Software Foundation; version 2.1 only. with the special
- * exception on linking described in file LICENSE.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser General Public License for more details.
- */
-
-#include "libxl_osdeps.h" /* must come before any other headers */
-
-#include "libxl_internal.h"
-
-extern const libxl__remus_device_instance_ops remus_device_nic;
-extern const libxl__remus_device_instance_ops remus_device_drbd_disk;
-static const libxl__remus_device_instance_ops *remus_ops[] = {
- &remus_device_nic,
- &remus_device_drbd_disk,
- NULL,
-};
-
-/*----- helper functions -----*/
-
-static int init_device_subkind(libxl__remus_devices_state *rds)
-{
- /* init device subkind-specific state in the libxl ctx */
- int rc;
- STATE_AO_GC(rds->ao);
-
- if (libxl__netbuffer_enabled(gc)) {
- rc = init_subkind_nic(rds);
- if (rc) goto out;
- }
-
- rc = init_subkind_drbd_disk(rds);
- if (rc) goto out;
-
- rc = 0;
-out:
- return rc;
-}
-
-static void cleanup_device_subkind(libxl__remus_devices_state *rds)
-{
- /* cleanup device subkind-specific state in the libxl ctx */
- STATE_AO_GC(rds->ao);
-
- if (libxl__netbuffer_enabled(gc))
- cleanup_subkind_nic(rds);
-
- cleanup_subkind_drbd_disk(rds);
-}
-
-/*----- setup() and teardown() -----*/
-
-/* callbacks */
-
-static void all_devices_setup_cb(libxl__egc *egc,
- libxl__multidev *multidev,
- int rc);
-static void device_setup_iterate(libxl__egc *egc,
- libxl__ao_device *aodev);
-static void devices_teardown_cb(libxl__egc *egc,
- libxl__multidev *multidev,
- int rc);
-
-/* remus device setup and teardown */
-
-static libxl__remus_device* remus_device_init(libxl__egc *egc,
- libxl__remus_devices_state *rds,
- libxl__device_kind kind,
- void *libxl_dev)
-{
- libxl__remus_device *dev = NULL;
-
- STATE_AO_GC(rds->ao);
- GCNEW(dev);
- dev->backend_dev = libxl_dev;
- dev->kind = kind;
- dev->rds = rds;
-
- return dev;
-}
-
-static void remus_devices_setup(libxl__egc *egc,
- libxl__remus_devices_state *rds);
-
-void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_devices_state *rds)
-{
- int i, rc;
-
- STATE_AO_GC(rds->ao);
-
- rc = init_device_subkind(rds);
- if (rc)
- goto out;
-
- rds->num_devices = 0;
- rds->num_nics = 0;
- rds->num_disks = 0;
-
- if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF))
- rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics);
-
- if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD))
- rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks);
-
- if (rds->num_nics == 0 && rds->num_disks == 0)
- goto out;
-
- GCNEW_ARRAY(rds->devs, rds->num_nics + rds->num_disks);
-
- for (i = 0; i < rds->num_nics; i++) {
- rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
- LIBXL__DEVICE_KIND_VIF,
- &rds->nics[i]);
- }
-
- for (i = 0; i < rds->num_disks; i++) {
- rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
- LIBXL__DEVICE_KIND_VBD,
- &rds->disks[i]);
- }
-
- remus_devices_setup(egc, rds);
-
- return;
-
-out:
- rds->callback(egc, rds, rc);
-}
-
-static void remus_devices_setup(libxl__egc *egc,
- libxl__remus_devices_state *rds)
-{
- int i, rc;
-
- STATE_AO_GC(rds->ao);
-
- libxl__multidev_begin(ao, &rds->multidev);
- rds->multidev.callback = all_devices_setup_cb;
- for (i = 0; i < rds->num_devices; i++) {
- libxl__remus_device *dev = rds->devs[i];
- dev->ops_index = -1;
- libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
-
- dev->aodev.rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
- dev->aodev.callback = device_setup_iterate;
- device_setup_iterate(egc,&dev->aodev);
- }
-
- rc = 0;
- libxl__multidev_prepared(egc, &rds->multidev, rc);
-}
-
-
-static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
-{
- libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
- EGC_GC;
-
- if (aodev->rc != ERROR_REMUS_DEVICE_NOT_SUPPORTED &&
- aodev->rc != ERROR_REMUS_DEVOPS_DOES_NOT_MATCH)
- /* might be success or disaster */
- goto out;
-
- do {
- dev->ops = remus_ops[++dev->ops_index];
- if (!dev->ops) {
- libxl_device_nic * nic = NULL;
- libxl_device_disk * disk = NULL;
- uint32_t domid;
- int devid;
- if (dev->kind == LIBXL__DEVICE_KIND_VIF) {
- nic = (libxl_device_nic *)dev->backend_dev;
- domid = nic->backend_domid;
- devid = nic->devid;
- } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) {
- disk = (libxl_device_disk *)dev->backend_dev;
- domid = disk->backend_domid;
- devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL);
- } else {
- LOG(ERROR,"device kind not handled by remus: %s",
- libxl__device_kind_to_string(dev->kind));
- aodev->rc = ERROR_FAIL;
- goto out;
- }
- LOG(ERROR,"device not handled by remus"
- " (device=%s:%"PRId32"/%"PRId32")",
- libxl__device_kind_to_string(dev->kind),
- domid, devid);
- aodev->rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
- goto out;
- }
- } while (dev->ops->kind != dev->kind);
-
- /* found the next ops_index to try */
- assert(dev->aodev.callback == device_setup_iterate);
- dev->ops->setup(egc,dev);
- return;
-
- out:
- libxl__multidev_one_callback(egc,aodev);
-}
-
-static void all_devices_setup_cb(libxl__egc *egc,
- libxl__multidev *multidev,
- int rc)
-{
- STATE_AO_GC(multidev->ao);
-
- /* Convenience aliases */
- libxl__remus_devices_state *const rds =
- CONTAINER_OF(multidev, *rds, multidev);
-
- rds->callback(egc, rds, rc);
-}
-
-void libxl__remus_devices_teardown(libxl__egc *egc,
- libxl__remus_devices_state *rds)
-{
- int i;
- libxl__remus_device *dev;
-
- STATE_AO_GC(rds->ao);
-
- libxl__multidev_begin(ao, &rds->multidev);
- rds->multidev.callback = devices_teardown_cb;
- for (i = 0; i < rds->num_devices; i++) {
- dev = rds->devs[i];
- if (!dev->ops || !dev->matched)
- continue;
-
- libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
- dev->ops->teardown(egc,dev);
- }
-
- libxl__multidev_prepared(egc, &rds->multidev, 0);
-}
-
-static void devices_teardown_cb(libxl__egc *egc,
- libxl__multidev *multidev,
- int rc)
-{
- int i;
-
- STATE_AO_GC(multidev->ao);
-
- /* Convenience aliases */
- libxl__remus_devices_state *const rds =
- CONTAINER_OF(multidev, *rds, multidev);
-
- /* clean nic */
- for (i = 0; i < rds->num_nics; i++)
- libxl_device_nic_dispose(&rds->nics[i]);
- free(rds->nics);
- rds->nics = NULL;
- rds->num_nics = 0;
-
- /* clean disk */
- for (i = 0; i < rds->num_disks; i++)
- libxl_device_disk_dispose(&rds->disks[i]);
- free(rds->disks);
- rds->disks = NULL;
- rds->num_disks = 0;
-
- cleanup_device_subkind(rds);
-
- rds->callback(egc, rds, rc);
-}
-
-/*----- checkpointing APIs -----*/
-
-/* callbacks */
-
-static void devices_checkpoint_cb(libxl__egc *egc,
- libxl__multidev *multidev,
- int rc);
-
-/* API implementations */
-
-#define define_remus_checkpoint_api(api) \
-void libxl__remus_devices_##api(libxl__egc *egc, \
- libxl__remus_devices_state *rds) \
-{ \
- int i; \
- libxl__remus_device *dev; \
- \
- STATE_AO_GC(rds->ao); \
- \
- libxl__multidev_begin(ao, &rds->multidev); \
- rds->multidev.callback = devices_checkpoint_cb; \
- for (i = 0; i < rds->num_devices; i++) { \
- dev = rds->devs[i]; \
- if (!dev->matched || !dev->ops->api) \
- continue; \
- libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);\
- dev->ops->api(egc,dev); \
- } \
- \
- libxl__multidev_prepared(egc, &rds->multidev, 0); \
-}
-
-define_remus_checkpoint_api(postsuspend);
-
-define_remus_checkpoint_api(preresume);
-
-define_remus_checkpoint_api(commit);
-
-static void devices_checkpoint_cb(libxl__egc *egc,
- libxl__multidev *multidev,
- int rc)
-{
- STATE_AO_GC(multidev->ao);
-
- /* Convenience aliases */
- libxl__remus_devices_state *const rds =
- CONTAINER_OF(multidev, *rds, multidev);
-
- rds->callback(egc, rds, rc);
-}
diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
index afe9b61..50b897d 100644
--- a/tools/libxl/libxl_remus_disk_drbd.c
+++ b/tools/libxl/libxl_remus_disk_drbd.c
@@ -26,30 +26,30 @@ typedef struct libxl__remus_drbd_disk {
int ackwait;
} libxl__remus_drbd_disk;
-int init_subkind_drbd_disk(libxl__remus_devices_state *rds)
+int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
{
- STATE_AO_GC(rds->ao);
+ STATE_AO_GC(cds->ao);
- rds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
+ cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
libxl__xen_script_dir_path());
return 0;
}
-void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds)
+void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
{
return;
}
/*----- helper functions, for async calls -----*/
static void drbd_async_call(libxl__egc *egc,
- libxl__remus_device *dev,
- void func(libxl__remus_device *),
+ libxl__checkpoint_device *dev,
+ void func(libxl__checkpoint_device *),
libxl__ev_child_callback callback)
{
int pid = -1, rc;
libxl__ao_device *aodev = &dev->aodev;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
/* Fork and call */
pid = libxl__ev_child_fork(gc, &aodev->child, callback);
@@ -82,21 +82,21 @@ static void match_async_exec_cb(libxl__egc *egc,
/* implementations */
-static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev);
+static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev);
-static void drbd_setup(libxl__egc *egc, libxl__remus_device *dev)
+static void drbd_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
{
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
match_async_exec(egc, dev);
}
-static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev)
+static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
{
int arraysize, nr = 0, rc;
const libxl_device_disk *disk = dev->backend_dev;
libxl__async_exec_state *aes = &dev->aodev.aes;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
/* setup env & args */
arraysize = 1;
@@ -107,12 +107,12 @@ static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev)
arraysize = 3;
nr = 0;
GCNEW_ARRAY(aes->args, arraysize);
- aes->args[nr++] = dev->rds->drbd_probe_script;
+ aes->args[nr++] = dev->cds->drbd_probe_script;
aes->args[nr++] = disk->pdev_path;
aes->args[nr++] = NULL;
assert(nr <= arraysize);
- aes->ao = dev->rds->ao;
+ aes->ao = dev->cds->ao;
aes->what = GCSPRINTF("%s %s", aes->args[0], aes->args[1]);
aes->timeout_ms = LIBXL_HOTPLUG_TIMEOUT * 1000;
aes->callback = match_async_exec_cb;
@@ -137,14 +137,14 @@ static void match_async_exec_cb(libxl__egc *egc,
{
int rc;
libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
- libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
+ libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
libxl__remus_drbd_disk *drbd_disk;
const libxl_device_disk *disk = dev->backend_dev;
STATE_AO_GC(aodev->ao);
if (status) {
- rc = ERROR_REMUS_DEVOPS_DOES_NOT_MATCH;
+ rc = ERROR_CHECKPOINT_DEVOPS_DOES_NOT_MATCH;
/* BUG: seems to assume that any exit status means `no match' */
/* BUG: exit status will have been logged as an error */
goto out;
@@ -169,10 +169,10 @@ out:
aodev->callback(egc, aodev);
}
-static void drbd_teardown(libxl__egc *egc, libxl__remus_device *dev)
+static void drbd_teardown(libxl__egc *egc, libxl__checkpoint_device *dev)
{
libxl__remus_drbd_disk *drbd_disk = dev->concrete_data;
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
close(drbd_disk->ctl_fd);
dev->aodev.rc = 0;
@@ -189,9 +189,9 @@ static void checkpoint_async_call_done(libxl__egc *egc,
/* API implementations */
/* this op will not wait and block, so implement as sync op */
-static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
+static void drbd_postsuspend(libxl__egc *egc, libxl__checkpoint_device *dev)
{
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
libxl__remus_drbd_disk *rdd = dev->concrete_data;
@@ -205,16 +205,16 @@ static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
}
-static void drbd_preresume_async(libxl__remus_device *dev);
+static void drbd_preresume_async(libxl__checkpoint_device *dev);
-static void drbd_preresume(libxl__egc *egc, libxl__remus_device *dev)
+static void drbd_preresume(libxl__egc *egc, libxl__checkpoint_device *dev)
{
- STATE_AO_GC(dev->rds->ao);
+ STATE_AO_GC(dev->cds->ao);
drbd_async_call(egc, dev, drbd_preresume_async, checkpoint_async_call_done);
}
-static void drbd_preresume_async(libxl__remus_device *dev)
+static void drbd_preresume_async(libxl__checkpoint_device *dev)
{
libxl__remus_drbd_disk *rdd = dev->concrete_data;
int ackwait = rdd->ackwait;
@@ -233,7 +233,7 @@ static void checkpoint_async_call_done(libxl__egc *egc,
{
int rc;
libxl__ao_device *aodev = CONTAINER_OF(child, *aodev, child);
- libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
+ libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
libxl__remus_drbd_disk *rdd = dev->concrete_data;
STATE_AO_GC(aodev->ao);
@@ -251,7 +251,7 @@ out:
aodev->callback(egc, aodev);
}
-const libxl__remus_device_instance_ops remus_device_drbd_disk = {
+const libxl__checkpoint_device_instance_ops remus_device_drbd_disk = {
.kind = LIBXL__DEVICE_KIND_VBD,
.setup = drbd_setup,
.teardown = drbd_teardown,
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 5c7b82d..e05d12b 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -61,8 +61,8 @@ libxl_error = Enumeration("error", [
(-15, "LOCK_FAIL"),
(-16, "JSON_CONFIG_EMPTY"),
(-17, "DEVICE_EXISTS"),
- (-18, "REMUS_DEVOPS_DOES_NOT_MATCH"),
- (-19, "REMUS_DEVICE_NOT_SUPPORTED"),
+ (-18, "CHECKPOINT_DEVOPS_DOES_NOT_MATCH"),
+ (-19, "CHECKPOINT_DEVICE_NOT_SUPPORTED"),
(-20, "VNUMA_CONFIG_INVALID"),
(-21, "DOMAIN_NOTFOUND"),
], value_namespace = "")
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 24/26] tools/libxl: rename remus device to checkpoint device
2015-06-25 6:25 ` [PATCH v3 COLOPre 24/26] tools/libxl: rename remus device to checkpoint device Yang Hongyang
@ 2015-06-30 10:43 ` Ian Campbell
2015-07-01 3:11 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:43 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> This patch is auto generated by the following commands:
> 1. git mv tools/libxl/libxl_remus_device.c tools/libxl/libxl_checkpoint_device.c
Please use the -M option to git format-patch/send-email, which will make
the move and subsequent changes far more apparent.
> 2. perl -pi -e 's/libxl_remus_device/libxl_checkpoint_device/g' tools/libxl/Makefile
> 3. perl -pi -e 's/\blibxl__remus_devices/libxl__checkpoint_devices/g' tools/libxl/*.[ch]
> 4. perl -pi -e 's/\blibxl__remus_device\b/libxl__checkpoint_device/g' tools/libxl/*.[ch]
> 5. perl -pi -e 's/\blibxl__remus_device_instance_ops\b/libxl__checkpoint_device_instance_ops/g' tools/libxl/*.[ch]
> 6. perl -pi -e 's/\blibxl__remus_callback\b/libxl__checkpoint_callback/g' tools/libxl/*.[ch]
> 7. perl -pi -e 's/\bremus_device_init\b/checkpoint_device_init/g' tools/libxl/*.[ch]
> 8. perl -pi -e 's/\bremus_devices_setup\b/checkpoint_devices_setup/g' tools/libxl/*.[ch]
> 9. perl -pi -e 's/\bdefine_remus_checkpoint_api\b/define_checkpoint_api/g' tools/libxl/*.[ch]
> 10. perl -pi -e 's/\brds\b/cds/g' tools/libxl/*.[ch]
> 11. perl -pi -e 's/REMUS_DEVICE/CHECKPOINT_DEVICE/g' tools/libxl/*.[ch] tools/libxl/*.idl
> 12. perl -pi -e 's/REMUS_DEVOPS/CHECKPOINT_DEVOPS/g' tools/libxl/*.[ch] tools/libxl/*.idl
> 13. perl -pi -e 's/\bremus\b/checkpoint/g' tools/libxl/libxl_checkpoint_device.[ch]
> 14. perl -pi -e 's/\bremus device/checkpoint device/g' tools/libxl/libxl_internal.h
> 15. perl -pi -e 's/\bRemus device/checkpoint device/g' tools/libxl/libxl_internal.h
> 16. perl -pi -e 's/\bremus abstract/checkpoint abstract/g' tools/libxl/libxl_internal.h
> 17. perl -pi -e 's/\bremus invocation/checkpoint invocation/g' tools/libxl/libxl_internal.h
> 18. perl -pi -e 's/\blibxl__remus_device_\(/libxl__checkpoint_device_(/g' tools/libxl/libxl_internal.h
Wowza!
>
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
> tools/libxl/Makefile | 2 +-
> tools/libxl/libxl_checkpoint_device.c | 327 ++++++++++++++++++++++++++++++++++
> tools/libxl/libxl_internal.h | 112 ++++++------
> tools/libxl/libxl_netbuffer.c | 108 +++++------
> tools/libxl/libxl_nonetbuffer.c | 10 +-
> tools/libxl/libxl_remus.c | 76 ++++----
> tools/libxl/libxl_remus_device.c | 327 ----------------------------------
> tools/libxl/libxl_remus_disk_drbd.c | 52 +++---
> tools/libxl/libxl_types.idl | 4 +-
> 9 files changed, 509 insertions(+), 509 deletions(-)
> create mode 100644 tools/libxl/libxl_checkpoint_device.c
> delete mode 100644 tools/libxl/libxl_remus_device.c
>
> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
> index d61c191..2f4efd4 100644
> --- a/tools/libxl/Makefile
> +++ b/tools/libxl/Makefile
> @@ -56,7 +56,7 @@ else
> LIBXL_OBJS-y += libxl_nonetbuffer.o
> endif
>
> -LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
> +LIBXL_OBJS-y += libxl_remus.o libxl_checkpoint_device.o libxl_remus_disk_drbd.o
>
> LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
> LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
> diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
> new file mode 100644
> index 0000000..109cd23
> --- /dev/null
> +++ b/tools/libxl/libxl_checkpoint_device.c
> @@ -0,0 +1,327 @@
> +/*
> + * Copyright (C) 2014 FUJITSU LIMITED
> + * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2.1 only. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#include "libxl_osdeps.h" /* must come before any other headers */
> +
> +#include "libxl_internal.h"
> +
> +extern const libxl__checkpoint_device_instance_ops remus_device_nic;
> +extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
> +static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
> + &remus_device_nic,
> + &remus_device_drbd_disk,
> + NULL,
> +};
> +
> +/*----- helper functions -----*/
> +
> +static int init_device_subkind(libxl__checkpoint_devices_state *cds)
> +{
> + /* init device subkind-specific state in the libxl ctx */
> + int rc;
> + STATE_AO_GC(cds->ao);
> +
> + if (libxl__netbuffer_enabled(gc)) {
> + rc = init_subkind_nic(cds);
> + if (rc) goto out;
> + }
> +
> + rc = init_subkind_drbd_disk(cds);
> + if (rc) goto out;
> +
> + rc = 0;
> +out:
> + return rc;
> +}
> +
> +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
> +{
> + /* cleanup device subkind-specific state in the libxl ctx */
> + STATE_AO_GC(cds->ao);
> +
> + if (libxl__netbuffer_enabled(gc))
> + cleanup_subkind_nic(cds);
> +
> + cleanup_subkind_drbd_disk(cds);
> +}
> +
> +/*----- setup() and teardown() -----*/
> +
> +/* callbacks */
> +
> +static void all_devices_setup_cb(libxl__egc *egc,
> + libxl__multidev *multidev,
> + int rc);
> +static void device_setup_iterate(libxl__egc *egc,
> + libxl__ao_device *aodev);
> +static void devices_teardown_cb(libxl__egc *egc,
> + libxl__multidev *multidev,
> + int rc);
> +
> +/* checkpoint device setup and teardown */
> +
> +static libxl__checkpoint_device* checkpoint_device_init(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds,
> + libxl__device_kind kind,
> + void *libxl_dev)
> +{
> + libxl__checkpoint_device *dev = NULL;
> +
> + STATE_AO_GC(cds->ao);
> + GCNEW(dev);
> + dev->backend_dev = libxl_dev;
> + dev->kind = kind;
> + dev->cds = cds;
> +
> + return dev;
> +}
> +
> +static void checkpoint_devices_setup(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds);
> +
> +void libxl__checkpoint_devices_setup(libxl__egc *egc, libxl__checkpoint_devices_state *cds)
> +{
> + int i, rc;
> +
> + STATE_AO_GC(cds->ao);
> +
> + rc = init_device_subkind(cds);
> + if (rc)
> + goto out;
> +
> + cds->num_devices = 0;
> + cds->num_nics = 0;
> + cds->num_disks = 0;
> +
> + if (cds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF))
> + cds->nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
> +
> + if (cds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD))
> + cds->disks = libxl_device_disk_list(CTX, cds->domid, &cds->num_disks);
> +
> + if (cds->num_nics == 0 && cds->num_disks == 0)
> + goto out;
> +
> + GCNEW_ARRAY(cds->devs, cds->num_nics + cds->num_disks);
> +
> + for (i = 0; i < cds->num_nics; i++) {
> + cds->devs[cds->num_devices++] = checkpoint_device_init(egc, cds,
> + LIBXL__DEVICE_KIND_VIF,
> + &cds->nics[i]);
> + }
> +
> + for (i = 0; i < cds->num_disks; i++) {
> + cds->devs[cds->num_devices++] = checkpoint_device_init(egc, cds,
> + LIBXL__DEVICE_KIND_VBD,
> + &cds->disks[i]);
> + }
> +
> + checkpoint_devices_setup(egc, cds);
> +
> + return;
> +
> +out:
> + cds->callback(egc, cds, rc);
> +}
> +
> +static void checkpoint_devices_setup(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds)
> +{
> + int i, rc;
> +
> + STATE_AO_GC(cds->ao);
> +
> + libxl__multidev_begin(ao, &cds->multidev);
> + cds->multidev.callback = all_devices_setup_cb;
> + for (i = 0; i < cds->num_devices; i++) {
> + libxl__checkpoint_device *dev = cds->devs[i];
> + dev->ops_index = -1;
> + libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);
> +
> + dev->aodev.rc = ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED;
> + dev->aodev.callback = device_setup_iterate;
> + device_setup_iterate(egc,&dev->aodev);
> + }
> +
> + rc = 0;
> + libxl__multidev_prepared(egc, &cds->multidev, rc);
> +}
> +
> +
> +static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
> +{
> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> + EGC_GC;
> +
> + if (aodev->rc != ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED &&
> + aodev->rc != ERROR_CHECKPOINT_DEVOPS_DOES_NOT_MATCH)
> + /* might be success or disaster */
> + goto out;
> +
> + do {
> + dev->ops = remus_ops[++dev->ops_index];
> + if (!dev->ops) {
> + libxl_device_nic * nic = NULL;
> + libxl_device_disk * disk = NULL;
> + uint32_t domid;
> + int devid;
> + if (dev->kind == LIBXL__DEVICE_KIND_VIF) {
> + nic = (libxl_device_nic *)dev->backend_dev;
> + domid = nic->backend_domid;
> + devid = nic->devid;
> + } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) {
> + disk = (libxl_device_disk *)dev->backend_dev;
> + domid = disk->backend_domid;
> + devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL);
> + } else {
> + LOG(ERROR,"device kind not handled by checkpoint: %s",
> + libxl__device_kind_to_string(dev->kind));
> + aodev->rc = ERROR_FAIL;
> + goto out;
> + }
> + LOG(ERROR,"device not handled by checkpoint"
> + " (device=%s:%"PRId32"/%"PRId32")",
> + libxl__device_kind_to_string(dev->kind),
> + domid, devid);
> + aodev->rc = ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED;
> + goto out;
> + }
> + } while (dev->ops->kind != dev->kind);
> +
> + /* found the next ops_index to try */
> + assert(dev->aodev.callback == device_setup_iterate);
> + dev->ops->setup(egc,dev);
> + return;
> +
> + out:
> + libxl__multidev_one_callback(egc,aodev);
> +}
> +
> +static void all_devices_setup_cb(libxl__egc *egc,
> + libxl__multidev *multidev,
> + int rc)
> +{
> + STATE_AO_GC(multidev->ao);
> +
> + /* Convenience aliases */
> + libxl__checkpoint_devices_state *const cds =
> + CONTAINER_OF(multidev, *cds, multidev);
> +
> + cds->callback(egc, cds, rc);
> +}
> +
> +void libxl__checkpoint_devices_teardown(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds)
> +{
> + int i;
> + libxl__checkpoint_device *dev;
> +
> + STATE_AO_GC(cds->ao);
> +
> + libxl__multidev_begin(ao, &cds->multidev);
> + cds->multidev.callback = devices_teardown_cb;
> + for (i = 0; i < cds->num_devices; i++) {
> + dev = cds->devs[i];
> + if (!dev->ops || !dev->matched)
> + continue;
> +
> + libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);
> + dev->ops->teardown(egc,dev);
> + }
> +
> + libxl__multidev_prepared(egc, &cds->multidev, 0);
> +}
> +
> +static void devices_teardown_cb(libxl__egc *egc,
> + libxl__multidev *multidev,
> + int rc)
> +{
> + int i;
> +
> + STATE_AO_GC(multidev->ao);
> +
> + /* Convenience aliases */
> + libxl__checkpoint_devices_state *const cds =
> + CONTAINER_OF(multidev, *cds, multidev);
> +
> + /* clean nic */
> + for (i = 0; i < cds->num_nics; i++)
> + libxl_device_nic_dispose(&cds->nics[i]);
> + free(cds->nics);
> + cds->nics = NULL;
> + cds->num_nics = 0;
> +
> + /* clean disk */
> + for (i = 0; i < cds->num_disks; i++)
> + libxl_device_disk_dispose(&cds->disks[i]);
> + free(cds->disks);
> + cds->disks = NULL;
> + cds->num_disks = 0;
> +
> + cleanup_device_subkind(cds);
> +
> + cds->callback(egc, cds, rc);
> +}
> +
> +/*----- checkpointing APIs -----*/
> +
> +/* callbacks */
> +
> +static void devices_checkpoint_cb(libxl__egc *egc,
> + libxl__multidev *multidev,
> + int rc);
> +
> +/* API implementations */
> +
> +#define define_checkpoint_api(api) \
> +void libxl__checkpoint_devices_##api(libxl__egc *egc, \
> + libxl__checkpoint_devices_state *cds) \
> +{ \
> + int i; \
> + libxl__checkpoint_device *dev; \
> + \
> + STATE_AO_GC(cds->ao); \
> + \
> + libxl__multidev_begin(ao, &cds->multidev); \
> + cds->multidev.callback = devices_checkpoint_cb; \
> + for (i = 0; i < cds->num_devices; i++) { \
> + dev = cds->devs[i]; \
> + if (!dev->matched || !dev->ops->api) \
> + continue; \
> + libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);\
> + dev->ops->api(egc,dev); \
> + } \
> + \
> + libxl__multidev_prepared(egc, &cds->multidev, 0); \
> +}
> +
> +define_checkpoint_api(postsuspend);
> +
> +define_checkpoint_api(preresume);
> +
> +define_checkpoint_api(commit);
> +
> +static void devices_checkpoint_cb(libxl__egc *egc,
> + libxl__multidev *multidev,
> + int rc)
> +{
> + STATE_AO_GC(multidev->ao);
> +
> + /* Convenience aliases */
> + libxl__checkpoint_devices_state *const cds =
> + CONTAINER_OF(multidev, *cds, multidev);
> +
> + cds->callback(egc, cds, rc);
> +}
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index bd369f2..ae3b964 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -2651,9 +2651,9 @@ typedef struct libxl__save_helper_state {
> * marshalling and xc callback functions */
> } libxl__save_helper_state;
>
> -/*----- remus device related state structure -----*/
> +/*----- checkpoint device related state structure -----*/
> /*
> - * The abstract Remus device layer exposes a common
> + * The abstract checkpoint device layer exposes a common
> * set of API to [external] libxl for manipulating devices attached to
> * a guest protected by Remus. The device layer also exposes a set of
> * [internal] interfaces that every device type must implement.
> @@ -2661,34 +2661,34 @@ typedef struct libxl__save_helper_state {
> * The following API are exposed to libxl:
> *
> * One-time configuration operations:
> - * +libxl__remus_devices_setup
> + * +libxl__checkpoint_devices_setup
> * > Enable output buffering for NICs, setup disk replication, etc.
> - * +libxl__remus_devices_teardown
> + * +libxl__checkpoint_devices_teardown
> * > Disable output buffering and disk replication; teardown any
> * associated external setups like qdiscs for NICs.
> *
> * Operations executed every checkpoint (in order of invocation):
> - * +libxl__remus_devices_postsuspend
> - * +libxl__remus_devices_preresume
> - * +libxl__remus_devices_commit
> + * +libxl__checkpoint_devices_postsuspend
> + * +libxl__checkpoint_devices_preresume
> + * +libxl__checkpoint_devices_commit
> *
> * Each device type needs to implement the interfaces specified in
> - * the libxl__remus_device_instance_ops if it wishes to support Remus.
> + * the libxl__checkpoint_device_instance_ops if it wishes to support Remus.
> *
> - * The high-level control flow through the Remus device layer is shown below:
> + * The high-level control flow through the checkpoint device layer is shown below:
> *
> * xl remus
> * |-> libxl_domain_remus_start
> - * |-> libxl__remus_devices_setup
> - * |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit]
> + * |-> libxl__checkpoint_devices_setup
> + * |-> Per-checkpoint libxl__checkpoint_devices_[postsuspend,preresume,commit]
> * ...
> * |-> On backup failure, network error or other internal errors:
> - * libxl__remus_devices_teardown
> + * libxl__checkpoint_devices_teardown
> */
>
> -typedef struct libxl__remus_device libxl__remus_device;
> -typedef struct libxl__remus_devices_state libxl__remus_devices_state;
> -typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops;
> +typedef struct libxl__checkpoint_device libxl__checkpoint_device;
> +typedef struct libxl__checkpoint_devices_state libxl__checkpoint_devices_state;
> +typedef struct libxl__checkpoint_device_instance_ops libxl__checkpoint_device_instance_ops;
>
> /*
> * Interfaces to be implemented by every device subkind that wishes to
> @@ -2698,7 +2698,7 @@ typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops
> * synchronous and call dev->aodev.callback directly (as the last
> * thing they do).
> */
> -struct libxl__remus_device_instance_ops {
> +struct libxl__checkpoint_device_instance_ops {
> /* the device kind this ops belongs to... */
> libxl__device_kind kind;
>
> @@ -2709,12 +2709,12 @@ struct libxl__remus_device_instance_ops {
> * Asynchronous.
> */
>
> - void (*postsuspend)(libxl__egc *egc, libxl__remus_device *dev);
> - void (*preresume)(libxl__egc *egc, libxl__remus_device *dev);
> - void (*commit)(libxl__egc *egc, libxl__remus_device *dev);
> + void (*postsuspend)(libxl__egc *egc, libxl__checkpoint_device *dev);
> + void (*preresume)(libxl__egc *egc, libxl__checkpoint_device *dev);
> + void (*commit)(libxl__egc *egc, libxl__checkpoint_device *dev);
>
> /*
> - * setup() and teardown() are refer to the actual remus device.
> + * setup() and teardown() are refer to the actual checkpoint device.
> * Asynchronous.
> * teardown is called even if setup fails.
> */
> @@ -2723,45 +2723,45 @@ struct libxl__remus_device_instance_ops {
> * device. If matched, the device will then be managed with this set of
> * subkind operations.
> * Yields 0 if the device successfully set up.
> - * REMUS_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
> + * CHECKPOINT_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
> * any other rc indicates failure.
> */
> - void (*setup)(libxl__egc *egc, libxl__remus_device *dev);
> - void (*teardown)(libxl__egc *egc, libxl__remus_device *dev);
> + void (*setup)(libxl__egc *egc, libxl__checkpoint_device *dev);
> + void (*teardown)(libxl__egc *egc, libxl__checkpoint_device *dev);
> };
>
> -int init_subkind_nic(libxl__remus_devices_state *rds);
> -void cleanup_subkind_nic(libxl__remus_devices_state *rds);
> -int init_subkind_drbd_disk(libxl__remus_devices_state *rds);
> -void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds);
> +int init_subkind_nic(libxl__checkpoint_devices_state *cds);
> +void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds);
> +int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
> +void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
>
> -typedef void libxl__remus_callback(libxl__egc *,
> - libxl__remus_devices_state *, int rc);
> +typedef void libxl__checkpoint_callback(libxl__egc *,
> + libxl__checkpoint_devices_state *, int rc);
>
> /*
> - * State associated with a remus invocation, including parameters
> - * passed to the remus abstract device layer by the remus
> + * State associated with a checkpoint invocation, including parameters
> + * passed to the checkpoint abstract device layer by the remus
> * save/restore machinery.
> */
> -struct libxl__remus_devices_state {
> - /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/
> +struct libxl__checkpoint_devices_state {
> + /*---- must be set by caller of libxl__checkpoint_device_(setup|teardown) ----*/
>
> libxl__ao *ao;
> uint32_t domid;
> - libxl__remus_callback *callback;
> + libxl__checkpoint_callback *callback;
> int device_kind_flags;
>
> /*----- private for abstract layer only -----*/
>
> int num_devices;
> /*
> - * this array is allocated before setup the remus devices by the
> - * remus abstract layer.
> - * devs may be NULL, means there's no remus devices that has been set up.
> + * this array is allocated before setup the checkpoint devices by the
> + * checkpoint abstract layer.
> + * devs may be NULL, means there's no checkpoint devices that has been set up.
> * the size of this array is 'num_devices', which is the total number
> * of libxl nic devices and disk devices(num_nics + num_disks).
> */
> - libxl__remus_device **devs;
> + libxl__checkpoint_device **devs;
>
> libxl_device_nic *nics;
> int num_nics;
> @@ -2783,20 +2783,20 @@ struct libxl__remus_devices_state {
>
> /*
> * Information about a single device being handled by remus.
> - * Allocated by the remus abstract layer.
> + * Allocated by the checkpoint abstract layer.
> */
> -struct libxl__remus_device {
> +struct libxl__checkpoint_device {
> /*----- shared between abstract and concrete layers -----*/
> /*
> * if this is true, that means the subkind ops match the device
> */
> bool matched;
>
> - /*----- set by remus device abstruct layer -----*/
> - /* libxl__device_* which this remus device related to */
> + /*----- set by checkpoint device abstruct layer -----*/
> + /* libxl__device_* which this checkpoint device related to */
> const void *backend_dev;
> libxl__device_kind kind;
> - libxl__remus_devices_state *rds;
> + libxl__checkpoint_devices_state *cds;
> libxl__ao_device aodev;
>
> /*----- private for abstract layer only -----*/
> @@ -2807,7 +2807,7 @@ struct libxl__remus_device {
> * individual devices.
> */
> int ops_index;
> - const libxl__remus_device_instance_ops *ops;
> + const libxl__checkpoint_device_instance_ops *ops;
>
> /*----- private for concrete (device-specific) layer -----*/
>
> @@ -2815,17 +2815,17 @@ struct libxl__remus_device {
> void *concrete_data;
> };
>
> -/* the following 5 APIs are async ops, call rds->callback when done */
> -_hidden void libxl__remus_devices_setup(libxl__egc *egc,
> - libxl__remus_devices_state *rds);
> -_hidden void libxl__remus_devices_teardown(libxl__egc *egc,
> - libxl__remus_devices_state *rds);
> -_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc,
> - libxl__remus_devices_state *rds);
> -_hidden void libxl__remus_devices_preresume(libxl__egc *egc,
> - libxl__remus_devices_state *rds);
> -_hidden void libxl__remus_devices_commit(libxl__egc *egc,
> - libxl__remus_devices_state *rds);
> +/* the following 5 APIs are async ops, call cds->callback when done */
> +_hidden void libxl__checkpoint_devices_setup(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds);
> +_hidden void libxl__checkpoint_devices_teardown(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds);
> +_hidden void libxl__checkpoint_devices_postsuspend(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds);
> +_hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds);
> +_hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
> + libxl__checkpoint_devices_state *cds);
> _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
>
> /*----- Legacy conversion helper -----*/
> @@ -2959,7 +2959,7 @@ struct libxl__domain_save_state {
> libxl__domain_suspend_state dsps;
> int hvm;
> int xcflags;
> - libxl__remus_devices_state rds;
> + libxl__checkpoint_devices_state cds;
> libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
> int interval; /* checkpoint interval (for Remus) */
> libxl__save_helper_state shs;
> diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
> index 71c6531..86afba6 100644
> --- a/tools/libxl/libxl_netbuffer.c
> +++ b/tools/libxl/libxl_netbuffer.c
> @@ -38,21 +38,21 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
> return 1;
> }
>
> -int init_subkind_nic(libxl__remus_devices_state *rds)
> +int init_subkind_nic(libxl__checkpoint_devices_state *cds)
> {
> int rc, ret;
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> - rds->nlsock = nl_socket_alloc();
> - if (!rds->nlsock) {
> + cds->nlsock = nl_socket_alloc();
> + if (!cds->nlsock) {
> LOG(ERROR, "cannot allocate nl socket");
> rc = ERROR_FAIL;
> goto out;
> }
>
> - ret = nl_connect(rds->nlsock, NETLINK_ROUTE);
> + ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
> if (ret) {
> LOG(ERROR, "failed to open netlink socket: %s",
> nl_geterror(ret));
> @@ -61,7 +61,7 @@ int init_subkind_nic(libxl__remus_devices_state *rds)
> }
>
> /* get list of all qdiscs installed on network devs. */
> - ret = rtnl_qdisc_alloc_cache(rds->nlsock, &rds->qdisc_cache);
> + ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
> if (ret) {
> LOG(ERROR, "failed to allocate qdisc cache: %s",
> nl_geterror(ret));
> @@ -70,9 +70,9 @@ int init_subkind_nic(libxl__remus_devices_state *rds)
> }
>
> if (dss->remus->netbufscript) {
> - rds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
> + cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
> } else {
> - rds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
> + cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
> libxl__xen_script_dir_path());
> }
>
> @@ -82,22 +82,22 @@ out:
> return rc;
> }
>
> -void cleanup_subkind_nic(libxl__remus_devices_state *rds)
> +void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
> {
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> /* free qdisc cache */
> - if (rds->qdisc_cache) {
> - nl_cache_clear(rds->qdisc_cache);
> - nl_cache_free(rds->qdisc_cache);
> - rds->qdisc_cache = NULL;
> + if (cds->qdisc_cache) {
> + nl_cache_clear(cds->qdisc_cache);
> + nl_cache_free(cds->qdisc_cache);
> + cds->qdisc_cache = NULL;
> }
>
> /* close & free nlsock */
> - if (rds->nlsock) {
> - nl_close(rds->nlsock);
> - nl_socket_free(rds->nlsock);
> - rds->nlsock = NULL;
> + if (cds->nlsock) {
> + nl_close(cds->nlsock);
> + nl_socket_free(cds->nlsock);
> + cds->nlsock = NULL;
> }
> }
>
> @@ -111,17 +111,17 @@ void cleanup_subkind_nic(libxl__remus_devices_state *rds)
> * it must ONLY be used for remus because if driver domains
> * were in use it would constitute a security vulnerability.
> */
> -static const char *get_vifname(libxl__remus_device *dev,
> +static const char *get_vifname(libxl__checkpoint_device *dev,
> const libxl_device_nic *nic)
> {
> const char *vifname = NULL;
> const char *path;
> int rc;
>
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> /* Convenience aliases */
> - const uint32_t domid = dev->rds->domid;
> + const uint32_t domid = dev->cds->domid;
>
> path = GCSPRINTF("%s/backend/vif/%d/%d/vifname",
> libxl__xs_get_dompath(gc, 0), domid, nic->devid);
> @@ -144,19 +144,19 @@ static void free_qdisc(libxl__remus_device_nic *remus_nic)
> remus_nic->qdisc = NULL;
> }
>
> -static int init_qdisc(libxl__remus_devices_state *rds,
> +static int init_qdisc(libxl__checkpoint_devices_state *cds,
> libxl__remus_device_nic *remus_nic)
> {
> int rc, ret, ifindex;
> struct rtnl_link *ifb = NULL;
> struct rtnl_qdisc *qdisc = NULL;
>
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> /* Now that we have brought up REMUS_IFB device with plug qdisc for
> * this vif, so we need to refill the qdisc cache.
> */
> - ret = nl_cache_refill(rds->nlsock, rds->qdisc_cache);
> + ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
> if (ret) {
> LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
> rc = ERROR_FAIL;
> @@ -164,7 +164,7 @@ static int init_qdisc(libxl__remus_devices_state *rds,
> }
>
> /* get a handle to the REMUS_IFB interface */
> - ret = rtnl_link_get_kernel(rds->nlsock, 0, remus_nic->ifb, &ifb);
> + ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
> if (ret) {
> LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
> nl_geterror(ret));
> @@ -187,7 +187,7 @@ static int init_qdisc(libxl__remus_devices_state *rds,
> * There is no need to explicitly free this qdisc as its just a
> * reference from the qdisc cache we allocated earlier.
> */
> - qdisc = rtnl_qdisc_get_by_parent(rds->qdisc_cache, ifindex, TC_H_ROOT);
> + qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
> if (qdisc) {
> const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
> /* Sanity check: Ensure that the root qdisc is a plug qdisc. */
> @@ -231,19 +231,19 @@ static void netbuf_teardown_script_cb(libxl__egc *egc,
> * $REMUS_IFB (for teardown)
> * setup/teardown as command line arg.
> */
> -static void setup_async_exec(libxl__remus_device *dev, char *op)
> +static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
> {
> int arraysize, nr = 0;
> char **env = NULL, **args = NULL;
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
> - libxl__remus_devices_state *rds = dev->rds;
> + libxl__checkpoint_devices_state *cds = dev->cds;
> libxl__async_exec_state *aes = &dev->aodev.aes;
>
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> /* Convenience aliases */
> - char *const script = libxl__strdup(gc, rds->netbufscript);
> - const uint32_t domid = rds->domid;
> + char *const script = libxl__strdup(gc, cds->netbufscript);
> + const uint32_t domid = cds->domid;
> const int dev_id = remus_nic->devid;
> const char *const vif = remus_nic->vif;
> const char *const ifb = remus_nic->ifb;
> @@ -269,7 +269,7 @@ static void setup_async_exec(libxl__remus_device *dev, char *op)
> args[nr++] = NULL;
> assert(nr == arraysize);
>
> - aes->ao = dev->rds->ao;
> + aes->ao = dev->cds->ao;
> aes->what = GCSPRINTF("%s %s", args[0], args[1]);
> aes->env = env;
> aes->args = args;
> @@ -286,13 +286,13 @@ static void setup_async_exec(libxl__remus_device *dev, char *op)
>
> /* setup() and teardown() */
>
> -static void nic_setup(libxl__egc *egc, libxl__remus_device *dev)
> +static void nic_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> int rc;
> libxl__remus_device_nic *remus_nic;
> const libxl_device_nic *nic = dev->backend_dev;
>
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> /*
> * thers's no subkind of nic devices, so nic ops is always matched
> @@ -330,16 +330,16 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
> int status)
> {
> libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
> - libxl__remus_devices_state *rds = dev->rds;
> + libxl__checkpoint_devices_state *cds = dev->cds;
> const char *out_path_base, *hotplug_error = NULL;
> int rc;
>
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> /* Convenience aliases */
> - const uint32_t domid = rds->domid;
> + const uint32_t domid = cds->domid;
> const int devid = remus_nic->devid;
> const char *const vif = remus_nic->vif;
> const char **const ifb = &remus_nic->ifb;
> @@ -373,7 +373,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>
> if (hotplug_error) {
> LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
> - rds->netbufscript, vif, hotplug_error);
> + cds->netbufscript, vif, hotplug_error);
> rc = ERROR_FAIL;
> goto out;
> }
> @@ -384,17 +384,17 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
> }
>
> LOG(DEBUG, "%s will buffer packets from vif %s", *ifb, vif);
> - rc = init_qdisc(rds, remus_nic);
> + rc = init_qdisc(cds, remus_nic);
>
> out:
> aodev->rc = rc;
> aodev->callback(egc, aodev);
> }
>
> -static void nic_teardown(libxl__egc *egc, libxl__remus_device *dev)
> +static void nic_teardown(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> int rc;
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> setup_async_exec(dev, "teardown");
>
> @@ -415,7 +415,7 @@ static void netbuf_teardown_script_cb(libxl__egc *egc,
> {
> int rc;
> libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>
> if (status)
> @@ -440,12 +440,12 @@ enum {
> /* API implementations */
>
> static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int buffer_op)
> {
> int rc, ret;
>
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> if (buffer_op == tc_buffer_start)
> ret = rtnl_qdisc_plug_buffer(remus_nic->qdisc);
> @@ -457,7 +457,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
> goto out;
> }
>
> - ret = rtnl_qdisc_add(rds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
> + ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
> if (ret) {
> rc = ERROR_FAIL;
> goto out;
> @@ -474,33 +474,33 @@ out:
> return rc;
> }
>
> -static void nic_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
> +static void nic_postsuspend(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> int rc;
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> - rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_start);
> + rc = remus_netbuf_op(remus_nic, dev->cds, tc_buffer_start);
>
> dev->aodev.rc = rc;
> dev->aodev.callback(egc, &dev->aodev);
> }
>
> -static void nic_commit(libxl__egc *egc, libxl__remus_device *dev)
> +static void nic_commit(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> int rc;
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> - rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_release);
> + rc = remus_netbuf_op(remus_nic, dev->cds, tc_buffer_release);
>
> dev->aodev.rc = rc;
> dev->aodev.callback(egc, &dev->aodev);
> }
>
> -const libxl__remus_device_instance_ops remus_device_nic = {
> +const libxl__checkpoint_device_instance_ops remus_device_nic = {
> .kind = LIBXL__DEVICE_KIND_VIF,
> .setup = nic_setup,
> .teardown = nic_teardown,
> diff --git a/tools/libxl/libxl_nonetbuffer.c b/tools/libxl/libxl_nonetbuffer.c
> index 3c659c2..4b68152 100644
> --- a/tools/libxl/libxl_nonetbuffer.c
> +++ b/tools/libxl/libxl_nonetbuffer.c
> @@ -22,25 +22,25 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
> return 0;
> }
>
> -int init_subkind_nic(libxl__remus_devices_state *rds)
> +int init_subkind_nic(libxl__checkpoint_devices_state *cds)
> {
> return 0;
> }
>
> -void cleanup_subkind_nic(libxl__remus_devices_state *rds)
> +void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
> {
> return;
> }
>
> -static void nic_setup(libxl__egc *egc, libxl__remus_device *dev)
> +static void nic_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> dev->aodev.rc = ERROR_FAIL;
> dev->aodev.callback(egc, &dev->aodev);
> }
>
> -const libxl__remus_device_instance_ops remus_device_nic = {
> +const libxl__checkpoint_device_instance_ops remus_device_nic = {
> .kind = LIBXL__DEVICE_KIND_VIF,
> .setup = nic_setup,
> };
> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
> index 0c449ad..fe4acc8 100644
> --- a/tools/libxl/libxl_remus.c
> +++ b/tools/libxl/libxl_remus.c
> @@ -20,14 +20,14 @@
> /*----- Remus setup and teardown -----*/
>
> static void remus_setup_done(libxl__egc *egc,
> - libxl__remus_devices_state *rds, int rc);
> + libxl__checkpoint_devices_state *cds, int rc);
> static void remus_setup_failed(libxl__egc *egc,
> - libxl__remus_devices_state *rds, int rc);
> + libxl__checkpoint_devices_state *cds, int rc);
>
> void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
> {
> /* Convenience aliases */
> - libxl__remus_devices_state *const rds = &dss->rds;
> + libxl__checkpoint_devices_state *const cds = &dss->cds;
> const libxl_domain_remus_info *const info = dss->remus;
>
> STATE_AO_GC(dss->ao);
> @@ -37,17 +37,17 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
> LOG(ERROR, "Remus: No support for network buffering");
> goto out;
> }
> - rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
> + cds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
> }
>
> if (libxl_defbool_val(info->diskbuf))
> - rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
> + cds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
>
> - rds->ao = ao;
> - rds->domid = dss->domid;
> - rds->callback = remus_setup_done;
> + cds->ao = ao;
> + cds->domid = dss->domid;
> + cds->callback = remus_setup_done;
>
> - libxl__remus_devices_setup(egc, rds);
> + libxl__checkpoint_devices_setup(egc, cds);
> return;
>
> out:
> @@ -55,9 +55,9 @@ out:
> }
>
> static void remus_setup_done(libxl__egc *egc,
> - libxl__remus_devices_state *rds, int rc)
> + libxl__checkpoint_devices_state *cds, int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> STATE_AO_GC(dss->ao);
>
> if (!rc) {
> @@ -67,14 +67,14 @@ static void remus_setup_done(libxl__egc *egc,
>
> LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
> dss->domid, rc);
> - rds->callback = remus_setup_failed;
> - libxl__remus_devices_teardown(egc, rds);
> + cds->callback = remus_setup_failed;
> + libxl__checkpoint_devices_teardown(egc, cds);
> }
>
> static void remus_setup_failed(libxl__egc *egc,
> - libxl__remus_devices_state *rds, int rc)
> + libxl__checkpoint_devices_state *cds, int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> STATE_AO_GC(dss->ao);
>
> if (rc)
> @@ -85,7 +85,7 @@ static void remus_setup_failed(libxl__egc *egc,
> }
>
> static void remus_teardown_done(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc);
> void libxl__remus_teardown(libxl__egc *egc,
> libxl__domain_save_state *dss,
> @@ -95,15 +95,15 @@ void libxl__remus_teardown(libxl__egc *egc,
>
> LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
> " teardown Remus devices...", rc);
> - dss->rds.callback = remus_teardown_done;
> - libxl__remus_devices_teardown(egc, &dss->rds);
> + dss->cds.callback = remus_teardown_done;
> + libxl__checkpoint_devices_teardown(egc, &dss->cds);
> }
>
> static void remus_teardown_done(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> STATE_AO_GC(dss->ao);
>
> if (rc)
> @@ -118,10 +118,10 @@ static void remus_teardown_done(libxl__egc *egc,
> static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> libxl__domain_suspend_state *dsps, int ok);
> static void remus_devices_postsuspend_cb(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc);
> static void remus_devices_preresume_cb(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc);
>
> void libxl__remus_domain_suspend_callback(void *data)
> @@ -143,9 +143,9 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> if (!ok)
> goto out;
>
> - libxl__remus_devices_state *const rds = &dss->rds;
> - rds->callback = remus_devices_postsuspend_cb;
> - libxl__remus_devices_postsuspend(egc, rds);
> + libxl__checkpoint_devices_state *const cds = &dss->cds;
> + cds->callback = remus_devices_postsuspend_cb;
> + libxl__checkpoint_devices_postsuspend(egc, cds);
> return;
>
> out:
> @@ -153,11 +153,11 @@ out:
> }
>
> static void remus_devices_postsuspend_cb(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc)
> {
> int ok = 0;
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>
> if (rc)
> goto out;
> @@ -175,17 +175,17 @@ void libxl__remus_domain_resume_callback(void *data)
> libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
> STATE_AO_GC(dss->ao);
>
> - libxl__remus_devices_state *const rds = &dss->rds;
> - rds->callback = remus_devices_preresume_cb;
> - libxl__remus_devices_preresume(egc, rds);
> + libxl__checkpoint_devices_state *const cds = &dss->cds;
> + cds->callback = remus_devices_preresume_cb;
> + libxl__checkpoint_devices_preresume(egc, cds);
> }
>
> static void remus_devices_preresume_cb(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc)
> {
> int ok = 0;
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> STATE_AO_GC(dss->ao);
>
> if (rc)
> @@ -207,7 +207,7 @@ out:
> static void remus_checkpoint_stream_written(
> libxl__egc *egc, libxl__stream_write_state *stream, int rc);
> static void remus_devices_commit_cb(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc);
> static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
> const struct timeval *requested_abs);
> @@ -229,7 +229,7 @@ static void remus_checkpoint_stream_written(
> libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
>
> /* Convenience aliases */
> - libxl__remus_devices_state *const rds = &dss->rds;
> + libxl__checkpoint_devices_state *const cds = &dss->cds;
>
> STATE_AO_GC(dss->ao);
>
> @@ -238,8 +238,8 @@ static void remus_checkpoint_stream_written(
> goto out;
> }
>
> - rds->callback = remus_devices_commit_cb;
> - libxl__remus_devices_commit(egc, rds);
> + cds->callback = remus_devices_commit_cb;
> + libxl__checkpoint_devices_commit(egc, cds);
>
> return;
>
> @@ -248,10 +248,10 @@ out:
> }
>
> static void remus_devices_commit_cb(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> + libxl__checkpoint_devices_state *cds,
> int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>
> STATE_AO_GC(dss->ao);
>
> diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c
> deleted file mode 100644
> index a6cb7f6..0000000
> --- a/tools/libxl/libxl_remus_device.c
> +++ /dev/null
> @@ -1,327 +0,0 @@
> -/*
> - * Copyright (C) 2014 FUJITSU LIMITED
> - * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
> - *
> - * This program is free software; you can redistribute it and/or modify
> - * it under the terms of the GNU Lesser General Public License as published
> - * by the Free Software Foundation; version 2.1 only. with the special
> - * exception on linking described in file LICENSE.
> - *
> - * This program is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> - * GNU Lesser General Public License for more details.
> - */
> -
> -#include "libxl_osdeps.h" /* must come before any other headers */
> -
> -#include "libxl_internal.h"
> -
> -extern const libxl__remus_device_instance_ops remus_device_nic;
> -extern const libxl__remus_device_instance_ops remus_device_drbd_disk;
> -static const libxl__remus_device_instance_ops *remus_ops[] = {
> - &remus_device_nic,
> - &remus_device_drbd_disk,
> - NULL,
> -};
> -
> -/*----- helper functions -----*/
> -
> -static int init_device_subkind(libxl__remus_devices_state *rds)
> -{
> - /* init device subkind-specific state in the libxl ctx */
> - int rc;
> - STATE_AO_GC(rds->ao);
> -
> - if (libxl__netbuffer_enabled(gc)) {
> - rc = init_subkind_nic(rds);
> - if (rc) goto out;
> - }
> -
> - rc = init_subkind_drbd_disk(rds);
> - if (rc) goto out;
> -
> - rc = 0;
> -out:
> - return rc;
> -}
> -
> -static void cleanup_device_subkind(libxl__remus_devices_state *rds)
> -{
> - /* cleanup device subkind-specific state in the libxl ctx */
> - STATE_AO_GC(rds->ao);
> -
> - if (libxl__netbuffer_enabled(gc))
> - cleanup_subkind_nic(rds);
> -
> - cleanup_subkind_drbd_disk(rds);
> -}
> -
> -/*----- setup() and teardown() -----*/
> -
> -/* callbacks */
> -
> -static void all_devices_setup_cb(libxl__egc *egc,
> - libxl__multidev *multidev,
> - int rc);
> -static void device_setup_iterate(libxl__egc *egc,
> - libxl__ao_device *aodev);
> -static void devices_teardown_cb(libxl__egc *egc,
> - libxl__multidev *multidev,
> - int rc);
> -
> -/* remus device setup and teardown */
> -
> -static libxl__remus_device* remus_device_init(libxl__egc *egc,
> - libxl__remus_devices_state *rds,
> - libxl__device_kind kind,
> - void *libxl_dev)
> -{
> - libxl__remus_device *dev = NULL;
> -
> - STATE_AO_GC(rds->ao);
> - GCNEW(dev);
> - dev->backend_dev = libxl_dev;
> - dev->kind = kind;
> - dev->rds = rds;
> -
> - return dev;
> -}
> -
> -static void remus_devices_setup(libxl__egc *egc,
> - libxl__remus_devices_state *rds);
> -
> -void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_devices_state *rds)
> -{
> - int i, rc;
> -
> - STATE_AO_GC(rds->ao);
> -
> - rc = init_device_subkind(rds);
> - if (rc)
> - goto out;
> -
> - rds->num_devices = 0;
> - rds->num_nics = 0;
> - rds->num_disks = 0;
> -
> - if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF))
> - rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics);
> -
> - if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD))
> - rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks);
> -
> - if (rds->num_nics == 0 && rds->num_disks == 0)
> - goto out;
> -
> - GCNEW_ARRAY(rds->devs, rds->num_nics + rds->num_disks);
> -
> - for (i = 0; i < rds->num_nics; i++) {
> - rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
> - LIBXL__DEVICE_KIND_VIF,
> - &rds->nics[i]);
> - }
> -
> - for (i = 0; i < rds->num_disks; i++) {
> - rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
> - LIBXL__DEVICE_KIND_VBD,
> - &rds->disks[i]);
> - }
> -
> - remus_devices_setup(egc, rds);
> -
> - return;
> -
> -out:
> - rds->callback(egc, rds, rc);
> -}
> -
> -static void remus_devices_setup(libxl__egc *egc,
> - libxl__remus_devices_state *rds)
> -{
> - int i, rc;
> -
> - STATE_AO_GC(rds->ao);
> -
> - libxl__multidev_begin(ao, &rds->multidev);
> - rds->multidev.callback = all_devices_setup_cb;
> - for (i = 0; i < rds->num_devices; i++) {
> - libxl__remus_device *dev = rds->devs[i];
> - dev->ops_index = -1;
> - libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
> -
> - dev->aodev.rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
> - dev->aodev.callback = device_setup_iterate;
> - device_setup_iterate(egc,&dev->aodev);
> - }
> -
> - rc = 0;
> - libxl__multidev_prepared(egc, &rds->multidev, rc);
> -}
> -
> -
> -static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
> -{
> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> - EGC_GC;
> -
> - if (aodev->rc != ERROR_REMUS_DEVICE_NOT_SUPPORTED &&
> - aodev->rc != ERROR_REMUS_DEVOPS_DOES_NOT_MATCH)
> - /* might be success or disaster */
> - goto out;
> -
> - do {
> - dev->ops = remus_ops[++dev->ops_index];
> - if (!dev->ops) {
> - libxl_device_nic * nic = NULL;
> - libxl_device_disk * disk = NULL;
> - uint32_t domid;
> - int devid;
> - if (dev->kind == LIBXL__DEVICE_KIND_VIF) {
> - nic = (libxl_device_nic *)dev->backend_dev;
> - domid = nic->backend_domid;
> - devid = nic->devid;
> - } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) {
> - disk = (libxl_device_disk *)dev->backend_dev;
> - domid = disk->backend_domid;
> - devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL);
> - } else {
> - LOG(ERROR,"device kind not handled by remus: %s",
> - libxl__device_kind_to_string(dev->kind));
> - aodev->rc = ERROR_FAIL;
> - goto out;
> - }
> - LOG(ERROR,"device not handled by remus"
> - " (device=%s:%"PRId32"/%"PRId32")",
> - libxl__device_kind_to_string(dev->kind),
> - domid, devid);
> - aodev->rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
> - goto out;
> - }
> - } while (dev->ops->kind != dev->kind);
> -
> - /* found the next ops_index to try */
> - assert(dev->aodev.callback == device_setup_iterate);
> - dev->ops->setup(egc,dev);
> - return;
> -
> - out:
> - libxl__multidev_one_callback(egc,aodev);
> -}
> -
> -static void all_devices_setup_cb(libxl__egc *egc,
> - libxl__multidev *multidev,
> - int rc)
> -{
> - STATE_AO_GC(multidev->ao);
> -
> - /* Convenience aliases */
> - libxl__remus_devices_state *const rds =
> - CONTAINER_OF(multidev, *rds, multidev);
> -
> - rds->callback(egc, rds, rc);
> -}
> -
> -void libxl__remus_devices_teardown(libxl__egc *egc,
> - libxl__remus_devices_state *rds)
> -{
> - int i;
> - libxl__remus_device *dev;
> -
> - STATE_AO_GC(rds->ao);
> -
> - libxl__multidev_begin(ao, &rds->multidev);
> - rds->multidev.callback = devices_teardown_cb;
> - for (i = 0; i < rds->num_devices; i++) {
> - dev = rds->devs[i];
> - if (!dev->ops || !dev->matched)
> - continue;
> -
> - libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
> - dev->ops->teardown(egc,dev);
> - }
> -
> - libxl__multidev_prepared(egc, &rds->multidev, 0);
> -}
> -
> -static void devices_teardown_cb(libxl__egc *egc,
> - libxl__multidev *multidev,
> - int rc)
> -{
> - int i;
> -
> - STATE_AO_GC(multidev->ao);
> -
> - /* Convenience aliases */
> - libxl__remus_devices_state *const rds =
> - CONTAINER_OF(multidev, *rds, multidev);
> -
> - /* clean nic */
> - for (i = 0; i < rds->num_nics; i++)
> - libxl_device_nic_dispose(&rds->nics[i]);
> - free(rds->nics);
> - rds->nics = NULL;
> - rds->num_nics = 0;
> -
> - /* clean disk */
> - for (i = 0; i < rds->num_disks; i++)
> - libxl_device_disk_dispose(&rds->disks[i]);
> - free(rds->disks);
> - rds->disks = NULL;
> - rds->num_disks = 0;
> -
> - cleanup_device_subkind(rds);
> -
> - rds->callback(egc, rds, rc);
> -}
> -
> -/*----- checkpointing APIs -----*/
> -
> -/* callbacks */
> -
> -static void devices_checkpoint_cb(libxl__egc *egc,
> - libxl__multidev *multidev,
> - int rc);
> -
> -/* API implementations */
> -
> -#define define_remus_checkpoint_api(api) \
> -void libxl__remus_devices_##api(libxl__egc *egc, \
> - libxl__remus_devices_state *rds) \
> -{ \
> - int i; \
> - libxl__remus_device *dev; \
> - \
> - STATE_AO_GC(rds->ao); \
> - \
> - libxl__multidev_begin(ao, &rds->multidev); \
> - rds->multidev.callback = devices_checkpoint_cb; \
> - for (i = 0; i < rds->num_devices; i++) { \
> - dev = rds->devs[i]; \
> - if (!dev->matched || !dev->ops->api) \
> - continue; \
> - libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);\
> - dev->ops->api(egc,dev); \
> - } \
> - \
> - libxl__multidev_prepared(egc, &rds->multidev, 0); \
> -}
> -
> -define_remus_checkpoint_api(postsuspend);
> -
> -define_remus_checkpoint_api(preresume);
> -
> -define_remus_checkpoint_api(commit);
> -
> -static void devices_checkpoint_cb(libxl__egc *egc,
> - libxl__multidev *multidev,
> - int rc)
> -{
> - STATE_AO_GC(multidev->ao);
> -
> - /* Convenience aliases */
> - libxl__remus_devices_state *const rds =
> - CONTAINER_OF(multidev, *rds, multidev);
> -
> - rds->callback(egc, rds, rc);
> -}
> diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
> index afe9b61..50b897d 100644
> --- a/tools/libxl/libxl_remus_disk_drbd.c
> +++ b/tools/libxl/libxl_remus_disk_drbd.c
> @@ -26,30 +26,30 @@ typedef struct libxl__remus_drbd_disk {
> int ackwait;
> } libxl__remus_drbd_disk;
>
> -int init_subkind_drbd_disk(libxl__remus_devices_state *rds)
> +int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
> {
> - STATE_AO_GC(rds->ao);
> + STATE_AO_GC(cds->ao);
>
> - rds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
> + cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
> libxl__xen_script_dir_path());
>
> return 0;
> }
>
> -void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds)
> +void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
> {
> return;
> }
>
> /*----- helper functions, for async calls -----*/
> static void drbd_async_call(libxl__egc *egc,
> - libxl__remus_device *dev,
> - void func(libxl__remus_device *),
> + libxl__checkpoint_device *dev,
> + void func(libxl__checkpoint_device *),
> libxl__ev_child_callback callback)
> {
> int pid = -1, rc;
> libxl__ao_device *aodev = &dev->aodev;
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> /* Fork and call */
> pid = libxl__ev_child_fork(gc, &aodev->child, callback);
> @@ -82,21 +82,21 @@ static void match_async_exec_cb(libxl__egc *egc,
>
> /* implementations */
>
> -static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev);
> +static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev);
>
> -static void drbd_setup(libxl__egc *egc, libxl__remus_device *dev)
> +static void drbd_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> match_async_exec(egc, dev);
> }
>
> -static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev)
> +static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> int arraysize, nr = 0, rc;
> const libxl_device_disk *disk = dev->backend_dev;
> libxl__async_exec_state *aes = &dev->aodev.aes;
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> /* setup env & args */
> arraysize = 1;
> @@ -107,12 +107,12 @@ static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev)
> arraysize = 3;
> nr = 0;
> GCNEW_ARRAY(aes->args, arraysize);
> - aes->args[nr++] = dev->rds->drbd_probe_script;
> + aes->args[nr++] = dev->cds->drbd_probe_script;
> aes->args[nr++] = disk->pdev_path;
> aes->args[nr++] = NULL;
> assert(nr <= arraysize);
>
> - aes->ao = dev->rds->ao;
> + aes->ao = dev->cds->ao;
> aes->what = GCSPRINTF("%s %s", aes->args[0], aes->args[1]);
> aes->timeout_ms = LIBXL_HOTPLUG_TIMEOUT * 1000;
> aes->callback = match_async_exec_cb;
> @@ -137,14 +137,14 @@ static void match_async_exec_cb(libxl__egc *egc,
> {
> int rc;
> libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> libxl__remus_drbd_disk *drbd_disk;
> const libxl_device_disk *disk = dev->backend_dev;
>
> STATE_AO_GC(aodev->ao);
>
> if (status) {
> - rc = ERROR_REMUS_DEVOPS_DOES_NOT_MATCH;
> + rc = ERROR_CHECKPOINT_DEVOPS_DOES_NOT_MATCH;
> /* BUG: seems to assume that any exit status means `no match' */
> /* BUG: exit status will have been logged as an error */
> goto out;
> @@ -169,10 +169,10 @@ out:
> aodev->callback(egc, aodev);
> }
>
> -static void drbd_teardown(libxl__egc *egc, libxl__remus_device *dev)
> +static void drbd_teardown(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> libxl__remus_drbd_disk *drbd_disk = dev->concrete_data;
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> close(drbd_disk->ctl_fd);
> dev->aodev.rc = 0;
> @@ -189,9 +189,9 @@ static void checkpoint_async_call_done(libxl__egc *egc,
> /* API implementations */
>
> /* this op will not wait and block, so implement as sync op */
> -static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
> +static void drbd_postsuspend(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> libxl__remus_drbd_disk *rdd = dev->concrete_data;
>
> @@ -205,16 +205,16 @@ static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
> }
>
>
> -static void drbd_preresume_async(libxl__remus_device *dev);
> +static void drbd_preresume_async(libxl__checkpoint_device *dev);
>
> -static void drbd_preresume(libxl__egc *egc, libxl__remus_device *dev)
> +static void drbd_preresume(libxl__egc *egc, libxl__checkpoint_device *dev)
> {
> - STATE_AO_GC(dev->rds->ao);
> + STATE_AO_GC(dev->cds->ao);
>
> drbd_async_call(egc, dev, drbd_preresume_async, checkpoint_async_call_done);
> }
>
> -static void drbd_preresume_async(libxl__remus_device *dev)
> +static void drbd_preresume_async(libxl__checkpoint_device *dev)
> {
> libxl__remus_drbd_disk *rdd = dev->concrete_data;
> int ackwait = rdd->ackwait;
> @@ -233,7 +233,7 @@ static void checkpoint_async_call_done(libxl__egc *egc,
> {
> int rc;
> libxl__ao_device *aodev = CONTAINER_OF(child, *aodev, child);
> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> libxl__remus_drbd_disk *rdd = dev->concrete_data;
>
> STATE_AO_GC(aodev->ao);
> @@ -251,7 +251,7 @@ out:
> aodev->callback(egc, aodev);
> }
>
> -const libxl__remus_device_instance_ops remus_device_drbd_disk = {
> +const libxl__checkpoint_device_instance_ops remus_device_drbd_disk = {
> .kind = LIBXL__DEVICE_KIND_VBD,
> .setup = drbd_setup,
> .teardown = drbd_teardown,
> diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
> index 5c7b82d..e05d12b 100644
> --- a/tools/libxl/libxl_types.idl
> +++ b/tools/libxl/libxl_types.idl
> @@ -61,8 +61,8 @@ libxl_error = Enumeration("error", [
> (-15, "LOCK_FAIL"),
> (-16, "JSON_CONFIG_EMPTY"),
> (-17, "DEVICE_EXISTS"),
> - (-18, "REMUS_DEVOPS_DOES_NOT_MATCH"),
> - (-19, "REMUS_DEVICE_NOT_SUPPORTED"),
> + (-18, "CHECKPOINT_DEVOPS_DOES_NOT_MATCH"),
> + (-19, "CHECKPOINT_DEVICE_NOT_SUPPORTED"),
> (-20, "VNUMA_CONFIG_INVALID"),
> (-21, "DOMAIN_NOTFOUND"),
> ], value_namespace = "")
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 24/26] tools/libxl: rename remus device to checkpoint device
2015-06-30 10:43 ` Ian Campbell
@ 2015-07-01 3:11 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 3:11 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:43 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> This patch is auto generated by the following commands:
>> 1. git mv tools/libxl/libxl_remus_device.c tools/libxl/libxl_checkpoint_device.c
>
> Please use the -M option to git format-patch/send-email, which will make
> the move and subsequent changes far more apparent.
Ok.
>
>> 2. perl -pi -e 's/libxl_remus_device/libxl_checkpoint_device/g' tools/libxl/Makefile
>> 3. perl -pi -e 's/\blibxl__remus_devices/libxl__checkpoint_devices/g' tools/libxl/*.[ch]
>> 4. perl -pi -e 's/\blibxl__remus_device\b/libxl__checkpoint_device/g' tools/libxl/*.[ch]
>> 5. perl -pi -e 's/\blibxl__remus_device_instance_ops\b/libxl__checkpoint_device_instance_ops/g' tools/libxl/*.[ch]
>> 6. perl -pi -e 's/\blibxl__remus_callback\b/libxl__checkpoint_callback/g' tools/libxl/*.[ch]
>> 7. perl -pi -e 's/\bremus_device_init\b/checkpoint_device_init/g' tools/libxl/*.[ch]
>> 8. perl -pi -e 's/\bremus_devices_setup\b/checkpoint_devices_setup/g' tools/libxl/*.[ch]
>> 9. perl -pi -e 's/\bdefine_remus_checkpoint_api\b/define_checkpoint_api/g' tools/libxl/*.[ch]
>> 10. perl -pi -e 's/\brds\b/cds/g' tools/libxl/*.[ch]
>> 11. perl -pi -e 's/REMUS_DEVICE/CHECKPOINT_DEVICE/g' tools/libxl/*.[ch] tools/libxl/*.idl
>> 12. perl -pi -e 's/REMUS_DEVOPS/CHECKPOINT_DEVOPS/g' tools/libxl/*.[ch] tools/libxl/*.idl
>> 13. perl -pi -e 's/\bremus\b/checkpoint/g' tools/libxl/libxl_checkpoint_device.[ch]
>> 14. perl -pi -e 's/\bremus device/checkpoint device/g' tools/libxl/libxl_internal.h
>> 15. perl -pi -e 's/\bRemus device/checkpoint device/g' tools/libxl/libxl_internal.h
>> 16. perl -pi -e 's/\bremus abstract/checkpoint abstract/g' tools/libxl/libxl_internal.h
>> 17. perl -pi -e 's/\bremus invocation/checkpoint invocation/g' tools/libxl/libxl_internal.h
>> 18. perl -pi -e 's/\blibxl__remus_device_\(/libxl__checkpoint_device_(/g' tools/libxl/libxl_internal.h
>
> Wowza!
>
>>
>> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> ---
>> tools/libxl/Makefile | 2 +-
>> tools/libxl/libxl_checkpoint_device.c | 327 ++++++++++++++++++++++++++++++++++
>> tools/libxl/libxl_internal.h | 112 ++++++------
>> tools/libxl/libxl_netbuffer.c | 108 +++++------
>> tools/libxl/libxl_nonetbuffer.c | 10 +-
>> tools/libxl/libxl_remus.c | 76 ++++----
>> tools/libxl/libxl_remus_device.c | 327 ----------------------------------
>> tools/libxl/libxl_remus_disk_drbd.c | 52 +++---
>> tools/libxl/libxl_types.idl | 4 +-
>> 9 files changed, 509 insertions(+), 509 deletions(-)
>> create mode 100644 tools/libxl/libxl_checkpoint_device.c
>> delete mode 100644 tools/libxl/libxl_remus_device.c
>>
>> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
>> index d61c191..2f4efd4 100644
>> --- a/tools/libxl/Makefile
>> +++ b/tools/libxl/Makefile
>> @@ -56,7 +56,7 @@ else
>> LIBXL_OBJS-y += libxl_nonetbuffer.o
>> endif
>>
>> -LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o
>> +LIBXL_OBJS-y += libxl_remus.o libxl_checkpoint_device.o libxl_remus_disk_drbd.o
>>
>> LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
>> LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
>> diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
>> new file mode 100644
>> index 0000000..109cd23
>> --- /dev/null
>> +++ b/tools/libxl/libxl_checkpoint_device.c
>> @@ -0,0 +1,327 @@
>> +/*
>> + * Copyright (C) 2014 FUJITSU LIMITED
>> + * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU Lesser General Public License as published
>> + * by the Free Software Foundation; version 2.1 only. with the special
>> + * exception on linking described in file LICENSE.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> + * GNU Lesser General Public License for more details.
>> + */
>> +
>> +#include "libxl_osdeps.h" /* must come before any other headers */
>> +
>> +#include "libxl_internal.h"
>> +
>> +extern const libxl__checkpoint_device_instance_ops remus_device_nic;
>> +extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
>> +static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
>> + &remus_device_nic,
>> + &remus_device_drbd_disk,
>> + NULL,
>> +};
>> +
>> +/*----- helper functions -----*/
>> +
>> +static int init_device_subkind(libxl__checkpoint_devices_state *cds)
>> +{
>> + /* init device subkind-specific state in the libxl ctx */
>> + int rc;
>> + STATE_AO_GC(cds->ao);
>> +
>> + if (libxl__netbuffer_enabled(gc)) {
>> + rc = init_subkind_nic(cds);
>> + if (rc) goto out;
>> + }
>> +
>> + rc = init_subkind_drbd_disk(cds);
>> + if (rc) goto out;
>> +
>> + rc = 0;
>> +out:
>> + return rc;
>> +}
>> +
>> +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
>> +{
>> + /* cleanup device subkind-specific state in the libxl ctx */
>> + STATE_AO_GC(cds->ao);
>> +
>> + if (libxl__netbuffer_enabled(gc))
>> + cleanup_subkind_nic(cds);
>> +
>> + cleanup_subkind_drbd_disk(cds);
>> +}
>> +
>> +/*----- setup() and teardown() -----*/
>> +
>> +/* callbacks */
>> +
>> +static void all_devices_setup_cb(libxl__egc *egc,
>> + libxl__multidev *multidev,
>> + int rc);
>> +static void device_setup_iterate(libxl__egc *egc,
>> + libxl__ao_device *aodev);
>> +static void devices_teardown_cb(libxl__egc *egc,
>> + libxl__multidev *multidev,
>> + int rc);
>> +
>> +/* checkpoint device setup and teardown */
>> +
>> +static libxl__checkpoint_device* checkpoint_device_init(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds,
>> + libxl__device_kind kind,
>> + void *libxl_dev)
>> +{
>> + libxl__checkpoint_device *dev = NULL;
>> +
>> + STATE_AO_GC(cds->ao);
>> + GCNEW(dev);
>> + dev->backend_dev = libxl_dev;
>> + dev->kind = kind;
>> + dev->cds = cds;
>> +
>> + return dev;
>> +}
>> +
>> +static void checkpoint_devices_setup(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds);
>> +
>> +void libxl__checkpoint_devices_setup(libxl__egc *egc, libxl__checkpoint_devices_state *cds)
>> +{
>> + int i, rc;
>> +
>> + STATE_AO_GC(cds->ao);
>> +
>> + rc = init_device_subkind(cds);
>> + if (rc)
>> + goto out;
>> +
>> + cds->num_devices = 0;
>> + cds->num_nics = 0;
>> + cds->num_disks = 0;
>> +
>> + if (cds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF))
>> + cds->nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
>> +
>> + if (cds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD))
>> + cds->disks = libxl_device_disk_list(CTX, cds->domid, &cds->num_disks);
>> +
>> + if (cds->num_nics == 0 && cds->num_disks == 0)
>> + goto out;
>> +
>> + GCNEW_ARRAY(cds->devs, cds->num_nics + cds->num_disks);
>> +
>> + for (i = 0; i < cds->num_nics; i++) {
>> + cds->devs[cds->num_devices++] = checkpoint_device_init(egc, cds,
>> + LIBXL__DEVICE_KIND_VIF,
>> + &cds->nics[i]);
>> + }
>> +
>> + for (i = 0; i < cds->num_disks; i++) {
>> + cds->devs[cds->num_devices++] = checkpoint_device_init(egc, cds,
>> + LIBXL__DEVICE_KIND_VBD,
>> + &cds->disks[i]);
>> + }
>> +
>> + checkpoint_devices_setup(egc, cds);
>> +
>> + return;
>> +
>> +out:
>> + cds->callback(egc, cds, rc);
>> +}
>> +
>> +static void checkpoint_devices_setup(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds)
>> +{
>> + int i, rc;
>> +
>> + STATE_AO_GC(cds->ao);
>> +
>> + libxl__multidev_begin(ao, &cds->multidev);
>> + cds->multidev.callback = all_devices_setup_cb;
>> + for (i = 0; i < cds->num_devices; i++) {
>> + libxl__checkpoint_device *dev = cds->devs[i];
>> + dev->ops_index = -1;
>> + libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);
>> +
>> + dev->aodev.rc = ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED;
>> + dev->aodev.callback = device_setup_iterate;
>> + device_setup_iterate(egc,&dev->aodev);
>> + }
>> +
>> + rc = 0;
>> + libxl__multidev_prepared(egc, &cds->multidev, rc);
>> +}
>> +
>> +
>> +static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
>> +{
>> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> + EGC_GC;
>> +
>> + if (aodev->rc != ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED &&
>> + aodev->rc != ERROR_CHECKPOINT_DEVOPS_DOES_NOT_MATCH)
>> + /* might be success or disaster */
>> + goto out;
>> +
>> + do {
>> + dev->ops = remus_ops[++dev->ops_index];
>> + if (!dev->ops) {
>> + libxl_device_nic * nic = NULL;
>> + libxl_device_disk * disk = NULL;
>> + uint32_t domid;
>> + int devid;
>> + if (dev->kind == LIBXL__DEVICE_KIND_VIF) {
>> + nic = (libxl_device_nic *)dev->backend_dev;
>> + domid = nic->backend_domid;
>> + devid = nic->devid;
>> + } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) {
>> + disk = (libxl_device_disk *)dev->backend_dev;
>> + domid = disk->backend_domid;
>> + devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL);
>> + } else {
>> + LOG(ERROR,"device kind not handled by checkpoint: %s",
>> + libxl__device_kind_to_string(dev->kind));
>> + aodev->rc = ERROR_FAIL;
>> + goto out;
>> + }
>> + LOG(ERROR,"device not handled by checkpoint"
>> + " (device=%s:%"PRId32"/%"PRId32")",
>> + libxl__device_kind_to_string(dev->kind),
>> + domid, devid);
>> + aodev->rc = ERROR_CHECKPOINT_DEVICE_NOT_SUPPORTED;
>> + goto out;
>> + }
>> + } while (dev->ops->kind != dev->kind);
>> +
>> + /* found the next ops_index to try */
>> + assert(dev->aodev.callback == device_setup_iterate);
>> + dev->ops->setup(egc,dev);
>> + return;
>> +
>> + out:
>> + libxl__multidev_one_callback(egc,aodev);
>> +}
>> +
>> +static void all_devices_setup_cb(libxl__egc *egc,
>> + libxl__multidev *multidev,
>> + int rc)
>> +{
>> + STATE_AO_GC(multidev->ao);
>> +
>> + /* Convenience aliases */
>> + libxl__checkpoint_devices_state *const cds =
>> + CONTAINER_OF(multidev, *cds, multidev);
>> +
>> + cds->callback(egc, cds, rc);
>> +}
>> +
>> +void libxl__checkpoint_devices_teardown(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds)
>> +{
>> + int i;
>> + libxl__checkpoint_device *dev;
>> +
>> + STATE_AO_GC(cds->ao);
>> +
>> + libxl__multidev_begin(ao, &cds->multidev);
>> + cds->multidev.callback = devices_teardown_cb;
>> + for (i = 0; i < cds->num_devices; i++) {
>> + dev = cds->devs[i];
>> + if (!dev->ops || !dev->matched)
>> + continue;
>> +
>> + libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);
>> + dev->ops->teardown(egc,dev);
>> + }
>> +
>> + libxl__multidev_prepared(egc, &cds->multidev, 0);
>> +}
>> +
>> +static void devices_teardown_cb(libxl__egc *egc,
>> + libxl__multidev *multidev,
>> + int rc)
>> +{
>> + int i;
>> +
>> + STATE_AO_GC(multidev->ao);
>> +
>> + /* Convenience aliases */
>> + libxl__checkpoint_devices_state *const cds =
>> + CONTAINER_OF(multidev, *cds, multidev);
>> +
>> + /* clean nic */
>> + for (i = 0; i < cds->num_nics; i++)
>> + libxl_device_nic_dispose(&cds->nics[i]);
>> + free(cds->nics);
>> + cds->nics = NULL;
>> + cds->num_nics = 0;
>> +
>> + /* clean disk */
>> + for (i = 0; i < cds->num_disks; i++)
>> + libxl_device_disk_dispose(&cds->disks[i]);
>> + free(cds->disks);
>> + cds->disks = NULL;
>> + cds->num_disks = 0;
>> +
>> + cleanup_device_subkind(cds);
>> +
>> + cds->callback(egc, cds, rc);
>> +}
>> +
>> +/*----- checkpointing APIs -----*/
>> +
>> +/* callbacks */
>> +
>> +static void devices_checkpoint_cb(libxl__egc *egc,
>> + libxl__multidev *multidev,
>> + int rc);
>> +
>> +/* API implementations */
>> +
>> +#define define_checkpoint_api(api) \
>> +void libxl__checkpoint_devices_##api(libxl__egc *egc, \
>> + libxl__checkpoint_devices_state *cds) \
>> +{ \
>> + int i; \
>> + libxl__checkpoint_device *dev; \
>> + \
>> + STATE_AO_GC(cds->ao); \
>> + \
>> + libxl__multidev_begin(ao, &cds->multidev); \
>> + cds->multidev.callback = devices_checkpoint_cb; \
>> + for (i = 0; i < cds->num_devices; i++) { \
>> + dev = cds->devs[i]; \
>> + if (!dev->matched || !dev->ops->api) \
>> + continue; \
>> + libxl__multidev_prepare_with_aodev(&cds->multidev, &dev->aodev);\
>> + dev->ops->api(egc,dev); \
>> + } \
>> + \
>> + libxl__multidev_prepared(egc, &cds->multidev, 0); \
>> +}
>> +
>> +define_checkpoint_api(postsuspend);
>> +
>> +define_checkpoint_api(preresume);
>> +
>> +define_checkpoint_api(commit);
>> +
>> +static void devices_checkpoint_cb(libxl__egc *egc,
>> + libxl__multidev *multidev,
>> + int rc)
>> +{
>> + STATE_AO_GC(multidev->ao);
>> +
>> + /* Convenience aliases */
>> + libxl__checkpoint_devices_state *const cds =
>> + CONTAINER_OF(multidev, *cds, multidev);
>> +
>> + cds->callback(egc, cds, rc);
>> +}
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index bd369f2..ae3b964 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
>> @@ -2651,9 +2651,9 @@ typedef struct libxl__save_helper_state {
>> * marshalling and xc callback functions */
>> } libxl__save_helper_state;
>>
>> -/*----- remus device related state structure -----*/
>> +/*----- checkpoint device related state structure -----*/
>> /*
>> - * The abstract Remus device layer exposes a common
>> + * The abstract checkpoint device layer exposes a common
>> * set of API to [external] libxl for manipulating devices attached to
>> * a guest protected by Remus. The device layer also exposes a set of
>> * [internal] interfaces that every device type must implement.
>> @@ -2661,34 +2661,34 @@ typedef struct libxl__save_helper_state {
>> * The following API are exposed to libxl:
>> *
>> * One-time configuration operations:
>> - * +libxl__remus_devices_setup
>> + * +libxl__checkpoint_devices_setup
>> * > Enable output buffering for NICs, setup disk replication, etc.
>> - * +libxl__remus_devices_teardown
>> + * +libxl__checkpoint_devices_teardown
>> * > Disable output buffering and disk replication; teardown any
>> * associated external setups like qdiscs for NICs.
>> *
>> * Operations executed every checkpoint (in order of invocation):
>> - * +libxl__remus_devices_postsuspend
>> - * +libxl__remus_devices_preresume
>> - * +libxl__remus_devices_commit
>> + * +libxl__checkpoint_devices_postsuspend
>> + * +libxl__checkpoint_devices_preresume
>> + * +libxl__checkpoint_devices_commit
>> *
>> * Each device type needs to implement the interfaces specified in
>> - * the libxl__remus_device_instance_ops if it wishes to support Remus.
>> + * the libxl__checkpoint_device_instance_ops if it wishes to support Remus.
>> *
>> - * The high-level control flow through the Remus device layer is shown below:
>> + * The high-level control flow through the checkpoint device layer is shown below:
>> *
>> * xl remus
>> * |-> libxl_domain_remus_start
>> - * |-> libxl__remus_devices_setup
>> - * |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit]
>> + * |-> libxl__checkpoint_devices_setup
>> + * |-> Per-checkpoint libxl__checkpoint_devices_[postsuspend,preresume,commit]
>> * ...
>> * |-> On backup failure, network error or other internal errors:
>> - * libxl__remus_devices_teardown
>> + * libxl__checkpoint_devices_teardown
>> */
>>
>> -typedef struct libxl__remus_device libxl__remus_device;
>> -typedef struct libxl__remus_devices_state libxl__remus_devices_state;
>> -typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops;
>> +typedef struct libxl__checkpoint_device libxl__checkpoint_device;
>> +typedef struct libxl__checkpoint_devices_state libxl__checkpoint_devices_state;
>> +typedef struct libxl__checkpoint_device_instance_ops libxl__checkpoint_device_instance_ops;
>>
>> /*
>> * Interfaces to be implemented by every device subkind that wishes to
>> @@ -2698,7 +2698,7 @@ typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops
>> * synchronous and call dev->aodev.callback directly (as the last
>> * thing they do).
>> */
>> -struct libxl__remus_device_instance_ops {
>> +struct libxl__checkpoint_device_instance_ops {
>> /* the device kind this ops belongs to... */
>> libxl__device_kind kind;
>>
>> @@ -2709,12 +2709,12 @@ struct libxl__remus_device_instance_ops {
>> * Asynchronous.
>> */
>>
>> - void (*postsuspend)(libxl__egc *egc, libxl__remus_device *dev);
>> - void (*preresume)(libxl__egc *egc, libxl__remus_device *dev);
>> - void (*commit)(libxl__egc *egc, libxl__remus_device *dev);
>> + void (*postsuspend)(libxl__egc *egc, libxl__checkpoint_device *dev);
>> + void (*preresume)(libxl__egc *egc, libxl__checkpoint_device *dev);
>> + void (*commit)(libxl__egc *egc, libxl__checkpoint_device *dev);
>>
>> /*
>> - * setup() and teardown() are refer to the actual remus device.
>> + * setup() and teardown() are refer to the actual checkpoint device.
>> * Asynchronous.
>> * teardown is called even if setup fails.
>> */
>> @@ -2723,45 +2723,45 @@ struct libxl__remus_device_instance_ops {
>> * device. If matched, the device will then be managed with this set of
>> * subkind operations.
>> * Yields 0 if the device successfully set up.
>> - * REMUS_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
>> + * CHECKPOINT_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
>> * any other rc indicates failure.
>> */
>> - void (*setup)(libxl__egc *egc, libxl__remus_device *dev);
>> - void (*teardown)(libxl__egc *egc, libxl__remus_device *dev);
>> + void (*setup)(libxl__egc *egc, libxl__checkpoint_device *dev);
>> + void (*teardown)(libxl__egc *egc, libxl__checkpoint_device *dev);
>> };
>>
>> -int init_subkind_nic(libxl__remus_devices_state *rds);
>> -void cleanup_subkind_nic(libxl__remus_devices_state *rds);
>> -int init_subkind_drbd_disk(libxl__remus_devices_state *rds);
>> -void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds);
>> +int init_subkind_nic(libxl__checkpoint_devices_state *cds);
>> +void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds);
>> +int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
>> +void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
>>
>> -typedef void libxl__remus_callback(libxl__egc *,
>> - libxl__remus_devices_state *, int rc);
>> +typedef void libxl__checkpoint_callback(libxl__egc *,
>> + libxl__checkpoint_devices_state *, int rc);
>>
>> /*
>> - * State associated with a remus invocation, including parameters
>> - * passed to the remus abstract device layer by the remus
>> + * State associated with a checkpoint invocation, including parameters
>> + * passed to the checkpoint abstract device layer by the remus
>> * save/restore machinery.
>> */
>> -struct libxl__remus_devices_state {
>> - /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/
>> +struct libxl__checkpoint_devices_state {
>> + /*---- must be set by caller of libxl__checkpoint_device_(setup|teardown) ----*/
>>
>> libxl__ao *ao;
>> uint32_t domid;
>> - libxl__remus_callback *callback;
>> + libxl__checkpoint_callback *callback;
>> int device_kind_flags;
>>
>> /*----- private for abstract layer only -----*/
>>
>> int num_devices;
>> /*
>> - * this array is allocated before setup the remus devices by the
>> - * remus abstract layer.
>> - * devs may be NULL, means there's no remus devices that has been set up.
>> + * this array is allocated before setup the checkpoint devices by the
>> + * checkpoint abstract layer.
>> + * devs may be NULL, means there's no checkpoint devices that has been set up.
>> * the size of this array is 'num_devices', which is the total number
>> * of libxl nic devices and disk devices(num_nics + num_disks).
>> */
>> - libxl__remus_device **devs;
>> + libxl__checkpoint_device **devs;
>>
>> libxl_device_nic *nics;
>> int num_nics;
>> @@ -2783,20 +2783,20 @@ struct libxl__remus_devices_state {
>>
>> /*
>> * Information about a single device being handled by remus.
>> - * Allocated by the remus abstract layer.
>> + * Allocated by the checkpoint abstract layer.
>> */
>> -struct libxl__remus_device {
>> +struct libxl__checkpoint_device {
>> /*----- shared between abstract and concrete layers -----*/
>> /*
>> * if this is true, that means the subkind ops match the device
>> */
>> bool matched;
>>
>> - /*----- set by remus device abstruct layer -----*/
>> - /* libxl__device_* which this remus device related to */
>> + /*----- set by checkpoint device abstruct layer -----*/
>> + /* libxl__device_* which this checkpoint device related to */
>> const void *backend_dev;
>> libxl__device_kind kind;
>> - libxl__remus_devices_state *rds;
>> + libxl__checkpoint_devices_state *cds;
>> libxl__ao_device aodev;
>>
>> /*----- private for abstract layer only -----*/
>> @@ -2807,7 +2807,7 @@ struct libxl__remus_device {
>> * individual devices.
>> */
>> int ops_index;
>> - const libxl__remus_device_instance_ops *ops;
>> + const libxl__checkpoint_device_instance_ops *ops;
>>
>> /*----- private for concrete (device-specific) layer -----*/
>>
>> @@ -2815,17 +2815,17 @@ struct libxl__remus_device {
>> void *concrete_data;
>> };
>>
>> -/* the following 5 APIs are async ops, call rds->callback when done */
>> -_hidden void libxl__remus_devices_setup(libxl__egc *egc,
>> - libxl__remus_devices_state *rds);
>> -_hidden void libxl__remus_devices_teardown(libxl__egc *egc,
>> - libxl__remus_devices_state *rds);
>> -_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc,
>> - libxl__remus_devices_state *rds);
>> -_hidden void libxl__remus_devices_preresume(libxl__egc *egc,
>> - libxl__remus_devices_state *rds);
>> -_hidden void libxl__remus_devices_commit(libxl__egc *egc,
>> - libxl__remus_devices_state *rds);
>> +/* the following 5 APIs are async ops, call cds->callback when done */
>> +_hidden void libxl__checkpoint_devices_setup(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds);
>> +_hidden void libxl__checkpoint_devices_teardown(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds);
>> +_hidden void libxl__checkpoint_devices_postsuspend(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds);
>> +_hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds);
>> +_hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
>> + libxl__checkpoint_devices_state *cds);
>> _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
>>
>> /*----- Legacy conversion helper -----*/
>> @@ -2959,7 +2959,7 @@ struct libxl__domain_save_state {
>> libxl__domain_suspend_state dsps;
>> int hvm;
>> int xcflags;
>> - libxl__remus_devices_state rds;
>> + libxl__checkpoint_devices_state cds;
>> libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
>> int interval; /* checkpoint interval (for Remus) */
>> libxl__save_helper_state shs;
>> diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
>> index 71c6531..86afba6 100644
>> --- a/tools/libxl/libxl_netbuffer.c
>> +++ b/tools/libxl/libxl_netbuffer.c
>> @@ -38,21 +38,21 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
>> return 1;
>> }
>>
>> -int init_subkind_nic(libxl__remus_devices_state *rds)
>> +int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>> {
>> int rc, ret;
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>>
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> - rds->nlsock = nl_socket_alloc();
>> - if (!rds->nlsock) {
>> + cds->nlsock = nl_socket_alloc();
>> + if (!cds->nlsock) {
>> LOG(ERROR, "cannot allocate nl socket");
>> rc = ERROR_FAIL;
>> goto out;
>> }
>>
>> - ret = nl_connect(rds->nlsock, NETLINK_ROUTE);
>> + ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
>> if (ret) {
>> LOG(ERROR, "failed to open netlink socket: %s",
>> nl_geterror(ret));
>> @@ -61,7 +61,7 @@ int init_subkind_nic(libxl__remus_devices_state *rds)
>> }
>>
>> /* get list of all qdiscs installed on network devs. */
>> - ret = rtnl_qdisc_alloc_cache(rds->nlsock, &rds->qdisc_cache);
>> + ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
>> if (ret) {
>> LOG(ERROR, "failed to allocate qdisc cache: %s",
>> nl_geterror(ret));
>> @@ -70,9 +70,9 @@ int init_subkind_nic(libxl__remus_devices_state *rds)
>> }
>>
>> if (dss->remus->netbufscript) {
>> - rds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
>> + cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
>> } else {
>> - rds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
>> + cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
>> libxl__xen_script_dir_path());
>> }
>>
>> @@ -82,22 +82,22 @@ out:
>> return rc;
>> }
>>
>> -void cleanup_subkind_nic(libxl__remus_devices_state *rds)
>> +void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
>> {
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> /* free qdisc cache */
>> - if (rds->qdisc_cache) {
>> - nl_cache_clear(rds->qdisc_cache);
>> - nl_cache_free(rds->qdisc_cache);
>> - rds->qdisc_cache = NULL;
>> + if (cds->qdisc_cache) {
>> + nl_cache_clear(cds->qdisc_cache);
>> + nl_cache_free(cds->qdisc_cache);
>> + cds->qdisc_cache = NULL;
>> }
>>
>> /* close & free nlsock */
>> - if (rds->nlsock) {
>> - nl_close(rds->nlsock);
>> - nl_socket_free(rds->nlsock);
>> - rds->nlsock = NULL;
>> + if (cds->nlsock) {
>> + nl_close(cds->nlsock);
>> + nl_socket_free(cds->nlsock);
>> + cds->nlsock = NULL;
>> }
>> }
>>
>> @@ -111,17 +111,17 @@ void cleanup_subkind_nic(libxl__remus_devices_state *rds)
>> * it must ONLY be used for remus because if driver domains
>> * were in use it would constitute a security vulnerability.
>> */
>> -static const char *get_vifname(libxl__remus_device *dev,
>> +static const char *get_vifname(libxl__checkpoint_device *dev,
>> const libxl_device_nic *nic)
>> {
>> const char *vifname = NULL;
>> const char *path;
>> int rc;
>>
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> /* Convenience aliases */
>> - const uint32_t domid = dev->rds->domid;
>> + const uint32_t domid = dev->cds->domid;
>>
>> path = GCSPRINTF("%s/backend/vif/%d/%d/vifname",
>> libxl__xs_get_dompath(gc, 0), domid, nic->devid);
>> @@ -144,19 +144,19 @@ static void free_qdisc(libxl__remus_device_nic *remus_nic)
>> remus_nic->qdisc = NULL;
>> }
>>
>> -static int init_qdisc(libxl__remus_devices_state *rds,
>> +static int init_qdisc(libxl__checkpoint_devices_state *cds,
>> libxl__remus_device_nic *remus_nic)
>> {
>> int rc, ret, ifindex;
>> struct rtnl_link *ifb = NULL;
>> struct rtnl_qdisc *qdisc = NULL;
>>
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> /* Now that we have brought up REMUS_IFB device with plug qdisc for
>> * this vif, so we need to refill the qdisc cache.
>> */
>> - ret = nl_cache_refill(rds->nlsock, rds->qdisc_cache);
>> + ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
>> if (ret) {
>> LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
>> rc = ERROR_FAIL;
>> @@ -164,7 +164,7 @@ static int init_qdisc(libxl__remus_devices_state *rds,
>> }
>>
>> /* get a handle to the REMUS_IFB interface */
>> - ret = rtnl_link_get_kernel(rds->nlsock, 0, remus_nic->ifb, &ifb);
>> + ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
>> if (ret) {
>> LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
>> nl_geterror(ret));
>> @@ -187,7 +187,7 @@ static int init_qdisc(libxl__remus_devices_state *rds,
>> * There is no need to explicitly free this qdisc as its just a
>> * reference from the qdisc cache we allocated earlier.
>> */
>> - qdisc = rtnl_qdisc_get_by_parent(rds->qdisc_cache, ifindex, TC_H_ROOT);
>> + qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
>> if (qdisc) {
>> const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
>> /* Sanity check: Ensure that the root qdisc is a plug qdisc. */
>> @@ -231,19 +231,19 @@ static void netbuf_teardown_script_cb(libxl__egc *egc,
>> * $REMUS_IFB (for teardown)
>> * setup/teardown as command line arg.
>> */
>> -static void setup_async_exec(libxl__remus_device *dev, char *op)
>> +static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
>> {
>> int arraysize, nr = 0;
>> char **env = NULL, **args = NULL;
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>> - libxl__remus_devices_state *rds = dev->rds;
>> + libxl__checkpoint_devices_state *cds = dev->cds;
>> libxl__async_exec_state *aes = &dev->aodev.aes;
>>
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> /* Convenience aliases */
>> - char *const script = libxl__strdup(gc, rds->netbufscript);
>> - const uint32_t domid = rds->domid;
>> + char *const script = libxl__strdup(gc, cds->netbufscript);
>> + const uint32_t domid = cds->domid;
>> const int dev_id = remus_nic->devid;
>> const char *const vif = remus_nic->vif;
>> const char *const ifb = remus_nic->ifb;
>> @@ -269,7 +269,7 @@ static void setup_async_exec(libxl__remus_device *dev, char *op)
>> args[nr++] = NULL;
>> assert(nr == arraysize);
>>
>> - aes->ao = dev->rds->ao;
>> + aes->ao = dev->cds->ao;
>> aes->what = GCSPRINTF("%s %s", args[0], args[1]);
>> aes->env = env;
>> aes->args = args;
>> @@ -286,13 +286,13 @@ static void setup_async_exec(libxl__remus_device *dev, char *op)
>>
>> /* setup() and teardown() */
>>
>> -static void nic_setup(libxl__egc *egc, libxl__remus_device *dev)
>> +static void nic_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> int rc;
>> libxl__remus_device_nic *remus_nic;
>> const libxl_device_nic *nic = dev->backend_dev;
>>
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> /*
>> * thers's no subkind of nic devices, so nic ops is always matched
>> @@ -330,16 +330,16 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>> int status)
>> {
>> libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
>> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>> - libxl__remus_devices_state *rds = dev->rds;
>> + libxl__checkpoint_devices_state *cds = dev->cds;
>> const char *out_path_base, *hotplug_error = NULL;
>> int rc;
>>
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> /* Convenience aliases */
>> - const uint32_t domid = rds->domid;
>> + const uint32_t domid = cds->domid;
>> const int devid = remus_nic->devid;
>> const char *const vif = remus_nic->vif;
>> const char **const ifb = &remus_nic->ifb;
>> @@ -373,7 +373,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>>
>> if (hotplug_error) {
>> LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
>> - rds->netbufscript, vif, hotplug_error);
>> + cds->netbufscript, vif, hotplug_error);
>> rc = ERROR_FAIL;
>> goto out;
>> }
>> @@ -384,17 +384,17 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>> }
>>
>> LOG(DEBUG, "%s will buffer packets from vif %s", *ifb, vif);
>> - rc = init_qdisc(rds, remus_nic);
>> + rc = init_qdisc(cds, remus_nic);
>>
>> out:
>> aodev->rc = rc;
>> aodev->callback(egc, aodev);
>> }
>>
>> -static void nic_teardown(libxl__egc *egc, libxl__remus_device *dev)
>> +static void nic_teardown(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> int rc;
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> setup_async_exec(dev, "teardown");
>>
>> @@ -415,7 +415,7 @@ static void netbuf_teardown_script_cb(libxl__egc *egc,
>> {
>> int rc;
>> libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
>> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>>
>> if (status)
>> @@ -440,12 +440,12 @@ enum {
>> /* API implementations */
>>
>> static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int buffer_op)
>> {
>> int rc, ret;
>>
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> if (buffer_op == tc_buffer_start)
>> ret = rtnl_qdisc_plug_buffer(remus_nic->qdisc);
>> @@ -457,7 +457,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
>> goto out;
>> }
>>
>> - ret = rtnl_qdisc_add(rds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
>> + ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
>> if (ret) {
>> rc = ERROR_FAIL;
>> goto out;
>> @@ -474,33 +474,33 @@ out:
>> return rc;
>> }
>>
>> -static void nic_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
>> +static void nic_postsuspend(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> int rc;
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>>
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> - rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_start);
>> + rc = remus_netbuf_op(remus_nic, dev->cds, tc_buffer_start);
>>
>> dev->aodev.rc = rc;
>> dev->aodev.callback(egc, &dev->aodev);
>> }
>>
>> -static void nic_commit(libxl__egc *egc, libxl__remus_device *dev)
>> +static void nic_commit(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> int rc;
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>>
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> - rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_release);
>> + rc = remus_netbuf_op(remus_nic, dev->cds, tc_buffer_release);
>>
>> dev->aodev.rc = rc;
>> dev->aodev.callback(egc, &dev->aodev);
>> }
>>
>> -const libxl__remus_device_instance_ops remus_device_nic = {
>> +const libxl__checkpoint_device_instance_ops remus_device_nic = {
>> .kind = LIBXL__DEVICE_KIND_VIF,
>> .setup = nic_setup,
>> .teardown = nic_teardown,
>> diff --git a/tools/libxl/libxl_nonetbuffer.c b/tools/libxl/libxl_nonetbuffer.c
>> index 3c659c2..4b68152 100644
>> --- a/tools/libxl/libxl_nonetbuffer.c
>> +++ b/tools/libxl/libxl_nonetbuffer.c
>> @@ -22,25 +22,25 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
>> return 0;
>> }
>>
>> -int init_subkind_nic(libxl__remus_devices_state *rds)
>> +int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>> {
>> return 0;
>> }
>>
>> -void cleanup_subkind_nic(libxl__remus_devices_state *rds)
>> +void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
>> {
>> return;
>> }
>>
>> -static void nic_setup(libxl__egc *egc, libxl__remus_device *dev)
>> +static void nic_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> dev->aodev.rc = ERROR_FAIL;
>> dev->aodev.callback(egc, &dev->aodev);
>> }
>>
>> -const libxl__remus_device_instance_ops remus_device_nic = {
>> +const libxl__checkpoint_device_instance_ops remus_device_nic = {
>> .kind = LIBXL__DEVICE_KIND_VIF,
>> .setup = nic_setup,
>> };
>> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
>> index 0c449ad..fe4acc8 100644
>> --- a/tools/libxl/libxl_remus.c
>> +++ b/tools/libxl/libxl_remus.c
>> @@ -20,14 +20,14 @@
>> /*----- Remus setup and teardown -----*/
>>
>> static void remus_setup_done(libxl__egc *egc,
>> - libxl__remus_devices_state *rds, int rc);
>> + libxl__checkpoint_devices_state *cds, int rc);
>> static void remus_setup_failed(libxl__egc *egc,
>> - libxl__remus_devices_state *rds, int rc);
>> + libxl__checkpoint_devices_state *cds, int rc);
>>
>> void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
>> {
>> /* Convenience aliases */
>> - libxl__remus_devices_state *const rds = &dss->rds;
>> + libxl__checkpoint_devices_state *const cds = &dss->cds;
>> const libxl_domain_remus_info *const info = dss->remus;
>>
>> STATE_AO_GC(dss->ao);
>> @@ -37,17 +37,17 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
>> LOG(ERROR, "Remus: No support for network buffering");
>> goto out;
>> }
>> - rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
>> + cds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF);
>> }
>>
>> if (libxl_defbool_val(info->diskbuf))
>> - rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
>> + cds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD);
>>
>> - rds->ao = ao;
>> - rds->domid = dss->domid;
>> - rds->callback = remus_setup_done;
>> + cds->ao = ao;
>> + cds->domid = dss->domid;
>> + cds->callback = remus_setup_done;
>>
>> - libxl__remus_devices_setup(egc, rds);
>> + libxl__checkpoint_devices_setup(egc, cds);
>> return;
>>
>> out:
>> @@ -55,9 +55,9 @@ out:
>> }
>>
>> static void remus_setup_done(libxl__egc *egc,
>> - libxl__remus_devices_state *rds, int rc)
>> + libxl__checkpoint_devices_state *cds, int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (!rc) {
>> @@ -67,14 +67,14 @@ static void remus_setup_done(libxl__egc *egc,
>>
>> LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
>> dss->domid, rc);
>> - rds->callback = remus_setup_failed;
>> - libxl__remus_devices_teardown(egc, rds);
>> + cds->callback = remus_setup_failed;
>> + libxl__checkpoint_devices_teardown(egc, cds);
>> }
>>
>> static void remus_setup_failed(libxl__egc *egc,
>> - libxl__remus_devices_state *rds, int rc)
>> + libxl__checkpoint_devices_state *cds, int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (rc)
>> @@ -85,7 +85,7 @@ static void remus_setup_failed(libxl__egc *egc,
>> }
>>
>> static void remus_teardown_done(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc);
>> void libxl__remus_teardown(libxl__egc *egc,
>> libxl__domain_save_state *dss,
>> @@ -95,15 +95,15 @@ void libxl__remus_teardown(libxl__egc *egc,
>>
>> LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
>> " teardown Remus devices...", rc);
>> - dss->rds.callback = remus_teardown_done;
>> - libxl__remus_devices_teardown(egc, &dss->rds);
>> + dss->cds.callback = remus_teardown_done;
>> + libxl__checkpoint_devices_teardown(egc, &dss->cds);
>> }
>>
>> static void remus_teardown_done(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (rc)
>> @@ -118,10 +118,10 @@ static void remus_teardown_done(libxl__egc *egc,
>> static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> libxl__domain_suspend_state *dsps, int ok);
>> static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc);
>> static void remus_devices_preresume_cb(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc);
>>
>> void libxl__remus_domain_suspend_callback(void *data)
>> @@ -143,9 +143,9 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> if (!ok)
>> goto out;
>>
>> - libxl__remus_devices_state *const rds = &dss->rds;
>> - rds->callback = remus_devices_postsuspend_cb;
>> - libxl__remus_devices_postsuspend(egc, rds);
>> + libxl__checkpoint_devices_state *const cds = &dss->cds;
>> + cds->callback = remus_devices_postsuspend_cb;
>> + libxl__checkpoint_devices_postsuspend(egc, cds);
>> return;
>>
>> out:
>> @@ -153,11 +153,11 @@ out:
>> }
>>
>> static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc)
>> {
>> int ok = 0;
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>>
>> if (rc)
>> goto out;
>> @@ -175,17 +175,17 @@ void libxl__remus_domain_resume_callback(void *data)
>> libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
>> STATE_AO_GC(dss->ao);
>>
>> - libxl__remus_devices_state *const rds = &dss->rds;
>> - rds->callback = remus_devices_preresume_cb;
>> - libxl__remus_devices_preresume(egc, rds);
>> + libxl__checkpoint_devices_state *const cds = &dss->cds;
>> + cds->callback = remus_devices_preresume_cb;
>> + libxl__checkpoint_devices_preresume(egc, cds);
>> }
>>
>> static void remus_devices_preresume_cb(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc)
>> {
>> int ok = 0;
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (rc)
>> @@ -207,7 +207,7 @@ out:
>> static void remus_checkpoint_stream_written(
>> libxl__egc *egc, libxl__stream_write_state *stream, int rc);
>> static void remus_devices_commit_cb(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc);
>> static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>> const struct timeval *requested_abs);
>> @@ -229,7 +229,7 @@ static void remus_checkpoint_stream_written(
>> libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
>>
>> /* Convenience aliases */
>> - libxl__remus_devices_state *const rds = &dss->rds;
>> + libxl__checkpoint_devices_state *const cds = &dss->cds;
>>
>> STATE_AO_GC(dss->ao);
>>
>> @@ -238,8 +238,8 @@ static void remus_checkpoint_stream_written(
>> goto out;
>> }
>>
>> - rds->callback = remus_devices_commit_cb;
>> - libxl__remus_devices_commit(egc, rds);
>> + cds->callback = remus_devices_commit_cb;
>> + libxl__checkpoint_devices_commit(egc, cds);
>>
>> return;
>>
>> @@ -248,10 +248,10 @@ out:
>> }
>>
>> static void remus_devices_commit_cb(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> + libxl__checkpoint_devices_state *cds,
>> int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(rds, *dss, rds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>>
>> STATE_AO_GC(dss->ao);
>>
>> diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c
>> deleted file mode 100644
>> index a6cb7f6..0000000
>> --- a/tools/libxl/libxl_remus_device.c
>> +++ /dev/null
>> @@ -1,327 +0,0 @@
>> -/*
>> - * Copyright (C) 2014 FUJITSU LIMITED
>> - * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
>> - *
>> - * This program is free software; you can redistribute it and/or modify
>> - * it under the terms of the GNU Lesser General Public License as published
>> - * by the Free Software Foundation; version 2.1 only. with the special
>> - * exception on linking described in file LICENSE.
>> - *
>> - * This program is distributed in the hope that it will be useful,
>> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> - * GNU Lesser General Public License for more details.
>> - */
>> -
>> -#include "libxl_osdeps.h" /* must come before any other headers */
>> -
>> -#include "libxl_internal.h"
>> -
>> -extern const libxl__remus_device_instance_ops remus_device_nic;
>> -extern const libxl__remus_device_instance_ops remus_device_drbd_disk;
>> -static const libxl__remus_device_instance_ops *remus_ops[] = {
>> - &remus_device_nic,
>> - &remus_device_drbd_disk,
>> - NULL,
>> -};
>> -
>> -/*----- helper functions -----*/
>> -
>> -static int init_device_subkind(libxl__remus_devices_state *rds)
>> -{
>> - /* init device subkind-specific state in the libxl ctx */
>> - int rc;
>> - STATE_AO_GC(rds->ao);
>> -
>> - if (libxl__netbuffer_enabled(gc)) {
>> - rc = init_subkind_nic(rds);
>> - if (rc) goto out;
>> - }
>> -
>> - rc = init_subkind_drbd_disk(rds);
>> - if (rc) goto out;
>> -
>> - rc = 0;
>> -out:
>> - return rc;
>> -}
>> -
>> -static void cleanup_device_subkind(libxl__remus_devices_state *rds)
>> -{
>> - /* cleanup device subkind-specific state in the libxl ctx */
>> - STATE_AO_GC(rds->ao);
>> -
>> - if (libxl__netbuffer_enabled(gc))
>> - cleanup_subkind_nic(rds);
>> -
>> - cleanup_subkind_drbd_disk(rds);
>> -}
>> -
>> -/*----- setup() and teardown() -----*/
>> -
>> -/* callbacks */
>> -
>> -static void all_devices_setup_cb(libxl__egc *egc,
>> - libxl__multidev *multidev,
>> - int rc);
>> -static void device_setup_iterate(libxl__egc *egc,
>> - libxl__ao_device *aodev);
>> -static void devices_teardown_cb(libxl__egc *egc,
>> - libxl__multidev *multidev,
>> - int rc);
>> -
>> -/* remus device setup and teardown */
>> -
>> -static libxl__remus_device* remus_device_init(libxl__egc *egc,
>> - libxl__remus_devices_state *rds,
>> - libxl__device_kind kind,
>> - void *libxl_dev)
>> -{
>> - libxl__remus_device *dev = NULL;
>> -
>> - STATE_AO_GC(rds->ao);
>> - GCNEW(dev);
>> - dev->backend_dev = libxl_dev;
>> - dev->kind = kind;
>> - dev->rds = rds;
>> -
>> - return dev;
>> -}
>> -
>> -static void remus_devices_setup(libxl__egc *egc,
>> - libxl__remus_devices_state *rds);
>> -
>> -void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_devices_state *rds)
>> -{
>> - int i, rc;
>> -
>> - STATE_AO_GC(rds->ao);
>> -
>> - rc = init_device_subkind(rds);
>> - if (rc)
>> - goto out;
>> -
>> - rds->num_devices = 0;
>> - rds->num_nics = 0;
>> - rds->num_disks = 0;
>> -
>> - if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF))
>> - rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics);
>> -
>> - if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD))
>> - rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks);
>> -
>> - if (rds->num_nics == 0 && rds->num_disks == 0)
>> - goto out;
>> -
>> - GCNEW_ARRAY(rds->devs, rds->num_nics + rds->num_disks);
>> -
>> - for (i = 0; i < rds->num_nics; i++) {
>> - rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
>> - LIBXL__DEVICE_KIND_VIF,
>> - &rds->nics[i]);
>> - }
>> -
>> - for (i = 0; i < rds->num_disks; i++) {
>> - rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
>> - LIBXL__DEVICE_KIND_VBD,
>> - &rds->disks[i]);
>> - }
>> -
>> - remus_devices_setup(egc, rds);
>> -
>> - return;
>> -
>> -out:
>> - rds->callback(egc, rds, rc);
>> -}
>> -
>> -static void remus_devices_setup(libxl__egc *egc,
>> - libxl__remus_devices_state *rds)
>> -{
>> - int i, rc;
>> -
>> - STATE_AO_GC(rds->ao);
>> -
>> - libxl__multidev_begin(ao, &rds->multidev);
>> - rds->multidev.callback = all_devices_setup_cb;
>> - for (i = 0; i < rds->num_devices; i++) {
>> - libxl__remus_device *dev = rds->devs[i];
>> - dev->ops_index = -1;
>> - libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
>> -
>> - dev->aodev.rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
>> - dev->aodev.callback = device_setup_iterate;
>> - device_setup_iterate(egc,&dev->aodev);
>> - }
>> -
>> - rc = 0;
>> - libxl__multidev_prepared(egc, &rds->multidev, rc);
>> -}
>> -
>> -
>> -static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
>> -{
>> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> - EGC_GC;
>> -
>> - if (aodev->rc != ERROR_REMUS_DEVICE_NOT_SUPPORTED &&
>> - aodev->rc != ERROR_REMUS_DEVOPS_DOES_NOT_MATCH)
>> - /* might be success or disaster */
>> - goto out;
>> -
>> - do {
>> - dev->ops = remus_ops[++dev->ops_index];
>> - if (!dev->ops) {
>> - libxl_device_nic * nic = NULL;
>> - libxl_device_disk * disk = NULL;
>> - uint32_t domid;
>> - int devid;
>> - if (dev->kind == LIBXL__DEVICE_KIND_VIF) {
>> - nic = (libxl_device_nic *)dev->backend_dev;
>> - domid = nic->backend_domid;
>> - devid = nic->devid;
>> - } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) {
>> - disk = (libxl_device_disk *)dev->backend_dev;
>> - domid = disk->backend_domid;
>> - devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL);
>> - } else {
>> - LOG(ERROR,"device kind not handled by remus: %s",
>> - libxl__device_kind_to_string(dev->kind));
>> - aodev->rc = ERROR_FAIL;
>> - goto out;
>> - }
>> - LOG(ERROR,"device not handled by remus"
>> - " (device=%s:%"PRId32"/%"PRId32")",
>> - libxl__device_kind_to_string(dev->kind),
>> - domid, devid);
>> - aodev->rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
>> - goto out;
>> - }
>> - } while (dev->ops->kind != dev->kind);
>> -
>> - /* found the next ops_index to try */
>> - assert(dev->aodev.callback == device_setup_iterate);
>> - dev->ops->setup(egc,dev);
>> - return;
>> -
>> - out:
>> - libxl__multidev_one_callback(egc,aodev);
>> -}
>> -
>> -static void all_devices_setup_cb(libxl__egc *egc,
>> - libxl__multidev *multidev,
>> - int rc)
>> -{
>> - STATE_AO_GC(multidev->ao);
>> -
>> - /* Convenience aliases */
>> - libxl__remus_devices_state *const rds =
>> - CONTAINER_OF(multidev, *rds, multidev);
>> -
>> - rds->callback(egc, rds, rc);
>> -}
>> -
>> -void libxl__remus_devices_teardown(libxl__egc *egc,
>> - libxl__remus_devices_state *rds)
>> -{
>> - int i;
>> - libxl__remus_device *dev;
>> -
>> - STATE_AO_GC(rds->ao);
>> -
>> - libxl__multidev_begin(ao, &rds->multidev);
>> - rds->multidev.callback = devices_teardown_cb;
>> - for (i = 0; i < rds->num_devices; i++) {
>> - dev = rds->devs[i];
>> - if (!dev->ops || !dev->matched)
>> - continue;
>> -
>> - libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
>> - dev->ops->teardown(egc,dev);
>> - }
>> -
>> - libxl__multidev_prepared(egc, &rds->multidev, 0);
>> -}
>> -
>> -static void devices_teardown_cb(libxl__egc *egc,
>> - libxl__multidev *multidev,
>> - int rc)
>> -{
>> - int i;
>> -
>> - STATE_AO_GC(multidev->ao);
>> -
>> - /* Convenience aliases */
>> - libxl__remus_devices_state *const rds =
>> - CONTAINER_OF(multidev, *rds, multidev);
>> -
>> - /* clean nic */
>> - for (i = 0; i < rds->num_nics; i++)
>> - libxl_device_nic_dispose(&rds->nics[i]);
>> - free(rds->nics);
>> - rds->nics = NULL;
>> - rds->num_nics = 0;
>> -
>> - /* clean disk */
>> - for (i = 0; i < rds->num_disks; i++)
>> - libxl_device_disk_dispose(&rds->disks[i]);
>> - free(rds->disks);
>> - rds->disks = NULL;
>> - rds->num_disks = 0;
>> -
>> - cleanup_device_subkind(rds);
>> -
>> - rds->callback(egc, rds, rc);
>> -}
>> -
>> -/*----- checkpointing APIs -----*/
>> -
>> -/* callbacks */
>> -
>> -static void devices_checkpoint_cb(libxl__egc *egc,
>> - libxl__multidev *multidev,
>> - int rc);
>> -
>> -/* API implementations */
>> -
>> -#define define_remus_checkpoint_api(api) \
>> -void libxl__remus_devices_##api(libxl__egc *egc, \
>> - libxl__remus_devices_state *rds) \
>> -{ \
>> - int i; \
>> - libxl__remus_device *dev; \
>> - \
>> - STATE_AO_GC(rds->ao); \
>> - \
>> - libxl__multidev_begin(ao, &rds->multidev); \
>> - rds->multidev.callback = devices_checkpoint_cb; \
>> - for (i = 0; i < rds->num_devices; i++) { \
>> - dev = rds->devs[i]; \
>> - if (!dev->matched || !dev->ops->api) \
>> - continue; \
>> - libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);\
>> - dev->ops->api(egc,dev); \
>> - } \
>> - \
>> - libxl__multidev_prepared(egc, &rds->multidev, 0); \
>> -}
>> -
>> -define_remus_checkpoint_api(postsuspend);
>> -
>> -define_remus_checkpoint_api(preresume);
>> -
>> -define_remus_checkpoint_api(commit);
>> -
>> -static void devices_checkpoint_cb(libxl__egc *egc,
>> - libxl__multidev *multidev,
>> - int rc)
>> -{
>> - STATE_AO_GC(multidev->ao);
>> -
>> - /* Convenience aliases */
>> - libxl__remus_devices_state *const rds =
>> - CONTAINER_OF(multidev, *rds, multidev);
>> -
>> - rds->callback(egc, rds, rc);
>> -}
>> diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
>> index afe9b61..50b897d 100644
>> --- a/tools/libxl/libxl_remus_disk_drbd.c
>> +++ b/tools/libxl/libxl_remus_disk_drbd.c
>> @@ -26,30 +26,30 @@ typedef struct libxl__remus_drbd_disk {
>> int ackwait;
>> } libxl__remus_drbd_disk;
>>
>> -int init_subkind_drbd_disk(libxl__remus_devices_state *rds)
>> +int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
>> {
>> - STATE_AO_GC(rds->ao);
>> + STATE_AO_GC(cds->ao);
>>
>> - rds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
>> + cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
>> libxl__xen_script_dir_path());
>>
>> return 0;
>> }
>>
>> -void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds)
>> +void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
>> {
>> return;
>> }
>>
>> /*----- helper functions, for async calls -----*/
>> static void drbd_async_call(libxl__egc *egc,
>> - libxl__remus_device *dev,
>> - void func(libxl__remus_device *),
>> + libxl__checkpoint_device *dev,
>> + void func(libxl__checkpoint_device *),
>> libxl__ev_child_callback callback)
>> {
>> int pid = -1, rc;
>> libxl__ao_device *aodev = &dev->aodev;
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> /* Fork and call */
>> pid = libxl__ev_child_fork(gc, &aodev->child, callback);
>> @@ -82,21 +82,21 @@ static void match_async_exec_cb(libxl__egc *egc,
>>
>> /* implementations */
>>
>> -static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev);
>> +static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev);
>>
>> -static void drbd_setup(libxl__egc *egc, libxl__remus_device *dev)
>> +static void drbd_setup(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> match_async_exec(egc, dev);
>> }
>>
>> -static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev)
>> +static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> int arraysize, nr = 0, rc;
>> const libxl_device_disk *disk = dev->backend_dev;
>> libxl__async_exec_state *aes = &dev->aodev.aes;
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> /* setup env & args */
>> arraysize = 1;
>> @@ -107,12 +107,12 @@ static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev)
>> arraysize = 3;
>> nr = 0;
>> GCNEW_ARRAY(aes->args, arraysize);
>> - aes->args[nr++] = dev->rds->drbd_probe_script;
>> + aes->args[nr++] = dev->cds->drbd_probe_script;
>> aes->args[nr++] = disk->pdev_path;
>> aes->args[nr++] = NULL;
>> assert(nr <= arraysize);
>>
>> - aes->ao = dev->rds->ao;
>> + aes->ao = dev->cds->ao;
>> aes->what = GCSPRINTF("%s %s", aes->args[0], aes->args[1]);
>> aes->timeout_ms = LIBXL_HOTPLUG_TIMEOUT * 1000;
>> aes->callback = match_async_exec_cb;
>> @@ -137,14 +137,14 @@ static void match_async_exec_cb(libxl__egc *egc,
>> {
>> int rc;
>> libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes);
>> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> libxl__remus_drbd_disk *drbd_disk;
>> const libxl_device_disk *disk = dev->backend_dev;
>>
>> STATE_AO_GC(aodev->ao);
>>
>> if (status) {
>> - rc = ERROR_REMUS_DEVOPS_DOES_NOT_MATCH;
>> + rc = ERROR_CHECKPOINT_DEVOPS_DOES_NOT_MATCH;
>> /* BUG: seems to assume that any exit status means `no match' */
>> /* BUG: exit status will have been logged as an error */
>> goto out;
>> @@ -169,10 +169,10 @@ out:
>> aodev->callback(egc, aodev);
>> }
>>
>> -static void drbd_teardown(libxl__egc *egc, libxl__remus_device *dev)
>> +static void drbd_teardown(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> libxl__remus_drbd_disk *drbd_disk = dev->concrete_data;
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> close(drbd_disk->ctl_fd);
>> dev->aodev.rc = 0;
>> @@ -189,9 +189,9 @@ static void checkpoint_async_call_done(libxl__egc *egc,
>> /* API implementations */
>>
>> /* this op will not wait and block, so implement as sync op */
>> -static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
>> +static void drbd_postsuspend(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> libxl__remus_drbd_disk *rdd = dev->concrete_data;
>>
>> @@ -205,16 +205,16 @@ static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev)
>> }
>>
>>
>> -static void drbd_preresume_async(libxl__remus_device *dev);
>> +static void drbd_preresume_async(libxl__checkpoint_device *dev);
>>
>> -static void drbd_preresume(libxl__egc *egc, libxl__remus_device *dev)
>> +static void drbd_preresume(libxl__egc *egc, libxl__checkpoint_device *dev)
>> {
>> - STATE_AO_GC(dev->rds->ao);
>> + STATE_AO_GC(dev->cds->ao);
>>
>> drbd_async_call(egc, dev, drbd_preresume_async, checkpoint_async_call_done);
>> }
>>
>> -static void drbd_preresume_async(libxl__remus_device *dev)
>> +static void drbd_preresume_async(libxl__checkpoint_device *dev)
>> {
>> libxl__remus_drbd_disk *rdd = dev->concrete_data;
>> int ackwait = rdd->ackwait;
>> @@ -233,7 +233,7 @@ static void checkpoint_async_call_done(libxl__egc *egc,
>> {
>> int rc;
>> libxl__ao_device *aodev = CONTAINER_OF(child, *aodev, child);
>> - libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> + libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> libxl__remus_drbd_disk *rdd = dev->concrete_data;
>>
>> STATE_AO_GC(aodev->ao);
>> @@ -251,7 +251,7 @@ out:
>> aodev->callback(egc, aodev);
>> }
>>
>> -const libxl__remus_device_instance_ops remus_device_drbd_disk = {
>> +const libxl__checkpoint_device_instance_ops remus_device_drbd_disk = {
>> .kind = LIBXL__DEVICE_KIND_VBD,
>> .setup = drbd_setup,
>> .teardown = drbd_teardown,
>> diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
>> index 5c7b82d..e05d12b 100644
>> --- a/tools/libxl/libxl_types.idl
>> +++ b/tools/libxl/libxl_types.idl
>> @@ -61,8 +61,8 @@ libxl_error = Enumeration("error", [
>> (-15, "LOCK_FAIL"),
>> (-16, "JSON_CONFIG_EMPTY"),
>> (-17, "DEVICE_EXISTS"),
>> - (-18, "REMUS_DEVOPS_DOES_NOT_MATCH"),
>> - (-19, "REMUS_DEVICE_NOT_SUPPORTED"),
>> + (-18, "CHECKPOINT_DEVOPS_DOES_NOT_MATCH"),
>> + (-19, "CHECKPOINT_DEVICE_NOT_SUPPORTED"),
>> (-20, "VNUMA_CONFIG_INVALID"),
>> (-21, "DOMAIN_NOTFOUND"),
>> ], value_namespace = "")
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread
* [PATCH v3 COLOPre 25/26] tools/libxl: adjust the indentation
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (23 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 24/26] tools/libxl: rename remus device to checkpoint device Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-25 6:25 ` [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device Yang Hongyang
25 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
This is just tidying up after the previous automatic renaming.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
tools/libxl/libxl_checkpoint_device.c | 21 +++++++++++----------
tools/libxl/libxl_internal.h | 19 +++++++++++--------
2 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
index 109cd23..226f159 100644
--- a/tools/libxl/libxl_checkpoint_device.c
+++ b/tools/libxl/libxl_checkpoint_device.c
@@ -73,9 +73,9 @@ static void devices_teardown_cb(libxl__egc *egc,
/* checkpoint device setup and teardown */
static libxl__checkpoint_device* checkpoint_device_init(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds,
- libxl__device_kind kind,
- void *libxl_dev)
+ libxl__checkpoint_devices_state *cds,
+ libxl__device_kind kind,
+ void *libxl_dev)
{
libxl__checkpoint_device *dev = NULL;
@@ -89,9 +89,10 @@ static libxl__checkpoint_device* checkpoint_device_init(libxl__egc *egc,
}
static void checkpoint_devices_setup(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds);
+ libxl__checkpoint_devices_state *cds);
-void libxl__checkpoint_devices_setup(libxl__egc *egc, libxl__checkpoint_devices_state *cds)
+void libxl__checkpoint_devices_setup(libxl__egc *egc,
+ libxl__checkpoint_devices_state *cds)
{
int i, rc;
@@ -137,7 +138,7 @@ out:
}
static void checkpoint_devices_setup(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds)
+ libxl__checkpoint_devices_state *cds)
{
int i, rc;
@@ -285,12 +286,12 @@ static void devices_checkpoint_cb(libxl__egc *egc,
/* API implementations */
-#define define_checkpoint_api(api) \
-void libxl__checkpoint_devices_##api(libxl__egc *egc, \
- libxl__checkpoint_devices_state *cds) \
+#define define_checkpoint_api(api) \
+void libxl__checkpoint_devices_##api(libxl__egc *egc, \
+ libxl__checkpoint_devices_state *cds) \
{ \
int i; \
- libxl__checkpoint_device *dev; \
+ libxl__checkpoint_device *dev; \
\
STATE_AO_GC(cds->ao); \
\
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index ae3b964..f3a139b 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2675,7 +2675,8 @@ typedef struct libxl__save_helper_state {
* Each device type needs to implement the interfaces specified in
* the libxl__checkpoint_device_instance_ops if it wishes to support Remus.
*
- * The high-level control flow through the checkpoint device layer is shown below:
+ * The high-level control flow through the checkpoint device layer is shown
+ * below:
*
* xl remus
* |-> libxl_domain_remus_start
@@ -2736,7 +2737,8 @@ int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
void cleanup_subkind_drbd_disk(libxl__checkpoint_devices_state *cds);
typedef void libxl__checkpoint_callback(libxl__egc *,
- libxl__checkpoint_devices_state *, int rc);
+ libxl__checkpoint_devices_state *,
+ int rc);
/*
* State associated with a checkpoint invocation, including parameters
@@ -2744,7 +2746,7 @@ typedef void libxl__checkpoint_callback(libxl__egc *,
* save/restore machinery.
*/
struct libxl__checkpoint_devices_state {
- /*---- must be set by caller of libxl__checkpoint_device_(setup|teardown) ----*/
+ /*-- must be set by caller of libxl__checkpoint_device_(setup|teardown) --*/
libxl__ao *ao;
uint32_t domid;
@@ -2757,7 +2759,8 @@ struct libxl__checkpoint_devices_state {
/*
* this array is allocated before setup the checkpoint devices by the
* checkpoint abstract layer.
- * devs may be NULL, means there's no checkpoint devices that has been set up.
+ * devs may be NULL, means there's no checkpoint devices that has been
+ * set up.
* the size of this array is 'num_devices', which is the total number
* of libxl nic devices and disk devices(num_nics + num_disks).
*/
@@ -2819,13 +2822,13 @@ struct libxl__checkpoint_device {
_hidden void libxl__checkpoint_devices_setup(libxl__egc *egc,
libxl__checkpoint_devices_state *cds);
_hidden void libxl__checkpoint_devices_teardown(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds);
+ libxl__checkpoint_devices_state *cds);
_hidden void libxl__checkpoint_devices_postsuspend(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds);
+ libxl__checkpoint_devices_state *cds);
_hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds);
+ libxl__checkpoint_devices_state *cds);
_hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
- libxl__checkpoint_devices_state *cds);
+ libxl__checkpoint_devices_state *cds);
_hidden int libxl__netbuffer_enabled(libxl__gc *gc);
/*----- Legacy conversion helper -----*/
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device
2015-06-25 6:25 [PATCH v3 COLOPre 00/26] Prerequisite patches for COLO Yang Hongyang
` (24 preceding siblings ...)
2015-06-25 6:25 ` [PATCH v3 COLOPre 25/26] tools/libxl: adjust the indentation Yang Hongyang
@ 2015-06-25 6:25 ` Yang Hongyang
2015-06-30 10:50 ` Ian Campbell
25 siblings, 1 reply; 103+ messages in thread
From: Yang Hongyang @ 2015-06-25 6:25 UTC (permalink / raw)
To: xen-devel
Cc: wei.liu2, ian.campbell, wency, andrew.cooper3, yunhong.jiang,
eddie.dong, guijianfeng, rshriram, ian.jackson
Checkpoint device is an abstract layer to do checkpoint.
COLO can also use it to do checkpoint. But there are
still some codes in checkpoint device which touch remus:
1. remus_ops: we use remus ops directly in checkpoint
device. Store it in checkpoint device state.
2. concrete layer's private member: add a new structure
remus state, and move them to remus state.
3. init/cleanup device subkind: we call (init|cleanup)_subkind_nic
and (init|cleanup)_subkind_drbd_disk directly in checkpoint
device. Call them before calling libxl__checkpoint_devices_setup()
or after calling libxl__checkpoint_devices_teardown().
it is pure refactoring and no functional changes.
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
tools/libxl/libxl.c | 2 +-
tools/libxl/libxl_checkpoint_device.c | 52 ++-------------------
tools/libxl/libxl_dom_save.c | 3 +-
tools/libxl/libxl_internal.h | 40 ++++++++++------
tools/libxl/libxl_netbuffer.c | 51 +++++++++++---------
tools/libxl/libxl_remus.c | 88 ++++++++++++++++++++++++++++-------
tools/libxl/libxl_remus_disk_drbd.c | 8 ++--
7 files changed, 135 insertions(+), 109 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index a837ff2..f851957 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -842,7 +842,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
assert(info);
/* Point of no return */
- libxl__remus_setup(egc, dss);
+ libxl__remus_setup(egc, &dss->rs);
return AO_INPROGRESS;
out:
diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
index 226f159..0a16dbb 100644
--- a/tools/libxl/libxl_checkpoint_device.c
+++ b/tools/libxl/libxl_checkpoint_device.c
@@ -17,46 +17,6 @@
#include "libxl_internal.h"
-extern const libxl__checkpoint_device_instance_ops remus_device_nic;
-extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
-static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
- &remus_device_nic,
- &remus_device_drbd_disk,
- NULL,
-};
-
-/*----- helper functions -----*/
-
-static int init_device_subkind(libxl__checkpoint_devices_state *cds)
-{
- /* init device subkind-specific state in the libxl ctx */
- int rc;
- STATE_AO_GC(cds->ao);
-
- if (libxl__netbuffer_enabled(gc)) {
- rc = init_subkind_nic(cds);
- if (rc) goto out;
- }
-
- rc = init_subkind_drbd_disk(cds);
- if (rc) goto out;
-
- rc = 0;
-out:
- return rc;
-}
-
-static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
-{
- /* cleanup device subkind-specific state in the libxl ctx */
- STATE_AO_GC(cds->ao);
-
- if (libxl__netbuffer_enabled(gc))
- cleanup_subkind_nic(cds);
-
- cleanup_subkind_drbd_disk(cds);
-}
-
/*----- setup() and teardown() -----*/
/* callbacks */
@@ -94,14 +54,10 @@ static void checkpoint_devices_setup(libxl__egc *egc,
void libxl__checkpoint_devices_setup(libxl__egc *egc,
libxl__checkpoint_devices_state *cds)
{
- int i, rc;
+ int i;
STATE_AO_GC(cds->ao);
- rc = init_device_subkind(cds);
- if (rc)
- goto out;
-
cds->num_devices = 0;
cds->num_nics = 0;
cds->num_disks = 0;
@@ -134,7 +90,7 @@ void libxl__checkpoint_devices_setup(libxl__egc *egc,
return;
out:
- cds->callback(egc, cds, rc);
+ cds->callback(egc, cds, 0);
}
static void checkpoint_devices_setup(libxl__egc *egc,
@@ -172,7 +128,7 @@ static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
goto out;
do {
- dev->ops = remus_ops[++dev->ops_index];
+ dev->ops = dev->cds->ops[++dev->ops_index];
if (!dev->ops) {
libxl_device_nic * nic = NULL;
libxl_device_disk * disk = NULL;
@@ -271,8 +227,6 @@ static void devices_teardown_cb(libxl__egc *egc,
cds->disks = NULL;
cds->num_disks = 0;
- cleanup_device_subkind(cds);
-
cds->callback(egc, cds, rc);
}
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 9a3b33d..9a3d009 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -410,7 +410,6 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
- dss->interval = r_info->interval;
if (libxl_defbool_val(r_info->compression))
dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
}
@@ -581,7 +580,7 @@ static void domain_save_done(libxl__egc *egc,
* from sending checkpoints. Teardown the network buffers and
* release netlink resources. This is an async op.
*/
- libxl__remus_teardown(egc, dss, rc);
+ libxl__remus_teardown(egc, &dss->rs, rc);
}
/*========================= Domain restore ============================*/
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index f3a139b..840734d 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2752,6 +2752,8 @@ struct libxl__checkpoint_devices_state {
uint32_t domid;
libxl__checkpoint_callback *callback;
int device_kind_flags;
+ /* The ops must be pointer array, and the last ops must be NULL */
+ const libxl__checkpoint_device_instance_ops **ops;
/*----- private for abstract layer only -----*/
@@ -2772,16 +2774,6 @@ struct libxl__checkpoint_devices_state {
int num_disks;
libxl__multidev multidev;
-
- /*----- private for concrete (device-specific) layer only -----*/
-
- /* private for nic device subkind ops */
- char *netbufscript;
- struct nl_sock *nlsock;
- struct nl_cache *qdisc_cache;
-
- /* private for drbd disk subkind ops */
- char *drbd_probe_script;
};
/*
@@ -2829,6 +2821,26 @@ _hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
libxl__checkpoint_devices_state *cds);
_hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
libxl__checkpoint_devices_state *cds);
+
+/*----- Remus related state structure -----*/
+typedef struct libxl__remus_state libxl__remus_state;
+struct libxl__remus_state {
+ /* private */
+ libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
+ int interval; /* checkpoint interval */
+
+ /* abstract layer */
+ libxl__checkpoint_devices_state cds;
+
+ /*----- private for concrete (device-specific) layer only -----*/
+ /* private for nic device subkind ops */
+ char *netbufscript;
+ struct nl_sock *nlsock;
+ struct nl_cache *qdisc_cache;
+
+ /* private for drbd disk subkind ops */
+ char *drbd_probe_script;
+};
_hidden int libxl__netbuffer_enabled(libxl__gc *gc);
/*----- Legacy conversion helper -----*/
@@ -2962,9 +2974,7 @@ struct libxl__domain_save_state {
libxl__domain_suspend_state dsps;
int hvm;
int xcflags;
- libxl__checkpoint_devices_state cds;
- libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
- int interval; /* checkpoint interval (for Remus) */
+ libxl__remus_state rs;
libxl__save_helper_state shs;
libxl__logdirty_switch logdirty;
/* private for libxl__domain_save_device_model */
@@ -3364,9 +3374,9 @@ _hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
_hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
/* Remus setup and teardown*/
_hidden void libxl__remus_setup(libxl__egc *egc,
- libxl__domain_save_state *dss);
+ libxl__remus_state *rs);
_hidden void libxl__remus_teardown(libxl__egc *egc,
- libxl__domain_save_state *dss,
+ libxl__remus_state *rs,
int rc);
/*
diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
index 86afba6..1d01e10 100644
--- a/tools/libxl/libxl_netbuffer.c
+++ b/tools/libxl/libxl_netbuffer.c
@@ -41,18 +41,19 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
int init_subkind_nic(libxl__checkpoint_devices_state *cds)
{
int rc, ret;
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
STATE_AO_GC(cds->ao);
- cds->nlsock = nl_socket_alloc();
- if (!cds->nlsock) {
+ rs->nlsock = nl_socket_alloc();
+ if (!rs->nlsock) {
LOG(ERROR, "cannot allocate nl socket");
rc = ERROR_FAIL;
goto out;
}
- ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
+ ret = nl_connect(rs->nlsock, NETLINK_ROUTE);
if (ret) {
LOG(ERROR, "failed to open netlink socket: %s",
nl_geterror(ret));
@@ -61,7 +62,7 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
}
/* get list of all qdiscs installed on network devs. */
- ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
+ ret = rtnl_qdisc_alloc_cache(rs->nlsock, &rs->qdisc_cache);
if (ret) {
LOG(ERROR, "failed to allocate qdisc cache: %s",
nl_geterror(ret));
@@ -70,10 +71,10 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
}
if (dss->remus->netbufscript) {
- cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
+ rs->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
} else {
- cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
- libxl__xen_script_dir_path());
+ rs->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
+ libxl__xen_script_dir_path());
}
rc = 0;
@@ -84,20 +85,22 @@ out:
void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
{
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
+
STATE_AO_GC(cds->ao);
/* free qdisc cache */
- if (cds->qdisc_cache) {
- nl_cache_clear(cds->qdisc_cache);
- nl_cache_free(cds->qdisc_cache);
- cds->qdisc_cache = NULL;
+ if (rs->qdisc_cache) {
+ nl_cache_clear(rs->qdisc_cache);
+ nl_cache_free(rs->qdisc_cache);
+ rs->qdisc_cache = NULL;
}
/* close & free nlsock */
- if (cds->nlsock) {
- nl_close(cds->nlsock);
- nl_socket_free(cds->nlsock);
- cds->nlsock = NULL;
+ if (rs->nlsock) {
+ nl_close(rs->nlsock);
+ nl_socket_free(rs->nlsock);
+ rs->nlsock = NULL;
}
}
@@ -150,13 +153,14 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
int rc, ret, ifindex;
struct rtnl_link *ifb = NULL;
struct rtnl_qdisc *qdisc = NULL;
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
STATE_AO_GC(cds->ao);
/* Now that we have brought up REMUS_IFB device with plug qdisc for
* this vif, so we need to refill the qdisc cache.
*/
- ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
+ ret = nl_cache_refill(rs->nlsock, rs->qdisc_cache);
if (ret) {
LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
rc = ERROR_FAIL;
@@ -164,7 +168,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
}
/* get a handle to the REMUS_IFB interface */
- ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
+ ret = rtnl_link_get_kernel(rs->nlsock, 0, remus_nic->ifb, &ifb);
if (ret) {
LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
nl_geterror(ret));
@@ -187,7 +191,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
* There is no need to explicitly free this qdisc as its just a
* reference from the qdisc cache we allocated earlier.
*/
- qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
+ qdisc = rtnl_qdisc_get_by_parent(rs->qdisc_cache, ifindex, TC_H_ROOT);
if (qdisc) {
const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
/* Sanity check: Ensure that the root qdisc is a plug qdisc. */
@@ -238,11 +242,12 @@ static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
libxl__remus_device_nic *remus_nic = dev->concrete_data;
libxl__checkpoint_devices_state *cds = dev->cds;
libxl__async_exec_state *aes = &dev->aodev.aes;
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
STATE_AO_GC(cds->ao);
/* Convenience aliases */
- char *const script = libxl__strdup(gc, cds->netbufscript);
+ char *const script = libxl__strdup(gc, rs->netbufscript);
const uint32_t domid = cds->domid;
const int dev_id = remus_nic->devid;
const char *const vif = remus_nic->vif;
@@ -333,6 +338,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
libxl__remus_device_nic *remus_nic = dev->concrete_data;
libxl__checkpoint_devices_state *cds = dev->cds;
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
const char *out_path_base, *hotplug_error = NULL;
int rc;
@@ -373,7 +379,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
if (hotplug_error) {
LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
- cds->netbufscript, vif, hotplug_error);
+ rs->netbufscript, vif, hotplug_error);
rc = ERROR_FAIL;
goto out;
}
@@ -444,6 +450,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
int buffer_op)
{
int rc, ret;
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
STATE_AO_GC(cds->ao);
@@ -457,7 +464,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
goto out;
}
- ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
+ ret = rtnl_qdisc_add(rs->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
if (ret) {
rc = ERROR_FAIL;
goto out;
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index fe4acc8..2a427a4 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -17,6 +17,46 @@
#include "libxl_internal.h"
+extern const libxl__checkpoint_device_instance_ops remus_device_nic;
+extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
+static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
+ &remus_device_nic,
+ &remus_device_drbd_disk,
+ NULL,
+};
+
+/*----- helper functions -----*/
+
+static int init_device_subkind(libxl__checkpoint_devices_state *cds)
+{
+ /* init device subkind-specific state in the libxl ctx */
+ int rc;
+ STATE_AO_GC(cds->ao);
+
+ if (libxl__netbuffer_enabled(gc)) {
+ rc = init_subkind_nic(cds);
+ if (rc) goto out;
+ }
+
+ rc = init_subkind_drbd_disk(cds);
+ if (rc) goto out;
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
+{
+ /* cleanup device subkind-specific state in the libxl ctx */
+ STATE_AO_GC(cds->ao);
+
+ if (libxl__netbuffer_enabled(gc))
+ cleanup_subkind_nic(cds);
+
+ cleanup_subkind_drbd_disk(cds);
+}
+
/*----- Remus setup and teardown -----*/
static void remus_setup_done(libxl__egc *egc,
@@ -24,10 +64,12 @@ static void remus_setup_done(libxl__egc *egc,
static void remus_setup_failed(libxl__egc *egc,
libxl__checkpoint_devices_state *cds, int rc);
-void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
+void libxl__remus_setup(libxl__egc *egc, libxl__remus_state *rs)
{
+ libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
+
/* Convenience aliases */
- libxl__checkpoint_devices_state *const cds = &dss->cds;
+ libxl__checkpoint_devices_state *const cds = &rs->cds;
const libxl_domain_remus_info *const info = dss->remus;
STATE_AO_GC(dss->ao);
@@ -46,6 +88,14 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
cds->ao = ao;
cds->domid = dss->domid;
cds->callback = remus_setup_done;
+ cds->ops = remus_ops;
+ rs->interval = info->interval;
+
+ if (init_device_subkind(cds)) {
+ LOG(ERROR, "Remus: failed to init device subkind for guest %u",
+ dss->domid);
+ goto out;
+ }
libxl__checkpoint_devices_setup(egc, cds);
return;
@@ -57,7 +107,7 @@ out:
static void remus_setup_done(libxl__egc *egc,
libxl__checkpoint_devices_state *cds, int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
STATE_AO_GC(dss->ao);
if (!rc) {
@@ -74,13 +124,15 @@ static void remus_setup_done(libxl__egc *egc,
static void remus_setup_failed(libxl__egc *egc,
libxl__checkpoint_devices_state *cds, int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
STATE_AO_GC(dss->ao);
if (rc)
LOG(ERROR, "Remus: failed to teardown device after setup failed"
" for guest with domid %u, rc %d", dss->domid, rc);
+ cleanup_device_subkind(cds);
+
dss->callback(egc, dss, rc);
}
@@ -88,28 +140,30 @@ static void remus_teardown_done(libxl__egc *egc,
libxl__checkpoint_devices_state *cds,
int rc);
void libxl__remus_teardown(libxl__egc *egc,
- libxl__domain_save_state *dss,
+ libxl__remus_state *rs,
int rc)
{
EGC_GC;
LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
" teardown Remus devices...", rc);
- dss->cds.callback = remus_teardown_done;
- libxl__checkpoint_devices_teardown(egc, &dss->cds);
+ rs->cds.callback = remus_teardown_done;
+ libxl__checkpoint_devices_teardown(egc, &rs->cds);
}
static void remus_teardown_done(libxl__egc *egc,
libxl__checkpoint_devices_state *cds,
int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
STATE_AO_GC(dss->ao);
if (rc)
LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
" rc %d", dss->domid, rc);
+ cleanup_device_subkind(cds);
+
dss->callback(egc, dss, rc);
}
@@ -143,7 +197,7 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
if (!ok)
goto out;
- libxl__checkpoint_devices_state *const cds = &dss->cds;
+ libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
cds->callback = remus_devices_postsuspend_cb;
libxl__checkpoint_devices_postsuspend(egc, cds);
return;
@@ -157,7 +211,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
int rc)
{
int ok = 0;
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
if (rc)
goto out;
@@ -175,7 +229,7 @@ void libxl__remus_domain_resume_callback(void *data)
libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
STATE_AO_GC(dss->ao);
- libxl__checkpoint_devices_state *const cds = &dss->cds;
+ libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
cds->callback = remus_devices_preresume_cb;
libxl__checkpoint_devices_preresume(egc, cds);
}
@@ -185,7 +239,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
int rc)
{
int ok = 0;
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
STATE_AO_GC(dss->ao);
if (rc)
@@ -229,7 +283,7 @@ static void remus_checkpoint_stream_written(
libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
/* Convenience aliases */
- libxl__checkpoint_devices_state *const cds = &dss->cds;
+ libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
STATE_AO_GC(dss->ao);
@@ -251,7 +305,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
libxl__checkpoint_devices_state *cds,
int rc)
{
- libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
+ libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
STATE_AO_GC(dss->ao);
@@ -269,9 +323,9 @@ static void remus_devices_commit_cb(libxl__egc *egc,
*/
/* Set checkpoint interval timeout */
- rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
+ rc = libxl__ev_time_register_rel(gc, &dss->rs.checkpoint_timeout,
remus_next_checkpoint,
- dss->interval);
+ dss->rs.interval);
if (rc)
goto out;
@@ -286,7 +340,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
const struct timeval *requested_abs)
{
libxl__domain_save_state *dss =
- CONTAINER_OF(ev, *dss, checkpoint_timeout);
+ CONTAINER_OF(ev, *dss, rs.checkpoint_timeout);
STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
index 50b897d..a8d8949 100644
--- a/tools/libxl/libxl_remus_disk_drbd.c
+++ b/tools/libxl/libxl_remus_disk_drbd.c
@@ -28,10 +28,11 @@ typedef struct libxl__remus_drbd_disk {
int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
{
+ libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
STATE_AO_GC(cds->ao);
- cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
- libxl__xen_script_dir_path());
+ rs->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
+ libxl__xen_script_dir_path());
return 0;
}
@@ -96,6 +97,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
int arraysize, nr = 0, rc;
const libxl_device_disk *disk = dev->backend_dev;
libxl__async_exec_state *aes = &dev->aodev.aes;
+ libxl__remus_state *rs = CONTAINER_OF(dev->cds, *rs, cds);
STATE_AO_GC(dev->cds->ao);
/* setup env & args */
@@ -107,7 +109,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
arraysize = 3;
nr = 0;
GCNEW_ARRAY(aes->args, arraysize);
- aes->args[nr++] = dev->cds->drbd_probe_script;
+ aes->args[nr++] = rs->drbd_probe_script;
aes->args[nr++] = disk->pdev_path;
aes->args[nr++] = NULL;
assert(nr <= arraysize);
--
1.9.1
^ permalink raw reply related [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device
2015-06-25 6:25 ` [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device Yang Hongyang
@ 2015-06-30 10:50 ` Ian Campbell
2015-07-01 3:11 ` Yang Hongyang
0 siblings, 1 reply; 103+ messages in thread
From: Ian Campbell @ 2015-06-30 10:50 UTC (permalink / raw)
To: Yang Hongyang
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Checkpoint device is an abstract layer to do checkpoint.
> COLO can also use it to do checkpoint. But there are
> still some codes in checkpoint device which touch remus:
> 1. remus_ops: we use remus ops directly in checkpoint
> device. Store it in checkpoint device state.
> 2. concrete layer's private member: add a new structure
> remus state, and move them to remus state.
> 3. init/cleanup device subkind: we call (init|cleanup)_subkind_nic
> and (init|cleanup)_subkind_drbd_disk directly in checkpoint
> device. Call them before calling libxl__checkpoint_devices_setup()
> or after calling libxl__checkpoint_devices_teardown().
This is quite a lot to think about in one go, can any of it be split up?
> it is pure refactoring and no functional changes.
>
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> ---
> tools/libxl/libxl.c | 2 +-
> tools/libxl/libxl_checkpoint_device.c | 52 ++-------------------
> tools/libxl/libxl_dom_save.c | 3 +-
> tools/libxl/libxl_internal.h | 40 ++++++++++------
> tools/libxl/libxl_netbuffer.c | 51 +++++++++++---------
> tools/libxl/libxl_remus.c | 88 ++++++++++++++++++++++++++++-------
> tools/libxl/libxl_remus_disk_drbd.c | 8 ++--
> 7 files changed, 135 insertions(+), 109 deletions(-)
>
> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> index a837ff2..f851957 100644
> --- a/tools/libxl/libxl.c
> +++ b/tools/libxl/libxl.c
> @@ -842,7 +842,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
> assert(info);
>
> /* Point of no return */
> - libxl__remus_setup(egc, dss);
> + libxl__remus_setup(egc, &dss->rs);
> return AO_INPROGRESS;
>
> out:
> diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
> index 226f159..0a16dbb 100644
> --- a/tools/libxl/libxl_checkpoint_device.c
> +++ b/tools/libxl/libxl_checkpoint_device.c
> @@ -17,46 +17,6 @@
>
> #include "libxl_internal.h"
>
> -extern const libxl__checkpoint_device_instance_ops remus_device_nic;
> -extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
> -static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
> - &remus_device_nic,
> - &remus_device_drbd_disk,
> - NULL,
> -};
> -
> -/*----- helper functions -----*/
> -
> -static int init_device_subkind(libxl__checkpoint_devices_state *cds)
> -{
> - /* init device subkind-specific state in the libxl ctx */
> - int rc;
> - STATE_AO_GC(cds->ao);
> -
> - if (libxl__netbuffer_enabled(gc)) {
> - rc = init_subkind_nic(cds);
> - if (rc) goto out;
> - }
> -
> - rc = init_subkind_drbd_disk(cds);
> - if (rc) goto out;
> -
> - rc = 0;
> -out:
> - return rc;
> -}
> -
> -static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
> -{
> - /* cleanup device subkind-specific state in the libxl ctx */
> - STATE_AO_GC(cds->ao);
> -
> - if (libxl__netbuffer_enabled(gc))
> - cleanup_subkind_nic(cds);
> -
> - cleanup_subkind_drbd_disk(cds);
> -}
> -
> /*----- setup() and teardown() -----*/
>
> /* callbacks */
> @@ -94,14 +54,10 @@ static void checkpoint_devices_setup(libxl__egc *egc,
> void libxl__checkpoint_devices_setup(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds)
> {
> - int i, rc;
> + int i;
>
> STATE_AO_GC(cds->ao);
>
> - rc = init_device_subkind(cds);
> - if (rc)
> - goto out;
> -
> cds->num_devices = 0;
> cds->num_nics = 0;
> cds->num_disks = 0;
> @@ -134,7 +90,7 @@ void libxl__checkpoint_devices_setup(libxl__egc *egc,
> return;
>
> out:
> - cds->callback(egc, cds, rc);
> + cds->callback(egc, cds, 0);
> }
>
> static void checkpoint_devices_setup(libxl__egc *egc,
> @@ -172,7 +128,7 @@ static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
> goto out;
>
> do {
> - dev->ops = remus_ops[++dev->ops_index];
> + dev->ops = dev->cds->ops[++dev->ops_index];
> if (!dev->ops) {
> libxl_device_nic * nic = NULL;
> libxl_device_disk * disk = NULL;
> @@ -271,8 +227,6 @@ static void devices_teardown_cb(libxl__egc *egc,
> cds->disks = NULL;
> cds->num_disks = 0;
>
> - cleanup_device_subkind(cds);
> -
> cds->callback(egc, cds, rc);
> }
>
> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
> index 9a3b33d..9a3d009 100644
> --- a/tools/libxl/libxl_dom_save.c
> +++ b/tools/libxl/libxl_dom_save.c
> @@ -410,7 +410,6 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
> dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
>
> if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
> - dss->interval = r_info->interval;
> if (libxl_defbool_val(r_info->compression))
> dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
> }
> @@ -581,7 +580,7 @@ static void domain_save_done(libxl__egc *egc,
> * from sending checkpoints. Teardown the network buffers and
> * release netlink resources. This is an async op.
> */
> - libxl__remus_teardown(egc, dss, rc);
> + libxl__remus_teardown(egc, &dss->rs, rc);
> }
>
> /*========================= Domain restore ============================*/
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index f3a139b..840734d 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -2752,6 +2752,8 @@ struct libxl__checkpoint_devices_state {
> uint32_t domid;
> libxl__checkpoint_callback *callback;
> int device_kind_flags;
> + /* The ops must be pointer array, and the last ops must be NULL */
> + const libxl__checkpoint_device_instance_ops **ops;
>
> /*----- private for abstract layer only -----*/
>
> @@ -2772,16 +2774,6 @@ struct libxl__checkpoint_devices_state {
> int num_disks;
>
> libxl__multidev multidev;
> -
> - /*----- private for concrete (device-specific) layer only -----*/
> -
> - /* private for nic device subkind ops */
> - char *netbufscript;
> - struct nl_sock *nlsock;
> - struct nl_cache *qdisc_cache;
> -
> - /* private for drbd disk subkind ops */
> - char *drbd_probe_script;
> };
>
> /*
> @@ -2829,6 +2821,26 @@ _hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds);
> _hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds);
> +
> +/*----- Remus related state structure -----*/
> +typedef struct libxl__remus_state libxl__remus_state;
> +struct libxl__remus_state {
> + /* private */
> + libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
> + int interval; /* checkpoint interval */
> +
> + /* abstract layer */
> + libxl__checkpoint_devices_state cds;
> +
> + /*----- private for concrete (device-specific) layer only -----*/
> + /* private for nic device subkind ops */
> + char *netbufscript;
> + struct nl_sock *nlsock;
> + struct nl_cache *qdisc_cache;
> +
> + /* private for drbd disk subkind ops */
> + char *drbd_probe_script;
> +};
> _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
>
> /*----- Legacy conversion helper -----*/
> @@ -2962,9 +2974,7 @@ struct libxl__domain_save_state {
> libxl__domain_suspend_state dsps;
> int hvm;
> int xcflags;
> - libxl__checkpoint_devices_state cds;
> - libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
> - int interval; /* checkpoint interval (for Remus) */
> + libxl__remus_state rs;
> libxl__save_helper_state shs;
> libxl__logdirty_switch logdirty;
> /* private for libxl__domain_save_device_model */
> @@ -3364,9 +3374,9 @@ _hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
> _hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
> /* Remus setup and teardown*/
> _hidden void libxl__remus_setup(libxl__egc *egc,
> - libxl__domain_save_state *dss);
> + libxl__remus_state *rs);
> _hidden void libxl__remus_teardown(libxl__egc *egc,
> - libxl__domain_save_state *dss,
> + libxl__remus_state *rs,
> int rc);
>
> /*
> diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
> index 86afba6..1d01e10 100644
> --- a/tools/libxl/libxl_netbuffer.c
> +++ b/tools/libxl/libxl_netbuffer.c
> @@ -41,18 +41,19 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
> int init_subkind_nic(libxl__checkpoint_devices_state *cds)
> {
> int rc, ret;
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
>
> STATE_AO_GC(cds->ao);
>
> - cds->nlsock = nl_socket_alloc();
> - if (!cds->nlsock) {
> + rs->nlsock = nl_socket_alloc();
> + if (!rs->nlsock) {
> LOG(ERROR, "cannot allocate nl socket");
> rc = ERROR_FAIL;
> goto out;
> }
>
> - ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
> + ret = nl_connect(rs->nlsock, NETLINK_ROUTE);
> if (ret) {
> LOG(ERROR, "failed to open netlink socket: %s",
> nl_geterror(ret));
> @@ -61,7 +62,7 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
> }
>
> /* get list of all qdiscs installed on network devs. */
> - ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
> + ret = rtnl_qdisc_alloc_cache(rs->nlsock, &rs->qdisc_cache);
> if (ret) {
> LOG(ERROR, "failed to allocate qdisc cache: %s",
> nl_geterror(ret));
> @@ -70,10 +71,10 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
> }
>
> if (dss->remus->netbufscript) {
> - cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
> + rs->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
> } else {
> - cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
> - libxl__xen_script_dir_path());
> + rs->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
> + libxl__xen_script_dir_path());
> }
>
> rc = 0;
> @@ -84,20 +85,22 @@ out:
>
> void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
> {
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
> +
> STATE_AO_GC(cds->ao);
>
> /* free qdisc cache */
> - if (cds->qdisc_cache) {
> - nl_cache_clear(cds->qdisc_cache);
> - nl_cache_free(cds->qdisc_cache);
> - cds->qdisc_cache = NULL;
> + if (rs->qdisc_cache) {
> + nl_cache_clear(rs->qdisc_cache);
> + nl_cache_free(rs->qdisc_cache);
> + rs->qdisc_cache = NULL;
> }
>
> /* close & free nlsock */
> - if (cds->nlsock) {
> - nl_close(cds->nlsock);
> - nl_socket_free(cds->nlsock);
> - cds->nlsock = NULL;
> + if (rs->nlsock) {
> + nl_close(rs->nlsock);
> + nl_socket_free(rs->nlsock);
> + rs->nlsock = NULL;
> }
> }
>
> @@ -150,13 +153,14 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
> int rc, ret, ifindex;
> struct rtnl_link *ifb = NULL;
> struct rtnl_qdisc *qdisc = NULL;
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>
> STATE_AO_GC(cds->ao);
>
> /* Now that we have brought up REMUS_IFB device with plug qdisc for
> * this vif, so we need to refill the qdisc cache.
> */
> - ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
> + ret = nl_cache_refill(rs->nlsock, rs->qdisc_cache);
> if (ret) {
> LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
> rc = ERROR_FAIL;
> @@ -164,7 +168,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
> }
>
> /* get a handle to the REMUS_IFB interface */
> - ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
> + ret = rtnl_link_get_kernel(rs->nlsock, 0, remus_nic->ifb, &ifb);
> if (ret) {
> LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
> nl_geterror(ret));
> @@ -187,7 +191,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
> * There is no need to explicitly free this qdisc as its just a
> * reference from the qdisc cache we allocated earlier.
> */
> - qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
> + qdisc = rtnl_qdisc_get_by_parent(rs->qdisc_cache, ifindex, TC_H_ROOT);
> if (qdisc) {
> const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
> /* Sanity check: Ensure that the root qdisc is a plug qdisc. */
> @@ -238,11 +242,12 @@ static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
> libxl__checkpoint_devices_state *cds = dev->cds;
> libxl__async_exec_state *aes = &dev->aodev.aes;
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>
> STATE_AO_GC(cds->ao);
>
> /* Convenience aliases */
> - char *const script = libxl__strdup(gc, cds->netbufscript);
> + char *const script = libxl__strdup(gc, rs->netbufscript);
> const uint32_t domid = cds->domid;
> const int dev_id = remus_nic->devid;
> const char *const vif = remus_nic->vif;
> @@ -333,6 +338,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
> libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
> libxl__remus_device_nic *remus_nic = dev->concrete_data;
> libxl__checkpoint_devices_state *cds = dev->cds;
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
> const char *out_path_base, *hotplug_error = NULL;
> int rc;
>
> @@ -373,7 +379,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>
> if (hotplug_error) {
> LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
> - cds->netbufscript, vif, hotplug_error);
> + rs->netbufscript, vif, hotplug_error);
> rc = ERROR_FAIL;
> goto out;
> }
> @@ -444,6 +450,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
> int buffer_op)
> {
> int rc, ret;
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>
> STATE_AO_GC(cds->ao);
>
> @@ -457,7 +464,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
> goto out;
> }
>
> - ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
> + ret = rtnl_qdisc_add(rs->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
> if (ret) {
> rc = ERROR_FAIL;
> goto out;
> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
> index fe4acc8..2a427a4 100644
> --- a/tools/libxl/libxl_remus.c
> +++ b/tools/libxl/libxl_remus.c
> @@ -17,6 +17,46 @@
>
> #include "libxl_internal.h"
>
> +extern const libxl__checkpoint_device_instance_ops remus_device_nic;
> +extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
> +static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
> + &remus_device_nic,
> + &remus_device_drbd_disk,
> + NULL,
> +};
> +
> +/*----- helper functions -----*/
> +
> +static int init_device_subkind(libxl__checkpoint_devices_state *cds)
> +{
> + /* init device subkind-specific state in the libxl ctx */
> + int rc;
> + STATE_AO_GC(cds->ao);
> +
> + if (libxl__netbuffer_enabled(gc)) {
> + rc = init_subkind_nic(cds);
> + if (rc) goto out;
> + }
> +
> + rc = init_subkind_drbd_disk(cds);
> + if (rc) goto out;
> +
> + rc = 0;
> +out:
> + return rc;
> +}
> +
> +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
> +{
> + /* cleanup device subkind-specific state in the libxl ctx */
> + STATE_AO_GC(cds->ao);
> +
> + if (libxl__netbuffer_enabled(gc))
> + cleanup_subkind_nic(cds);
> +
> + cleanup_subkind_drbd_disk(cds);
> +}
> +
> /*----- Remus setup and teardown -----*/
>
> static void remus_setup_done(libxl__egc *egc,
> @@ -24,10 +64,12 @@ static void remus_setup_done(libxl__egc *egc,
> static void remus_setup_failed(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds, int rc);
>
> -void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
> +void libxl__remus_setup(libxl__egc *egc, libxl__remus_state *rs)
> {
> + libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
> +
> /* Convenience aliases */
> - libxl__checkpoint_devices_state *const cds = &dss->cds;
> + libxl__checkpoint_devices_state *const cds = &rs->cds;
> const libxl_domain_remus_info *const info = dss->remus;
>
> STATE_AO_GC(dss->ao);
> @@ -46,6 +88,14 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
> cds->ao = ao;
> cds->domid = dss->domid;
> cds->callback = remus_setup_done;
> + cds->ops = remus_ops;
> + rs->interval = info->interval;
> +
> + if (init_device_subkind(cds)) {
> + LOG(ERROR, "Remus: failed to init device subkind for guest %u",
> + dss->domid);
> + goto out;
> + }
>
> libxl__checkpoint_devices_setup(egc, cds);
> return;
> @@ -57,7 +107,7 @@ out:
> static void remus_setup_done(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds, int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
> STATE_AO_GC(dss->ao);
>
> if (!rc) {
> @@ -74,13 +124,15 @@ static void remus_setup_done(libxl__egc *egc,
> static void remus_setup_failed(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds, int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
> STATE_AO_GC(dss->ao);
>
> if (rc)
> LOG(ERROR, "Remus: failed to teardown device after setup failed"
> " for guest with domid %u, rc %d", dss->domid, rc);
>
> + cleanup_device_subkind(cds);
> +
> dss->callback(egc, dss, rc);
> }
>
> @@ -88,28 +140,30 @@ static void remus_teardown_done(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds,
> int rc);
> void libxl__remus_teardown(libxl__egc *egc,
> - libxl__domain_save_state *dss,
> + libxl__remus_state *rs,
> int rc)
> {
> EGC_GC;
>
> LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
> " teardown Remus devices...", rc);
> - dss->cds.callback = remus_teardown_done;
> - libxl__checkpoint_devices_teardown(egc, &dss->cds);
> + rs->cds.callback = remus_teardown_done;
> + libxl__checkpoint_devices_teardown(egc, &rs->cds);
> }
>
> static void remus_teardown_done(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds,
> int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
> STATE_AO_GC(dss->ao);
>
> if (rc)
> LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
> " rc %d", dss->domid, rc);
>
> + cleanup_device_subkind(cds);
> +
> dss->callback(egc, dss, rc);
> }
>
> @@ -143,7 +197,7 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
> if (!ok)
> goto out;
>
> - libxl__checkpoint_devices_state *const cds = &dss->cds;
> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
> cds->callback = remus_devices_postsuspend_cb;
> libxl__checkpoint_devices_postsuspend(egc, cds);
> return;
> @@ -157,7 +211,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
> int rc)
> {
> int ok = 0;
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>
> if (rc)
> goto out;
> @@ -175,7 +229,7 @@ void libxl__remus_domain_resume_callback(void *data)
> libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
> STATE_AO_GC(dss->ao);
>
> - libxl__checkpoint_devices_state *const cds = &dss->cds;
> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
> cds->callback = remus_devices_preresume_cb;
> libxl__checkpoint_devices_preresume(egc, cds);
> }
> @@ -185,7 +239,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
> int rc)
> {
> int ok = 0;
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
> STATE_AO_GC(dss->ao);
>
> if (rc)
> @@ -229,7 +283,7 @@ static void remus_checkpoint_stream_written(
> libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
>
> /* Convenience aliases */
> - libxl__checkpoint_devices_state *const cds = &dss->cds;
> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>
> STATE_AO_GC(dss->ao);
>
> @@ -251,7 +305,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
> libxl__checkpoint_devices_state *cds,
> int rc)
> {
> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>
> STATE_AO_GC(dss->ao);
>
> @@ -269,9 +323,9 @@ static void remus_devices_commit_cb(libxl__egc *egc,
> */
>
> /* Set checkpoint interval timeout */
> - rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
> + rc = libxl__ev_time_register_rel(gc, &dss->rs.checkpoint_timeout,
> remus_next_checkpoint,
> - dss->interval);
> + dss->rs.interval);
>
> if (rc)
> goto out;
> @@ -286,7 +340,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
> const struct timeval *requested_abs)
> {
> libxl__domain_save_state *dss =
> - CONTAINER_OF(ev, *dss, checkpoint_timeout);
> + CONTAINER_OF(ev, *dss, rs.checkpoint_timeout);
>
> STATE_AO_GC(dss->ao);
>
> diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
> index 50b897d..a8d8949 100644
> --- a/tools/libxl/libxl_remus_disk_drbd.c
> +++ b/tools/libxl/libxl_remus_disk_drbd.c
> @@ -28,10 +28,11 @@ typedef struct libxl__remus_drbd_disk {
>
> int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
> {
> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
> STATE_AO_GC(cds->ao);
>
> - cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
> - libxl__xen_script_dir_path());
> + rs->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
> + libxl__xen_script_dir_path());
>
> return 0;
> }
> @@ -96,6 +97,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
> int arraysize, nr = 0, rc;
> const libxl_device_disk *disk = dev->backend_dev;
> libxl__async_exec_state *aes = &dev->aodev.aes;
> + libxl__remus_state *rs = CONTAINER_OF(dev->cds, *rs, cds);
> STATE_AO_GC(dev->cds->ao);
>
> /* setup env & args */
> @@ -107,7 +109,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
> arraysize = 3;
> nr = 0;
> GCNEW_ARRAY(aes->args, arraysize);
> - aes->args[nr++] = dev->cds->drbd_probe_script;
> + aes->args[nr++] = rs->drbd_probe_script;
> aes->args[nr++] = disk->pdev_path;
> aes->args[nr++] = NULL;
> assert(nr <= arraysize);
^ permalink raw reply [flat|nested] 103+ messages in thread* Re: [PATCH v3 COLOPre 26/26] tools/libxl: don't touch remus in checkpoint_device
2015-06-30 10:50 ` Ian Campbell
@ 2015-07-01 3:11 ` Yang Hongyang
0 siblings, 0 replies; 103+ messages in thread
From: Yang Hongyang @ 2015-07-01 3:11 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, wency, andrew.cooper3, yunhong.jiang, eddie.dong,
xen-devel, guijianfeng, rshriram, ian.jackson
On 06/30/2015 06:50 PM, Ian Campbell wrote:
> On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
>> Checkpoint device is an abstract layer to do checkpoint.
>> COLO can also use it to do checkpoint. But there are
>> still some codes in checkpoint device which touch remus:
>> 1. remus_ops: we use remus ops directly in checkpoint
>> device. Store it in checkpoint device state.
>> 2. concrete layer's private member: add a new structure
>> remus state, and move them to remus state.
>> 3. init/cleanup device subkind: we call (init|cleanup)_subkind_nic
>> and (init|cleanup)_subkind_drbd_disk directly in checkpoint
>> device. Call them before calling libxl__checkpoint_devices_setup()
>> or after calling libxl__checkpoint_devices_teardown().
>
> This is quite a lot to think about in one go, can any of it be split up?
I will try to split this patch in the next version...
>
>> it is pure refactoring and no functional changes.
>>
>> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
>> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
>> ---
>> tools/libxl/libxl.c | 2 +-
>> tools/libxl/libxl_checkpoint_device.c | 52 ++-------------------
>> tools/libxl/libxl_dom_save.c | 3 +-
>> tools/libxl/libxl_internal.h | 40 ++++++++++------
>> tools/libxl/libxl_netbuffer.c | 51 +++++++++++---------
>> tools/libxl/libxl_remus.c | 88 ++++++++++++++++++++++++++++-------
>> tools/libxl/libxl_remus_disk_drbd.c | 8 ++--
>> 7 files changed, 135 insertions(+), 109 deletions(-)
>>
>> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
>> index a837ff2..f851957 100644
>> --- a/tools/libxl/libxl.c
>> +++ b/tools/libxl/libxl.c
>> @@ -842,7 +842,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
>> assert(info);
>>
>> /* Point of no return */
>> - libxl__remus_setup(egc, dss);
>> + libxl__remus_setup(egc, &dss->rs);
>> return AO_INPROGRESS;
>>
>> out:
>> diff --git a/tools/libxl/libxl_checkpoint_device.c b/tools/libxl/libxl_checkpoint_device.c
>> index 226f159..0a16dbb 100644
>> --- a/tools/libxl/libxl_checkpoint_device.c
>> +++ b/tools/libxl/libxl_checkpoint_device.c
>> @@ -17,46 +17,6 @@
>>
>> #include "libxl_internal.h"
>>
>> -extern const libxl__checkpoint_device_instance_ops remus_device_nic;
>> -extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
>> -static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
>> - &remus_device_nic,
>> - &remus_device_drbd_disk,
>> - NULL,
>> -};
>> -
>> -/*----- helper functions -----*/
>> -
>> -static int init_device_subkind(libxl__checkpoint_devices_state *cds)
>> -{
>> - /* init device subkind-specific state in the libxl ctx */
>> - int rc;
>> - STATE_AO_GC(cds->ao);
>> -
>> - if (libxl__netbuffer_enabled(gc)) {
>> - rc = init_subkind_nic(cds);
>> - if (rc) goto out;
>> - }
>> -
>> - rc = init_subkind_drbd_disk(cds);
>> - if (rc) goto out;
>> -
>> - rc = 0;
>> -out:
>> - return rc;
>> -}
>> -
>> -static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
>> -{
>> - /* cleanup device subkind-specific state in the libxl ctx */
>> - STATE_AO_GC(cds->ao);
>> -
>> - if (libxl__netbuffer_enabled(gc))
>> - cleanup_subkind_nic(cds);
>> -
>> - cleanup_subkind_drbd_disk(cds);
>> -}
>> -
>> /*----- setup() and teardown() -----*/
>>
>> /* callbacks */
>> @@ -94,14 +54,10 @@ static void checkpoint_devices_setup(libxl__egc *egc,
>> void libxl__checkpoint_devices_setup(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds)
>> {
>> - int i, rc;
>> + int i;
>>
>> STATE_AO_GC(cds->ao);
>>
>> - rc = init_device_subkind(cds);
>> - if (rc)
>> - goto out;
>> -
>> cds->num_devices = 0;
>> cds->num_nics = 0;
>> cds->num_disks = 0;
>> @@ -134,7 +90,7 @@ void libxl__checkpoint_devices_setup(libxl__egc *egc,
>> return;
>>
>> out:
>> - cds->callback(egc, cds, rc);
>> + cds->callback(egc, cds, 0);
>> }
>>
>> static void checkpoint_devices_setup(libxl__egc *egc,
>> @@ -172,7 +128,7 @@ static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev)
>> goto out;
>>
>> do {
>> - dev->ops = remus_ops[++dev->ops_index];
>> + dev->ops = dev->cds->ops[++dev->ops_index];
>> if (!dev->ops) {
>> libxl_device_nic * nic = NULL;
>> libxl_device_disk * disk = NULL;
>> @@ -271,8 +227,6 @@ static void devices_teardown_cb(libxl__egc *egc,
>> cds->disks = NULL;
>> cds->num_disks = 0;
>>
>> - cleanup_device_subkind(cds);
>> -
>> cds->callback(egc, cds, rc);
>> }
>>
>> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
>> index 9a3b33d..9a3d009 100644
>> --- a/tools/libxl/libxl_dom_save.c
>> +++ b/tools/libxl/libxl_dom_save.c
>> @@ -410,7 +410,6 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>> dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
>>
>> if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
>> - dss->interval = r_info->interval;
>> if (libxl_defbool_val(r_info->compression))
>> dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
>> }
>> @@ -581,7 +580,7 @@ static void domain_save_done(libxl__egc *egc,
>> * from sending checkpoints. Teardown the network buffers and
>> * release netlink resources. This is an async op.
>> */
>> - libxl__remus_teardown(egc, dss, rc);
>> + libxl__remus_teardown(egc, &dss->rs, rc);
>> }
>>
>> /*========================= Domain restore ============================*/
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index f3a139b..840734d 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
>> @@ -2752,6 +2752,8 @@ struct libxl__checkpoint_devices_state {
>> uint32_t domid;
>> libxl__checkpoint_callback *callback;
>> int device_kind_flags;
>> + /* The ops must be pointer array, and the last ops must be NULL */
>> + const libxl__checkpoint_device_instance_ops **ops;
>>
>> /*----- private for abstract layer only -----*/
>>
>> @@ -2772,16 +2774,6 @@ struct libxl__checkpoint_devices_state {
>> int num_disks;
>>
>> libxl__multidev multidev;
>> -
>> - /*----- private for concrete (device-specific) layer only -----*/
>> -
>> - /* private for nic device subkind ops */
>> - char *netbufscript;
>> - struct nl_sock *nlsock;
>> - struct nl_cache *qdisc_cache;
>> -
>> - /* private for drbd disk subkind ops */
>> - char *drbd_probe_script;
>> };
>>
>> /*
>> @@ -2829,6 +2821,26 @@ _hidden void libxl__checkpoint_devices_preresume(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds);
>> _hidden void libxl__checkpoint_devices_commit(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds);
>> +
>> +/*----- Remus related state structure -----*/
>> +typedef struct libxl__remus_state libxl__remus_state;
>> +struct libxl__remus_state {
>> + /* private */
>> + libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
>> + int interval; /* checkpoint interval */
>> +
>> + /* abstract layer */
>> + libxl__checkpoint_devices_state cds;
>> +
>> + /*----- private for concrete (device-specific) layer only -----*/
>> + /* private for nic device subkind ops */
>> + char *netbufscript;
>> + struct nl_sock *nlsock;
>> + struct nl_cache *qdisc_cache;
>> +
>> + /* private for drbd disk subkind ops */
>> + char *drbd_probe_script;
>> +};
>> _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
>>
>> /*----- Legacy conversion helper -----*/
>> @@ -2962,9 +2974,7 @@ struct libxl__domain_save_state {
>> libxl__domain_suspend_state dsps;
>> int hvm;
>> int xcflags;
>> - libxl__checkpoint_devices_state cds;
>> - libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
>> - int interval; /* checkpoint interval (for Remus) */
>> + libxl__remus_state rs;
>> libxl__save_helper_state shs;
>> libxl__logdirty_switch logdirty;
>> /* private for libxl__domain_save_device_model */
>> @@ -3364,9 +3374,9 @@ _hidden void libxl__remus_domain_save_checkpoint_callback(void *data);
>> _hidden void libxl__remus_domain_restore_checkpoint_callback(void *data);
>> /* Remus setup and teardown*/
>> _hidden void libxl__remus_setup(libxl__egc *egc,
>> - libxl__domain_save_state *dss);
>> + libxl__remus_state *rs);
>> _hidden void libxl__remus_teardown(libxl__egc *egc,
>> - libxl__domain_save_state *dss,
>> + libxl__remus_state *rs,
>> int rc);
>>
>> /*
>> diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c
>> index 86afba6..1d01e10 100644
>> --- a/tools/libxl/libxl_netbuffer.c
>> +++ b/tools/libxl/libxl_netbuffer.c
>> @@ -41,18 +41,19 @@ int libxl__netbuffer_enabled(libxl__gc *gc)
>> int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>> {
>> int rc, ret;
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
>>
>> STATE_AO_GC(cds->ao);
>>
>> - cds->nlsock = nl_socket_alloc();
>> - if (!cds->nlsock) {
>> + rs->nlsock = nl_socket_alloc();
>> + if (!rs->nlsock) {
>> LOG(ERROR, "cannot allocate nl socket");
>> rc = ERROR_FAIL;
>> goto out;
>> }
>>
>> - ret = nl_connect(cds->nlsock, NETLINK_ROUTE);
>> + ret = nl_connect(rs->nlsock, NETLINK_ROUTE);
>> if (ret) {
>> LOG(ERROR, "failed to open netlink socket: %s",
>> nl_geterror(ret));
>> @@ -61,7 +62,7 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>> }
>>
>> /* get list of all qdiscs installed on network devs. */
>> - ret = rtnl_qdisc_alloc_cache(cds->nlsock, &cds->qdisc_cache);
>> + ret = rtnl_qdisc_alloc_cache(rs->nlsock, &rs->qdisc_cache);
>> if (ret) {
>> LOG(ERROR, "failed to allocate qdisc cache: %s",
>> nl_geterror(ret));
>> @@ -70,10 +71,10 @@ int init_subkind_nic(libxl__checkpoint_devices_state *cds)
>> }
>>
>> if (dss->remus->netbufscript) {
>> - cds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
>> + rs->netbufscript = libxl__strdup(gc, dss->remus->netbufscript);
>> } else {
>> - cds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
>> - libxl__xen_script_dir_path());
>> + rs->netbufscript = GCSPRINTF("%s/remus-netbuf-setup",
>> + libxl__xen_script_dir_path());
>> }
>>
>> rc = 0;
>> @@ -84,20 +85,22 @@ out:
>>
>> void cleanup_subkind_nic(libxl__checkpoint_devices_state *cds)
>> {
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>> +
>> STATE_AO_GC(cds->ao);
>>
>> /* free qdisc cache */
>> - if (cds->qdisc_cache) {
>> - nl_cache_clear(cds->qdisc_cache);
>> - nl_cache_free(cds->qdisc_cache);
>> - cds->qdisc_cache = NULL;
>> + if (rs->qdisc_cache) {
>> + nl_cache_clear(rs->qdisc_cache);
>> + nl_cache_free(rs->qdisc_cache);
>> + rs->qdisc_cache = NULL;
>> }
>>
>> /* close & free nlsock */
>> - if (cds->nlsock) {
>> - nl_close(cds->nlsock);
>> - nl_socket_free(cds->nlsock);
>> - cds->nlsock = NULL;
>> + if (rs->nlsock) {
>> + nl_close(rs->nlsock);
>> + nl_socket_free(rs->nlsock);
>> + rs->nlsock = NULL;
>> }
>> }
>>
>> @@ -150,13 +153,14 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
>> int rc, ret, ifindex;
>> struct rtnl_link *ifb = NULL;
>> struct rtnl_qdisc *qdisc = NULL;
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>>
>> STATE_AO_GC(cds->ao);
>>
>> /* Now that we have brought up REMUS_IFB device with plug qdisc for
>> * this vif, so we need to refill the qdisc cache.
>> */
>> - ret = nl_cache_refill(cds->nlsock, cds->qdisc_cache);
>> + ret = nl_cache_refill(rs->nlsock, rs->qdisc_cache);
>> if (ret) {
>> LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret));
>> rc = ERROR_FAIL;
>> @@ -164,7 +168,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
>> }
>>
>> /* get a handle to the REMUS_IFB interface */
>> - ret = rtnl_link_get_kernel(cds->nlsock, 0, remus_nic->ifb, &ifb);
>> + ret = rtnl_link_get_kernel(rs->nlsock, 0, remus_nic->ifb, &ifb);
>> if (ret) {
>> LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb,
>> nl_geterror(ret));
>> @@ -187,7 +191,7 @@ static int init_qdisc(libxl__checkpoint_devices_state *cds,
>> * There is no need to explicitly free this qdisc as its just a
>> * reference from the qdisc cache we allocated earlier.
>> */
>> - qdisc = rtnl_qdisc_get_by_parent(cds->qdisc_cache, ifindex, TC_H_ROOT);
>> + qdisc = rtnl_qdisc_get_by_parent(rs->qdisc_cache, ifindex, TC_H_ROOT);
>> if (qdisc) {
>> const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
>> /* Sanity check: Ensure that the root qdisc is a plug qdisc. */
>> @@ -238,11 +242,12 @@ static void setup_async_exec(libxl__checkpoint_device *dev, char *op)
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>> libxl__checkpoint_devices_state *cds = dev->cds;
>> libxl__async_exec_state *aes = &dev->aodev.aes;
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>>
>> STATE_AO_GC(cds->ao);
>>
>> /* Convenience aliases */
>> - char *const script = libxl__strdup(gc, cds->netbufscript);
>> + char *const script = libxl__strdup(gc, rs->netbufscript);
>> const uint32_t domid = cds->domid;
>> const int dev_id = remus_nic->devid;
>> const char *const vif = remus_nic->vif;
>> @@ -333,6 +338,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>> libxl__checkpoint_device *dev = CONTAINER_OF(aodev, *dev, aodev);
>> libxl__remus_device_nic *remus_nic = dev->concrete_data;
>> libxl__checkpoint_devices_state *cds = dev->cds;
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>> const char *out_path_base, *hotplug_error = NULL;
>> int rc;
>>
>> @@ -373,7 +379,7 @@ static void netbuf_setup_script_cb(libxl__egc *egc,
>>
>> if (hotplug_error) {
>> LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
>> - cds->netbufscript, vif, hotplug_error);
>> + rs->netbufscript, vif, hotplug_error);
>> rc = ERROR_FAIL;
>> goto out;
>> }
>> @@ -444,6 +450,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
>> int buffer_op)
>> {
>> int rc, ret;
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>>
>> STATE_AO_GC(cds->ao);
>>
>> @@ -457,7 +464,7 @@ static int remus_netbuf_op(libxl__remus_device_nic *remus_nic,
>> goto out;
>> }
>>
>> - ret = rtnl_qdisc_add(cds->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
>> + ret = rtnl_qdisc_add(rs->nlsock, remus_nic->qdisc, NLM_F_REQUEST);
>> if (ret) {
>> rc = ERROR_FAIL;
>> goto out;
>> diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
>> index fe4acc8..2a427a4 100644
>> --- a/tools/libxl/libxl_remus.c
>> +++ b/tools/libxl/libxl_remus.c
>> @@ -17,6 +17,46 @@
>>
>> #include "libxl_internal.h"
>>
>> +extern const libxl__checkpoint_device_instance_ops remus_device_nic;
>> +extern const libxl__checkpoint_device_instance_ops remus_device_drbd_disk;
>> +static const libxl__checkpoint_device_instance_ops *remus_ops[] = {
>> + &remus_device_nic,
>> + &remus_device_drbd_disk,
>> + NULL,
>> +};
>> +
>> +/*----- helper functions -----*/
>> +
>> +static int init_device_subkind(libxl__checkpoint_devices_state *cds)
>> +{
>> + /* init device subkind-specific state in the libxl ctx */
>> + int rc;
>> + STATE_AO_GC(cds->ao);
>> +
>> + if (libxl__netbuffer_enabled(gc)) {
>> + rc = init_subkind_nic(cds);
>> + if (rc) goto out;
>> + }
>> +
>> + rc = init_subkind_drbd_disk(cds);
>> + if (rc) goto out;
>> +
>> + rc = 0;
>> +out:
>> + return rc;
>> +}
>> +
>> +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
>> +{
>> + /* cleanup device subkind-specific state in the libxl ctx */
>> + STATE_AO_GC(cds->ao);
>> +
>> + if (libxl__netbuffer_enabled(gc))
>> + cleanup_subkind_nic(cds);
>> +
>> + cleanup_subkind_drbd_disk(cds);
>> +}
>> +
>> /*----- Remus setup and teardown -----*/
>>
>> static void remus_setup_done(libxl__egc *egc,
>> @@ -24,10 +64,12 @@ static void remus_setup_done(libxl__egc *egc,
>> static void remus_setup_failed(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds, int rc);
>>
>> -void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
>> +void libxl__remus_setup(libxl__egc *egc, libxl__remus_state *rs)
>> {
>> + libxl__domain_save_state *dss = CONTAINER_OF(rs, *dss, rs);
>> +
>> /* Convenience aliases */
>> - libxl__checkpoint_devices_state *const cds = &dss->cds;
>> + libxl__checkpoint_devices_state *const cds = &rs->cds;
>> const libxl_domain_remus_info *const info = dss->remus;
>>
>> STATE_AO_GC(dss->ao);
>> @@ -46,6 +88,14 @@ void libxl__remus_setup(libxl__egc *egc, libxl__domain_save_state *dss)
>> cds->ao = ao;
>> cds->domid = dss->domid;
>> cds->callback = remus_setup_done;
>> + cds->ops = remus_ops;
>> + rs->interval = info->interval;
>> +
>> + if (init_device_subkind(cds)) {
>> + LOG(ERROR, "Remus: failed to init device subkind for guest %u",
>> + dss->domid);
>> + goto out;
>> + }
>>
>> libxl__checkpoint_devices_setup(egc, cds);
>> return;
>> @@ -57,7 +107,7 @@ out:
>> static void remus_setup_done(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds, int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (!rc) {
>> @@ -74,13 +124,15 @@ static void remus_setup_done(libxl__egc *egc,
>> static void remus_setup_failed(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds, int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (rc)
>> LOG(ERROR, "Remus: failed to teardown device after setup failed"
>> " for guest with domid %u, rc %d", dss->domid, rc);
>>
>> + cleanup_device_subkind(cds);
>> +
>> dss->callback(egc, dss, rc);
>> }
>>
>> @@ -88,28 +140,30 @@ static void remus_teardown_done(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds,
>> int rc);
>> void libxl__remus_teardown(libxl__egc *egc,
>> - libxl__domain_save_state *dss,
>> + libxl__remus_state *rs,
>> int rc)
>> {
>> EGC_GC;
>>
>> LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
>> " teardown Remus devices...", rc);
>> - dss->cds.callback = remus_teardown_done;
>> - libxl__checkpoint_devices_teardown(egc, &dss->cds);
>> + rs->cds.callback = remus_teardown_done;
>> + libxl__checkpoint_devices_teardown(egc, &rs->cds);
>> }
>>
>> static void remus_teardown_done(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds,
>> int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (rc)
>> LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
>> " rc %d", dss->domid, rc);
>>
>> + cleanup_device_subkind(cds);
>> +
>> dss->callback(egc, dss, rc);
>> }
>>
>> @@ -143,7 +197,7 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
>> if (!ok)
>> goto out;
>>
>> - libxl__checkpoint_devices_state *const cds = &dss->cds;
>> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>> cds->callback = remus_devices_postsuspend_cb;
>> libxl__checkpoint_devices_postsuspend(egc, cds);
>> return;
>> @@ -157,7 +211,7 @@ static void remus_devices_postsuspend_cb(libxl__egc *egc,
>> int rc)
>> {
>> int ok = 0;
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>>
>> if (rc)
>> goto out;
>> @@ -175,7 +229,7 @@ void libxl__remus_domain_resume_callback(void *data)
>> libxl__domain_save_state *dss = CONTAINER_OF(shs, *dss, shs);
>> STATE_AO_GC(dss->ao);
>>
>> - libxl__checkpoint_devices_state *const cds = &dss->cds;
>> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>> cds->callback = remus_devices_preresume_cb;
>> libxl__checkpoint_devices_preresume(egc, cds);
>> }
>> @@ -185,7 +239,7 @@ static void remus_devices_preresume_cb(libxl__egc *egc,
>> int rc)
>> {
>> int ok = 0;
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>> STATE_AO_GC(dss->ao);
>>
>> if (rc)
>> @@ -229,7 +283,7 @@ static void remus_checkpoint_stream_written(
>> libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
>>
>> /* Convenience aliases */
>> - libxl__checkpoint_devices_state *const cds = &dss->cds;
>> + libxl__checkpoint_devices_state *const cds = &dss->rs.cds;
>>
>> STATE_AO_GC(dss->ao);
>>
>> @@ -251,7 +305,7 @@ static void remus_devices_commit_cb(libxl__egc *egc,
>> libxl__checkpoint_devices_state *cds,
>> int rc)
>> {
>> - libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, cds);
>> + libxl__domain_save_state *dss = CONTAINER_OF(cds, *dss, rs.cds);
>>
>> STATE_AO_GC(dss->ao);
>>
>> @@ -269,9 +323,9 @@ static void remus_devices_commit_cb(libxl__egc *egc,
>> */
>>
>> /* Set checkpoint interval timeout */
>> - rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
>> + rc = libxl__ev_time_register_rel(gc, &dss->rs.checkpoint_timeout,
>> remus_next_checkpoint,
>> - dss->interval);
>> + dss->rs.interval);
>>
>> if (rc)
>> goto out;
>> @@ -286,7 +340,7 @@ static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
>> const struct timeval *requested_abs)
>> {
>> libxl__domain_save_state *dss =
>> - CONTAINER_OF(ev, *dss, checkpoint_timeout);
>> + CONTAINER_OF(ev, *dss, rs.checkpoint_timeout);
>>
>> STATE_AO_GC(dss->ao);
>>
>> diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c
>> index 50b897d..a8d8949 100644
>> --- a/tools/libxl/libxl_remus_disk_drbd.c
>> +++ b/tools/libxl/libxl_remus_disk_drbd.c
>> @@ -28,10 +28,11 @@ typedef struct libxl__remus_drbd_disk {
>>
>> int init_subkind_drbd_disk(libxl__checkpoint_devices_state *cds)
>> {
>> + libxl__remus_state *rs = CONTAINER_OF(cds, *rs, cds);
>> STATE_AO_GC(cds->ao);
>>
>> - cds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
>> - libxl__xen_script_dir_path());
>> + rs->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe",
>> + libxl__xen_script_dir_path());
>>
>> return 0;
>> }
>> @@ -96,6 +97,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
>> int arraysize, nr = 0, rc;
>> const libxl_device_disk *disk = dev->backend_dev;
>> libxl__async_exec_state *aes = &dev->aodev.aes;
>> + libxl__remus_state *rs = CONTAINER_OF(dev->cds, *rs, cds);
>> STATE_AO_GC(dev->cds->ao);
>>
>> /* setup env & args */
>> @@ -107,7 +109,7 @@ static void match_async_exec(libxl__egc *egc, libxl__checkpoint_device *dev)
>> arraysize = 3;
>> nr = 0;
>> GCNEW_ARRAY(aes->args, arraysize);
>> - aes->args[nr++] = dev->cds->drbd_probe_script;
>> + aes->args[nr++] = rs->drbd_probe_script;
>> aes->args[nr++] = disk->pdev_path;
>> aes->args[nr++] = NULL;
>> assert(nr <= arraysize);
>
>
> .
>
--
Thanks,
Yang.
^ permalink raw reply [flat|nested] 103+ messages in thread