* [PATCH RFC v2 1/8] Added QMP commands for adding NBD server and disk migration
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 2/8] Modified xl stack to receieve mirror QEMU disk option Bruno Alvisio
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/libxl/libxl_internal.h | 18 +++++++
tools/libxl/libxl_qmp.c | 125 ++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 142 insertions(+), 1 deletion(-)
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 7247509..1349a8f 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1835,6 +1835,24 @@ _hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
/* Start replication */
_hidden int libxl__qmp_start_replication(libxl__gc *gc, int domid,
bool primary);
+
+/* Add a disk to NBD server */
+_hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
+ const char *disk);
+
+/* Mirror disk drive */
+_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* device,
+ const char* target, const char* format);
+
+/* Query block devices */
+_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid, char *device_names);
+
+/* Query existing block jobs*/
+_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid, bool *is_ready);
+
+/* Resume QEMU process started with -incoming defer */
+_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid, const char* uri);
+
/* Get replication error that occurs when the vm is running */
_hidden int libxl__qmp_query_xen_replication_status(libxl__gc *gc, int domid);
/* Do checkpoint */
diff --git a/tools/libxl/libxl_qmp.c b/tools/libxl/libxl_qmp.c
index eab993a..fe6f076 100644
--- a/tools/libxl/libxl_qmp.c
+++ b/tools/libxl/libxl_qmp.c
@@ -347,7 +347,10 @@ static libxl__qmp_handler *qmp_init_handler(libxl__gc *gc, uint32_t domid)
}
qmp->ctx = CTX;
qmp->domid = domid;
- qmp->timeout = 5;
+
+ //TODO: Changed default timeout because drive-mirror command takes a long
+ //TODO: to return. Consider timeout to be passed as param.
+ qmp->timeout = 600;
LIBXL_STAILQ_INIT(&qmp->callback_list);
@@ -1069,6 +1072,126 @@ int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid, const char *disk)
return qmp_run_command(gc, domid, "nbd-server-add", args, NULL, NULL);
}
+int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* device, const char* target, const char* format)
+{
+ libxl__json_object *args = NULL;
+ //TODO: Allow method to receive "sync", "speed", "mode", "granurality", "buf-size"
+ qmp_parameters_add_string(gc, &args, "device", device);
+ qmp_parameters_add_string(gc, &args, "target", target);
+ qmp_parameters_add_string(gc, &args, "sync", "full");
+ qmp_parameters_add_string(gc, &args, "format", format);
+ qmp_parameters_add_string(gc, &args, "mode", "existing");
+ qmp_parameters_add_integer(gc, &args, "granularity", 0);
+ qmp_parameters_add_integer(gc, &args, "buf-size", 0);
+
+ return qmp_run_command(gc, domid, "drive-mirror", args, NULL, NULL);
+}
+
+static int query_block_callback(libxl__qmp_handler *qmp,
+ const libxl__json_object *response,
+ void *opaque)
+{
+ const libxl__json_object *blockinfo = NULL;
+ GC_INIT(qmp->ctx);
+ int i, rc = -1;
+
+ for (i = 0; (blockinfo = libxl__json_array_get(response, i)); i++) {
+ const libxl__json_object *d;
+ const char* device_name;
+ d = libxl__json_map_get("device", blockinfo, JSON_STRING);
+ if(!d){
+ goto out;
+ }
+ device_name = libxl__json_object_get_string(d);
+ }
+
+ rc = 0;
+out:
+ GC_FREE;
+ return rc;
+}
+
+static int query_block_jobs_callback(libxl__qmp_handler *qmp,
+ const libxl__json_object *response,
+ void *opaque)
+{
+ const libxl__json_object *blockjobinfo = NULL;
+ GC_INIT(qmp->ctx);
+ int i, rc = -1;
+ bool empty = true;
+
+ for (i = 0; (blockjobinfo = libxl__json_array_get(response, i)); i++) {
+ empty = false;
+ const char *type;
+ const char *device;
+ unsigned int len;
+ unsigned int offset;
+ bool busy;
+ bool paused;
+ const char *iostatus;
+ bool ready;
+
+ const libxl__json_object *type_o = NULL;
+ const libxl__json_object *device_o = NULL;
+ const libxl__json_object *len_o = NULL;
+ const libxl__json_object *offset_o = NULL;
+ const libxl__json_object *busy_o = NULL;
+ const libxl__json_object *paused_o = NULL;
+ const libxl__json_object *io_status_o = NULL;
+ const libxl__json_object *ready_o = NULL;
+
+ type_o = libxl__json_map_get("type", blockjobinfo, JSON_STRING);
+ device_o = libxl__json_map_get("device", blockjobinfo, JSON_STRING);
+ len_o = libxl__json_map_get("len", blockjobinfo, JSON_INTEGER);
+ offset_o = libxl__json_map_get("offset", blockjobinfo, JSON_INTEGER);
+ busy_o = libxl__json_map_get("busy", blockjobinfo, JSON_BOOL);
+ paused_o = libxl__json_map_get("type", blockjobinfo, JSON_BOOL);
+ io_status_o = libxl__json_map_get("io-status", blockjobinfo, JSON_STRING);
+ ready_o = libxl__json_map_get("ready", blockjobinfo, JSON_BOOL);
+
+ type = libxl__json_object_get_string(type_o);
+ device = libxl__json_object_get_string(device_o);
+ len = libxl__json_object_get_integer(len_o);
+ offset = libxl__json_object_get_integer(offset_o);
+ busy = libxl__json_object_get_bool(len_o);
+ paused = libxl__json_object_get_bool(paused_o);
+ iostatus = libxl__json_object_get_string(io_status_o);
+ ready = libxl__json_object_get_bool(ready_o);
+
+ bool *is_ready = opaque;
+ *is_ready = ready;
+ }
+
+ if(empty){
+ bool *is_ready = opaque;
+ *is_ready = true;
+ }
+
+ rc = 0;
+
+ GC_FREE;
+ return rc;
+}
+
+int libxl__qmp_query_block(libxl__gc *gc, int domid, char *device_names)
+{
+ return qmp_run_command(gc, domid, "query-block", NULL, query_block_callback, device_names);
+}
+
+int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid, bool *is_ready)
+{
+ return qmp_run_command(gc, domid, "query-block-jobs", NULL, query_block_jobs_callback, is_ready);
+}
+
+int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid, const char* uri)
+{
+ libxl__json_object *args = NULL;
+
+ qmp_parameters_add_string(gc, &args, "uri", uri);
+
+ return qmp_run_command(gc, domid, "migrate-incoming", args, NULL, NULL);
+}
+
int libxl__qmp_start_replication(libxl__gc *gc, int domid, bool primary)
{
libxl__json_object *args = NULL;
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 2/8] Modified xl stack to receieve mirror QEMU disk option
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 1/8] Added QMP commands for adding NBD server and disk migration Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 3/8] Adapted libxl to handle migration of instance with qemu based disks Bruno Alvisio
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/ocaml/libs/xl/xenlight_stubs.c | 4 +--
tools/xl/xl.h | 1 +
tools/xl/xl_migrate.c | 47 +++++++++++++++++++++++++++++-------
tools/xl/xl_saverestore.c | 2 +-
tools/xl/xl_vmcontrol.c | 5 ++--
5 files changed, 45 insertions(+), 14 deletions(-)
diff --git a/tools/ocaml/libs/xl/xenlight_stubs.c b/tools/ocaml/libs/xl/xenlight_stubs.c
index 98b52b9..9e19009 100644
--- a/tools/ocaml/libs/xl/xenlight_stubs.c
+++ b/tools/ocaml/libs/xl/xenlight_stubs.c
@@ -538,7 +538,7 @@ value stub_libxl_domain_create_restore(value ctx, value domain_config, value par
caml_enter_blocking_section();
ret = libxl_domain_create_restore(CTX, &c_dconfig, &c_domid, restore_fd,
- -1, &c_params, ao_how, NULL);
+ -1, 0, &c_params, ao_how, NULL);
caml_leave_blocking_section();
free(ao_how);
@@ -611,7 +611,7 @@ value stub_libxl_domain_suspend(value ctx, value domid, value fd, value async, v
libxl_asyncop_how *ao_how = aohow_val(async);
caml_enter_blocking_section();
- ret = libxl_domain_suspend(CTX, c_domid, c_fd, 0, ao_how);
+ ret = libxl_domain_suspend(CTX, c_domid, c_fd, 0, NULL, ao_how);
caml_leave_blocking_section();
free(ao_how);
diff --git a/tools/xl/xl.h b/tools/xl/xl.h
index 01c2af6..070bac1 100644
--- a/tools/xl/xl.h
+++ b/tools/xl/xl.h
@@ -35,6 +35,7 @@ struct domain_create {
int daemonize;
int monitor; /* handle guest reboots etc */
int paused;
+ int mirror_qemu_disk;
int dryrun;
int quiet;
int vnc;
diff --git a/tools/xl/xl_migrate.c b/tools/xl/xl_migrate.c
index 1f0e87d..fee726f 100644
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -177,7 +177,8 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
}
static void migrate_domain(uint32_t domid, const char *rune, int debug,
- const char *override_config_file)
+ const char *override_config_file,
+ int mirror_qemu_disks, const char* hostname)
{
pid_t child = -1;
int rc;
@@ -205,7 +206,9 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
if (debug)
flags |= LIBXL_SUSPEND_DEBUG;
- rc = libxl_domain_suspend(ctx, domid, send_fd, flags, NULL);
+ if(mirror_qemu_disks)
+ flags |= LIBXL_SUSPEND_MIRROR_QEMU_DISKS;
+ rc = libxl_domain_suspend(ctx, domid, send_fd, flags, hostname, NULL);
if (rc) {
fprintf(stderr, "migration sender: libxl_domain_suspend failed"
" (rc=%d)\n", rc);
@@ -316,7 +319,7 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
}
static void migrate_receive(int debug, int daemonize, int monitor,
- int pause_after_migration,
+ int pause_after_migration, int mirror_qemu_disks,
int send_fd, int recv_fd,
libxl_checkpointed_stream checkpointed,
char *colo_proxy_script,
@@ -343,6 +346,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
dom_info.daemonize = daemonize;
dom_info.monitor = monitor;
dom_info.paused = 1;
+ dom_info.mirror_qemu_disks = mirror_qemu_disks;
dom_info.migrate_fd = recv_fd;
dom_info.send_back_fd = send_fd;
dom_info.migration_domname_r = &migration_domname;
@@ -423,6 +427,17 @@ static void migrate_receive(int debug, int daemonize, int monitor,
fprintf(stderr, "migration target: Got permission, starting domain.\n");
+ if(mirror_qemu_disks){
+ fprintf(stderr, "migration target: Stopping NBD server\n");
+ rc = libxl__nbd_server_stop(ctx, domid);
+ if (rc){
+ rc = 0; //For now, mask the error if NBD server fails to stop
+ fprintf(stderr, "Failed to stop NBD server\n");
+ }else{
+ fprintf(stderr, "Stopped NBD server successfully\n");
+ }
+ }
+
if (migration_domname) {
rc = libxl_domain_rename(ctx, domid, migration_domname, common_domname);
if (rc) goto perhaps_destroy_notify_rc;
@@ -478,6 +493,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
int main_migrate_receive(int argc, char **argv)
{
int debug = 0, daemonize = 1, monitor = 1, pause_after_migration = 0;
+ int mirror_qemu_disks = 0;
libxl_checkpointed_stream checkpointed = LIBXL_CHECKPOINTED_STREAM_NONE;
int opt;
bool userspace_colo_proxy = false;
@@ -490,7 +506,7 @@ int main_migrate_receive(int argc, char **argv)
COMMON_LONG_OPTS
};
- SWITCH_FOREACH_OPT(opt, "Fedrp", opts, "migrate-receive", 0) {
+ SWITCH_FOREACH_OPT(opt, "Fedrpq", opts, "migrate-receive", 0) {
case 'F':
daemonize = 0;
break;
@@ -516,6 +532,9 @@ int main_migrate_receive(int argc, char **argv)
case 'p':
pause_after_migration = 1;
break;
+ case 'q':
+ mirror_qemu_disks = 1;
+ break;
}
if (argc-optind != 0) {
@@ -523,7 +542,7 @@ int main_migrate_receive(int argc, char **argv)
return EXIT_FAILURE;
}
migrate_receive(debug, daemonize, monitor, pause_after_migration,
- STDOUT_FILENO, STDIN_FILENO,
+ mirror_qemu_disks, STDOUT_FILENO, STDIN_FILENO,
checkpointed, script, userspace_colo_proxy);
return EXIT_SUCCESS;
@@ -536,14 +555,16 @@ int main_migrate(int argc, char **argv)
const char *ssh_command = "ssh";
char *rune = NULL;
char *host;
+ char *hostname;
int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
+ int mirror_qemu_disks = 0;
static struct option opts[] = {
{"debug", 0, 0, 0x100},
{"live", 0, 0, 0x200},
COMMON_LONG_OPTS
};
- SWITCH_FOREACH_OPT(opt, "FC:s:ep", opts, "migrate", 2) {
+ SWITCH_FOREACH_OPT(opt, "FC:s:epq", opts, "migrate", 2) {
case 'C':
config_filename = optarg;
break;
@@ -560,6 +581,9 @@ int main_migrate(int argc, char **argv)
case 'p':
pause_after_migration = 1;
break;
+ case 'q':
+ mirror_qemu_disks = 1;
+ break;
case 0x100: /* --debug */
debug = 1;
break;
@@ -571,6 +595,9 @@ int main_migrate(int argc, char **argv)
domid = find_domain(argv[optind]);
host = argv[optind + 1];
+ hostname = strchr(host, '@');
+ hostname++;
+
bool pass_tty_arg = progress_use_cr || (isatty(2) > 0);
if (!ssh_command[0]) {
@@ -587,16 +614,18 @@ int main_migrate(int argc, char **argv)
} else {
verbose_len = (minmsglevel_default - minmsglevel) + 2;
}
- xasprintf(&rune, "exec %s %s xl%s%.*s migrate-receive%s%s%s",
+ xasprintf(&rune, "exec %s %s xl%s%.*s migrate-receive%s%s%s%s",
ssh_command, host,
pass_tty_arg ? " -t" : "",
verbose_len, verbose_buf,
daemonize ? "" : " -e",
debug ? " -d" : "",
- pause_after_migration ? " -p" : "");
+ pause_after_migration ? " -p" : "",
+ mirror_qemu_disks ? "-q" :"");
}
- migrate_domain(domid, rune, debug, config_filename);
+ migrate_domain(domid, rune, debug, config_filename, mirror_qemu_disks,
+ hostname);
return EXIT_SUCCESS;
}
diff --git a/tools/xl/xl_saverestore.c b/tools/xl/xl_saverestore.c
index 9afeade..b1679ba 100644
--- a/tools/xl/xl_saverestore.c
+++ b/tools/xl/xl_saverestore.c
@@ -141,7 +141,7 @@ static int save_domain(uint32_t domid, const char *filename, int checkpoint,
save_domain_core_writeconfig(fd, filename, config_data, config_len);
- int rc = libxl_domain_suspend(ctx, domid, fd, 0, NULL);
+ int rc = libxl_domain_suspend(ctx, domid, fd, 0, NULL, NULL);
close(fd);
if (rc < 0) {
diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c
index 89c2b25..47ec546 100644
--- a/tools/xl/xl_vmcontrol.c
+++ b/tools/xl/xl_vmcontrol.c
@@ -882,8 +882,9 @@ start:
ret = libxl_domain_create_restore(ctx, &d_config,
&domid, restore_fd,
- send_back_fd, ¶ms,
- 0, autoconnect_console_how);
+ send_back_fd,
+ dom_info->mirror_qemu_disks,
+ ¶ms, 0, autoconnect_console_how);
libxl_domain_restore_params_dispose(¶ms);
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 3/8] Adapted libxl to handle migration of instance with qemu based disks
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 1/8] Added QMP commands for adding NBD server and disk migration Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 2/8] Modified xl stack to receieve mirror QEMU disk option Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 4/8] Remove stop NBD server command from xl Bruno Alvisio
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/libxl/libxl.h | 11 ++-
tools/libxl/libxl_create.c | 191 ++++++++++++++++++++++++++++++++++++---
tools/libxl/libxl_dm.c | 26 ++++--
tools/libxl/libxl_dom_save.c | 85 ++++++++++++++++-
tools/libxl/libxl_domain.c | 4 +-
tools/libxl/libxl_internal.h | 60 ++++++++++--
tools/libxl/libxl_save_callout.c | 38 +++++---
tools/libxl/libxl_save_helper.c | 4 +-
tools/libxl/libxl_stream_read.c | 17 +++-
tools/libxl/libxl_stream_write.c | 28 +++++-
10 files changed, 406 insertions(+), 58 deletions(-)
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 91408b4..4022c37 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1333,7 +1333,7 @@ int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
LIBXL_EXTERNAL_CALLERS_ONLY;
int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid, int restore_fd,
- int send_back_fd,
+ int send_back_fd, int mirror_qemu_disks,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1373,8 +1373,9 @@ static inline int libxl_domain_create_restore_0x040400(
const libxl_asyncprogress_how *aop_console_how)
LIBXL_EXTERNAL_CALLERS_ONLY
{
+ //TODO: balvisio: Review
return libxl_domain_create_restore(ctx, d_config, domid, restore_fd,
- -1, params, ao_how, aop_console_how);
+ -1, 0, params, ao_how, aop_console_how);
}
#define libxl_domain_create_restore libxl_domain_create_restore_0x040400
@@ -1408,10 +1409,16 @@ int libxl_retrieve_domain_configuration(libxl_ctx *ctx, uint32_t domid,
int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
int flags, /* LIBXL_SUSPEND_* */
+ const char* hostname,
const libxl_asyncop_how *ao_how)
LIBXL_EXTERNAL_CALLERS_ONLY;
#define LIBXL_SUSPEND_DEBUG 1
#define LIBXL_SUSPEND_LIVE 2
+#define LIBXL_SUSPEND_MIRROR_QEMU_DISKS 4
+
+//TODO: balvisio: DO NOT HARD-CODE THIS PARAMS
+#define QEMU_DRIVE_MIRROR_PORT "11000"
+#define QEMU_DRIVE_MIRROR_DEVICE "ide0-hd0"
/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
* If this parameter is true, use co-operative resume. The guest
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 9123585..f10f2ce 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -744,6 +744,10 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
static void domcreate_devmodel_started(libxl__egc *egc,
libxl__dm_spawn_state *dmss,
int rc);
+
+static void start_nbd_server(libxl__egc *egc, libxl__dm_spawn_state *dmss,
+ int ret);
+
static void domcreate_bootloader_console_available(libxl__egc *egc,
libxl__bootloader_state *bl);
static void domcreate_bootloader_done(libxl__egc *egc,
@@ -760,10 +764,17 @@ static void domcreate_stream_done(libxl__egc *egc,
libxl__stream_read_state *srs,
int ret);
+static void domcreate_pre_build(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int ret);
+
static void domcreate_rebuild_done(libxl__egc *egc,
libxl__domain_create_state *dcs,
int ret);
+static void domcreate_multidev_begin(libxl__egc *egc,
+ libxl__domain_create_state *dcs);
+
/* Our own function to clean up and call the user's callback.
* The final call in the sequence. */
static void domcreate_complete(libxl__egc *egc,
@@ -1016,6 +1027,64 @@ static void libxl__colo_restore_setup_done(libxl__egc *egc,
libxl__stream_read_start(egc, &dcs->srs);
}
+static void start_nbd_server(libxl__egc *egc, libxl__dm_spawn_state *dmss,
+ int ret){
+
+ libxl__domain_create_state *dcs = CONTAINER_OF(dmss, *dcs, sdss.dm);
+ STATE_AO_GC(dmss->spawn.ao);
+ const uint32_t domid = dcs->guest_domid;
+ dcs->sdss.dm.guest_domid = domid;
+
+ if (ret) {
+ LOGD(ERROR, domid, "device model did not start: %d", ret);
+ goto error_out;
+ }
+
+ if(dcs->restore_fd >= 0 && dcs->mirror_qemu_disks) {
+ /*
+ * Start and add the NBD server
+ * Host is set it to "::" for now
+ * Port we hard code a port for now
+
+ * This code just handles the case when -M pc is used.
+ * (The config xen_platform_pci = 0)
+ *
+ * TODO: Current implementation only works with upstream qemu
+ * TODO: consider the case when qemu-xen-traditional is used.
+ * TODO: Check and copy only those disks which are local
+ * TODO: Assign port dynamically
+ */
+
+ LOGD(DEBUG, "Starting NBD Server\n");
+ ret = libxl__qmp_nbd_server_start(gc, domid, "::", QEMU_DRIVE_MIRROR_PORT);
+ if (ret) {
+ ret = ERROR_FAIL;
+ LOGD(ERROR, "Failed to start NBD Server\n");
+ goto skip_nbd;
+ }else{
+ LOGD(INFO, "Started NBD Server Successfully\n");
+ }
+
+ ret = libxl__qmp_nbd_server_add(gc, domid, QEMU_DRIVE_MIRROR_DEVICE);
+
+ if (ret) {
+ ret = ERROR_FAIL;
+ LOGD(ERROR, "Failed to add NBD Server\n");
+ goto skip_nbd;
+ } else {
+ LOGD(INFO, "NBD Add Successful\n");
+ }
+ }
+
+skip_nbd:
+ libxl__stream_read_start(egc, &dcs->srs);
+ return;
+
+error_out:
+ assert(ret);
+ domcreate_complete(egc, dcs, ret);
+}
+
static void domcreate_bootloader_done(libxl__egc *egc,
libxl__bootloader_state *bl,
int rc)
@@ -1033,6 +1102,8 @@ static void domcreate_bootloader_done(libxl__egc *egc,
libxl_domain_build_info *const info = &d_config->b_info;
libxl__srm_restore_autogen_callbacks *const callbacks =
&dcs->srs.shs.callbacks.restore.a;
+ libxl__srm_restore_autogen_callbacks *const callbacks_mirror_qemu_disks =
+ &dcs->srs_local_disks.shs.callbacks.restore.a;
if (rc) {
domcreate_rebuild_done(egc, dcs, rc);
@@ -1050,8 +1121,41 @@ static void domcreate_bootloader_done(libxl__egc *egc,
dcs->sdss.dm.spawn.ao = ao;
dcs->sdss.dm.guest_config = dcs->guest_config;
dcs->sdss.dm.build_state = &dcs->build_state;
- dcs->sdss.dm.callback = domcreate_devmodel_started;
- dcs->sdss.callback = domcreate_devmodel_started;
+ if(!dcs->mirror_qemu_disks) {
+ dcs->sdss.dm.callback = domcreate_devmodel_started;
+ dcs->sdss.callback = domcreate_devmodel_started;
+ }else{
+ /*
+ Original calling sequence:
+ 0. domcreate_bootloader_done
+ 1. MEMORY STREAM
+ 2. domcreate_stream_done
+ 3. domcreate_pre_build
+ 4. domcreate_rebuild_done
+ 5. domcreate_multidev_begin
+ 6. domcreate_launch_dm
+ 7. domcreate_devmodel_started
+
+ In the case of local_disk domcreate_devmodel_started will be called:
+ through the following sequence:
+ 0. domcreate_bootloader_done
+ 1. 1st MEMORY STREAM
+ 2. domcreate_stream_done
+ 3. domcreate_pre_build
+ 4. domcreate_rebuild_done
+ 5. domcreate_multidev_begin
+ 6. domcreate_launch_dm
+ 7. start_nbd_server
+ 8. 2nd MEMORY STREAM
+ 9. domcreate_stream_done
+ 9.a. Calls -incoming to start QEMU process
+ 10. domcreate_devmodel_started
+ */
+
+ dcs->sdss.dm.callback = start_nbd_server;
+ dcs->sdss.callback = start_nbd_server;
+ }
+
if (restore_fd < 0 && dcs->domid_soft_reset == INVALID_DOMID) {
rc = libxl__domain_build(gc, d_config, domid, state);
@@ -1061,6 +1165,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
/* Restore */
callbacks->restore_results = libxl__srm_callout_callback_restore_results;
+ callbacks_mirror_qemu_disks->restore_results = libxl__srm_callout_callback_restore_results;
/* COLO only supports HVM now because it does not work very
* well with pv drivers:
@@ -1087,6 +1192,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
dcs->srs.back_channel = false;
dcs->srs.completion_callback = domcreate_stream_done;
+ dcs->srs.mirror_qemu_disks = 0;
if (restore_fd >= 0) {
switch (checkpointed_stream) {
@@ -1104,7 +1210,20 @@ static void domcreate_bootloader_done(libxl__egc *egc,
libxl__remus_restore_setup(egc, dcs);
/* fall through */
case LIBXL_CHECKPOINTED_STREAM_NONE:
- libxl__stream_read_start(egc, &dcs->srs);
+ if (dcs->mirror_qemu_disks) {
+ /* Spawn the local_dm process before reading the memory stream */
+ dcs->srs_mirror_qemu_disks.ao = ao;
+ dcs->srs_mirror_qemu_disks.dcs = dcs;
+ dcs->srs_mirror_qemu_disks.fd = restore_fd;
+ dcs->srs_mirror_qemu_disks.legacy = (dcs->restore_params.stream_version == 1);
+ dcs->srs_mirror_qemu_disks.back_channel = false;
+ dcs->srs_mirror_qemu_disks.completion_callback = domcreate_stream_done;
+ dcs->srs_mirror_qemu_disks.mirror_qemu_disks = 1;
+
+ libxl__stream_read_start(egc, &dcs->srs_mirror_qemu_disks);
+ } else {
+ libxl__stream_read_start(egc, &dcs->srs);
+ }
}
return;
}
@@ -1132,6 +1251,35 @@ static void domcreate_stream_done(libxl__egc *egc,
{
libxl__domain_create_state *dcs = srs->dcs;
STATE_AO_GC(dcs->ao);
+
+ const uint32_t domid = dcs->guest_domid;
+ const char* uri;
+ const char* state_file = GCSPRINTF(
+ LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", domid);
+
+ if(!dcs->mirror_qemu_disks || srs->mirror_qemu_disks){
+ domcreate_pre_build(egc, dcs, ret);
+ }else{
+ //Stop NBD server
+ fprintf(stderr, "Stopping NBD server\n");
+ rc = libxl__qmp_nbd_server_stop(gc, domid);
+ if (rc){
+ fprintf(stderr, "Failed to stop NBD server\n");
+ }else{
+ fprintf(stderr, "Stopped NBD server successfully\n");
+ }
+ uri = GCSPRINTF("exec: /bin/cat %s", (&dcs->sdss.dm)->build_state->saved_state);
+ libxl__qmp_migrate_incoming(gc, domid, uri);
+ domcreate_devmodel_started(egc, &dcs->sdss.dm, 0);
+ }
+ return;
+}
+
+static void domcreate_pre_build(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int ret)
+{
+ STATE_AO_GC(dcs->ao);
libxl_ctx *ctx = libxl__gc_owner(gc);
char **vments = NULL, **localents = NULL;
struct timeval start_time;
@@ -1185,7 +1333,7 @@ static void domcreate_stream_done(libxl__egc *egc,
if (ret)
goto out;
- if (info->type == LIBXL_DOMAIN_TYPE_HVM) {
+ if (info->type == LIBXL_DOMAIN_TYPE_HVM && !dcs->mirror_qemu_disks) {
state->saved_state = GCSPRINTF(
LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", domid);
}
@@ -1223,10 +1371,7 @@ static void domcreate_rebuild_done(libxl__egc *egc,
store_libxl_entry(gc, domid, &d_config->b_info);
- libxl__multidev_begin(ao, &dcs->multidev);
- dcs->multidev.callback = domcreate_launch_dm;
- libxl__add_disks(egc, ao, domid, d_config, &dcs->multidev);
- libxl__multidev_prepared(egc, &dcs->multidev, 0);
+ domcreate_multidev_begin(egc, dcs);
return;
@@ -1235,6 +1380,24 @@ static void domcreate_rebuild_done(libxl__egc *egc,
domcreate_complete(egc, dcs, ret);
}
+static void domcreate_multidev_begin(libxl__egc *egc,
+ libxl__domain_create_state *dcs){
+
+ STATE_AO_GC(dcs->ao);
+
+ /* convenience aliases */
+ const uint32_t domid = dcs->guest_domid;
+ libxl_domain_config *const d_config = dcs->guest_config;
+
+ libxl__multidev_begin(ao, &dcs->multidev);
+ dcs->multidev.callback = domcreate_launch_dm;
+
+ libxl__add_disks(egc, ao, domid, d_config, &dcs->multidev);
+ libxl__multidev_prepared(egc, &dcs->multidev, 0);
+
+ return;
+}
+
static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
int ret)
{
@@ -1355,7 +1518,7 @@ static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
if (libxl_defbool_val(d_config->b_info.device_model_stubdomain))
libxl__spawn_stub_dm(egc, &dcs->sdss);
else
- libxl__spawn_local_dm(egc, &dcs->sdss.dm);
+ libxl__spawn_local_dm(egc, &dcs->sdss.dm, dcs->mirror_qemu_disks);
/*
* Handle the domain's (and the related stubdomain's) access to
@@ -1387,7 +1550,7 @@ static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
goto error_out;
if (ret) {
dcs->sdss.dm.guest_domid = domid;
- libxl__spawn_local_dm(egc, &dcs->sdss.dm);
+ libxl__spawn_local_dm(egc, &dcs->sdss.dm, 0);
return;
} else {
assert(!dcs->sdss.dm.guest_domid);
@@ -1604,6 +1767,7 @@ static void domain_create_cb(libxl__egc *egc,
static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid, int restore_fd, int send_back_fd,
+ int mirror_qemu_disks,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1619,6 +1783,7 @@ static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
libxl_domain_config_copy(ctx, &cdcs->dcs.guest_config_saved, d_config);
cdcs->dcs.restore_fd = cdcs->dcs.libxc_fd = restore_fd;
cdcs->dcs.send_back_fd = send_back_fd;
+ cdcs->dcs.mirror_qemu_disks = mirror_qemu_disks;
if (restore_fd > -1) {
cdcs->dcs.restore_params = *params;
rc = libxl__fd_flags_modify_save(gc, cdcs->dcs.restore_fd,
@@ -1847,13 +2012,13 @@ int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
const libxl_asyncprogress_how *aop_console_how)
{
unset_disk_colo_restore(d_config);
- return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
+ return do_domain_create(ctx, d_config, domid, -1, -1, 0, NULL,
ao_how, aop_console_how);
}
int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid, int restore_fd,
- int send_back_fd,
+ int send_back_fd, int mirror_qemu_disks,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1865,7 +2030,7 @@ int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
}
return do_domain_create(ctx, d_config, domid, restore_fd, send_back_fd,
- params, ao_how, aop_console_how);
+ mirror_qemu_disks, params, ao_how, aop_console_how);
}
int libxl_domain_soft_reset(libxl_ctx *ctx,
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index e0e6a99..ff6721d 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -915,7 +915,7 @@ static int libxl__build_device_model_args_new(libxl__gc *gc,
const libxl_domain_config *guest_config,
char ***args, char ***envs,
const libxl__domain_build_state *state,
- int *dm_state_fd)
+ int *dm_state_fd, int mirror_qemu_disks)
{
const libxl_domain_create_info *c_info = &guest_config->c_info;
const libxl_domain_build_info *b_info = &guest_config->b_info;
@@ -1396,12 +1396,18 @@ static int libxl__build_device_model_args_new(libxl__gc *gc,
}
}
- if (state->saved_state) {
+ if (state->saved_state && !mirror_qemu_disks) {
/* This file descriptor is meant to be used by QEMU */
*dm_state_fd = open(state->saved_state, O_RDONLY);
flexarray_append(dm_args, "-incoming");
flexarray_append(dm_args, GCSPRINTF("fd:%d",*dm_state_fd));
}
+
+ if(mirror_qemu_disks) {
+ flexarray_append(dm_args, "-incoming");
+ flexarray_append(dm_args, "defer");
+ }
+
for (i = 0; b_info->extra && b_info->extra[i] != NULL; i++)
flexarray_append(dm_args, b_info->extra[i]);
@@ -1664,7 +1670,7 @@ static int libxl__build_device_model_args(libxl__gc *gc,
const libxl_domain_config *guest_config,
char ***args, char ***envs,
const libxl__domain_build_state *state,
- int *dm_state_fd)
+ int *dm_state_fd, int mirror_qemu_disks)
/* dm_state_fd may be NULL iff caller knows we are using old stubdom
* and therefore will be passing a filename rather than a fd. */
{
@@ -1680,7 +1686,8 @@ static int libxl__build_device_model_args(libxl__gc *gc,
return libxl__build_device_model_args_new(gc, dm,
guest_domid, guest_config,
args, envs,
- state, dm_state_fd);
+ state, dm_state_fd,
+ mirror_qemu_disks);
default:
LOGED(ERROR, guest_domid, "unknown device model version %d",
guest_config->b_info.device_model_version);
@@ -1897,7 +1904,7 @@ void libxl__spawn_stub_dm(libxl__egc *egc, libxl__stub_dm_spawn_state *sdss)
ret = libxl__build_device_model_args(gc, "stubdom-dm", guest_domid,
guest_config, &args, NULL,
- d_state, NULL);
+ d_state, NULL, 0);
if (ret) {
ret = ERROR_FAIL;
goto out;
@@ -2055,7 +2062,7 @@ static void spawn_stub_launch_dm(libxl__egc *egc,
/* If dom0 qemu not needed, do not launch it */
spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, 0);
} else {
- libxl__spawn_local_dm(egc, &sdss->pvqemu);
+ libxl__spawn_local_dm(egc, &sdss->pvqemu, 0);
}
return;
@@ -2160,7 +2167,8 @@ static void device_model_spawn_outcome(libxl__egc *egc,
libxl__dm_spawn_state *dmss,
int rc);
-void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss)
+void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss,
+ int mirror_qemu_disks)
{
/* convenience aliases */
const int domid = dmss->guest_domid;
@@ -2200,7 +2208,7 @@ void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss)
}
rc = libxl__build_device_model_args(gc, dm, domid, guest_config,
&args, &envs, state,
- &dm_state_fd);
+ &dm_state_fd, mirror_qemu_disks);
if (rc)
goto out;
@@ -2330,7 +2338,7 @@ static void device_model_confirm(libxl__egc *egc, libxl__spawn_state *spawn,
if (!xsdata)
return;
- if (strcmp(xsdata, "running"))
+ if (strcmp(xsdata, "running") && strcmp(xsdata, "inmigrate"))
return;
libxl__spawn_initiate_detach(gc, spawn);
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 77fe30e..a2730f5 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -408,18 +408,97 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
dss->sws.fd = dss->fd;
dss->sws.back_channel = false;
dss->sws.completion_callback = stream_done;
-
- libxl__stream_write_start(egc, &dss->sws);
+ dss->sws.mirror_qemu_disks = 0;
+
+ if(!dss->mirror_qemu_disks) {
+ libxl__stream_write_start(egc, &dss->sws);
+ } else {
+ dss->sws_mirror_qemu_disks.ao = dss->ao;
+ dss->sws_mirror_qemu_disks.dss = dss;
+ dss->sws_mirror_qemu_disks.fd = dss->fd;
+ dss->sws_mirror_qemu_disks.back_channel = false;
+ dss->sws_mirror_qemu_disks.mirror_qemu_disks = 1;
+ dss->sws_mirror_qemu_disks.completion_callback = stream_done;
+ libxl__stream_write_start(egc, &dss->sws_mirror_qemu_disks);
+ }
return;
out:
domain_save_done(egc, dss, rc);
}
+static void mirror_qemu_disks(libxl__egc *egc, libxl__stream_write_state *sws,
+ int rc)
+{
+ int counter = 20;
+ char* target;
+ bool job_is_ready = false;
+ libxl__domain_save_state *dss = sws->dss;
+ const uint32_t domid = dss->domid;
+ STATE_AO_GC(dss->ao);
+
+ if (dss->mirror_qemu_disks) {
+ /*
+ * If the -q was provided, the drive-mirror job is started.
+ * TODO: Move the following code as part of the domain_suspend
+ * TODO: The port should be sent by the destination.
+ */
+start_mirror:
+ LOGD(DEBUG, domid, "Sleeping for a bit so that source can start NBD\n");
+ sleep(30);
+ LOGD(DEBUG, "Starting mirror-drive of device %s\n",
+ QEMU_DRIVE_MIRROR_DEVICE);
+ target = GCSPRINTF("nbd:%s:%s:exportname=%s", dss->hostname,
+ QEMU_DRIVE_MIRROR_PORT, QEMU_DRIVE_MIRROR_DEVICE);
+ rc = libxl__qmp_drive_mirror(gc, dss->domid, QEMU_DRIVE_MIRROR_DEVICE,
+ target, "raw");
+ if (!rc) {
+ LOGD(INFO, domid, "Drive mirror command returned successfully\n");
+ }else{
+ LOGD(ERROR, domid, "Sending drive mirror command failed\n");
+ if(counter > 0){
+ LOGD(INFO, domid, "Counter: %d. Sleeping for 10 sec and retry\n", counter);
+ sleep(10);
+ counter--;
+ goto start_mirror;
+ }else{
+ goto cont;
+ }
+ }
+
+ /*
+ * Query job status until it is ready
+ * TODO: This code is just an inefficient busy wait. QMP sends an
+ * TODO: asynchronous message when mirroring job is completed. Consider
+ * TODO: adding the capability to handle asynchronous QMP messages (already done?)
+ */
+ while(!job_is_ready) {
+ LOGD(INFO, domid, "Checking for drive-mirror job");
+ rc = libxl__qmp_query_block_jobs(gc, dss->domid, &job_is_ready);
+ if(rc){
+ LOGD(ERROR, domid, "Checking block job failed\n");
+ goto cont;
+ }else{
+ LOGD(INFO, domid, "Checking block job succeeded\n");
+ }
+ if(!job_is_ready){
+ LOGD(INFO, domid, "Sleeping 5 sec\n");
+ sleep(5);
+ }
+ }
+ }
+cont:
+ libxl__stream_write_start(egc, &sws->dss->sws);
+}
+
static void stream_done(libxl__egc *egc,
libxl__stream_write_state *sws, int rc)
{
- domain_save_done(egc, sws->dss, rc);
+ if(!sws->dss->mirror_qemu_disks || !sws->mirror_qemu_disks) {
+ domain_save_done(egc, sws->dss, rc);
+ } else {
+ mirror_qemu_disks(egc, sws, rc);
+ }
}
static void domain_save_done(libxl__egc *egc,
diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c
index 08eccd0..76c6d3d 100644
--- a/tools/libxl/libxl_domain.c
+++ b/tools/libxl/libxl_domain.c
@@ -487,7 +487,7 @@ static void domain_suspend_cb(libxl__egc *egc,
}
int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
- const libxl_asyncop_how *ao_how)
+ const char* hostname, const libxl_asyncop_how *ao_how)
{
AO_CREATE(ctx, domid, ao_how);
int rc;
@@ -509,6 +509,8 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
dss->type = type;
dss->live = flags & LIBXL_SUSPEND_LIVE;
dss->debug = flags & LIBXL_SUSPEND_DEBUG;
+ dss->mirror_qemu_disks = flags & LIBXL_SUSPEND_MIRROR_QEMU_DISKS;
+ dss->hostname = hostname;
dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
rc = libxl__fd_flags_modify_save(gc, dss->fd,
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 1349a8f..16a476f 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1840,19 +1840,42 @@ _hidden int libxl__qmp_start_replication(libxl__gc *gc, int domid,
_hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
const char *disk);
+/* Add a disk to NBD server */
+_hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
+ const char *disk);
+/* Start replication */
+_hidden int libxl__qmp_start_replication(libxl__gc *gc, int domid,
+ bool primary);
+/* Add a disk to NBD server */
+ _hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
+ const char *disk);
/* Mirror disk drive */
-_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* device,
- const char* target, const char* format);
-
+_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid,
+ const char* device, const char* target,
+ const char* format);
/* Query block devices */
-_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid, char *device_names);
-
+_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid,
+ char *device_names);
/* Query existing block jobs*/
-_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid, bool *is_ready);
-
+_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid,
+ bool *is_ready);
/* Resume QEMU process started with -incoming defer */
-_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid, const char* uri);
+_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid,
+ const char* uri);
+/* Mirror disk drive */
+_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid,
+ const char* device, const char* target,
+ const char* format);
+/* Query block devices */
+_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid,
+ char *device_names);
+/* Query existing block jobs*/
+_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid,
+ bool *is_ready);
+/* Resume QEMU process started with -incoming defer */
+_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid,
+ const char* uri);
/* Get replication error that occurs when the vm is running */
_hidden int libxl__qmp_query_xen_replication_status(libxl__gc *gc, int domid);
/* Do checkpoint */
@@ -3142,6 +3165,7 @@ struct libxl__stream_read_state {
int fd;
bool legacy;
bool back_channel;
+ int mirror_qemu_disks;
void (*completion_callback)(libxl__egc *egc,
libxl__stream_read_state *srs,
int rc);
@@ -3220,6 +3244,7 @@ struct libxl__stream_write_state {
libxl__domain_save_state *dss;
int fd;
bool back_channel;
+ int mirror_qemu_disks;
void (*completion_callback)(libxl__egc *egc,
libxl__stream_write_state *sws,
int rc);
@@ -3319,6 +3344,8 @@ struct libxl__domain_save_state {
libxl_domain_type type;
int live;
int debug;
+ int mirror_qemu_disks;
+ const char* hostname;
int checkpointed_stream;
const libxl_domain_remus_info *remus;
/* private */
@@ -3334,6 +3361,7 @@ struct libxl__domain_save_state {
};
libxl__checkpoint_devices_state cds;
libxl__stream_write_state sws;
+ libxl__stream_write_state sws_mirror_qemu_disks;
libxl__logdirty_switch logdirty;
};
@@ -3668,6 +3696,7 @@ struct libxl__dm_spawn_state {
libxl__spawn_state spawn;
/* filled in by user, must remain valid: */
uint32_t guest_domid; /* domain being served */
+ int mirror_qemu_disks;
libxl_domain_config *guest_config;
libxl__domain_build_state *build_state; /* relates to guest_domid */
libxl__dm_spawn_cb *callback;
@@ -3713,6 +3742,7 @@ struct libxl__domain_create_state {
int restore_fd, libxc_fd;
int restore_fdfl; /* original flags of restore_fd */
int send_back_fd;
+ int mirror_qemu_disks;
libxl_domain_restore_params restore_params;
uint32_t domid_soft_reset;
libxl__domain_create_cb *callback;
@@ -3729,6 +3759,7 @@ struct libxl__domain_create_state {
/* If we're not doing stubdom, we use only dmss.dm,
* for the non-stubdom device model. */
libxl__stream_read_state srs;
+ libxl__stream_read_state srs_local_disks;
/* necessary if the domain creation failed and we have to destroy it */
libxl__domain_destroy_state dds;
libxl__multidev multidev;
@@ -3748,13 +3779,17 @@ _hidden void libxl__domain_save(libxl__egc *egc,
/* calls libxl__xc_domain_suspend_done when done */
_hidden void libxl__xc_domain_save(libxl__egc *egc,
libxl__domain_save_state *dss,
- libxl__save_helper_state *shs);
+ libxl__save_helper_state *shs,
+ int mirror_qemu_disks);
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
_hidden void libxl__xc_domain_save_done(libxl__egc*, void *dss_void,
int rc, int retval, int errnoval);
+_hidden void libxl__xc_mirror_disks_save_done(libxl__egc *egc, void *dss_void,
+ int rc, int retval, int errnoval);
+
/* Used by asynchronous callbacks: ie ones which xc regards as
* returning a value, but which we want to handle asynchronously.
* Such functions' actual callback function return void in libxl
@@ -3778,13 +3813,18 @@ _hidden int libxl__restore_emulator_xenstore_data
_hidden void libxl__xc_domain_restore(libxl__egc *egc,
libxl__domain_create_state *dcs,
libxl__save_helper_state *shs,
- int hvm, int pae);
+ int hvm, int pae, int mirror_qemu_disks);
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
_hidden void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
int rc, int retval, int errnoval);
+_hidden void libxl__xc_mirror_disks_restore_done(libxl__egc *egc,
+ void *dcs_void,
+ int rc, int retval,
+ int errnoval);
+
_hidden void libxl__save_helper_init(libxl__save_helper_state *shs);
_hidden void libxl__save_helper_abort(libxl__egc *egc,
libxl__save_helper_state *shs);
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 891c669..290d91d 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -43,7 +43,7 @@ static void helper_done(libxl__egc *egc, libxl__save_helper_state *shs);
void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
libxl__save_helper_state *shs,
- int hvm, int pae)
+ int hvm, int pae, int mirror_qemu_disks)
{
STATE_AO_GC(dcs->ao);
@@ -63,16 +63,22 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
state->console_domid,
hvm, pae,
cbflags, dcs->restore_params.checkpointed_stream,
+ mirror_qemu_disks,
};
shs->ao = ao;
shs->domid = domid;
shs->recv_callback = libxl__srm_callout_received_restore;
if (dcs->restore_params.checkpointed_stream ==
- LIBXL_CHECKPOINTED_STREAM_COLO)
+ LIBXL_CHECKPOINTED_STREAM_COLO) {
shs->completion_callback = libxl__colo_restore_teardown;
- else
- shs->completion_callback = libxl__xc_domain_restore_done;
+ } else {
+ if(!mirror_qemu_disks){
+ shs->completion_callback = libxl__xc_domain_restore_done;
+ } else {
+ shs->completion_callback = libxl__xc_hvm_params_restore_done;
+ }
+ }
shs->caller_state = dcs;
shs->need_results = 1;
@@ -81,7 +87,7 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
}
void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss,
- libxl__save_helper_state *shs)
+ libxl__save_helper_state *shs, int mirror_qemu_disks)
{
STATE_AO_GC(dss->ao);
@@ -90,13 +96,17 @@ void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss,
const unsigned long argnums[] = {
dss->domid, 0, 0, dss->xcflags, dss->hvm,
- cbflags, dss->checkpointed_stream,
+ cbflags, dss->checkpointed_stream, mirror_qemu_disks,
};
shs->ao = ao;
shs->domid = dss->domid;
shs->recv_callback = libxl__srm_callout_received_save;
- shs->completion_callback = libxl__xc_domain_save_done;
+ if(!mirror_qemu_disks){
+ shs->completion_callback = libxl__xc_domain_save_done;
+ }else{
+ shs->completion_callback = libxl__xc_hvm_params_save_done;
+ }
shs->caller_state = dss;
shs->need_results = 0;
@@ -336,13 +346,13 @@ static void helper_exited(libxl__egc *egc, libxl__ev_child *ch,
if (!shs->rc)
shs->rc = ERROR_FAIL;
}
-
- if (shs->need_results) {
- if (!shs->rc) {
- LOGD(ERROR,shs->domid,"%s exited without providing results",what);
- shs->rc = ERROR_FAIL;
- }
- }
+// balvisio: TODO: FIX
+// if (shs->need_results) {
+// if (!shs->rc) {
+// LOGD(ERROR,shs->domid,"%s exited without providing results",what);
+// shs->rc = ERROR_FAIL;
+// }
+// }
if (!shs->completed) {
if (!shs->rc) {
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
index 1dece23..cfc9279 100644
--- a/tools/libxl/libxl_save_helper.c
+++ b/tools/libxl/libxl_save_helper.c
@@ -257,6 +257,7 @@ int main(int argc, char **argv)
int hvm = atoi(NEXTARG);
unsigned cbflags = strtoul(NEXTARG,0,10);
xc_migration_stream_t stream_type = strtoul(NEXTARG,0,10);
+ int mirror_qemu_disks = atoi(NEXTARG);
assert(!*++argv);
helper_setcallbacks_save(&helper_save_callbacks, cbflags);
@@ -266,7 +267,7 @@ int main(int argc, char **argv)
r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, flags,
&helper_save_callbacks, hvm, stream_type,
- recv_fd);
+ recv_fd, mirror_qemu_disks);
complete(r);
} else if (!strcmp(mode,"--restore-domain")) {
@@ -282,6 +283,7 @@ int main(int argc, char **argv)
unsigned int pae = strtoul(NEXTARG,0,10);
unsigned cbflags = strtoul(NEXTARG,0,10);
xc_migration_stream_t stream_type = strtoul(NEXTARG,0,10);
+ int mirror_qemu_disks = atoi(NEXTARG);
assert(!*++argv);
helper_setcallbacks_restore(&helper_restore_callbacks, cbflags);
diff --git a/tools/libxl/libxl_stream_read.c b/tools/libxl/libxl_stream_read.c
index 4838750..ea5ffd3 100644
--- a/tools/libxl/libxl_stream_read.c
+++ b/tools/libxl/libxl_stream_read.c
@@ -561,6 +561,7 @@ static bool process_record(libxl__egc *egc,
{
STATE_AO_GC(stream->ao);
libxl__domain_create_state *dcs = stream->dcs;
+ int create_mirror_qemu_disks = stream->dcs->mirror_qemu_disks;
libxl__sr_record_buf *rec;
libxl_sr_checkpoint_state *srcs;
bool further_action_needed = false;
@@ -580,7 +581,9 @@ static bool process_record(libxl__egc *egc,
break;
case REC_TYPE_LIBXC_CONTEXT:
- libxl__xc_domain_restore(egc, dcs, &stream->shs, 0, 0);
+ libxl__xc_domain_restore(egc, dcs, &stream->shs, 0, 0,
+ stream->mirror_qemu_disks +
+ create_mirror_qemu_disks);
break;
case REC_TYPE_EMULATOR_XENSTORE_DATA:
@@ -835,6 +838,18 @@ static void stream_done(libxl__egc *egc,
}
}
+void libxl__xc_mirror_disks_restore_done(libxl__egc *egc, void *dcs_void,
+ int rc, int retval, int errnoval)
+{
+ libxl__domain_create_state *dcs = dcs_void;
+ libxl__stream_read_state *stream = &dcs->srs_mirror_qemu_disks;
+ STATE_AO_GC(dcs->ao);
+
+ check_all_finished(egc, stream, rc);
+
+ stream_continue(egc, stream);
+}
+
void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
int rc, int retval, int errnoval)
{
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index c96a6a2..ad312ed 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -335,7 +335,24 @@ static void stream_header_done(libxl__egc *egc,
static void libxc_header_done(libxl__egc *egc,
libxl__stream_write_state *stream)
{
- libxl__xc_domain_save(egc, stream->dss, &stream->shs);
+ int save_mirror_qemu_disks = stream->dss->local_disks;
+ libxl__xc_domain_save(egc, stream->dss, &stream->shs,
+ save_mirror_qemu_disks + stream->mirror_qemu_disks);
+}
+
+void libxl__xc_mirror_disks_save_done(libxl__egc *egc, void *dss_void,
+ int rc, int retval, int errnoval)
+{
+ libxl__domain_save_state *dss = dss_void;
+ libxl__stream_write_state *stream = &dss->sws_mirror_qemu_disks;
+ STATE_AO_GC(dss->ao);
+
+ check_all_finished(egc, stream, rc);
+
+ if (stream->in_checkpoint)
+ write_checkpoint_end_record(egc, stream);
+ else
+ write_emulator_xenstore_record(egc, stream);
}
void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
@@ -429,9 +446,12 @@ static void emulator_xenstore_record_done(libxl__egc *egc,
{
libxl__domain_save_state *dss = stream->dss;
- if (dss->type == LIBXL_DOMAIN_TYPE_HVM)
- write_emulator_context_record(egc, stream);
- else {
+ if (dss->type == LIBXL_DOMAIN_TYPE_HVM) {
+ if(!stream->mirror_qemu_disks)
+ write_emulator_context_record(egc, stream);
+ else
+ write_end_record(egc, stream);
+ } else {
if (stream->in_checkpoint)
write_checkpoint_end_record(egc, stream);
else
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 4/8] Remove stop NBD server command from xl
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
` (2 preceding siblings ...)
2017-10-18 17:26 ` [PATCH RFC v2 3/8] Adapted libxl to handle migration of instance with qemu based disks Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 5/8] Improved migration flow syntax in libxl Bruno Alvisio
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/xl/xl_migrate.c | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/tools/xl/xl_migrate.c b/tools/xl/xl_migrate.c
index fee726f..9f43d96 100644
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -427,17 +427,6 @@ static void migrate_receive(int debug, int daemonize, int monitor,
fprintf(stderr, "migration target: Got permission, starting domain.\n");
- if(mirror_qemu_disks){
- fprintf(stderr, "migration target: Stopping NBD server\n");
- rc = libxl__nbd_server_stop(ctx, domid);
- if (rc){
- rc = 0; //For now, mask the error if NBD server fails to stop
- fprintf(stderr, "Failed to stop NBD server\n");
- }else{
- fprintf(stderr, "Stopped NBD server successfully\n");
- }
- }
-
if (migration_domname) {
rc = libxl_domain_rename(ctx, domid, migration_domname, common_domname);
if (rc) goto perhaps_destroy_notify_rc;
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 5/8] Improved migration flow syntax in libxl
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
` (3 preceding siblings ...)
2017-10-18 17:26 ` [PATCH RFC v2 4/8] Remove stop NBD server command from xl Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 6/8] Adapted libxc for migration of local disk Bruno Alvisio
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/libxl/libxl_create.c | 5 +++--
tools/libxl/libxl_dm.c | 10 +++++-----
tools/libxl/libxl_internal.h | 4 ++--
tools/libxl/libxl_save_callout.c | 41 +++++++++++++++++++++-------------------
tools/libxl/libxl_save_helper.c | 15 ++++++++++-----
tools/libxl/libxl_types.idl | 6 ++++++
6 files changed, 48 insertions(+), 33 deletions(-)
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index f10f2ce..6df2754 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1121,6 +1121,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
dcs->sdss.dm.spawn.ao = ao;
dcs->sdss.dm.guest_config = dcs->guest_config;
dcs->sdss.dm.build_state = &dcs->build_state;
+ dcs->sdss.dm.mirror_qemu_disks = dcs->mirror_qemu_disks;
if(!dcs->mirror_qemu_disks) {
dcs->sdss.dm.callback = domcreate_devmodel_started;
dcs->sdss.callback = domcreate_devmodel_started;
@@ -1518,7 +1519,7 @@ static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
if (libxl_defbool_val(d_config->b_info.device_model_stubdomain))
libxl__spawn_stub_dm(egc, &dcs->sdss);
else
- libxl__spawn_local_dm(egc, &dcs->sdss.dm, dcs->mirror_qemu_disks);
+ libxl__spawn_local_dm(egc, &dcs->sdss.dm);
/*
* Handle the domain's (and the related stubdomain's) access to
@@ -1550,7 +1551,7 @@ static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
goto error_out;
if (ret) {
dcs->sdss.dm.guest_domid = domid;
- libxl__spawn_local_dm(egc, &dcs->sdss.dm, 0);
+ libxl__spawn_local_dm(egc, &dcs->sdss.dm);
return;
} else {
assert(!dcs->sdss.dm.guest_domid);
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index ff6721d..001e14e 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -1396,7 +1396,7 @@ static int libxl__build_device_model_args_new(libxl__gc *gc,
}
}
- if (state->saved_state && !mirror_qemu_disks) {
+ if (state->saved_state) {
/* This file descriptor is meant to be used by QEMU */
*dm_state_fd = open(state->saved_state, O_RDONLY);
flexarray_append(dm_args, "-incoming");
@@ -2062,7 +2062,7 @@ static void spawn_stub_launch_dm(libxl__egc *egc,
/* If dom0 qemu not needed, do not launch it */
spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, 0);
} else {
- libxl__spawn_local_dm(egc, &sdss->pvqemu, 0);
+ libxl__spawn_local_dm(egc, &sdss->pvqemu);
}
return;
@@ -2167,8 +2167,7 @@ static void device_model_spawn_outcome(libxl__egc *egc,
libxl__dm_spawn_state *dmss,
int rc);
-void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss,
- int mirror_qemu_disks)
+void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss)
{
/* convenience aliases */
const int domid = dmss->guest_domid;
@@ -2208,7 +2207,8 @@ void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss,
}
rc = libxl__build_device_model_args(gc, dm, domid, guest_config,
&args, &envs, state,
- &dm_state_fd, mirror_qemu_disks);
+ &dm_state_fd,
+ dmss->mirror_qemu_disks);
if (rc)
goto out;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 16a476f..30862c6 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3780,7 +3780,7 @@ _hidden void libxl__domain_save(libxl__egc *egc,
_hidden void libxl__xc_domain_save(libxl__egc *egc,
libxl__domain_save_state *dss,
libxl__save_helper_state *shs,
- int mirror_qemu_disks);
+ int migration_phase);
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
@@ -3813,7 +3813,7 @@ _hidden int libxl__restore_emulator_xenstore_data
_hidden void libxl__xc_domain_restore(libxl__egc *egc,
libxl__domain_create_state *dcs,
libxl__save_helper_state *shs,
- int hvm, int pae, int mirror_qemu_disks);
+ int hvm, int pae, int migration_phase);
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 290d91d..48f96d8 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -43,7 +43,7 @@ static void helper_done(libxl__egc *egc, libxl__save_helper_state *shs);
void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
libxl__save_helper_state *shs,
- int hvm, int pae, int mirror_qemu_disks)
+ int hvm, int pae, int migration_phase)
{
STATE_AO_GC(dcs->ao);
@@ -62,8 +62,7 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
state->store_domid, state->console_port,
state->console_domid,
hvm, pae,
- cbflags, dcs->restore_params.checkpointed_stream,
- mirror_qemu_disks,
+ cbflags, dcs->restore_params.checkpointed_stream
};
shs->ao = ao;
@@ -73,21 +72,25 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
LIBXL_CHECKPOINTED_STREAM_COLO) {
shs->completion_callback = libxl__colo_restore_teardown;
} else {
- if(!mirror_qemu_disks){
+ if ( migration_phase != LIBXL_MIGRATION_PHASE_MIRROR_DISK ) {
shs->completion_callback = libxl__xc_domain_restore_done;
+ if( local_disks == LIBXL_MIGRATION_PHASE_VIRTUAL_RAM )
+ shs->need_results = 0;
+ else
+ shs->need_results = 1;
} else {
- shs->completion_callback = libxl__xc_hvm_params_restore_done;
+ shs->completion_callback = libxl__xc_mirror_disks_restore_done;
+ shs->need_results = 1;
}
}
shs->caller_state = dcs;
- shs->need_results = 1;
run_helper(egc, shs, "--restore-domain", restore_fd, send_back_fd, 0, 0,
argnums, ARRAY_SIZE(argnums));
}
void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss,
- libxl__save_helper_state *shs, int mirror_qemu_disks)
+ libxl__save_helper_state *shs, int migration_phase)
{
STATE_AO_GC(dss->ao);
@@ -96,17 +99,17 @@ void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss,
const unsigned long argnums[] = {
dss->domid, 0, 0, dss->xcflags, dss->hvm,
- cbflags, dss->checkpointed_stream, mirror_qemu_disks,
+ cbflags, dss->checkpointed_stream, migration_phase,
};
shs->ao = ao;
shs->domid = dss->domid;
shs->recv_callback = libxl__srm_callout_received_save;
- if(!mirror_qemu_disks){
+ if ( migration_phase != LIBXL_MIGRATION_PHASE_MIRROR_DISK )
shs->completion_callback = libxl__xc_domain_save_done;
- }else{
- shs->completion_callback = libxl__xc_hvm_params_save_done;
- }
+ else
+ shs->completion_callback = libxl__xc_mirror_disks_save_done;
+
shs->caller_state = dss;
shs->need_results = 0;
@@ -346,13 +349,13 @@ static void helper_exited(libxl__egc *egc, libxl__ev_child *ch,
if (!shs->rc)
shs->rc = ERROR_FAIL;
}
-// balvisio: TODO: FIX
-// if (shs->need_results) {
-// if (!shs->rc) {
-// LOGD(ERROR,shs->domid,"%s exited without providing results",what);
-// shs->rc = ERROR_FAIL;
-// }
-// }
+
+ if (shs->need_results) {
+ if (!shs->rc) {
+ LOGD(ERROR,shs->domid,"%s exited without providing results",what);
+ shs->rc = ERROR_FAIL;
+ }
+ }
if (!shs->completed) {
if (!shs->rc) {
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
index cfc9279..9a57b28 100644
--- a/tools/libxl/libxl_save_helper.c
+++ b/tools/libxl/libxl_save_helper.c
@@ -257,7 +257,7 @@ int main(int argc, char **argv)
int hvm = atoi(NEXTARG);
unsigned cbflags = strtoul(NEXTARG,0,10);
xc_migration_stream_t stream_type = strtoul(NEXTARG,0,10);
- int mirror_qemu_disks = atoi(NEXTARG);
+ int migration_phase = atoi(NEXTARG);
assert(!*++argv);
helper_setcallbacks_save(&helper_save_callbacks, cbflags);
@@ -267,7 +267,7 @@ int main(int argc, char **argv)
r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, flags,
&helper_save_callbacks, hvm, stream_type,
- recv_fd, mirror_qemu_disks);
+ recv_fd, migration_phase);
complete(r);
} else if (!strcmp(mode,"--restore-domain")) {
@@ -283,7 +283,7 @@ int main(int argc, char **argv)
unsigned int pae = strtoul(NEXTARG,0,10);
unsigned cbflags = strtoul(NEXTARG,0,10);
xc_migration_stream_t stream_type = strtoul(NEXTARG,0,10);
- int mirror_qemu_disks = atoi(NEXTARG);
+ int migration_phase = atoi(NEXTARG);
assert(!*++argv);
helper_setcallbacks_restore(&helper_restore_callbacks, cbflags);
@@ -298,8 +298,13 @@ int main(int argc, char **argv)
store_domid, console_evtchn, &console_mfn,
console_domid, hvm, pae,
stream_type,
- &helper_restore_callbacks, send_back_fd);
- helper_stub_restore_results(store_mfn,console_mfn,0);
+ &helper_restore_callbacks, send_back_fd,
+ migration_phase);
+
+ if (migration_phase == LIBXL_MIGRATION_PHASE_NON_LOCAL_DISK ||
+ migration_phase == LIBXL_MIGRATION_PHASE_MIRROR_DISK) {
+ helper_stub_restore_results(store_mfn,console_mfn,0);
+ }
complete(r);
} else {
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 173d70a..de5f536 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -240,6 +240,12 @@ libxl_checkpointed_stream = Enumeration("checkpointed_stream", [
(2, "COLO"),
])
+libxl_stream_phase = Enumeration("migration_phase", [
+ (0, "NON_LOCAL_DISK"),
+ (1, "VIRTUAL_RAM"),
+ (2, "MIRROR_DISK"),
+ ])
+
#
# Complex libxl types
#
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 6/8] Adapted libxc for migration of local disk
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
` (4 preceding siblings ...)
2017-10-18 17:26 ` [PATCH RFC v2 5/8] Improved migration flow syntax in libxl Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 7/8] Fixed bugs in the migration flow Bruno Alvisio
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/libxc/include/xenguest.h | 6 +-
tools/libxc/xc_nomigrate.c | 6 +-
tools/libxc/xc_sr_common.h | 3 +
tools/libxc/xc_sr_restore.c | 14 +++--
tools/libxc/xc_sr_save.c | 118 +++++++++++++++++++++++++++++++++++++-
tools/libxc/xc_sr_save_x86_hvm.c | 7 ++-
tools/libxc/xc_sr_stream_format.h | 4 ++
7 files changed, 144 insertions(+), 14 deletions(-)
diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
index 5cd8111..a6f52f1 100644
--- a/tools/libxc/include/xenguest.h
+++ b/tools/libxc/include/xenguest.h
@@ -103,7 +103,8 @@ typedef enum {
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */,
struct save_callbacks* callbacks, int hvm,
- xc_migration_stream_t stream_type, int recv_fd);
+ xc_migration_stream_t stream_type, int recv_fd,
+ int migration_phase);
/* callbacks provided by xc_domain_restore */
struct restore_callbacks {
@@ -168,7 +169,8 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_mfn, domid_t console_domid,
unsigned int hvm, unsigned int pae,
xc_migration_stream_t stream_type,
- struct restore_callbacks *callbacks, int send_back_fd);
+ struct restore_callbacks *callbacks, int send_back_fd,
+ int migration_phase);
/**
* This function will create a domain for a paravirtualized Linux
diff --git a/tools/libxc/xc_nomigrate.c b/tools/libxc/xc_nomigrate.c
index 317c8ce..c75411b 100644
--- a/tools/libxc/xc_nomigrate.c
+++ b/tools/libxc/xc_nomigrate.c
@@ -23,7 +23,8 @@
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
uint32_t max_factor, uint32_t flags,
struct save_callbacks* callbacks, int hvm,
- xc_migration_stream_t stream_type, int recv_fd)
+ xc_migration_stream_t stream_type, int recv_fd,
+ int migration_phase)
{
errno = ENOSYS;
return -1;
@@ -35,7 +36,8 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_mfn, domid_t console_domid,
unsigned int hvm, unsigned int pae,
xc_migration_stream_t stream_type,
- struct restore_callbacks *callbacks, int send_back_fd)
+ struct restore_callbacks *callbacks, int send_back_fd,
+ int migration_phase)
{
errno = ENOSYS;
return -1;
diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
index a83f22a..903f18a 100644
--- a/tools/libxc/xc_sr_common.h
+++ b/tools/libxc/xc_sr_common.h
@@ -96,6 +96,8 @@ struct xc_sr_save_ops
* after a successful save, or upon encountering an error.
*/
int (*cleanup)(struct xc_sr_context *ctx);
+
+ int (*local_disks)(struct xc_sr_context *ctx);
};
@@ -177,6 +179,7 @@ struct xc_sr_context
xc_interface *xch;
uint32_t domid;
int fd;
+ int migration_phase;
xc_dominfo_t dominfo;
diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
index a016678..13e6abc 100644
--- a/tools/libxc/xc_sr_restore.c
+++ b/tools/libxc/xc_sr_restore.c
@@ -799,11 +799,13 @@ static int restore(struct xc_sr_context *ctx)
* With Remus, if we reach here, there must be some error on primary,
* failover from the last checkpoint state.
*/
- rc = ctx->restore.ops.stream_complete(ctx);
- if ( rc )
- goto err;
+ if ( !ctx->migration_phase != MIGRATION_PHASE_MIRROR_DISK ) {
+ rc = ctx->restore.ops.stream_complete(ctx);
+ if ( rc )
+ goto err;
- IPRINTF("Restore successful");
+ IPRINTF("Restore successful");
+ }
goto done;
err:
@@ -829,13 +831,15 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
unsigned long *console_gfn, domid_t console_domid,
unsigned int hvm, unsigned int pae,
xc_migration_stream_t stream_type,
- struct restore_callbacks *callbacks, int send_back_fd)
+ struct restore_callbacks *callbacks, int send_back_fd,
+ int migration_phase)
{
xen_pfn_t nr_pfns;
struct xc_sr_context ctx =
{
.xch = xch,
.fd = io_fd,
+ .migration_phase = migration_phase
};
/* GCC 4.4 (of CentOS 6.x vintage) can' t initialise anonymous unions. */
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index ca6913b..181a0c8 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -412,6 +412,96 @@ static int send_all_pages(struct xc_sr_context *ctx)
return send_dirty_pages(ctx, ctx->save.p2m_size);
}
+static void clear_virtual_devices_memory(struct xc_sr_context *ctx)
+{
+ xen_pfn_t p;
+ DECLARE_HYPERCALL_BUFFER_SHADOW(unsigned long, dirty_bitmap,
+ &ctx->save.dirty_bitmap_hbuf);
+
+ for ( p = 0x7800; p < 0xfeff2; p++ ){
+ if ( test_bit(p, dirty_bitmap) ){
+ clear_bit(p, dirty_bitmap);
+ }
+ }
+ return;
+}
+
+static int send_virtual_ram(struct xc_sr_context *ctx)
+{
+ DECLARE_HYPERCALL_BUFFER_SHADOW(unsigned long, dirty_bitmap,
+ &ctx->save.dirty_bitmap_hbuf);
+
+ bitmap_set(dirty_bitmap, ctx->save.p2m_size);
+
+ /*
+ * On the second stream of a migration with local disk,
+ * don't send the vfb, virtual devices. Only virtual RAM
+ */
+ clear_virtual_devices_memory(ctx);
+
+ return send_dirty_pages(ctx, ctx->save.p2m_size);
+}
+
+static int send_specific_pages(struct xc_sr_context *ctx, uint64_t value)
+{
+
+ int rc = 0;
+ DECLARE_HYPERCALL_BUFFER_SHADOW(unsigned long, dirty_bitmap,
+ &ctx->save.dirty_bitmap_hbuf);
+
+ bitmap_clear(dirty_bitmap, ctx->save.p2m_size);
+ set_bit(value, dirty_bitmap);
+
+ rc = send_dirty_pages(ctx, 1);
+ bitmap_clear(dirty_bitmap, ctx->save.p2m_size);
+ return rc;
+
+}
+
+static int send_virtual_devices_and_params(struct xc_sr_context *ctx)
+{
+ xc_interface *xch = ctx->xch;
+ uint64_t i = 0;
+ int rc = 0;
+
+ fprintf(stderr, "BRUNO: SEND VIRTUAL DEVICES AND PARAMS\n");
+ xc_set_progress_prefix(xch, "Frames");
+
+ //FOR RTL AND VGA IN 128MB VM . Might change on size of VM
+ for( i = 0x8000; i < 0x8050; i++ )
+ {
+ rc = send_specific_pages(ctx, i);
+ if( rc )
+ goto out;
+ }
+ //VGA
+ for( i = 0xf0000; i < 0xf0800; i++ )
+ {
+ rc = send_specific_pages(ctx, i);
+ if( rc )
+ goto out;
+ }
+
+ //Virtual Device
+ for( i = 0xfc000; i < 0xfc00b; i++ )
+ {
+ rc = send_specific_pages(ctx, i);
+ if( rc )
+ goto out;
+ }
+
+ for( i = 0xfeff2; i < 0xff000; i++ )
+ {
+ rc = send_specific_pages(ctx, i);
+ if( rc )
+ goto out;
+ }
+
+ rc = ctx->save.ops.local_disks(ctx);
+ out:
+ return rc;
+}
+
static int enable_logdirty(struct xc_sr_context *ctx)
{
xc_interface *xch = ctx->xch;
@@ -481,7 +571,11 @@ static int send_memory_live(struct xc_sr_context *ctx)
if ( rc )
goto out;
- rc = send_all_pages(ctx);
+ if ( !ctx->migration_phase )
+ rc = send_all_pages(ctx);
+ else
+ rc = send_virtual_ram(ctx);
+
if ( rc )
goto out;
@@ -499,6 +593,9 @@ static int send_memory_live(struct xc_sr_context *ctx)
goto out;
}
+ if ( ctx->migration_phase )
+ clear_virtual_devices_memory(ctx);
+
if ( stats.dirty_count == 0 )
break;
@@ -620,6 +717,9 @@ static int suspend_and_send_dirty(struct xc_sr_context *ctx)
}
}
+ if ( ctx->migration_phase )
+ clear_virtual_devices_memory(ctx);
+
rc = send_dirty_pages(ctx, stats.dirty_count + ctx->save.nr_deferred_pages);
if ( rc )
goto out;
@@ -805,6 +905,14 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
if ( rc )
goto err;
+ /* First pass of QEMU disk migration */
+ if ( ctx->migration_phase == MIGRATION_PHASE_MIRROR_DISK ) {
+ rc = send_virtual_devices_and_params(ctx);
+ if ( rc )
+ goto err;
+ goto end;
+ }
+
rc = ctx->save.ops.start_of_stream(ctx);
if ( rc )
goto err;
@@ -889,6 +997,7 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
}
} while ( ctx->save.checkpointed != XC_MIG_STREAM_NONE );
+ end:
xc_report_progress_single(xch, "End of stream");
rc = write_end_record(ctx);
@@ -918,12 +1027,14 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom,
uint32_t max_iters, uint32_t max_factor, uint32_t flags,
struct save_callbacks* callbacks, int hvm,
- xc_migration_stream_t stream_type, int recv_fd)
+ xc_migration_stream_t stream_type, int recv_fd,
+ int migration_phase)
{
struct xc_sr_context ctx =
{
.xch = xch,
.fd = io_fd,
+ .migration_phase = migration_phase
};
/* GCC 4.4 (of CentOS 6.x vintage) can' t initialise anonymous unions. */
@@ -948,7 +1059,8 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom,
ctx.save.dirty_threshold = 50;
/* Sanity checks for callbacks. */
- if ( hvm )
+ /* Now the mirror qemu stream doesn't enable/disable qemu log */
+ if ( hvm && ctx.migration_phase != MIGRATION_PHASE_MIRROR_DISK )
assert(callbacks->switch_qemu_logdirty);
if ( ctx.save.checkpointed )
assert(callbacks->checkpoint && callbacks->postcopy);
diff --git a/tools/libxc/xc_sr_save_x86_hvm.c b/tools/libxc/xc_sr_save_x86_hvm.c
index e17bb59..e417da2 100644
--- a/tools/libxc/xc_sr_save_x86_hvm.c
+++ b/tools/libxc/xc_sr_save_x86_hvm.c
@@ -157,7 +157,8 @@ static int x86_hvm_setup(struct xc_sr_context *ctx)
ctx->save.p2m_size = nr_pfns;
- if ( ctx->save.callbacks->switch_qemu_logdirty(
+ if ( ctx->migration_phase != MIGRATION_PHASE_MIRROR_DISK &&
+ ctx->save.callbacks->switch_qemu_logdirty(
ctx->domid, 1, ctx->save.callbacks->data) )
{
PERROR("Couldn't enable qemu log-dirty mode");
@@ -214,7 +215,8 @@ static int x86_hvm_cleanup(struct xc_sr_context *ctx)
xc_interface *xch = ctx->xch;
/* If qemu successfully enabled logdirty mode, attempt to disable. */
- if ( ctx->x86_hvm.save.qemu_enabled_logdirty &&
+ if ( ctx->migration_phase != MIGRATION_PHASE_MIRROR_DISK &&
+ ctx->x86_hvm.save.qemu_enabled_logdirty &&
ctx->save.callbacks->switch_qemu_logdirty(
ctx->domid, 0, ctx->save.callbacks->data) )
{
@@ -235,6 +237,7 @@ struct xc_sr_save_ops save_ops_x86_hvm =
.end_of_checkpoint = x86_hvm_end_of_checkpoint,
.check_vm_state = x86_hvm_check_vm_state,
.cleanup = x86_hvm_cleanup,
+ .local_disks = x86_hvm_end_of_checkpoint,
};
/*
diff --git a/tools/libxc/xc_sr_stream_format.h b/tools/libxc/xc_sr_stream_format.h
index 15ff1c7..1f0b274 100644
--- a/tools/libxc/xc_sr_stream_format.h
+++ b/tools/libxc/xc_sr_stream_format.h
@@ -8,6 +8,10 @@
#include <inttypes.h>
+#define MIGRATION_PHASE_NON_LOCAL_DISK 0
+#define MIGRATION_PHASE_VIRTUAL_RAM 1
+#define MIGRATION_PHASE_MIRROR_DISK 2
+
/*
* Image Header
*/
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 7/8] Fixed bugs in the migration flow
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
` (5 preceding siblings ...)
2017-10-18 17:26 ` [PATCH RFC v2 6/8] Adapted libxc for migration of local disk Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-18 17:26 ` [PATCH RFC v2 8/8] Added support to handle QMP events Bruno Alvisio
2017-10-30 17:03 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Wei Liu
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/libxc/xc_sr_save.c | 1 -
tools/libxl/libxl_create.c | 15 ++++++++-------
tools/libxl/libxl_dom_save.c | 2 +-
tools/libxl/libxl_domain.c | 2 +-
tools/libxl/libxl_internal.h | 2 +-
tools/libxl/libxl_save_callout.c | 5 +++--
tools/libxl/libxl_stream_write.c | 2 +-
tools/xl/xl.h | 2 +-
tools/xl/xl_cmdtable.c | 3 ++-
tools/xl/xl_migrate.c | 2 +-
10 files changed, 19 insertions(+), 17 deletions(-)
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index 181a0c8..f3e162f 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -464,7 +464,6 @@ static int send_virtual_devices_and_params(struct xc_sr_context *ctx)
uint64_t i = 0;
int rc = 0;
- fprintf(stderr, "BRUNO: SEND VIRTUAL DEVICES AND PARAMS\n");
xc_set_progress_prefix(xch, "Frames");
//FOR RTL AND VGA IN 128MB VM . Might change on size of VM
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 6df2754..0579671 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1055,24 +1055,24 @@ static void start_nbd_server(libxl__egc *egc, libxl__dm_spawn_state *dmss,
* TODO: Assign port dynamically
*/
- LOGD(DEBUG, "Starting NBD Server\n");
+ LOGD(DEBUG, domid, "Starting NBD Server\n");
ret = libxl__qmp_nbd_server_start(gc, domid, "::", QEMU_DRIVE_MIRROR_PORT);
if (ret) {
ret = ERROR_FAIL;
- LOGD(ERROR, "Failed to start NBD Server\n");
+ LOGD(ERROR, domid, "Failed to start NBD Server\n");
goto skip_nbd;
}else{
- LOGD(INFO, "Started NBD Server Successfully\n");
+ LOGD(INFO, domid, "Started NBD Server Successfully\n");
}
ret = libxl__qmp_nbd_server_add(gc, domid, QEMU_DRIVE_MIRROR_DEVICE);
if (ret) {
ret = ERROR_FAIL;
- LOGD(ERROR, "Failed to add NBD Server\n");
+ LOGD(ERROR, domid, "Failed to add NBD Server\n");
goto skip_nbd;
} else {
- LOGD(INFO, "NBD Add Successful\n");
+ LOGD(INFO, domid, "NBD Add Successful\n");
}
}
@@ -1103,7 +1103,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
libxl__srm_restore_autogen_callbacks *const callbacks =
&dcs->srs.shs.callbacks.restore.a;
libxl__srm_restore_autogen_callbacks *const callbacks_mirror_qemu_disks =
- &dcs->srs_local_disks.shs.callbacks.restore.a;
+ &dcs->srs_mirror_qemu_disks.shs.callbacks.restore.a;
if (rc) {
domcreate_rebuild_done(egc, dcs, rc);
@@ -1252,6 +1252,7 @@ static void domcreate_stream_done(libxl__egc *egc,
{
libxl__domain_create_state *dcs = srs->dcs;
STATE_AO_GC(dcs->ao);
+ int rc;
const uint32_t domid = dcs->guest_domid;
const char* uri;
@@ -1269,7 +1270,7 @@ static void domcreate_stream_done(libxl__egc *egc,
}else{
fprintf(stderr, "Stopped NBD server successfully\n");
}
- uri = GCSPRINTF("exec: /bin/cat %s", (&dcs->sdss.dm)->build_state->saved_state);
+ uri = GCSPRINTF("exec: /bin/cat %s", state_file);
libxl__qmp_migrate_incoming(gc, domid, uri);
domcreate_devmodel_started(egc, &dcs->sdss.dm, 0);
}
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index a2730f5..ddfe2f8 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -446,7 +446,7 @@ static void mirror_qemu_disks(libxl__egc *egc, libxl__stream_write_state *sws,
start_mirror:
LOGD(DEBUG, domid, "Sleeping for a bit so that source can start NBD\n");
sleep(30);
- LOGD(DEBUG, "Starting mirror-drive of device %s\n",
+ LOGD(DEBUG, domid, "Starting mirror-drive of device %s\n",
QEMU_DRIVE_MIRROR_DEVICE);
target = GCSPRINTF("nbd:%s:%s:exportname=%s", dss->hostname,
QEMU_DRIVE_MIRROR_PORT, QEMU_DRIVE_MIRROR_DEVICE);
diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c
index 76c6d3d..9b512b9 100644
--- a/tools/libxl/libxl_domain.c
+++ b/tools/libxl/libxl_domain.c
@@ -509,7 +509,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
dss->type = type;
dss->live = flags & LIBXL_SUSPEND_LIVE;
dss->debug = flags & LIBXL_SUSPEND_DEBUG;
- dss->mirror_qemu_disks = flags & LIBXL_SUSPEND_MIRROR_QEMU_DISKS;
+ dss->mirror_qemu_disks = (flags & LIBXL_SUSPEND_MIRROR_QEMU_DISKS) ? 1 : 0;
dss->hostname = hostname;
dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 30862c6..d7b338b 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3759,7 +3759,7 @@ struct libxl__domain_create_state {
/* If we're not doing stubdom, we use only dmss.dm,
* for the non-stubdom device model. */
libxl__stream_read_state srs;
- libxl__stream_read_state srs_local_disks;
+ libxl__stream_read_state srs_mirror_qemu_disks;
/* necessary if the domain creation failed and we have to destroy it */
libxl__domain_destroy_state dds;
libxl__multidev multidev;
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 48f96d8..64c891a 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -62,7 +62,8 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
state->store_domid, state->console_port,
state->console_domid,
hvm, pae,
- cbflags, dcs->restore_params.checkpointed_stream
+ cbflags, dcs->restore_params.checkpointed_stream,
+ migration_phase,
};
shs->ao = ao;
@@ -74,7 +75,7 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
} else {
if ( migration_phase != LIBXL_MIGRATION_PHASE_MIRROR_DISK ) {
shs->completion_callback = libxl__xc_domain_restore_done;
- if( local_disks == LIBXL_MIGRATION_PHASE_VIRTUAL_RAM )
+ if( migration_phase == LIBXL_MIGRATION_PHASE_VIRTUAL_RAM )
shs->need_results = 0;
else
shs->need_results = 1;
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index ad312ed..f5269f4 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -335,7 +335,7 @@ static void stream_header_done(libxl__egc *egc,
static void libxc_header_done(libxl__egc *egc,
libxl__stream_write_state *stream)
{
- int save_mirror_qemu_disks = stream->dss->local_disks;
+ int save_mirror_qemu_disks = stream->dss->mirror_qemu_disks;
libxl__xc_domain_save(egc, stream->dss, &stream->shs,
save_mirror_qemu_disks + stream->mirror_qemu_disks);
}
diff --git a/tools/xl/xl.h b/tools/xl/xl.h
index 070bac1..2bff52f 100644
--- a/tools/xl/xl.h
+++ b/tools/xl/xl.h
@@ -35,7 +35,7 @@ struct domain_create {
int daemonize;
int monitor; /* handle guest reboots etc */
int paused;
- int mirror_qemu_disk;
+ int mirror_qemu_disks;
int dryrun;
int quiet;
int vnc;
diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c
index 5cfc7e8..5fa5720 100644
--- a/tools/xl/xl_cmdtable.c
+++ b/tools/xl/xl_cmdtable.c
@@ -165,7 +165,8 @@ struct cmd_spec cmd_table[] = {
"-e Do not wait in the background (on <host>) for the death\n"
" of the domain.\n"
"--debug Print huge (!) amount of debug during the migration process.\n"
- "-p Do not unpause domain after migrating it."
+ "-p Do not unpause domain after migrating it.\n"
+ "-q Migrate local disks (Copy all the storage)"
},
{ "restore",
&main_restore, 0, 1,
diff --git a/tools/xl/xl_migrate.c b/tools/xl/xl_migrate.c
index 9f43d96..0a01cad 100644
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -610,7 +610,7 @@ int main_migrate(int argc, char **argv)
daemonize ? "" : " -e",
debug ? " -d" : "",
pause_after_migration ? " -p" : "",
- mirror_qemu_disks ? "-q" :"");
+ mirror_qemu_disks ? " -q" : "");
}
migrate_domain(domid, rune, debug, config_filename, mirror_qemu_disks,
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH RFC v2 8/8] Added support to handle QMP events
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
` (6 preceding siblings ...)
2017-10-18 17:26 ` [PATCH RFC v2 7/8] Fixed bugs in the migration flow Bruno Alvisio
@ 2017-10-18 17:26 ` Bruno Alvisio
2017-10-30 17:03 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Wei Liu
8 siblings, 0 replies; 10+ messages in thread
From: Bruno Alvisio @ 2017-10-18 17:26 UTC (permalink / raw)
To: xen-devel; +Cc: wei.liu2, ian.jackson, bruno.alvisio, dave
---
tools/libxl/libxl_dom_save.c | 71 ++++++++------------
tools/libxl/libxl_qmp.c | 150 +++++++++++++++++++++++++++++++++++++++++--
2 files changed, 170 insertions(+), 51 deletions(-)
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index ddfe2f8..d188dd2 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -432,63 +432,44 @@ static void mirror_qemu_disks(libxl__egc *egc, libxl__stream_write_state *sws,
{
int counter = 20;
char* target;
- bool job_is_ready = false;
libxl__domain_save_state *dss = sws->dss;
const uint32_t domid = dss->domid;
STATE_AO_GC(dss->ao);
- if (dss->mirror_qemu_disks) {
+ if (rc)
+ goto err;
/*
* If the -q was provided, the drive-mirror job is started.
- * TODO: Move the following code as part of the domain_suspend
* TODO: The port should be sent by the destination.
- */
-start_mirror:
- LOGD(DEBUG, domid, "Sleeping for a bit so that source can start NBD\n");
- sleep(30);
- LOGD(DEBUG, domid, "Starting mirror-drive of device %s\n",
- QEMU_DRIVE_MIRROR_DEVICE);
- target = GCSPRINTF("nbd:%s:%s:exportname=%s", dss->hostname,
- QEMU_DRIVE_MIRROR_PORT, QEMU_DRIVE_MIRROR_DEVICE);
- rc = libxl__qmp_drive_mirror(gc, dss->domid, QEMU_DRIVE_MIRROR_DEVICE,
+ */
+ start_mirror:
+ LOGD(DEBUG, domid, "Sleeping for a bit so that source can start NBD\n");
+ sleep(30);
+ LOGD(DEBUG, domid, "Starting mirror-drive of device %s\n",
+ QEMU_DRIVE_MIRROR_DEVICE);
+ target = GCSPRINTF("nbd:%s:%s:exportname=%s", dss->hostname,
+ QEMU_DRIVE_MIRROR_PORT, QEMU_DRIVE_MIRROR_DEVICE);
+ rc = libxl__qmp_drive_mirror(gc, dss->domid, QEMU_DRIVE_MIRROR_DEVICE,
target, "raw");
- if (!rc) {
- LOGD(INFO, domid, "Drive mirror command returned successfully\n");
+ if (!rc) {
+ LOGD(DEBUG, domid, "Drive mirror command returned successfully\n");
+ }else{
+ LOGD(ERROR, domid, "Sending drive mirror command failed\n");
+ if(counter > 0){
+ LOGD(INFO, domid, "Counter: %d. Sleeping for 10 sec and retry\n", counter);
+ sleep(10);
+ counter--;
+ goto start_mirror;
}else{
- LOGD(ERROR, domid, "Sending drive mirror command failed\n");
- if(counter > 0){
- LOGD(INFO, domid, "Counter: %d. Sleeping for 10 sec and retry\n", counter);
- sleep(10);
- counter--;
- goto start_mirror;
- }else{
- goto cont;
- }
- }
-
- /*
- * Query job status until it is ready
- * TODO: This code is just an inefficient busy wait. QMP sends an
- * TODO: asynchronous message when mirroring job is completed. Consider
- * TODO: adding the capability to handle asynchronous QMP messages (already done?)
- */
- while(!job_is_ready) {
- LOGD(INFO, domid, "Checking for drive-mirror job");
- rc = libxl__qmp_query_block_jobs(gc, dss->domid, &job_is_ready);
- if(rc){
- LOGD(ERROR, domid, "Checking block job failed\n");
- goto cont;
- }else{
- LOGD(INFO, domid, "Checking block job succeeded\n");
- }
- if(!job_is_ready){
- LOGD(INFO, domid, "Sleeping 5 sec\n");
- sleep(5);
- }
+ goto err;
}
}
-cont:
+
libxl__stream_write_start(egc, &sws->dss->sws);
+ return;
+
+ err:
+ dss->callback(egc, dss, rc);
}
static void stream_done(libxl__egc *egc,
diff --git a/tools/libxl/libxl_qmp.c b/tools/libxl/libxl_qmp.c
index fe6f076..5ef5fb1 100644
--- a/tools/libxl/libxl_qmp.c
+++ b/tools/libxl/libxl_qmp.c
@@ -59,6 +59,13 @@ typedef struct callback_id_pair {
LIBXL_STAILQ_ENTRY(struct callback_id_pair) next;
} callback_id_pair;
+typedef struct handler_event_pair {
+ const char* event_type;
+ void *opaque;
+ qmp_request_context *context;
+ qmp_callback_t event_handler;
+} event_handler_pair;
+
struct libxl__qmp_handler {
struct sockaddr_un addr;
int qmp_fd;
@@ -66,6 +73,9 @@ struct libxl__qmp_handler {
time_t timeout;
/* wait_for_id will be used by the synchronous send function */
int wait_for_id;
+ /* wait_for_event_type is used to wait on QMP events */
+ const char* wait_for_event_type;
+ event_handler_pair *hep;
char buffer[QMP_RECEIVE_BUFFER_SIZE + 1];
libxl__yajl_ctx *yajl_ctx;
@@ -287,6 +297,25 @@ static void qmp_handle_error_response(libxl__gc *gc, libxl__qmp_handler *qmp,
libxl__json_object_get_string(resp));
}
+static void qmp_handle_event(libxl__gc *gc, libxl__qmp_handler *qmp,
+ const libxl__json_object *event)
+{
+ const char* event_type = NULL;
+ const libxl__json_object *event_o = NULL;
+ event_o = libxl__json_map_get("event", event, JSON_ANY);
+ event_type = libxl__json_object_get_string(event_o);
+ int rc;
+
+ if(qmp->wait_for_event_type &&
+ !strcmp(event_type, qmp->wait_for_event_type)) {
+ rc = qmp->hep->event_handler(qmp,
+ libxl__json_map_get("data", event, JSON_ANY),
+ qmp->hep->opaque);
+ qmp->hep->context->rc = rc;
+ qmp->wait_for_event_type = NULL;
+ }
+}
+
static int qmp_handle_response(libxl__gc *gc, libxl__qmp_handler *qmp,
const libxl__json_object *resp)
{
@@ -325,6 +354,7 @@ static int qmp_handle_response(libxl__gc *gc, libxl__qmp_handler *qmp,
qmp_handle_error_response(gc, qmp, resp);
return -1;
case LIBXL__QMP_MESSAGE_TYPE_EVENT:
+ qmp_handle_event(gc, qmp, resp);
return 0;
case LIBXL__QMP_MESSAGE_TYPE_INVALID:
return -1;
@@ -348,9 +378,7 @@ static libxl__qmp_handler *qmp_init_handler(libxl__gc *gc, uint32_t domid)
qmp->ctx = CTX;
qmp->domid = domid;
- //TODO: Changed default timeout because drive-mirror command takes a long
- //TODO: to return. Consider timeout to be passed as param.
- qmp->timeout = 600;
+ qmp->timeout = 5;
LIBXL_STAILQ_INIT(&qmp->callback_list);
@@ -627,6 +655,31 @@ static void qmp_free_handler(libxl__qmp_handler *qmp)
free(qmp);
}
+static int wait_for_event(libxl__qmp_handler *qmp, event_handler_pair *hep,
+ int timeout)
+{
+ int ret = 0;
+ GC_INIT(qmp->ctx);
+ qmp->timeout = timeout;
+ qmp_request_context context = { .rc = 0 };
+ qmp->hep = hep;
+ qmp->wait_for_event_type = hep->event_type;
+ hep->context = &context;
+
+ while (qmp->wait_for_event_type) {
+ if ((ret = qmp_next(gc, qmp)) < 0) {
+ break;
+ }
+ }
+
+ if (!qmp->wait_for_event_type && ret == 0) {
+ ret = context.rc;
+ }
+ GC_FREE;
+
+ return ret;
+}
+
/*
* QMP Parameters Helpers
*/
@@ -1072,10 +1125,82 @@ int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid, const char *disk)
return qmp_run_command(gc, domid, "nbd-server-add", args, NULL, NULL);
}
-int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* device, const char* target, const char* format)
+static int block_job_ready_handler(libxl__qmp_handler *qmp,
+ const libxl__json_object *data, void *opaque){
+
+ GC_INIT(qmp->ctx);
+ int rc = -1;
+
+ const char *type;
+ const char *device;
+ unsigned int len;
+ unsigned int offset;
+ unsigned int speed;
+
+ const libxl__json_object *obj = NULL;
+
+ obj = libxl__json_map_get("type", data, JSON_STRING);
+ if (!obj) {
+ LOGD(ERROR, qmp->domid, "Failed to retrieve job type.");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ type = libxl__json_object_get_string(obj);
+
+ obj = libxl__json_map_get("device", data, JSON_STRING);
+ if (!obj) {
+ LOGD(ERROR, qmp->domid, "Failed to retrieve device.");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ device = libxl__json_object_get_string(obj);
+
+ obj = libxl__json_map_get("len", data, JSON_INTEGER);
+ if (!obj) {
+ LOGD(ERROR, qmp->domid, "Failed to retrieve length.");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ len = libxl__json_object_get_integer(obj);
+
+ obj = libxl__json_map_get("offset", data, JSON_INTEGER);
+ if (!obj) {
+ LOGD(ERROR, qmp->domid, "Failed to retrieve offset.");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ offset = libxl__json_object_get_integer(obj);
+
+ obj = libxl__json_map_get("speed", data, JSON_INTEGER);
+ if (!obj) {
+ LOGD(ERROR, qmp->domid, "Failed to retrieve speed.");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ speed = libxl__json_object_get_integer(obj);
+
+ LOGD(INFO, qmp->domid, "Block Job Ready: Details: Device: %s, Type: %s, Len: %u, Offset: %u, Speed %u\n",
+ device, type, len, offset, speed);
+
+ rc = 0;
+out:
+ GC_FREE;
+ return rc;
+}
+
+int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* device,
+ const char* target, const char* format)
{
+ libxl__qmp_handler *qmp = NULL;
libxl__json_object *args = NULL;
- //TODO: Allow method to receive "sync", "speed", "mode", "granurality", "buf-size"
+ int rc = 0;
+ /* TODO: Allow method to receive "sync", "speed", "mode", "granurality"
+ * "buf-size"
+ */
+ qmp = libxl__qmp_initialize(gc, domid);
+ if (!qmp)
+ return -1;
+
qmp_parameters_add_string(gc, &args, "device", device);
qmp_parameters_add_string(gc, &args, "target", target);
qmp_parameters_add_string(gc, &args, "sync", "full");
@@ -1084,7 +1209,20 @@ int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* device, const
qmp_parameters_add_integer(gc, &args, "granularity", 0);
qmp_parameters_add_integer(gc, &args, "buf-size", 0);
- return qmp_run_command(gc, domid, "drive-mirror", args, NULL, NULL);
+ rc = qmp_synchronous_send(qmp, "drive-mirror", args,
+ NULL, NULL, qmp->timeout);
+
+ if ( !rc ) {
+ event_handler_pair hep =
+ {
+ .event_type = "BLOCK_JOB_READY",
+ .event_handler = block_job_ready_handler,
+ };
+
+ rc = wait_for_event(qmp, &hep, 600);
+ }
+ libxl__qmp_close(qmp);
+ return rc;
}
static int query_block_callback(libxl__qmp_handler *qmp,
--
2.3.2 (Apple Git-55)
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage
2017-10-18 17:26 ` [PATCH RFC v2 0/8] Live migration for VMs with QEMU backed local storage Bruno Alvisio
` (7 preceding siblings ...)
2017-10-18 17:26 ` [PATCH RFC v2 8/8] Added support to handle QMP events Bruno Alvisio
@ 2017-10-30 17:03 ` Wei Liu
8 siblings, 0 replies; 10+ messages in thread
From: Wei Liu @ 2017-10-30 17:03 UTC (permalink / raw)
To: Bruno Alvisio
Cc: Stefano Stabellini, wei.liu2, ian.jackson, xen-devel, dave,
Anthony PERARD
Also CC QEMU maintainers.
Is there any new QEMU patch required?
On Wed, Oct 18, 2017 at 10:26:27AM -0700, Bruno Alvisio wrote:
> I am reviving this thread about the migration of VMs with local storage. I have worked on a solution to be able to migrate VMs that use QEMU as the backend disk driver. I have adapted the migration flow and piggybacked on the “drive-mirroring” capability already provided by QEMU.
>
> Overview
> 1. The “xl migrate” command has an additional “-q” flag. When provided the local storage of the VM is mirrored to the destination during the migration process.
> 2. Internally, the modification consists on adding a new libxl__stream_read_state struct to the libxl__domain_create_state structure and libxl__stream_read_state structure to the libxl__domain_save_state struct.
> 3. Migration flow can now be divided into three phases:
> a. Phase One: Copies the necessary state to start a QEMU process on the destination. It is started with the “-incoming defer” option.
> b. Phase Two: Disk is mirrored using the QEMU embedded NBD server.
> c. Phase Three: Once the disk is completely mirrored, virtual RAM of the domain is live migrated to the destination. This phase most closely resembles to the current migration flow.
> 4. If the “-q” option is not provided the migration is equivalent to the current migration flow.
>
> The new migration flow has follows the following major sequence of steps:
> 1. 1st stream copies the QEMU devices RAM from source to destination.
> 2. QEMU process is started on the destination with the option “-incoming defer”. (This creates the QEMU process but it doesn’t start running the main loop until “migrate incoming” command is executed)
> 3. “drive mirror” QMP command is executed so that the disk is mirrored to the destination node.
> 4. An event listener waits for the QMP BLOCK_JOB_READY event sent by QEMU which signals that the "disk mirror job" is complete.
> 5. 2nd Stream copies the virtual RAM from source to destination including QEMU state. At this point, the VM is suspended on source.
> 6. “migrate incoming” QMP command is executed on destination.
> 7. VM is restored in destination.
>
> This is sample configuration file that I have used to test my branch:
>
> name="tinycore"
> disk=['/home/balvisio/tinycore.img,raw,xvda,w']
> memory=128
> builder='hvm'
> vcpus=1
> vfb = ['type=vnc']
> vif= ['bridge=xenbr0']
> boot='b'
> acpi=1
> device_model_version='qemu-xen'
> serial='pty'
> vnc=1
> xen_platform_pci=0
>
> Notes
>
> 1. Note that the configuration file uses "xen_platform_pci=0”. This is necessary so that the block device is seen by QEMU. Further modification should be made for the case "xen_platform_pci=1” if we still want to use NBD mirroring capability provided by QEMU.
> 2. The current branch has still many hardcoded values. Many of the can be easily removed:
> a. Port used for disk mirroring (11000)
> b. Name of the block devices. (ide0-hd0) Currently the branch only supports VM with on disk drive.
> c. Live migration memory transfer: The pages transferred by libxc is hardcoded. Only a VM with 128 MB of memory is supported.
>
> Here is a link to the branch in Github:
> https://github.com/balvisio/xen/tree/feature/local_storage_migration
>
> Any feedback/suggestion is appreciated.
>
I haven't had time to review this series in detail, but I think the
series could use some clean-up and improvement to the commit message.
Please refer to
https://wiki.xenproject.org/wiki/Submitting_Xen_Project_Patches
Currently all patches are missing commit message explaining why the
change is needed as well as your SoB. In case you're not aware, we also
have the requirement each individual patch should build.
There are coding style issues in your patches, please refer to
libxl/CODING_STYLE. The same style applies to xl, too.
Furthermore, patches should be cleaned up before posting -- I see a few
debug printf's got added and removed, as well as a few TODOs with open
questions. Preferably the open questions should be mentioned in the
cover letter to catch attentions.
How this helps. Any more questions, feel free to ask.
Wei.
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 10+ messages in thread