* [Qemu-devel] [PATCH COLO-Frame v7 01/34] configure: Add parameter for configure to enable/disable COLO support
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 02/34] migration: Introduce capability 'colo' to migration zhanghailiang
` (32 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
configure --enable-colo/--disable-colo to switch COLO
support on/off.
COLO support is off by default.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
configure | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/configure b/configure
index 33b9455..a2475db 100755
--- a/configure
+++ b/configure
@@ -258,6 +258,7 @@ xfs=""
vhost_net="no"
vhost_scsi="no"
kvm="no"
+colo="no"
rdma=""
gprof="no"
debug_tcg="no"
@@ -930,6 +931,10 @@ for opt do
;;
--enable-kvm) kvm="yes"
;;
+ --disable-colo) colo="no"
+ ;;
+ --enable-colo) colo="yes"
+ ;;
--disable-tcg-interpreter) tcg_interpreter="no"
;;
--enable-tcg-interpreter) tcg_interpreter="yes"
@@ -1347,6 +1352,7 @@ disabled with --disable-FEATURE, default is enabled if available:
fdt fdt device tree
bluez bluez stack connectivity
kvm KVM acceleration support
+ colo COarse-grain LOck-stepping VM for Non-stop Service
rdma RDMA-based migration support
uuid uuid support
vde support for vde network
@@ -4524,6 +4530,7 @@ echo "Linux AIO support $linux_aio"
echo "ATTR/XATTR support $attr"
echo "Install blobs $blobs"
echo "KVM support $kvm"
+echo "COLO support $colo"
echo "RDMA support $rdma"
echo "TCG interpreter $tcg_interpreter"
echo "fdt support $fdt"
@@ -5097,6 +5104,10 @@ if have_backend "ftrace"; then
fi
echo "CONFIG_TRACE_FILE=$trace_file" >> $config_host_mak
+if test "$colo" = "yes"; then
+ echo "CONFIG_COLO=y" >> $config_host_mak
+fi
+
if test "$rdma" = "yes" ; then
echo "CONFIG_RDMA=y" >> $config_host_mak
fi
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 02/34] migration: Introduce capability 'colo' to migration
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 01/34] configure: Add parameter for configure to enable/disable COLO support zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 03/34] COLO: migrate colo related info to slave zhanghailiang
` (31 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Markus Armbruster, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, amit.shah, Lai Jiangshan,
Yang Hongyang, zhanghailiang
We add helper function colo_supported() to indicate whether
colo is supported or not, with which we use to control whether or not
showing 'colo' string to users, they can use qmp command
'query-migrate-capabilities' or hmp command 'info migrate_capabilities'
to learn if colo is supported.
Cc: Juan Quintela <quintela@redhat.com>
Cc: Amit Shah <amit.shah@redhat.com>
Cc: Eric Blake <eblake@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
include/migration/colo.h | 20 ++++++++++++++++++++
include/migration/migration.h | 1 +
migration/Makefile.objs | 1 +
migration/colo.c | 18 ++++++++++++++++++
migration/migration.c | 17 +++++++++++++++++
qapi-schema.json | 5 ++++-
qmp-commands.hx | 1 +
stubs/Makefile.objs | 1 +
stubs/migration-colo.c | 18 ++++++++++++++++++
9 files changed, 81 insertions(+), 1 deletion(-)
create mode 100644 include/migration/colo.h
create mode 100644 migration/colo.c
create mode 100644 stubs/migration-colo.c
diff --git a/include/migration/colo.h b/include/migration/colo.h
new file mode 100644
index 0000000..c60a590
--- /dev/null
+++ b/include/migration/colo.h
@@ -0,0 +1,20 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_COLO_H
+#define QEMU_COLO_H
+
+#include "qemu-common.h"
+
+bool colo_supported(void);
+
+#endif
diff --git a/include/migration/migration.h b/include/migration/migration.h
index b2711ef..5c797d4 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -170,6 +170,7 @@ int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen);
int migrate_use_xbzrle(void);
int64_t migrate_xbzrle_cache_size(void);
+bool migrate_enable_colo(void);
int64_t xbzrle_cache_resize(int64_t new_size);
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
index d929e96..5a25d39 100644
--- a/migration/Makefile.objs
+++ b/migration/Makefile.objs
@@ -1,4 +1,5 @@
common-obj-y += migration.o tcp.o
+common-obj-$(CONFIG_COLO) += colo.o
common-obj-y += vmstate.o
common-obj-y += qemu-file.o qemu-file-buf.o qemu-file-unix.o qemu-file-stdio.o
common-obj-y += xbzrle.o
diff --git a/migration/colo.c b/migration/colo.c
new file mode 100644
index 0000000..2c40d2e
--- /dev/null
+++ b/migration/colo.c
@@ -0,0 +1,18 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#include "migration/colo.h"
+
+bool colo_supported(void)
+{
+ return true;
+}
diff --git a/migration/migration.c b/migration/migration.c
index 45719a0..e34b2c9 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -28,6 +28,7 @@
#include "trace.h"
#include "qapi/util.h"
#include "qapi-event.h"
+#include "migration/colo.h"
#define MAX_THROTTLE (32 << 20) /* Migration speed throttling */
@@ -332,6 +333,9 @@ MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
caps = NULL; /* silence compiler warning */
for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
+ if (i == MIGRATION_CAPABILITY_COLO && !colo_supported()) {
+ continue;
+ }
if (head == NULL) {
head = g_malloc0(sizeof(*caps));
caps = head;
@@ -472,6 +476,13 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
}
for (cap = params; cap; cap = cap->next) {
+ if (cap->value->capability == MIGRATION_CAPABILITY_COLO &&
+ cap->value->state && !colo_supported()) {
+ error_setg(errp, "COLO is not currently supported, please"
+ " configure with --enable-colo option in order to"
+ " support COLO feature");
+ continue;
+ }
s->enabled_capabilities[cap->value->capability] = cap->value->state;
}
}
@@ -899,6 +910,12 @@ int64_t migrate_xbzrle_cache_size(void)
return s->xbzrle_cache_size;
}
+bool migrate_enable_colo(void)
+{
+ MigrationState *s = migrate_get_current();
+ return s->enabled_capabilities[MIGRATION_CAPABILITY_COLO];
+}
+
/* migration thread support */
static void *migration_thread(void *opaque)
diff --git a/qapi-schema.json b/qapi-schema.json
index 1285b8c..58c09f2 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -529,11 +529,14 @@
# @auto-converge: If enabled, QEMU will automatically throttle down the guest
# to speed up convergence of RAM migration. (since 1.6)
#
+# @colo: If enabled, migration will never end, and the state of VM in primary side
+# will be migrated continuously to VM in secondary side. (since 2.4)
+#
# Since: 1.2
##
{ 'enum': 'MigrationCapability',
'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks',
- 'compress', 'events'] }
+ 'compress', 'events', 'colo'] }
##
# @MigrationCapabilityStatus
diff --git a/qmp-commands.hx b/qmp-commands.hx
index e1bcc60..6e441eb 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3433,6 +3433,7 @@ Query current migration capabilities
- "rdma-pin-all" : RDMA Pin Page state (json-bool)
- "auto-converge" : Auto Converge state (json-bool)
- "zero-blocks" : Zero Blocks state (json-bool)
+ - "colo" : COLO FT state (json-bool)
Arguments:
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index 9937a12..5b124a9 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -38,3 +38,4 @@ stub-obj-$(CONFIG_WIN32) += fd-register.o
stub-obj-y += cpus.o
stub-obj-y += kvm.o
stub-obj-y += qmp_pc_dimm_device_list.o
+stub-obj-y += migration-colo.o
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
new file mode 100644
index 0000000..3d817df
--- /dev/null
+++ b/stubs/migration-colo.c
@@ -0,0 +1,18 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#include "migration/colo.h"
+
+bool colo_supported(void)
+{
+ return false;
+}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 03/34] COLO: migrate colo related info to slave
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 01/34] configure: Add parameter for configure to enable/disable COLO support zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 02/34] migration: Introduce capability 'colo' to migration zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases zhanghailiang
` (30 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
We can know if VM in destination should go into COLO mode by refer to
the info that been migrated from PVM.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
---
include/migration/colo.h | 2 ++
migration/Makefile.objs | 1 +
migration/colo-comm.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++
vl.c | 3 ++-
4 files changed, 55 insertions(+), 1 deletion(-)
create mode 100644 migration/colo-comm.c
diff --git a/include/migration/colo.h b/include/migration/colo.h
index c60a590..9b6662d 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -14,7 +14,9 @@
#define QEMU_COLO_H
#include "qemu-common.h"
+#include "migration/migration.h"
bool colo_supported(void);
+void colo_info_mig_init(void);
#endif
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
index 5a25d39..cb7bd30 100644
--- a/migration/Makefile.objs
+++ b/migration/Makefile.objs
@@ -1,5 +1,6 @@
common-obj-y += migration.o tcp.o
common-obj-$(CONFIG_COLO) += colo.o
+common-obj-y += colo-comm.o
common-obj-y += vmstate.o
common-obj-y += qemu-file.o qemu-file-buf.o qemu-file-unix.o qemu-file-stdio.o
common-obj-y += xbzrle.o
diff --git a/migration/colo-comm.c b/migration/colo-comm.c
new file mode 100644
index 0000000..0a93672
--- /dev/null
+++ b/migration/colo-comm.c
@@ -0,0 +1,50 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <migration/colo.h>
+#include "trace.h"
+
+typedef struct {
+ bool skip;
+ uint32_t colo_requested;
+} COLOInfo;
+
+static COLOInfo colo_info;
+
+static void colo_info_pre_save(void *opaque)
+{
+ COLOInfo *s = opaque;
+
+ if (migrate_enable_colo()) {
+ s->colo_requested = 1;
+ } else {
+ s->colo_requested = 0;
+ }
+}
+
+
+static const VMStateDescription colo_state = {
+ .name = "COLOState",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .pre_save = colo_info_pre_save,
+ .fields = (VMStateField[]) {
+ VMSTATE_UINT32(colo_requested, COLOInfo),
+ VMSTATE_END_OF_LIST()
+ },
+};
+
+void colo_info_mig_init(void)
+{
+ vmstate_register(NULL, 0, &colo_state, &colo_info);
+}
diff --git a/vl.c b/vl.c
index 3f269dc..fc594d0 100644
--- a/vl.c
+++ b/vl.c
@@ -91,6 +91,7 @@ int main(int argc, char **argv)
#include "sysemu/dma.h"
#include "audio/audio.h"
#include "migration/migration.h"
+#include "migration/colo.h"
#include "sysemu/kvm.h"
#include "qapi/qmp/qjson.h"
#include "qemu/option.h"
@@ -4433,7 +4434,7 @@ int main(int argc, char **argv, char **envp)
blk_mig_init();
ram_mig_init();
-
+ colo_info_mig_init();
/* If the currently selected machine wishes to override the units-per-bus
* property of its default HBA interface type, do so now. */
if (machine_class->units_per_default_bus) {
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (2 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 03/34] COLO: migrate colo related info to slave zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-17 17:07 ` Dr. David Alan Gilbert
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 05/34] migration: Integrate COLO checkpoint process into migration zhanghailiang
` (29 subsequent siblings)
33 siblings, 1 reply; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, zhanghailiang
For older machine types, we skip the colo info section when do
migration, in this way, we can migrate successfully between older mainchine and
the new one.
We also skip this section if colo is not enabled (i.e.
migrate_set_capability colo on), so that, It not break compatibility with migration
however the --enable-colo/disable-colo on the source/destination;
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
---
hw/i386/pc_piix.c | 1 +
hw/i386/pc_q35.c | 1 +
hw/ppc/spapr.c | 1 +
include/migration/migration.h | 1 +
migration/colo-comm.c | 13 +++++++++++++
5 files changed, 17 insertions(+)
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index 8167b12..926b0d8 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -313,6 +313,7 @@ static void pc_compat_2_3(MachineState *machine)
}
global_state_set_optional();
savevm_skip_configuration();
+ savevm_skip_colo_state();
}
static void pc_compat_2_2(MachineState *machine)
diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index 974aead..b5c6c85 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -296,6 +296,7 @@ static void pc_compat_2_3(MachineState *machine)
}
global_state_set_optional();
savevm_skip_configuration();
+ savevm_skip_colo_state();
}
static void pc_compat_2_2(MachineState *machine)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index a6f1947..568de93 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1880,6 +1880,7 @@ static void spapr_compat_2_3(Object *obj)
{
savevm_skip_section_footers();
global_state_set_optional();
+ savevm_skip_colo_state();
}
static void spapr_compat_2_2(Object *obj)
diff --git a/include/migration/migration.h b/include/migration/migration.h
index 5c797d4..1b23517 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -203,4 +203,5 @@ void savevm_skip_section_footers(void);
void register_global_state(void);
void global_state_set_optional(void);
void savevm_skip_configuration(void);
+void savevm_skip_colo_state(void);
#endif
diff --git a/migration/colo-comm.c b/migration/colo-comm.c
index 0a93672..3c8e361 100644
--- a/migration/colo-comm.c
+++ b/migration/colo-comm.c
@@ -21,6 +21,11 @@ typedef struct {
static COLOInfo colo_info;
+void savevm_skip_colo_state(void)
+{
+ colo_info.skip = true;
+}
+
static void colo_info_pre_save(void *opaque)
{
COLOInfo *s = opaque;
@@ -32,12 +37,20 @@ static void colo_info_pre_save(void *opaque)
}
}
+static bool colo_info_need(void *opaque)
+{
+ if (migrate_enable_colo() && !colo_info.skip) {
+ return true;
+ }
+ return false;
+}
static const VMStateDescription colo_state = {
.name = "COLOState",
.version_id = 1,
.minimum_version_id = 1,
.pre_save = colo_info_pre_save,
+ .needed = colo_info_need,
.fields = (VMStateField[]) {
VMSTATE_UINT32(colo_requested, COLOInfo),
VMSTATE_END_OF_LIST()
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* Re: [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases zhanghailiang
@ 2015-07-17 17:07 ` Dr. David Alan Gilbert
2015-07-20 8:42 ` zhanghailiang
0 siblings, 1 reply; 37+ messages in thread
From: Dr. David Alan Gilbert @ 2015-07-17 17:07 UTC (permalink / raw)
To: zhanghailiang
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
qemu-devel, arei.gonglei, amit.shah
* zhanghailiang (zhang.zhanghailiang@huawei.com) wrote:
> For older machine types, we skip the colo info section when do
> migration, in this way, we can migrate successfully between older mainchine and
> the new one.
>
> We also skip this section if colo is not enabled (i.e.
> migrate_set_capability colo on), so that, It not break compatibility with migration
> however the --enable-colo/disable-colo on the source/destination;
>
> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> ---
> hw/i386/pc_piix.c | 1 +
> hw/i386/pc_q35.c | 1 +
> hw/ppc/spapr.c | 1 +
> include/migration/migration.h | 1 +
> migration/colo-comm.c | 13 +++++++++++++
> 5 files changed, 17 insertions(+)
>
> diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
> index 8167b12..926b0d8 100644
> --- a/hw/i386/pc_piix.c
> +++ b/hw/i386/pc_piix.c
> @@ -313,6 +313,7 @@ static void pc_compat_2_3(MachineState *machine)
> }
> global_state_set_optional();
> savevm_skip_configuration();
> + savevm_skip_colo_state();
> }
>
> static void pc_compat_2_2(MachineState *machine)
> diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
> index 974aead..b5c6c85 100644
> --- a/hw/i386/pc_q35.c
> +++ b/hw/i386/pc_q35.c
> @@ -296,6 +296,7 @@ static void pc_compat_2_3(MachineState *machine)
> }
> global_state_set_optional();
> savevm_skip_configuration();
> + savevm_skip_colo_state();
> }
>
> static void pc_compat_2_2(MachineState *machine)
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index a6f1947..568de93 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -1880,6 +1880,7 @@ static void spapr_compat_2_3(Object *obj)
> {
> savevm_skip_section_footers();
> global_state_set_optional();
> + savevm_skip_colo_state();
> }
>
> static void spapr_compat_2_2(Object *obj)
> diff --git a/include/migration/migration.h b/include/migration/migration.h
> index 5c797d4..1b23517 100644
> --- a/include/migration/migration.h
> +++ b/include/migration/migration.h
> @@ -203,4 +203,5 @@ void savevm_skip_section_footers(void);
> void register_global_state(void);
> void global_state_set_optional(void);
> void savevm_skip_configuration(void);
> +void savevm_skip_colo_state(void);
> #endif
> diff --git a/migration/colo-comm.c b/migration/colo-comm.c
> index 0a93672..3c8e361 100644
> --- a/migration/colo-comm.c
> +++ b/migration/colo-comm.c
> @@ -21,6 +21,11 @@ typedef struct {
>
> static COLOInfo colo_info;
>
> +void savevm_skip_colo_state(void)
> +{
> + colo_info.skip = true;
> +}
> +
> static void colo_info_pre_save(void *opaque)
> {
> COLOInfo *s = opaque;
> @@ -32,12 +37,20 @@ static void colo_info_pre_save(void *opaque)
> }
> }
>
> +static bool colo_info_need(void *opaque)
> +{
> + if (migrate_enable_colo() && !colo_info.skip) {
> + return true;
> + }
> + return false;
> +}
That will work, but I think (untested) this can just be:
+static bool colo_info_need(void *opaque)
+{
+ return migrate_enable_colo();
+}
and then you can get rid of the skip stuff (and merge it
back to the previous patch).
Old qemu's will never sent the section so we're safe.
New qemu's with that flag unset won't send the section, so
they're still migration compatible on the machine type.
New qemu's with the flag set will use it.
Dave
>
> static const VMStateDescription colo_state = {
> .name = "COLOState",
> .version_id = 1,
> .minimum_version_id = 1,
> .pre_save = colo_info_pre_save,
> + .needed = colo_info_need,
> .fields = (VMStateField[]) {
> VMSTATE_UINT32(colo_requested, COLOInfo),
> VMSTATE_END_OF_LIST()
> --
> 1.7.12.4
>
>
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
^ permalink raw reply [flat|nested] 37+ messages in thread* Re: [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases
2015-07-17 17:07 ` Dr. David Alan Gilbert
@ 2015-07-20 8:42 ` zhanghailiang
0 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-20 8:42 UTC (permalink / raw)
To: Dr. David Alan Gilbert
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
qemu-devel, arei.gonglei, amit.shah
On 2015/7/18 1:07, Dr. David Alan Gilbert wrote:
> * zhanghailiang (zhang.zhanghailiang@huawei.com) wrote:
>> For older machine types, we skip the colo info section when do
>> migration, in this way, we can migrate successfully between older mainchine and
>> the new one.
>>
>> We also skip this section if colo is not enabled (i.e.
>> migrate_set_capability colo on), so that, It not break compatibility with migration
>> however the --enable-colo/disable-colo on the source/destination;
>>
>> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
>> ---
>> hw/i386/pc_piix.c | 1 +
>> hw/i386/pc_q35.c | 1 +
>> hw/ppc/spapr.c | 1 +
>> include/migration/migration.h | 1 +
>> migration/colo-comm.c | 13 +++++++++++++
>> 5 files changed, 17 insertions(+)
>>
>> diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
>> index 8167b12..926b0d8 100644
>> --- a/hw/i386/pc_piix.c
>> +++ b/hw/i386/pc_piix.c
>> @@ -313,6 +313,7 @@ static void pc_compat_2_3(MachineState *machine)
>> }
>> global_state_set_optional();
>> savevm_skip_configuration();
>> + savevm_skip_colo_state();
>> }
>>
>> static void pc_compat_2_2(MachineState *machine)
>> diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
>> index 974aead..b5c6c85 100644
>> --- a/hw/i386/pc_q35.c
>> +++ b/hw/i386/pc_q35.c
>> @@ -296,6 +296,7 @@ static void pc_compat_2_3(MachineState *machine)
>> }
>> global_state_set_optional();
>> savevm_skip_configuration();
>> + savevm_skip_colo_state();
>> }
>>
>> static void pc_compat_2_2(MachineState *machine)
>> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
>> index a6f1947..568de93 100644
>> --- a/hw/ppc/spapr.c
>> +++ b/hw/ppc/spapr.c
>> @@ -1880,6 +1880,7 @@ static void spapr_compat_2_3(Object *obj)
>> {
>> savevm_skip_section_footers();
>> global_state_set_optional();
>> + savevm_skip_colo_state();
>> }
>>
>> static void spapr_compat_2_2(Object *obj)
>> diff --git a/include/migration/migration.h b/include/migration/migration.h
>> index 5c797d4..1b23517 100644
>> --- a/include/migration/migration.h
>> +++ b/include/migration/migration.h
>> @@ -203,4 +203,5 @@ void savevm_skip_section_footers(void);
>> void register_global_state(void);
>> void global_state_set_optional(void);
>> void savevm_skip_configuration(void);
>> +void savevm_skip_colo_state(void);
>> #endif
>> diff --git a/migration/colo-comm.c b/migration/colo-comm.c
>> index 0a93672..3c8e361 100644
>> --- a/migration/colo-comm.c
>> +++ b/migration/colo-comm.c
>> @@ -21,6 +21,11 @@ typedef struct {
>>
>> static COLOInfo colo_info;
>>
>> +void savevm_skip_colo_state(void)
>> +{
>> + colo_info.skip = true;
>> +}
>> +
>> static void colo_info_pre_save(void *opaque)
>> {
>> COLOInfo *s = opaque;
>> @@ -32,12 +37,20 @@ static void colo_info_pre_save(void *opaque)
>> }
>> }
>>
>> +static bool colo_info_need(void *opaque)
>> +{
>> + if (migrate_enable_colo() && !colo_info.skip) {
>> + return true;
>> + }
>> + return false;
>> +}
>
> That will work, but I think (untested) this can just be:
>
> +static bool colo_info_need(void *opaque)
> +{
> + return migrate_enable_colo();
> +}
>
> and then you can get rid of the skip stuff (and merge it
> back to the previous patch).
>
> Old qemu's will never sent the section so we're safe.
> New qemu's with that flag unset won't send the section, so
> they're still migration compatible on the machine type.
> New qemu's with the flag set will use it.
>
Yes, you are right, the 'skip stuff' is redundant, will fix it in next version, thanks.
>
>>
>> static const VMStateDescription colo_state = {
>> .name = "COLOState",
>> .version_id = 1,
>> .minimum_version_id = 1,
>> .pre_save = colo_info_pre_save,
>> + .needed = colo_info_need,
>> .fields = (VMStateField[]) {
>> VMSTATE_UINT32(colo_requested, COLOInfo),
>> VMSTATE_END_OF_LIST()
>> --
>> 1.7.12.4
>>
>>
> --
> Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
>
> .
>
^ permalink raw reply [flat|nested] 37+ messages in thread
* [Qemu-devel] [PATCH COLO-Frame v7 05/34] migration: Integrate COLO checkpoint process into migration
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (3 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 06/34] migration: Integrate COLO checkpoint process into loadvm zhanghailiang
` (28 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, zhanghailiang
Add a migrate state: MIGRATION_STATUS_COLO, enter this migration state
after the first live migration successfully finished.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
include/migration/colo.h | 3 +++
include/migration/migration.h | 2 ++
migration/colo.c | 55 +++++++++++++++++++++++++++++++++++++++++++
migration/migration.c | 25 +++++++++++++++-----
qapi-schema.json | 2 +-
stubs/migration-colo.c | 9 +++++++
trace-events | 3 +++
7 files changed, 92 insertions(+), 7 deletions(-)
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 9b6662d..2320594 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -19,4 +19,7 @@
bool colo_supported(void);
void colo_info_mig_init(void);
+void colo_init_checkpointer(MigrationState *s);
+bool migrate_in_colo_state(void);
+
#endif
diff --git a/include/migration/migration.h b/include/migration/migration.h
index 1b23517..0fc85cc 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -82,6 +82,8 @@ struct MigrationState
int64_t dirty_sync_count;
};
+void migrate_set_state(MigrationState *s, int old_state, int new_state);
+
void process_incoming_migration(QEMUFile *f);
void qemu_start_incoming_migration(const char *uri, Error **errp);
diff --git a/migration/colo.c b/migration/colo.c
index 2c40d2e..33a5a48 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -10,9 +10,64 @@
* later. See the COPYING file in the top-level directory.
*/
+#include "sysemu/sysemu.h"
#include "migration/colo.h"
+#include "trace.h"
+
+static QEMUBH *colo_bh;
bool colo_supported(void)
{
return true;
}
+
+bool migrate_in_colo_state(void)
+{
+ MigrationState *s = migrate_get_current();
+ return (s->state == MIGRATION_STATUS_COLO);
+}
+
+static void *colo_thread(void *opaque)
+{
+ MigrationState *s = opaque;
+
+ qemu_mutex_lock_iothread();
+ vm_start();
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("stop", "run");
+
+ /*TODO: COLO checkpoint savevm loop*/
+
+ migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+
+ qemu_mutex_lock_iothread();
+ qemu_bh_schedule(s->cleanup_bh);
+ qemu_mutex_unlock_iothread();
+
+ return NULL;
+}
+
+static void colo_start_checkpointer(void *opaque)
+{
+ MigrationState *s = opaque;
+
+ if (colo_bh) {
+ qemu_bh_delete(colo_bh);
+ colo_bh = NULL;
+ }
+
+ qemu_mutex_unlock_iothread();
+ qemu_thread_join(&s->thread);
+ qemu_mutex_lock_iothread();
+
+ migrate_set_state(s, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_COLO);
+
+ qemu_thread_create(&s->thread, "colo", colo_thread, s,
+ QEMU_THREAD_JOINABLE);
+}
+
+void colo_init_checkpointer(MigrationState *s)
+{
+ colo_bh = qemu_bh_new(colo_start_checkpointer, s);
+ qemu_bh_schedule(colo_bh);
+}
diff --git a/migration/migration.c b/migration/migration.c
index e34b2c9..fefa47b 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -428,6 +428,10 @@ MigrationInfo *qmp_query_migrate(Error **errp)
get_xbzrle_cache_stats(info);
break;
+ case MIGRATION_STATUS_COLO:
+ info->has_status = true;
+ /* TODO: display COLO specific information (checkpoint info etc.) */
+ break;
case MIGRATION_STATUS_COMPLETED:
get_xbzrle_cache_stats(info);
@@ -530,7 +534,7 @@ void qmp_migrate_set_parameters(bool has_compress_level,
/* shared migration helpers */
-static void migrate_set_state(MigrationState *s, int old_state, int new_state)
+void migrate_set_state(MigrationState *s, int old_state, int new_state)
{
if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) {
migrate_generate_event(new_state);
@@ -711,7 +715,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
if (s->state == MIGRATION_STATUS_ACTIVE ||
s->state == MIGRATION_STATUS_SETUP ||
- s->state == MIGRATION_STATUS_CANCELLING) {
+ s->state == MIGRATION_STATUS_CANCELLING ||
+ s->state == MIGRATION_STATUS_COLO) {
error_setg(errp, QERR_MIGRATION_ACTIVE);
return;
}
@@ -927,6 +932,7 @@ static void *migration_thread(void *opaque)
int64_t max_size = 0;
int64_t start_time = initial_time;
bool old_vm_running = false;
+ bool enable_colo = migrate_enable_colo();
qemu_savevm_state_header(s->file);
qemu_savevm_state_begin(s->file, &s->params);
@@ -968,8 +974,10 @@ static void *migration_thread(void *opaque)
}
if (!qemu_file_get_error(s->file)) {
- migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
- MIGRATION_STATUS_COMPLETED);
+ if (!enable_colo) {
+ migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
+ MIGRATION_STATUS_COMPLETED);
+ }
break;
}
}
@@ -1020,11 +1028,16 @@ static void *migration_thread(void *opaque)
}
runstate_set(RUN_STATE_POSTMIGRATE);
} else {
- if (old_vm_running) {
+ if (s->state == MIGRATION_STATUS_ACTIVE && enable_colo) {
+ colo_init_checkpointer(s);
+ } else if (old_vm_running) {
vm_start();
}
}
- qemu_bh_schedule(s->cleanup_bh);
+
+ if (!enable_colo) {
+ qemu_bh_schedule(s->cleanup_bh);
+ }
qemu_mutex_unlock_iothread();
return NULL;
diff --git a/qapi-schema.json b/qapi-schema.json
index 58c09f2..f16f482 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -433,7 +433,7 @@
##
{ 'enum': 'MigrationStatus',
'data': [ 'none', 'setup', 'cancelling', 'cancelled',
- 'active', 'completed', 'failed' ] }
+ 'active', 'completed', 'failed', 'colo' ] }
##
# @MigrationInfo
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index 3d817df..96aa9d7 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -16,3 +16,12 @@ bool colo_supported(void)
{
return false;
}
+
+bool migrate_in_colo_state(void)
+{
+ return false;
+}
+
+void colo_init_checkpointer(MigrationState *s)
+{
+}
diff --git a/trace-events b/trace-events
index 2d395c5..125a33b 100644
--- a/trace-events
+++ b/trace-events
@@ -1472,6 +1472,9 @@ rdma_start_incoming_migration_after_rdma_listen(void) ""
rdma_start_outgoing_migration_after_rdma_connect(void) ""
rdma_start_outgoing_migration_after_rdma_source_init(void) ""
+# migration/colo.c
+colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
+
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 06/34] migration: Integrate COLO checkpoint process into loadvm
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (4 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 05/34] migration: Integrate COLO checkpoint process into migration zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 07/34] COLO: Implement colo checkpoint protocol zhanghailiang
` (27 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
Switch from normal migration loadvm process into COLO checkpoint process if
COLO mode is enabled.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
include/migration/colo.h | 13 +++++++++++++
migration/colo-comm.c | 10 ++++++++++
migration/colo.c | 20 ++++++++++++++++++++
migration/migration.c | 44 +++++++++++++++++++++++++++++++++++---------
stubs/migration-colo.c | 10 ++++++++++
trace-events | 1 +
6 files changed, 89 insertions(+), 9 deletions(-)
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 2320594..4344d29 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -15,11 +15,24 @@
#include "qemu-common.h"
#include "migration/migration.h"
+#include "block/coroutine.h"
+#include "qemu/thread.h"
bool colo_supported(void);
void colo_info_mig_init(void);
+struct colo_incoming {
+ QEMUFile *file;
+ QemuThread thread;
+};
+
void colo_init_checkpointer(MigrationState *s);
bool migrate_in_colo_state(void);
+/* loadvm */
+extern Coroutine *migration_incoming_co;
+bool loadvm_enable_colo(void);
+void loadvm_exit_colo(void);
+void *colo_process_incoming_checkpoints(void *opaque);
+bool loadvm_in_colo_state(void);
#endif
diff --git a/migration/colo-comm.c b/migration/colo-comm.c
index 3c8e361..049e96c 100644
--- a/migration/colo-comm.c
+++ b/migration/colo-comm.c
@@ -61,3 +61,13 @@ void colo_info_mig_init(void)
{
vmstate_register(NULL, 0, &colo_state, &colo_info);
}
+
+bool loadvm_enable_colo(void)
+{
+ return colo_info.colo_requested;
+}
+
+void loadvm_exit_colo(void)
+{
+ colo_info.colo_requested = 0;
+}
diff --git a/migration/colo.c b/migration/colo.c
index 33a5a48..e1ffd1f 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -13,8 +13,10 @@
#include "sysemu/sysemu.h"
#include "migration/colo.h"
#include "trace.h"
+#include "qemu/error-report.h"
static QEMUBH *colo_bh;
+static Coroutine *colo;
bool colo_supported(void)
{
@@ -27,6 +29,11 @@ bool migrate_in_colo_state(void)
return (s->state == MIGRATION_STATUS_COLO);
}
+bool loadvm_in_colo_state(void)
+{
+ return colo != NULL;
+}
+
static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
@@ -71,3 +78,16 @@ void colo_init_checkpointer(MigrationState *s)
colo_bh = qemu_bh_new(colo_start_checkpointer, s);
qemu_bh_schedule(colo_bh);
}
+
+void *colo_process_incoming_checkpoints(void *opaque)
+{
+ colo = qemu_coroutine_self();
+ assert(colo != NULL);
+
+ /* TODO: COLO checkpoint restore loop */
+
+ colo = NULL;
+ loadvm_exit_colo();
+
+ return NULL;
+}
diff --git a/migration/migration.c b/migration/migration.c
index fefa47b..1140012 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -253,6 +253,7 @@ void qemu_start_incoming_migration(const char *uri, Error **errp)
}
}
+Coroutine *migration_incoming_co;
static void process_incoming_migration_co(void *opaque)
{
QEMUFile *f = opaque;
@@ -263,7 +264,33 @@ static void process_incoming_migration_co(void *opaque)
migrate_generate_event(MIGRATION_STATUS_ACTIVE);
ret = qemu_loadvm_state(f);
- qemu_fclose(f);
+ if (!ret) {
+ /* Make sure all file formats flush their mutable metadata */
+ bdrv_invalidate_cache_all(&local_err);
+ if (local_err) {
+ error_report_err(local_err);
+ migrate_decompress_threads_join();
+ exit(EXIT_FAILURE);
+ }
+ }
+ /* we get colo info, and know if we are in colo mode */
+ if (!ret && loadvm_enable_colo()) {
+ struct colo_incoming *colo_in = g_malloc0(sizeof(*colo_in));
+
+ colo_in->file = f;
+ migration_incoming_co = qemu_coroutine_self();
+ qemu_thread_create(&colo_in->thread, "colo incoming",
+ colo_process_incoming_checkpoints, colo_in, QEMU_THREAD_JOINABLE);
+ qemu_coroutine_yield();
+ migration_incoming_co = NULL;
+#if 0
+ /* FIXME wait checkpoint incoming thread exit, and free resource */
+ qemu_thread_join(&colo_in->thread);
+ g_free(colo_in);
+#endif
+ } else {
+ qemu_fclose(f);
+ }
free_xbzrle_decoded_buf();
migration_incoming_state_destroy();
@@ -276,14 +303,6 @@ static void process_incoming_migration_co(void *opaque)
migrate_generate_event(MIGRATION_STATUS_COMPLETED);
qemu_announce_self();
- /* Make sure all file formats flush their mutable metadata */
- bdrv_invalidate_cache_all(&local_err);
- if (local_err) {
- error_report_err(local_err);
- migrate_decompress_threads_join();
- exit(EXIT_FAILURE);
- }
-
/* runstate == "" means that we haven't received it through the
* wire, so we obey autostart. runstate == runing means that we
* need to run it, we need to make sure that we do it after
@@ -720,6 +739,13 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
error_setg(errp, QERR_MIGRATION_ACTIVE);
return;
}
+
+ if (loadvm_in_colo_state()) {
+ error_setg(errp, "Secondary VM is not allowed to do migration while"
+ "in COLO status");
+ return;
+ }
+
if (runstate_check(RUN_STATE_INMIGRATE)) {
error_setg(errp, "Guest is waiting for an incoming migration");
return;
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index 96aa9d7..a2edd96 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -22,6 +22,16 @@ bool migrate_in_colo_state(void)
return false;
}
+bool loadvm_in_colo_state(void)
+{
+ return false;
+}
+
void colo_init_checkpointer(MigrationState *s)
{
}
+
+void *colo_process_incoming_checkpoints(void *opaque)
+{
+ return NULL;
+}
diff --git a/trace-events b/trace-events
index 125a33b..c34c7c4 100644
--- a/trace-events
+++ b/trace-events
@@ -1474,6 +1474,7 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) ""
# migration/colo.c
colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
+colo_receive_message(const char *msg) "Receive '%s'"
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 07/34] COLO: Implement colo checkpoint protocol
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (5 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 06/34] migration: Integrate COLO checkpoint process into loadvm zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 08/34] COLO: Add a new RunState RUN_STATE_COLO zhanghailiang
` (26 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
We need communications protocol of user-defined to control the checkpoint
process.
The new checkpoint request is started by Primary VM, and the interactive process
like below:
Checkpoint synchronizing points,
Primary Secondary
NEW @
Suspend
SUSPENDED @
Suspend&Save state
SEND @
Send state Receive state
RECEIVED @
Flush network Load state
LOADED @
Resume Resume
Start Comparing
NOTE:
1) '@' who sends the message
2) Every sync-point is synchronized by two sides with only
one handshake(single direction) for low-latency.
If more strict synchronization is required, a opposite direction
sync-point should be added.
3) Since sync-points are single direction, the remote side may
go forward a lot when this side just receives the sync-point.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
---
migration/colo.c | 237 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 235 insertions(+), 2 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index e1ffd1f..f4a0b4e 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -15,6 +15,41 @@
#include "trace.h"
#include "qemu/error-report.h"
+enum {
+ COLO_CHECPOINT_READY = 0x46,
+
+ /*
+ * Checkpoint synchronizing points.
+ *
+ * Primary Secondary
+ * NEW @
+ * Suspend
+ * SUSPENDED @
+ * Suspend&Save state
+ * SEND @
+ * Send state Receive state
+ * RECEIVED @
+ * Flush network Load state
+ * LOADED @
+ * Resume Resume
+ *
+ * Start Comparing
+ * NOTE:
+ * 1) '@' who sends the message
+ * 2) Every sync-point is synchronized by two sides with only
+ * one handshake(single direction) for low-latency.
+ * If more strict synchronization is required, a opposite direction
+ * sync-point should be added.
+ * 3) Since sync-points are single direction, the remote side may
+ * go forward a lot when this side just receives the sync-point.
+ */
+ COLO_CHECKPOINT_NEW,
+ COLO_CHECKPOINT_SUSPENDED,
+ COLO_CHECKPOINT_SEND,
+ COLO_CHECKPOINT_RECEIVED,
+ COLO_CHECKPOINT_LOADED,
+};
+
static QEMUBH *colo_bh;
static Coroutine *colo;
@@ -34,19 +69,136 @@ bool loadvm_in_colo_state(void)
return colo != NULL;
}
+/* colo checkpoint control helper */
+static int colo_ctl_put(QEMUFile *f, uint64_t request)
+{
+ int ret = 0;
+
+ qemu_put_be64(f, request);
+ qemu_fflush(f);
+
+ ret = qemu_file_get_error(f);
+
+ return ret;
+}
+
+static int colo_ctl_get_value(QEMUFile *f, uint64_t *value)
+{
+ int ret = 0;
+ uint64_t temp;
+
+ temp = qemu_get_be64(f);
+
+ ret = qemu_file_get_error(f);
+ if (ret < 0) {
+ return -1;
+ }
+
+ *value = temp;
+ return 0;
+}
+
+static int colo_ctl_get(QEMUFile *f, uint64_t require)
+{
+ int ret;
+ uint64_t value;
+
+ ret = colo_ctl_get_value(f, &value);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (value != require) {
+ error_report("unexpected state! expected: %"PRIu64
+ ", received: %"PRIu64, require, value);
+ exit(1);
+ }
+
+ return ret;
+}
+
+static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
+{
+ int ret;
+
+ ret = colo_ctl_put(s->file, COLO_CHECKPOINT_NEW);
+ if (ret < 0) {
+ goto out;
+ }
+
+ ret = colo_ctl_get(control, COLO_CHECKPOINT_SUSPENDED);
+ if (ret < 0) {
+ goto out;
+ }
+
+ /* TODO: suspend and save vm state to colo buffer */
+
+ ret = colo_ctl_put(s->file, COLO_CHECKPOINT_SEND);
+ if (ret < 0) {
+ goto out;
+ }
+
+ /* TODO: send vmstate to Secondary */
+
+ ret = colo_ctl_get(control, COLO_CHECKPOINT_RECEIVED);
+ if (ret < 0) {
+ goto out;
+ }
+ trace_colo_receive_message("COLO_CHECKPOINT_RECEIVED");
+
+ ret = colo_ctl_get(control, COLO_CHECKPOINT_LOADED);
+ if (ret < 0) {
+ goto out;
+ }
+ trace_colo_receive_message("COLO_CHECKPOINT_LOADED");
+
+ /* TODO: resume Primary */
+
+out:
+ return ret;
+}
+
static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
+ QEMUFile *colo_control = NULL;
+ int ret;
+
+ colo_control = qemu_fopen_socket(qemu_get_fd(s->file), "rb");
+ if (!colo_control) {
+ error_report("Open colo_control failed!");
+ goto out;
+ }
+
+ /*
+ * Wait for Secondary finish loading vm states and enter COLO
+ * restore.
+ */
+ ret = colo_ctl_get(colo_control, COLO_CHECPOINT_READY);
+ if (ret < 0) {
+ goto out;
+ }
+ trace_colo_receive_message("COLO_CHECPOINT_READY");
qemu_mutex_lock_iothread();
vm_start();
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "run");
- /*TODO: COLO checkpoint savevm loop*/
+ while (s->state == MIGRATION_STATUS_COLO) {
+ /* start a colo checkpoint */
+ if (colo_do_checkpoint_transaction(s, colo_control)) {
+ goto out;
+ }
+ }
+out:
migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ if (colo_control) {
+ qemu_fclose(colo_control);
+ }
+
qemu_mutex_lock_iothread();
qemu_bh_schedule(s->cleanup_bh);
qemu_mutex_unlock_iothread();
@@ -79,14 +231,95 @@ void colo_init_checkpointer(MigrationState *s)
qemu_bh_schedule(colo_bh);
}
+/*
+ * return:
+ * 0: start a checkpoint
+ * -1: some error happened, exit colo restore
+ */
+static int colo_wait_handle_cmd(QEMUFile *f, int *checkpoint_request)
+{
+ int ret;
+ uint64_t cmd;
+
+ ret = colo_ctl_get_value(f, &cmd);
+ if (ret < 0) {
+ return -1;
+ }
+
+ switch (cmd) {
+ case COLO_CHECKPOINT_NEW:
+ *checkpoint_request = 1;
+ return 0;
+ default:
+ return -1;
+ }
+}
+
void *colo_process_incoming_checkpoints(void *opaque)
{
+ struct colo_incoming *colo_in = opaque;
+ QEMUFile *f = colo_in->file;
+ int fd = qemu_get_fd(f);
+ QEMUFile *ctl = NULL;
+ int ret;
colo = qemu_coroutine_self();
assert(colo != NULL);
- /* TODO: COLO checkpoint restore loop */
+ ctl = qemu_fopen_socket(fd, "wb");
+ if (!ctl) {
+ error_report("Can't open incoming channel!");
+ goto out;
+ }
+ ret = colo_ctl_put(ctl, COLO_CHECPOINT_READY);
+ if (ret < 0) {
+ goto out;
+ }
+ /* TODO: in COLO mode, Secondary is runing, so start the vm */
+ while (true) {
+ int request = 0;
+ int ret = colo_wait_handle_cmd(f, &request);
+
+ if (ret < 0) {
+ break;
+ } else {
+ if (!request) {
+ continue;
+ }
+ }
+ /* TODO: suspend guest */
+ ret = colo_ctl_put(ctl, COLO_CHECKPOINT_SUSPENDED);
+ if (ret < 0) {
+ goto out;
+ }
+
+ ret = colo_ctl_get(f, COLO_CHECKPOINT_SEND);
+ if (ret < 0) {
+ goto out;
+ }
+ trace_colo_receive_message("COLO_CHECKPOINT_SEND");
+
+ /* TODO: read migration data into colo buffer */
+
+ ret = colo_ctl_put(ctl, COLO_CHECKPOINT_RECEIVED);
+ if (ret < 0) {
+ goto out;
+ }
+ trace_colo_receive_message("COLO_CHECKPOINT_RECEIVED");
+
+ /* TODO: load vm state */
+
+ ret = colo_ctl_put(ctl, COLO_CHECKPOINT_LOADED);
+ if (ret < 0) {
+ goto out;
+ }
+}
+
+out:
colo = NULL;
+ if (ctl) {
+ qemu_fclose(ctl);
+ }
loadvm_exit_colo();
return NULL;
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 08/34] COLO: Add a new RunState RUN_STATE_COLO
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (6 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 07/34] COLO: Implement colo checkpoint protocol zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 09/34] QEMUSizedBuffer: Introduce two help functions for qsb zhanghailiang
` (25 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Markus Armbruster, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, amit.shah, Lai Jiangshan,
zhanghailiang
Guest will enter this state when paused to save/restore VM state
under colo checkpoint.
Cc: Eric Blake <eblake@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
qapi-schema.json | 5 ++++-
vl.c | 8 ++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/qapi-schema.json b/qapi-schema.json
index f16f482..b41243a 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -148,12 +148,15 @@
# @watchdog: the watchdog action is configured to pause and has been triggered
#
# @guest-panicked: guest has been panicked as a result of guest OS panic
+#
+# @colo: guest is paused to save/restore VM state under colo checkpoint (since
+# 2.4)
##
{ 'enum': 'RunState',
'data': [ 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused',
'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm',
'running', 'save-vm', 'shutdown', 'suspended', 'watchdog',
- 'guest-panicked' ] }
+ 'guest-panicked', 'colo' ] }
##
# @StatusInfo:
diff --git a/vl.c b/vl.c
index fc594d0..dece65b 100644
--- a/vl.c
+++ b/vl.c
@@ -583,6 +583,7 @@ static const RunStateTransition runstate_transitions_def[] = {
{ RUN_STATE_INMIGRATE, RUN_STATE_SUSPENDED },
{ RUN_STATE_INMIGRATE, RUN_STATE_WATCHDOG },
{ RUN_STATE_INMIGRATE, RUN_STATE_GUEST_PANICKED },
+ { RUN_STATE_INMIGRATE, RUN_STATE_COLO },
{ RUN_STATE_INTERNAL_ERROR, RUN_STATE_PAUSED },
{ RUN_STATE_INTERNAL_ERROR, RUN_STATE_FINISH_MIGRATE },
@@ -592,6 +593,7 @@ static const RunStateTransition runstate_transitions_def[] = {
{ RUN_STATE_PAUSED, RUN_STATE_RUNNING },
{ RUN_STATE_PAUSED, RUN_STATE_FINISH_MIGRATE },
+ { RUN_STATE_PAUSED, RUN_STATE_COLO},
{ RUN_STATE_POSTMIGRATE, RUN_STATE_RUNNING },
{ RUN_STATE_POSTMIGRATE, RUN_STATE_FINISH_MIGRATE },
@@ -602,9 +604,12 @@ static const RunStateTransition runstate_transitions_def[] = {
{ RUN_STATE_FINISH_MIGRATE, RUN_STATE_RUNNING },
{ RUN_STATE_FINISH_MIGRATE, RUN_STATE_POSTMIGRATE },
+ { RUN_STATE_FINISH_MIGRATE, RUN_STATE_COLO},
{ RUN_STATE_RESTORE_VM, RUN_STATE_RUNNING },
+ { RUN_STATE_COLO, RUN_STATE_RUNNING },
+
{ RUN_STATE_RUNNING, RUN_STATE_DEBUG },
{ RUN_STATE_RUNNING, RUN_STATE_INTERNAL_ERROR },
{ RUN_STATE_RUNNING, RUN_STATE_IO_ERROR },
@@ -615,6 +620,7 @@ static const RunStateTransition runstate_transitions_def[] = {
{ RUN_STATE_RUNNING, RUN_STATE_SHUTDOWN },
{ RUN_STATE_RUNNING, RUN_STATE_WATCHDOG },
{ RUN_STATE_RUNNING, RUN_STATE_GUEST_PANICKED },
+ { RUN_STATE_RUNNING, RUN_STATE_COLO},
{ RUN_STATE_SAVE_VM, RUN_STATE_RUNNING },
@@ -625,9 +631,11 @@ static const RunStateTransition runstate_transitions_def[] = {
{ RUN_STATE_RUNNING, RUN_STATE_SUSPENDED },
{ RUN_STATE_SUSPENDED, RUN_STATE_RUNNING },
{ RUN_STATE_SUSPENDED, RUN_STATE_FINISH_MIGRATE },
+ { RUN_STATE_SUSPENDED, RUN_STATE_COLO},
{ RUN_STATE_WATCHDOG, RUN_STATE_RUNNING },
{ RUN_STATE_WATCHDOG, RUN_STATE_FINISH_MIGRATE },
+ { RUN_STATE_WATCHDOG, RUN_STATE_COLO},
{ RUN_STATE_GUEST_PANICKED, RUN_STATE_RUNNING },
{ RUN_STATE_GUEST_PANICKED, RUN_STATE_FINISH_MIGRATE },
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 09/34] QEMUSizedBuffer: Introduce two help functions for qsb
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (7 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 08/34] COLO: Add a new RunState RUN_STATE_COLO zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 10/34] COLO: Save VM state to slave when do checkpoint zhanghailiang
` (24 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Yang Hongyang, zhanghailiang
Introduce two new QEMUSizedBuffer APIs which will be used by COLO to buffer
VM state:
One is qsb_put_buffer(), which put the content of a given QEMUSizedBuffer
into QEMUFile, this is used to send buffered VM state to secondary.
Another is qsb_fill_buffer(), read 'size' bytes of data from the file into
qsb, this is used to get VM state from socket into a buffer.
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
include/migration/qemu-file.h | 3 ++-
migration/qemu-file-buf.c | 58 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 60 insertions(+), 1 deletion(-)
diff --git a/include/migration/qemu-file.h b/include/migration/qemu-file.h
index ea49f33..62461b5 100644
--- a/include/migration/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -144,7 +144,8 @@ ssize_t qsb_get_buffer(const QEMUSizedBuffer *, off_t start, size_t count,
uint8_t *buf);
ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *buf,
off_t pos, size_t count);
-
+void qsb_put_buffer(QEMUFile *f, QEMUSizedBuffer *qsb, int size);
+int qsb_fill_buffer(QEMUSizedBuffer *qsb, QEMUFile *f, int size);
/*
* For use on files opened with qemu_bufopen
diff --git a/migration/qemu-file-buf.c b/migration/qemu-file-buf.c
index 2de9330..93e67ca 100644
--- a/migration/qemu-file-buf.c
+++ b/migration/qemu-file-buf.c
@@ -366,6 +366,64 @@ ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *source,
return count;
}
+
+/**
+ * Put the content of a given QEMUSizedBuffer into QEMUFile.
+ *
+ * @f: A QEMUFile
+ * @qsb: A QEMUSizedBuffer
+ * @size: size of content to write
+ */
+void qsb_put_buffer(QEMUFile *f, QEMUSizedBuffer *qsb, int size)
+{
+ int i, l;
+
+ for (i = 0; i < qsb->n_iov && size > 0; i++) {
+ l = MIN(qsb->iov[i].iov_len, size);
+ qemu_put_buffer(f, qsb->iov[i].iov_base, l);
+ size -= l;
+ }
+}
+
+/*
+ * Read 'size' bytes of data from the file into qsb.
+ * always fill from pos 0 and used after qsb_create().
+ *
+ * It will return size bytes unless there was an error, in which case it will
+ * return as many as it managed to read (assuming blocking fd's which
+ * all current QEMUFile are)
+ */
+int qsb_fill_buffer(QEMUSizedBuffer *qsb, QEMUFile *f, int size)
+{
+ ssize_t rc = qsb_grow(qsb, size);
+ int pending = size, i;
+ qsb->used = 0;
+ uint8_t *buf = NULL;
+
+ if (rc < 0) {
+ return rc;
+ }
+
+ for (i = 0; i < qsb->n_iov && pending > 0; i++) {
+ int doneone = 0;
+ /* read until iov full */
+ while (doneone < qsb->iov[i].iov_len && pending > 0) {
+ int readone = 0;
+ buf = qsb->iov[i].iov_base;
+ readone = qemu_get_buffer(f, buf,
+ MIN(qsb->iov[i].iov_len - doneone, pending));
+ if (readone == 0) {
+ return qsb->used;
+ }
+ buf += readone;
+ doneone += readone;
+ pending -= readone;
+ qsb->used += readone;
+ }
+ }
+ return qsb->used;
+}
+
typedef struct QEMUBuffer {
QEMUSizedBuffer *qsb;
QEMUFile *file;
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 10/34] COLO: Save VM state to slave when do checkpoint
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (8 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 09/34] QEMUSizedBuffer: Introduce two help functions for qsb zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 11/34] COLO RAM: Load PVM's dirty page into SVM's RAM cache temporarily zhanghailiang
` (23 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
We should save PVM's RAM/device to slave when needed.
For VM state, we will cache them in slave, we use QEMUSizedBuffer
to store the data, we need know the data size of VM state, so in master,
we use qsb to store VM state temporarily, and then migrate the data to
slave.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
migration/colo.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++---
migration/ram.c | 47 ++++++++++++++++++++++++++++++++---------
migration/savevm.c | 2 +-
3 files changed, 96 insertions(+), 14 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index f4a0b4e..18c0d9b 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -52,6 +52,9 @@ enum {
static QEMUBH *colo_bh;
static Coroutine *colo;
+/* colo buffer */
+#define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
+QEMUSizedBuffer *colo_buffer;
bool colo_supported(void)
{
@@ -120,6 +123,8 @@ static int colo_ctl_get(QEMUFile *f, uint64_t require)
static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
{
int ret;
+ size_t size;
+ QEMUFile *trans = NULL;
ret = colo_ctl_put(s->file, COLO_CHECKPOINT_NEW);
if (ret < 0) {
@@ -130,15 +135,47 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
if (ret < 0) {
goto out;
}
+ /* Reset colo buffer and open it for write */
+ qsb_set_length(colo_buffer, 0);
+ trans = qemu_bufopen("w", colo_buffer);
+ if (!trans) {
+ error_report("Open colo buffer for write failed");
+ goto out;
+ }
+
+ /* suspend and save vm state to colo buffer */
+ qemu_mutex_lock_iothread();
+ vm_stop_force_state(RUN_STATE_COLO);
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("run", "stop");
+
+ /* Disable block migration */
+ s->params.blk = 0;
+ s->params.shared = 0;
+ qemu_savevm_state_begin(trans, &s->params);
+ qemu_mutex_lock_iothread();
+ qemu_savevm_state_complete(trans);
+ qemu_mutex_unlock_iothread();
- /* TODO: suspend and save vm state to colo buffer */
+ qemu_fflush(trans);
ret = colo_ctl_put(s->file, COLO_CHECKPOINT_SEND);
if (ret < 0) {
goto out;
}
+ /* we send the total size of the vmstate first */
+ size = qsb_get_length(colo_buffer);
+ ret = colo_ctl_put(s->file, size);
+ if (ret < 0) {
+ goto out;
+ }
- /* TODO: send vmstate to Secondary */
+ qsb_put_buffer(s->file, colo_buffer, size);
+ qemu_fflush(s->file);
+ ret = qemu_file_get_error(s->file);
+ if (ret < 0) {
+ goto out;
+ }
ret = colo_ctl_get(control, COLO_CHECKPOINT_RECEIVED);
if (ret < 0) {
@@ -152,9 +189,18 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
}
trace_colo_receive_message("COLO_CHECKPOINT_LOADED");
- /* TODO: resume Primary */
+ ret = 0;
+ /* resume master */
+ qemu_mutex_lock_iothread();
+ vm_start();
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("stop", "run");
out:
+ if (trans) {
+ qemu_fclose(trans);
+ }
+
return ret;
}
@@ -180,6 +226,12 @@ static void *colo_thread(void *opaque)
}
trace_colo_receive_message("COLO_CHECPOINT_READY");
+ colo_buffer = qsb_create(NULL, COLO_BUFFER_BASE_SIZE);
+ if (colo_buffer == NULL) {
+ error_report("Failed to allocate colo buffer!");
+ goto out;
+ }
+
qemu_mutex_lock_iothread();
vm_start();
qemu_mutex_unlock_iothread();
@@ -195,6 +247,9 @@ static void *colo_thread(void *opaque)
out:
migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ qsb_free(colo_buffer);
+ colo_buffer = NULL;
+
if (colo_control) {
qemu_fclose(colo_control);
}
diff --git a/migration/ram.c b/migration/ram.c
index c696814..56e3256 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -38,6 +38,7 @@
#include "trace.h"
#include "exec/ram_addr.h"
#include "qemu/rcu_queue.h"
+#include "migration/colo.h"
#ifdef DEBUG_MIGRATION_RAM
#define DPRINTF(fmt, ...) \
@@ -1090,15 +1091,8 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
}
}
-/* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
- * long-running RCU critical section. When rcu-reclaims in the code
- * start to become numerous it will be necessary to reduce the
- * granularity of these critical sections.
- */
-
-static int ram_save_setup(QEMUFile *f, void *opaque)
+static int ram_save_init_globals(void)
{
- RAMBlock *block;
int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
mig_throttle_on = false;
@@ -1158,6 +1152,31 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
migration_bitmap_sync();
qemu_mutex_unlock_ramlist();
qemu_mutex_unlock_iothread();
+ rcu_read_unlock();
+
+ return 0;
+}
+
+/* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
+ * long-running RCU critical section. When rcu-reclaims in the code
+ * start to become numerous it will be necessary to reduce the
+ * granularity of these critical sections.
+ */
+
+static int ram_save_setup(QEMUFile *f, void *opaque)
+{
+ RAMBlock *block;
+
+ /*
+ * migration has already setup the bitmap, reuse it.
+ */
+ if (!migrate_in_colo_state()) {
+ if (ram_save_init_globals() < 0) {
+ return -1;
+ }
+ }
+
+ rcu_read_lock();
qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
@@ -1257,7 +1276,8 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
while (true) {
int pages;
- pages = ram_find_and_save_block(f, true, &bytes_transferred);
+ pages = ram_find_and_save_block(f, !migrate_in_colo_state(),
+ &bytes_transferred);
/* no more blocks to sent */
if (pages == 0) {
break;
@@ -1266,7 +1286,14 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
flush_compressed_data(f);
ram_control_after_iterate(f, RAM_CONTROL_FINISH);
- migration_end();
+
+ /*
+ * Since we need to reuse dirty bitmap in colo,
+ * don't cleanup the bitmap.
+ */
+ if (!migrate_enable_colo() || migration_has_failed(migrate_get_current())) {
+ migration_end();
+ }
rcu_read_unlock();
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
diff --git a/migration/savevm.c b/migration/savevm.c
index 86735fc..15661ca 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -48,7 +48,7 @@
#include "qemu/iov.h"
#include "block/snapshot.h"
#include "block/qapi.h"
-
+#include "migration/colo.h"
#ifndef ETH_P_RARP
#define ETH_P_RARP 0x8035
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 11/34] COLO RAM: Load PVM's dirty page into SVM's RAM cache temporarily
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (9 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 10/34] COLO: Save VM state to slave when do checkpoint zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 12/34] COLO VMstate: Load VM state into qsb before restore it zhanghailiang
` (22 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
The ram cache is initially the same as SVM/PVM's memory.
At checkpoint, we cache the dirty RAM of PVM into RAM cache in the slave
(so that RAM cache always the same as PVM's memory at every
checkpoint), we will flush cached RAM to SVM after we receive
all PVM's vmstate (RAM/device).
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/exec/cpu-all.h | 1 +
include/migration/colo.h | 3 ++
migration/colo.c | 31 +++++++++++++---
migration/ram.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++--
4 files changed, 122 insertions(+), 6 deletions(-)
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index ea6a9a6..f8964c2 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -281,6 +281,7 @@ struct RAMBlock {
struct rcu_head rcu;
struct MemoryRegion *mr;
uint8_t *host;
+ uint8_t *host_cache; /* For colo, VM's ram cache */
ram_addr_t offset;
ram_addr_t used_length;
ram_addr_t max_length;
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 4344d29..2fa9c73 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -35,4 +35,7 @@ bool loadvm_enable_colo(void);
void loadvm_exit_colo(void);
void *colo_process_incoming_checkpoints(void *opaque);
bool loadvm_in_colo_state(void);
+/* ram cache */
+int create_and_init_ram_cache(void);
+void release_ram_cache(void);
#endif
diff --git a/migration/colo.c b/migration/colo.c
index 18c0d9b..73e9dda 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -325,11 +325,22 @@ void *colo_process_incoming_checkpoints(void *opaque)
error_report("Can't open incoming channel!");
goto out;
}
+
+ if (create_and_init_ram_cache() < 0) {
+ error_report("Failed to initialize ram cache");
+ goto out;
+ }
+
ret = colo_ctl_put(ctl, COLO_CHECPOINT_READY);
if (ret < 0) {
goto out;
}
- /* TODO: in COLO mode, Secondary is runing, so start the vm */
+ qemu_mutex_lock_iothread();
+ /* in COLO mode, slave is runing, so start the vm */
+ vm_start();
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("stop", "run");
+
while (true) {
int request = 0;
int ret = colo_wait_handle_cmd(f, &request);
@@ -342,7 +353,12 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
}
- /* TODO: suspend guest */
+ /* suspend guest */
+ qemu_mutex_lock_iothread();
+ vm_stop_force_state(RUN_STATE_COLO);
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("run", "stop");
+
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_SUSPENDED);
if (ret < 0) {
goto out;
@@ -354,7 +370,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
trace_colo_receive_message("COLO_CHECKPOINT_SEND");
- /* TODO: read migration data into colo buffer */
+ /*TODO Load VM state */
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_RECEIVED);
if (ret < 0) {
@@ -362,16 +378,23 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
trace_colo_receive_message("COLO_CHECKPOINT_RECEIVED");
- /* TODO: load vm state */
+ /* TODO: flush vm state */
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_LOADED);
if (ret < 0) {
goto out;
}
+
+ /* resume guest */
+ qemu_mutex_lock_iothread();
+ vm_start();
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("stop", "start");
}
out:
colo = NULL;
+ release_ram_cache();
if (ctl) {
qemu_fclose(ctl);
}
diff --git a/migration/ram.c b/migration/ram.c
index 56e3256..deb1100 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -225,6 +225,7 @@ static ram_addr_t last_offset;
static unsigned long *migration_bitmap;
static QemuMutex migration_bitmap_mutex;
static uint64_t migration_dirty_pages;
+static bool ram_cache_enable;
static uint32_t last_version;
static bool ram_bulk_stage;
@@ -1353,6 +1354,8 @@ static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
return 0;
}
+static void *memory_region_get_ram_cache_ptr(MemoryRegion *mr, RAMBlock *block);
+
/* Must be called from within a rcu critical section.
* Returns a pointer from within the RCU-protected ram_list.
*/
@@ -1370,7 +1373,20 @@ static inline void *host_from_stream_offset(QEMUFile *f,
return NULL;
}
- return memory_region_get_ram_ptr(block->mr) + offset;
+ if (ram_cache_enable) {
+ /*
+ * During colo checkpoint, we need bitmap of these migrated pages.
+ * It help us to decide which pages in ram cache should be flushed
+ * into VM's RAM later.
+ */
+ long k = (block->mr->ram_addr + offset) >> TARGET_PAGE_BITS;
+ if (!test_and_set_bit(k, migration_bitmap)) {
+ migration_dirty_pages++;
+ }
+ return memory_region_get_ram_cache_ptr(block->mr, block) + offset;
+ } else {
+ return memory_region_get_ram_ptr(block->mr) + offset;
+ }
}
len = qemu_get_byte(f);
@@ -1380,7 +1396,16 @@ static inline void *host_from_stream_offset(QEMUFile *f,
QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
if (!strncmp(id, block->idstr, sizeof(id)) &&
block->max_length > offset) {
- return memory_region_get_ram_ptr(block->mr) + offset;
+ if (ram_cache_enable) {
+ long k = (block->mr->ram_addr + offset) >> TARGET_PAGE_BITS;
+ if (!test_and_set_bit(k, migration_bitmap)) {
+ migration_dirty_pages++;
+ }
+ return memory_region_get_ram_cache_ptr(block->mr, block)
+ + offset;
+ } else {
+ return memory_region_get_ram_ptr(block->mr) + offset;
+ }
}
}
@@ -1631,6 +1656,70 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
return ret;
}
+/*
+ * colo cache: this is for secondary VM, we cache the whole
+ * memory of the secondary VM, it will be called after first migration.
+ */
+int create_and_init_ram_cache(void)
+{
+ RAMBlock *block;
+
+ rcu_read_lock();
+ QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ block->host_cache = qemu_anon_ram_alloc(block->used_length, NULL);
+ if (!block->host_cache) {
+ goto out_locked;
+ }
+ memcpy(block->host_cache, block->host, block->used_length);
+ }
+ rcu_read_unlock();
+ ram_cache_enable = true;
+ return 0;
+
+out_locked:
+ QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ if (block->host_cache) {
+ qemu_anon_ram_free(block->host_cache, block->used_length);
+ block->host_cache = NULL;
+ }
+ }
+
+ rcu_read_unlock();
+ return -1;
+}
+
+void release_ram_cache(void)
+{
+ RAMBlock *block;
+
+ ram_cache_enable = false;
+
+ rcu_read_lock();
+ QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ if (block->host_cache) {
+ qemu_anon_ram_free(block->host_cache, block->used_length);
+ block->host_cache = NULL;
+ }
+ }
+ rcu_read_unlock();
+}
+
+static void *memory_region_get_ram_cache_ptr(MemoryRegion *mr, RAMBlock *block)
+{
+ if (mr->alias) {
+ return memory_region_get_ram_cache_ptr(mr->alias, block) +
+ mr->alias_offset;
+ }
+
+ assert(mr->terminates);
+
+ ram_addr_t addr = mr->ram_addr & TARGET_PAGE_MASK;
+
+ assert(addr - block->offset < block->used_length);
+
+ return block->host_cache + (addr - block->offset);
+}
+
static SaveVMHandlers savevm_ram_handlers = {
.save_live_setup = ram_save_setup,
.save_live_iterate = ram_save_iterate,
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 12/34] COLO VMstate: Load VM state into qsb before restore it
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (10 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 11/34] COLO RAM: Load PVM's dirty page into SVM's RAM cache temporarily zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 13/34] arch_init: Start to trace dirty pages of SVM zhanghailiang
` (21 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Yang Hongyang, zhanghailiang
We should not destroy the state of secondary until we receive the whole
state from the primary, in case the primary fails in the middle of sending
the state, so, here we cache the device state in Secondary before restore it.
Besides, we should call qemu_system_reset() before load VM state,
which can ensure the data is intact.
Note: If we discard qemu_system_reset(), there will be some odd error,
For exmple, qemu in slave side crashes and reports:
KVM: entry failed, hardware error 0x7
EAX=00000000 EBX=0000e000 ECX=00009578 EDX=0000434f
ESI=0000fc10 EDI=0000434f EBP=00000000 ESP=00001fca
EIP=00009594 EFL=00010246 [---Z-P-] CPL=0 II=0 A20=1 SMM=0 HLT=0
ES =0040 00000400 0000ffff 00009300
CS =f000 000f0000 0000ffff 00009b00
SS =434f 000434f0 0000ffff 00009300
DS =434f 000434f0 0000ffff 00009300
FS =0000 00000000 0000ffff 00009300
GS =0000 00000000 0000ffff 00009300
LDT=0000 00000000 0000ffff 00008200
TR =0000 00000000 0000ffff 00008b00
GDT= 0002dcc8 00000047
IDT= 00000000 0000ffff
CR0=00000010 CR2=ffffffff CR3=00000000 CR4=00000000
DR0=0000000000000000 DR1=0000000000000000 DR2=0000000000000000 DR3=0000000000000000
DR6=00000000ffff0ff0 DR7=0000000000000400
EFER=0000000000000000
Code=c0 74 0f 66 b9 78 95 00 00 66 31 d2 66 31 c0 e9 47 e0 fb 90 <f3> 90 fa fc 66 c3 66 53 66 89 c3 66 e8 9d e8 ff ff 66 01 c3 66 89 d8 66 e8 40 e9 ff ff 66
ERROR: invalid runstate transition: 'internal-error' -> 'colo'
The reason is, some of the device state will be ignored when saving device state to slave,
if the corresponding data is in its initial value, such as 0.
But the device state in slave maybe in initialized value, after a loop of checkpoint,
there will be inconsistent for the value of device state.
This will happen when the PVM reboot or SVM run ahead of PVM in the startup process.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/colo.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index 73e9dda..ba83966 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -315,8 +315,10 @@ void *colo_process_incoming_checkpoints(void *opaque)
struct colo_incoming *colo_in = opaque;
QEMUFile *f = colo_in->file;
int fd = qemu_get_fd(f);
- QEMUFile *ctl = NULL;
+ QEMUFile *ctl = NULL, *fb = NULL;
int ret;
+ uint64_t total_size;
+
colo = qemu_coroutine_self();
assert(colo != NULL);
@@ -331,10 +333,17 @@ void *colo_process_incoming_checkpoints(void *opaque)
goto out;
}
+ colo_buffer = qsb_create(NULL, COLO_BUFFER_BASE_SIZE);
+ if (colo_buffer == NULL) {
+ error_report("Failed to allocate colo buffer!");
+ goto out;
+ }
+
ret = colo_ctl_put(ctl, COLO_CHECPOINT_READY);
if (ret < 0) {
goto out;
}
+
qemu_mutex_lock_iothread();
/* in COLO mode, slave is runing, so start the vm */
vm_start();
@@ -370,7 +379,18 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
trace_colo_receive_message("COLO_CHECKPOINT_SEND");
- /*TODO Load VM state */
+ /* read the VM state total size first */
+ ret = colo_ctl_get_value(f, &total_size);
+ if (ret < 0) {
+ goto out;
+ }
+
+ /* read vm device state into colo buffer */
+ ret = qsb_fill_buffer(colo_buffer, f, total_size);
+ if (ret != total_size) {
+ error_report("can't get all migration data");
+ goto out;
+ }
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_RECEIVED);
if (ret < 0) {
@@ -378,6 +398,22 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
trace_colo_receive_message("COLO_CHECKPOINT_RECEIVED");
+ /* open colo buffer for read */
+ fb = qemu_bufopen("r", colo_buffer);
+ if (!fb) {
+ error_report("can't open colo buffer for read");
+ goto out;
+ }
+
+ qemu_mutex_lock_iothread();
+ qemu_system_reset(VMRESET_SILENT);
+ if (qemu_loadvm_state(fb) < 0) {
+ error_report("COLO: loadvm failed");
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
+ qemu_mutex_unlock_iothread();
+
/* TODO: flush vm state */
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_LOADED);
@@ -390,14 +426,25 @@ void *colo_process_incoming_checkpoints(void *opaque)
vm_start();
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "start");
-}
+
+ qemu_fclose(fb);
+ fb = NULL;
+ }
out:
colo = NULL;
+
+ if (fb) {
+ qemu_fclose(fb);
+ }
+
release_ram_cache();
if (ctl) {
qemu_fclose(ctl);
}
+
+ qsb_free(colo_buffer);
+
loadvm_exit_colo();
return NULL;
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 13/34] arch_init: Start to trace dirty pages of SVM
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (11 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 12/34] COLO VMstate: Load VM state into qsb before restore it zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 14/34] COLO RAM: Flush cached RAM into SVM's memory zhanghailiang
` (20 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, zhanghailiang
we will use this dirty bitmap together with VM's cache RAM dirty bitmap
to decide which page in cache should be flushed into VM's RAM.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
---
migration/ram.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/migration/ram.c b/migration/ram.c
index deb1100..7a1a1b6 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1663,6 +1663,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
int create_and_init_ram_cache(void)
{
RAMBlock *block;
+ int64_t ram_cache_pages = last_ram_offset() >> TARGET_PAGE_BITS;
rcu_read_lock();
QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
@@ -1674,6 +1675,15 @@ int create_and_init_ram_cache(void)
}
rcu_read_unlock();
ram_cache_enable = true;
+ /*
+ * Start dirty log for Secondary VM, we use this dirty bitmap together with
+ * VM's cache RAM dirty bitmap to decide which page in cache should be
+ * flushed into VM's RAM.
+ */
+ migration_bitmap = bitmap_new(ram_cache_pages);
+ migration_dirty_pages = 0;
+ memory_global_dirty_log_start();
+
return 0;
out_locked:
@@ -1694,6 +1704,12 @@ void release_ram_cache(void)
ram_cache_enable = false;
+ if (migration_bitmap) {
+ memory_global_dirty_log_stop();
+ g_free(migration_bitmap);
+ migration_bitmap = NULL;
+ }
+
rcu_read_lock();
QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
if (block->host_cache) {
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 14/34] COLO RAM: Flush cached RAM into SVM's memory
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (12 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 13/34] arch_init: Start to trace dirty pages of SVM zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 15/34] COLO failover: Introduce a new command to trigger a failover zhanghailiang
` (19 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, Yang Hongyang,
zhanghailiang
During the time of VM's running, PVM/SVM may dirty some pages, we will transfer
PVM's dirty pages to SVM and store them into SVM's RAM cache at next checkpoint
time. So, the content of SVM's RAM cache will always be some with PVM's memory
after checkpoint.
Instead of flushing all content of SVM's RAM cache into SVM's MEMORY,
we do this in a more efficient way:
Only flush any page that dirtied by PVM or SVM since last checkpoint.
In this way, we ensure SVM's memory same with PVM's.
Besides, we must ensure flush RAM cache before load device state.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Gonglei <arei.gonglei@huawei.com>
---
include/migration/colo.h | 1 +
migration/colo.c | 2 --
migration/ram.c | 92 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 93 insertions(+), 2 deletions(-)
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 2fa9c73..c01364e 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -37,5 +37,6 @@ void *colo_process_incoming_checkpoints(void *opaque);
bool loadvm_in_colo_state(void);
/* ram cache */
int create_and_init_ram_cache(void);
+void colo_flush_ram_cache(void);
void release_ram_cache(void);
#endif
diff --git a/migration/colo.c b/migration/colo.c
index ba83966..ece8cb4 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -414,8 +414,6 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
qemu_mutex_unlock_iothread();
- /* TODO: flush vm state */
-
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_LOADED);
if (ret < 0) {
goto out;
diff --git a/migration/ram.c b/migration/ram.c
index 7a1a1b6..d6593f3 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1522,6 +1522,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
int flags = 0, ret = 0;
static uint64_t seq_iter;
int len = 0;
+ bool need_flush = false;
seq_iter++;
@@ -1590,6 +1591,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
ret = -EINVAL;
break;
}
+
+ need_flush = true;
ch = qemu_get_byte(f);
ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
break;
@@ -1600,6 +1603,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
ret = -EINVAL;
break;
}
+
+ need_flush = true;
qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
break;
case RAM_SAVE_FLAG_COMPRESS_PAGE:
@@ -1632,6 +1637,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
ret = -EINVAL;
break;
}
+ need_flush = true;
break;
case RAM_SAVE_FLAG_EOS:
/* normal exit */
@@ -1651,6 +1657,11 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
}
rcu_read_unlock();
+
+ if (!ret && ram_cache_enable && need_flush) {
+ DPRINTF("Flush ram_cache\n");
+ colo_flush_ram_cache();
+ }
DPRINTF("Completed load of VM with exit code %d seq iteration "
"%" PRIu64 "\n", ret, seq_iter);
return ret;
@@ -1736,6 +1747,87 @@ static void *memory_region_get_ram_cache_ptr(MemoryRegion *mr, RAMBlock *block)
return block->host_cache + (addr - block->offset);
}
+/* fix me: should this helper function be merged with
+ * migration_bitmap_find_and_reset_dirty ?
+ */
+static inline
+ram_addr_t host_bitmap_find_and_reset_dirty(MemoryRegion *mr,
+ ram_addr_t start)
+{
+ unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
+ unsigned long nr = base + (start >> TARGET_PAGE_BITS);
+ uint64_t mr_size = TARGET_PAGE_ALIGN(memory_region_size(mr));
+ unsigned long size = base + (mr_size >> TARGET_PAGE_BITS);
+
+ unsigned long next;
+
+ next = find_next_bit(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION],
+ size, nr);
+ if (next < size) {
+ clear_bit(next, ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
+ }
+ return (next - base) << TARGET_PAGE_BITS;
+}
+
+/*
+* Flush content of RAM cache into SVM's memory.
+* Only flush the pages that be dirtied by PVM or SVM or both.
+*/
+void colo_flush_ram_cache(void)
+{
+ RAMBlock *block = NULL;
+ void *dst_host;
+ void *src_host;
+ ram_addr_t ca = 0, ha = 0;
+ bool got_ca = 0, got_ha = 0;
+ int64_t host_dirty = 0, both_dirty = 0;
+
+ address_space_sync_dirty_bitmap(&address_space_memory);
+ rcu_read_lock();
+ block = QLIST_FIRST_RCU(&ram_list.blocks);
+ while (true) {
+ if (ca < block->used_length && ca <= ha) {
+ ca = migration_bitmap_find_and_reset_dirty(block->mr, ca);
+ if (ca < block->used_length) {
+ got_ca = 1;
+ }
+ }
+ if (ha < block->used_length && ha <= ca) {
+ ha = host_bitmap_find_and_reset_dirty(block->mr, ha);
+ if (ha < block->used_length && ha != ca) {
+ got_ha = 1;
+ }
+ host_dirty += (ha < block->used_length ? 1 : 0);
+ both_dirty += (ha < block->used_length && ha == ca ? 1 : 0);
+ }
+ if (ca >= block->used_length && ha >= block->used_length) {
+ ca = 0;
+ ha = 0;
+ block = QLIST_NEXT_RCU(block, next);
+ if (!block) {
+ break;
+ }
+ } else {
+ if (got_ha) {
+ got_ha = 0;
+ dst_host = memory_region_get_ram_ptr(block->mr) + ha;
+ src_host = memory_region_get_ram_cache_ptr(block->mr, block)
+ + ha;
+ memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
+ }
+ if (got_ca) {
+ got_ca = 0;
+ dst_host = memory_region_get_ram_ptr(block->mr) + ca;
+ src_host = memory_region_get_ram_cache_ptr(block->mr, block)
+ + ca;
+ memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
+ }
+ }
+ }
+ rcu_read_unlock();
+ assert(migration_dirty_pages == 0);
+}
+
static SaveVMHandlers savevm_ram_handlers = {
.save_live_setup = ram_save_setup,
.save_live_iterate = ram_save_iterate,
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 15/34] COLO failover: Introduce a new command to trigger a failover
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (13 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 14/34] COLO RAM: Flush cached RAM into SVM's memory zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 16/34] COLO failover: Introduce state to record failover process zhanghailiang
` (18 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: Yang Hongyang, lizhijian, quintela, Markus Armbruster,
yunhong.jiang, eddie.dong, peter.huangpeng, dgilbert,
arei.gonglei, amit.shah, Lai Jiangshan, Luiz Capitulino,
zhanghailiang
We leave users to use whatever heartbeat solution they want, if the heartbeat
is lost, or other errors they detect, they can use command
'colo_lost_heartbeat' to tell COLO to do failover, COLO will do operations
accordingly.
For example,
If send the command to PVM, Primary will exit COLO mode, and takeover,
if to Secondary, Secondary will do failover work and at last takeover server.
Cc: Luiz Capitulino <lcapitulino@redhat.com>
Cc: Eric Blake <eblake@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
hmp-commands.hx | 15 +++++++++++++++
hmp.c | 8 ++++++++
hmp.h | 1 +
include/migration/colo.h | 4 ++++
include/migration/failover.h | 20 ++++++++++++++++++++
migration/Makefile.objs | 2 +-
migration/colo-comm.c | 11 +++++++++++
migration/colo-failover.c | 41 +++++++++++++++++++++++++++++++++++++++++
migration/colo.c | 1 +
qapi-schema.json | 25 +++++++++++++++++++++++++
qmp-commands.hx | 19 +++++++++++++++++++
stubs/migration-colo.c | 8 ++++++++
12 files changed, 154 insertions(+), 1 deletion(-)
create mode 100644 include/migration/failover.h
create mode 100644 migration/colo-failover.c
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 1d5b392..410637f 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1034,6 +1034,21 @@ Set the parameter @var{parameter} for migration.
ETEXI
{
+ .name = "colo_lost_heartbeat",
+ .args_type = "",
+ .params = "",
+ .help = "Tell COLO that heartbeat is lost,\n\t\t\t"
+ "a failover or takeover is needed.",
+ .mhandler.cmd = hmp_colo_lost_heartbeat,
+ },
+
+STEXI
+@item colo_lost_heartbeat
+@findex colo_lost_heartbeat
+Tell COLO that heartbeat is lost, a failover or takeover is needed.
+ETEXI
+
+ {
.name = "client_migrate_info",
.args_type = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
.params = "protocol hostname port tls-port cert-subject",
diff --git a/hmp.c b/hmp.c
index dcc66f1..7f3a8a9 100644
--- a/hmp.c
+++ b/hmp.c
@@ -1272,6 +1272,14 @@ void hmp_client_migrate_info(Monitor *mon, const QDict *qdict)
hmp_handle_error(mon, &err);
}
+void hmp_colo_lost_heartbeat(Monitor *mon, const QDict *qdict)
+{
+ Error *err = NULL;
+
+ qmp_colo_lost_heartbeat(&err);
+ hmp_handle_error(mon, &err);
+}
+
void hmp_set_password(Monitor *mon, const QDict *qdict)
{
const char *protocol = qdict_get_str(qdict, "protocol");
diff --git a/hmp.h b/hmp.h
index 0cf4f2a..c36c99c 100644
--- a/hmp.h
+++ b/hmp.h
@@ -68,6 +68,7 @@ void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict);
void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict);
void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict);
void hmp_client_migrate_info(Monitor *mon, const QDict *qdict);
+void hmp_colo_lost_heartbeat(Monitor *mon, const QDict *qdict);
void hmp_set_password(Monitor *mon, const QDict *qdict);
void hmp_expire_password(Monitor *mon, const QDict *qdict);
void hmp_eject(Monitor *mon, const QDict *qdict);
diff --git a/include/migration/colo.h b/include/migration/colo.h
index c01364e..815db2e 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -17,6 +17,7 @@
#include "migration/migration.h"
#include "block/coroutine.h"
#include "qemu/thread.h"
+#include "qemu/main-loop.h"
bool colo_supported(void);
void colo_info_mig_init(void);
@@ -35,6 +36,9 @@ bool loadvm_enable_colo(void);
void loadvm_exit_colo(void);
void *colo_process_incoming_checkpoints(void *opaque);
bool loadvm_in_colo_state(void);
+
+int get_colo_mode(void);
+
/* ram cache */
int create_and_init_ram_cache(void);
void colo_flush_ram_cache(void);
diff --git a/include/migration/failover.h b/include/migration/failover.h
new file mode 100644
index 0000000..1785b52
--- /dev/null
+++ b/include/migration/failover.h
@@ -0,0 +1,20 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO.,LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_FAILOVER_H
+#define QEMU_FAILOVER_H
+
+#include "qemu-common.h"
+
+void failover_request_active(Error **errp);
+
+#endif
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
index cb7bd30..50d8392 100644
--- a/migration/Makefile.objs
+++ b/migration/Makefile.objs
@@ -1,6 +1,6 @@
common-obj-y += migration.o tcp.o
-common-obj-$(CONFIG_COLO) += colo.o
common-obj-y += colo-comm.o
+common-obj-$(CONFIG_COLO) += colo.o colo-failover.o
common-obj-y += vmstate.o
common-obj-y += qemu-file.o qemu-file-buf.o qemu-file-unix.o qemu-file-stdio.o
common-obj-y += xbzrle.o
diff --git a/migration/colo-comm.c b/migration/colo-comm.c
index 049e96c..557c3c6 100644
--- a/migration/colo-comm.c
+++ b/migration/colo-comm.c
@@ -21,6 +21,17 @@ typedef struct {
static COLOInfo colo_info;
+int get_colo_mode(void)
+{
+ if (migrate_in_colo_state()) {
+ return COLO_MODE_PRIMARY;
+ } else if (loadvm_in_colo_state()) {
+ return COLO_MODE_SECONDARY;
+ } else {
+ return COLO_MODE_UNKNOWN;
+ }
+}
+
void savevm_skip_colo_state(void)
{
colo_info.skip = true;
diff --git a/migration/colo-failover.c b/migration/colo-failover.c
new file mode 100644
index 0000000..af068d3
--- /dev/null
+++ b/migration/colo-failover.c
@@ -0,0 +1,41 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#include "migration/colo.h"
+#include "migration/failover.h"
+#include "qmp-commands.h"
+#include "qapi/qmp/qerror.h"
+
+static QEMUBH *failover_bh;
+
+static void colo_failover_bh(void *opaque)
+{
+ qemu_bh_delete(failover_bh);
+ failover_bh = NULL;
+ /*TODO: Do failover work */
+}
+
+void failover_request_active(Error **errp)
+{
+ failover_bh = qemu_bh_new(colo_failover_bh, NULL);
+ qemu_bh_schedule(failover_bh);
+}
+
+void qmp_colo_lost_heartbeat(Error **errp)
+{
+ if (get_colo_mode() == COLO_MODE_UNKNOWN) {
+ error_setg(errp, QERR_FEATURE_DISABLED, "colo");
+ return;
+ }
+
+ failover_request_active(errp);
+}
diff --git a/migration/colo.c b/migration/colo.c
index ece8cb4..6990c0c 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -14,6 +14,7 @@
#include "migration/colo.h"
#include "trace.h"
#include "qemu/error-report.h"
+#include "migration/failover.h"
enum {
COLO_CHECPOINT_READY = 0x46,
diff --git a/qapi-schema.json b/qapi-schema.json
index b41243a..94d178a 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -666,6 +666,31 @@
'*tls-port': 'int', '*cert-subject': 'str' } }
##
+# @COLOMode
+#
+# The colo mode
+#
+# @unknown: unknown mode
+#
+# @primary: master side
+#
+# @secondary: slave side
+#
+# Since: 2.4
+##
+{ 'enum': 'COLOMode',
+ 'data': [ 'unknown', 'primary', 'secondary'] }
+
+##
+# @colo-lost-heartbeat
+#
+# Tell qemu that heartbeat is lost, request it to do takeover working.
+#
+# Since: 2.4
+##
+{ 'command': 'colo-lost-heartbeat' }
+
+##
# @MouseInfo:
#
# Information about a mouse device.
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 6e441eb..443ef8e 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -781,6 +781,25 @@ Example:
EQMP
{
+ .name = "colo-lost-heartbeat",
+ .args_type = "",
+ .mhandler.cmd_new = qmp_marshal_input_colo_lost_heartbeat,
+ },
+
+SQMP
+colo-lost-heartbeat
+--------------------
+
+Tell COLO that heartbeat is lost, a failover or takeover is needed.
+
+Example:
+
+-> { "execute": "colo-lost-heartbeat" }
+<- { "return": {} }
+
+EQMP
+
+ {
.name = "client_migrate_info",
.args_type = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
.params = "protocol hostname port tls-port cert-subject",
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index a2edd96..2f0b455 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -11,6 +11,7 @@
*/
#include "migration/colo.h"
+#include "qmp-commands.h"
bool colo_supported(void)
{
@@ -35,3 +36,10 @@ void *colo_process_incoming_checkpoints(void *opaque)
{
return NULL;
}
+
+void qmp_colo_lost_heartbeat(Error **errp)
+{
+ error_setg(errp, "COLO is not supported, please rerun configure"
+ " with --enable-colo option in order to support"
+ " COLO feature");
+}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 16/34] COLO failover: Introduce state to record failover process
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (14 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 15/34] COLO failover: Introduce a new command to trigger a failover zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO primary/secondary vm failover work zhanghailiang
` (17 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, zhanghailiang
When handling failover, we do different things according to the different stage
of failover process, here we introduce a global atomic variable to record the
status of failover.
We add four failover status to indicate the different stage of failover process.
You should use the helpers to get and set the value.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
---
include/migration/failover.h | 9 +++++++++
migration/colo-failover.c | 24 ++++++++++++++++++++++++
trace-events | 1 +
3 files changed, 34 insertions(+)
diff --git a/include/migration/failover.h b/include/migration/failover.h
index 1785b52..56d57d7 100644
--- a/include/migration/failover.h
+++ b/include/migration/failover.h
@@ -15,6 +15,15 @@
#include "qemu-common.h"
+typedef enum COLOFailoverStatus {
+ FAILOVER_STATUS_NONE = 0,
+ FAILOVER_STATUS_REQUEST = 1, /* Request but not handled */
+ FAILOVER_STATUS_HANDLING = 2, /* In the process of handling failover */
+ FAILOVER_STATUS_COMPLETED = 3, /* Finish the failover process */
+} COLOFailoverStatus;
+
+int failover_set_state(int old_state, int new_state);
+int failover_get_state(void);
void failover_request_active(Error **errp);
#endif
diff --git a/migration/colo-failover.c b/migration/colo-failover.c
index af068d3..1bf22f5 100644
--- a/migration/colo-failover.c
+++ b/migration/colo-failover.c
@@ -14,8 +14,11 @@
#include "migration/failover.h"
#include "qmp-commands.h"
#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+#include "trace.h"
static QEMUBH *failover_bh;
+static COLOFailoverStatus failover_state;
static void colo_failover_bh(void *opaque)
{
@@ -26,10 +29,31 @@ static void colo_failover_bh(void *opaque)
void failover_request_active(Error **errp)
{
+ if (failover_set_state(FAILOVER_STATUS_NONE, FAILOVER_STATUS_REQUEST)
+ != FAILOVER_STATUS_NONE) {
+ error_setg(errp, "COLO failover is already actived");
+ return;
+ }
failover_bh = qemu_bh_new(colo_failover_bh, NULL);
qemu_bh_schedule(failover_bh);
}
+int failover_set_state(int old_state, int new_state)
+{
+ int old;
+
+ old = atomic_cmpxchg(&failover_state, old_state, new_state);;
+ if (old == old_state) {
+ trace_failover_set_state(new_state);
+ }
+ return old;
+}
+
+int failover_get_state(void)
+{
+ return atomic_read(&failover_state);
+}
+
void qmp_colo_lost_heartbeat(Error **errp)
{
if (get_colo_mode() == COLO_MODE_UNKNOWN) {
diff --git a/trace-events b/trace-events
index c34c7c4..20551c8 100644
--- a/trace-events
+++ b/trace-events
@@ -1475,6 +1475,7 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) ""
# migration/colo.c
colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
colo_receive_message(const char *msg) "Receive '%s'"
+failover_set_state(int new_state) "new state %d"
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO primary/secondary vm failover work
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (15 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 16/34] COLO failover: Introduce state to record failover process zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 18/34] qmp event: Add event notification for COLO error zhanghailiang
` (16 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, zhanghailiang
If there are some errors happen, we will give users(administrators) time to
get involved in failover verdict, which they can decide
which side should take over the work by using 'colo_lost_heartbeat' command.
Note: The default verdict is primary VM takes over work while secondary VM exit.
So if users choose secondary VM to take over work, please make sure that
Primary VM is dead, or there will be 'split-brain' problem.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
include/migration/colo.h | 4 ++
include/migration/failover.h | 1 +
migration/colo-failover.c | 15 ++++-
migration/colo.c | 149 +++++++++++++++++++++++++++++++++++++++++--
trace-events | 1 +
5 files changed, 165 insertions(+), 5 deletions(-)
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 815db2e..0364bab 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -43,4 +43,8 @@ int get_colo_mode(void);
int create_and_init_ram_cache(void);
void colo_flush_ram_cache(void);
void release_ram_cache(void);
+
+/* failover */
+void colo_do_failover(MigrationState *s);
+
#endif
diff --git a/include/migration/failover.h b/include/migration/failover.h
index 56d57d7..f85f97d 100644
--- a/include/migration/failover.h
+++ b/include/migration/failover.h
@@ -25,5 +25,6 @@ typedef enum COLOFailoverStatus {
int failover_set_state(int old_state, int new_state);
int failover_get_state(void);
void failover_request_active(Error **errp);
+bool failover_request_is_active(void);
#endif
diff --git a/migration/colo-failover.c b/migration/colo-failover.c
index 1bf22f5..7b0fa03 100644
--- a/migration/colo-failover.c
+++ b/migration/colo-failover.c
@@ -22,9 +22,17 @@ static COLOFailoverStatus failover_state;
static void colo_failover_bh(void *opaque)
{
+ int old_state;
+
qemu_bh_delete(failover_bh);
failover_bh = NULL;
- /*TODO: Do failover work */
+ old_state = failover_set_state(FAILOVER_STATUS_REQUEST,
+ FAILOVER_STATUS_HANDLING);
+ if (old_state != FAILOVER_STATUS_REQUEST) {
+ error_report(" Unkown error for failover, old_state=%d", old_state);
+ return;
+ }
+ colo_do_failover(NULL);
}
void failover_request_active(Error **errp)
@@ -54,6 +62,11 @@ int failover_get_state(void)
return atomic_read(&failover_state);
}
+bool failover_request_is_active(void)
+{
+ return ((failover_get_state() != FAILOVER_STATUS_NONE));
+}
+
void qmp_colo_lost_heartbeat(Error **errp)
{
if (get_colo_mode() == COLO_MODE_UNKNOWN) {
diff --git a/migration/colo.c b/migration/colo.c
index 6990c0c..243aa32 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -73,6 +73,83 @@ bool loadvm_in_colo_state(void)
return colo != NULL;
}
+static bool colo_runstate_is_stopped(void)
+{
+ return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
+}
+
+/*
+ * there are two way to entry this function
+ * 1. From colo checkpoint incoming thread, in this case
+ * we should protect it by iothread lock
+ * 2. From user command, because hmp/qmp command
+ * was happened in main loop, iothread lock will cause a
+ * dead lock.
+ */
+static void secondary_vm_do_failover(void)
+{
+ int old_state;
+
+ /* It means that VM exit from COLO state */
+ colo = NULL;
+
+ if (!autostart) {
+ error_report("\"-S\" qemu option will be ignored in secondary side");
+ /* recover runstate to normal migration finish state */
+ autostart = true;
+ }
+
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_COMPLETED);
+ if (old_state != FAILOVER_STATUS_HANDLING) {
+ error_report("Serious error while do failover for secondary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+ /* For Secondary VM, jump to incoming co */
+ if (migration_incoming_co) {
+ qemu_coroutine_enter(migration_incoming_co, NULL);
+ }
+}
+
+static void primary_vm_do_failover(void)
+{
+ MigrationState *s = migrate_get_current();
+ int old_state;
+
+ if (!colo_runstate_is_stopped()) {
+ vm_stop_force_state(RUN_STATE_COLO);
+ }
+
+ if (s->state != MIGRATION_STATUS_FAILED) {
+ migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ }
+
+ vm_start();
+
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_COMPLETED);
+ if (old_state != FAILOVER_STATUS_COMPLETED) {
+ error_report("Serious error while do failover for Primary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+}
+
+void colo_do_failover(MigrationState *s)
+{
+ /* Make sure vm stopped while failover */
+ if (!colo_runstate_is_stopped()) {
+ vm_stop_force_state(RUN_STATE_COLO);
+ }
+
+ if (get_colo_mode() == COLO_MODE_SECONDARY) {
+ secondary_vm_do_failover();
+ } else {
+ primary_vm_do_failover();
+ }
+}
+
/* colo checkpoint control helper */
static int colo_ctl_put(QEMUFile *f, uint64_t request)
{
@@ -144,11 +221,23 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
goto out;
}
+ if (failover_request_is_active()) {
+ ret = -1;
+ goto out;
+ }
/* suspend and save vm state to colo buffer */
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("run", "stop");
+ /*
+ * failover request bh could be called after
+ * vm_stop_force_state so we check failover_request_is_active() again.
+ */
+ if (failover_request_is_active()) {
+ ret = -1;
+ goto out;
+ }
/* Disable block migration */
s->params.blk = 0;
@@ -209,7 +298,7 @@ static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
QEMUFile *colo_control = NULL;
- int ret;
+ int i, ret;
colo_control = qemu_fopen_socket(qemu_get_fd(s->file), "rb");
if (!colo_control) {
@@ -239,6 +328,11 @@ static void *colo_thread(void *opaque)
trace_colo_vm_state_change("stop", "run");
while (s->state == MIGRATION_STATUS_COLO) {
+ if (failover_request_is_active()) {
+ error_report("failover request");
+ goto out;
+ }
+
/* start a colo checkpoint */
if (colo_do_checkpoint_transaction(s, colo_control)) {
goto out;
@@ -246,7 +340,26 @@ static void *colo_thread(void *opaque)
}
out:
- migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ error_report("colo: some error happens in colo_thread");
+ /* Give users time (2s) to get involved in this verdict */
+ for (i = 0; i < 10; i++) {
+ if (failover_request_is_active()) {
+ error_report("Primary VM will take over work");
+ break;
+ }
+ usleep(200 * 1000);
+ }
+ qemu_mutex_lock_iothread();
+ if (!failover_request_is_active()) {
+ error_report("Primary VM will take over work in default");
+ failover_request_active(NULL);
+ }
+ qemu_mutex_unlock_iothread();
+
+ while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+ ;
+ }
+ failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
qsb_free(colo_buffer);
colo_buffer = NULL;
@@ -317,7 +430,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
QEMUFile *f = colo_in->file;
int fd = qemu_get_fd(f);
QEMUFile *ctl = NULL, *fb = NULL;
- int ret;
+ int i, ret;
uint64_t total_size;
colo = qemu_coroutine_self();
@@ -363,6 +476,11 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
}
+ if (failover_request_is_active()) {
+ error_report("failover request");
+ goto out;
+ }
+
/* suspend guest */
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
@@ -431,7 +549,30 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
out:
- colo = NULL;
+ error_report("Detect some error or get a failover request");
+ /* Give users time (2s) to get involved in this verdict */
+ for (i = 0; i < 10; i++) {
+ if (failover_request_is_active()) {
+ error_report("Secondary VM will take over work");
+ break;
+ }
+ usleep(200*1000);
+ }
+ /* check flag again*/
+ if (!failover_request_is_active()) {
+ /*
+ * We assume that Primary VM is still alive according to heartbeat,
+ * just kill Secondary VM
+ */
+ error_report("SVM is going to exit in default!");
+ exit(1);
+ } else {
+ /* if we went here, means Primary VM may dead, we are doing failover */
+ while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+ ;
+ }
+ failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
+ }
if (fb) {
qemu_fclose(fb);
diff --git a/trace-events b/trace-events
index 20551c8..66bcd08 100644
--- a/trace-events
+++ b/trace-events
@@ -1476,6 +1476,7 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) ""
colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
colo_receive_message(const char *msg) "Receive '%s'"
failover_set_state(int new_state) "new state %d"
+colo_rcv_pkt(int result) "Result of net packets comparing is different: %d"
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 18/34] qmp event: Add event notification for COLO error
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (16 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO primary/secondary vm failover work zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 19/34] COLO failover: Don't do failover during loading VM's state zhanghailiang
` (15 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Markus Armbruster, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, amit.shah, Michael Roth,
zhanghailiang
If some errors happen during VM's COLO FT stage, it's import to notify the users
this event, Togehter with 'colo_lost_heartbeat', users can intervene in COLO's
failover work immediately.
If users don't want to get involved in COLO's failover verdict,
it is still necessary to notify users that we exit COLO mode.
Cc: Markus Armbruster <armbru@redhat.com>
Cc: Michael Roth <mdroth@linux.vnet.ibm.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
docs/qmp/qmp-events.txt | 16 ++++++++++++++++
migration/colo.c | 10 +++++++++-
qapi/event.json | 15 +++++++++++++++
3 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/docs/qmp/qmp-events.txt b/docs/qmp/qmp-events.txt
index d92cc48..c7b4581 100644
--- a/docs/qmp/qmp-events.txt
+++ b/docs/qmp/qmp-events.txt
@@ -488,6 +488,22 @@ Example:
{"timestamp": {"seconds": 1432121972, "microseconds": 744001},
"event": "MIGRATION", "data": {"status": "completed"}}
+COLO_EXIT
+---------
+
+Emitted when VM finish COLO mode due to some errors happening or
+the request of users.
+
+Data: None.
+
+ - "mode": COLO mode, 'primary' or 'secondary'
+ - "error": Error message (json-string, optional)
+
+Example:
+
+{"timestamp": {"seconds": 2032141960, "microseconds": 417172},
+ "event": "COLO_EXIT", "data": {"mode": "primary"}}
+
STOP
----
diff --git a/migration/colo.c b/migration/colo.c
index 243aa32..70fcf25 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -15,6 +15,7 @@
#include "trace.h"
#include "qemu/error-report.h"
#include "migration/failover.h"
+#include "qapi-event.h"
enum {
COLO_CHECPOINT_READY = 0x46,
@@ -341,6 +342,7 @@ static void *colo_thread(void *opaque)
out:
error_report("colo: some error happens in colo_thread");
+ qapi_event_send_colo_exit("primary", true, "unknown", NULL);;
/* Give users time (2s) to get involved in this verdict */
for (i = 0; i < 10; i++) {
if (failover_request_is_active()) {
@@ -550,13 +552,19 @@ void *colo_process_incoming_checkpoints(void *opaque)
out:
error_report("Detect some error or get a failover request");
+ /*
+ * Here, we raise a qmp event to the user,
+ * It can help user to know what happens, and help deciding whether to
+ * do failover.
+ */
+ qapi_event_send_colo_exit("secondary", true, "unknown", NULL);;
/* Give users time (2s) to get involved in this verdict */
for (i = 0; i < 10; i++) {
if (failover_request_is_active()) {
error_report("Secondary VM will take over work");
break;
}
- usleep(200*1000);
+ usleep(200 * 1000);
}
/* check flag again*/
if (!failover_request_is_active()) {
diff --git a/qapi/event.json b/qapi/event.json
index f0cef01..21dfe3a 100644
--- a/qapi/event.json
+++ b/qapi/event.json
@@ -255,6 +255,21 @@
'data': {'status': 'MigrationStatus'}}
##
+# @COLO_EXIT
+#
+# Emitted when VM finish COLO mode due to some errors happening or
+# the request of users.
+#
+# @mode: 'primary' or 'secondeary'.
+#
+# @error: #optional, error message. Only present on error happening.
+#
+# Since: 2.4
+##
+{ 'event': 'COLO_EXIT',
+ 'data': {'mode': 'str', '*error':'str'}}
+
+##
# @ACPI_DEVICE_OST
#
# Emitted when guest executes ACPI _OST method.
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 19/34] COLO failover: Don't do failover during loading VM's state
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (17 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 18/34] qmp event: Add event notification for COLO error zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 20/34] COLO: Add new command parameter 'forward_nic' 'colo_script' for net zhanghailiang
` (14 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Lai Jiangshan, zhanghailiang
We should not do failover work while the main thread is loading
VM's state, otherwise it will destroy the consistent of VM's memory and
device state.
Here we add a new failover status 'RELAUNCH' which means we should
relaunch the process of failover.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
---
include/migration/failover.h | 2 ++
migration/colo.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/include/migration/failover.h b/include/migration/failover.h
index f85f97d..3f65f5d 100644
--- a/include/migration/failover.h
+++ b/include/migration/failover.h
@@ -20,6 +20,8 @@ typedef enum COLOFailoverStatus {
FAILOVER_STATUS_REQUEST = 1, /* Request but not handled */
FAILOVER_STATUS_HANDLING = 2, /* In the process of handling failover */
FAILOVER_STATUS_COMPLETED = 3, /* Finish the failover process */
+ /* Optional, Relaunch the failover process, again 'NONE' -> 'COMPLETED' */
+ FAILOVER_STATUS_RELAUNCH = 4,
} COLOFailoverStatus;
int failover_set_state(int old_state, int new_state);
diff --git a/migration/colo.c b/migration/colo.c
index 70fcf25..aab7b38 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -53,6 +53,7 @@ enum {
};
static QEMUBH *colo_bh;
+static bool vmstate_loading;
static Coroutine *colo;
/* colo buffer */
#define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
@@ -90,6 +91,18 @@ static bool colo_runstate_is_stopped(void)
static void secondary_vm_do_failover(void)
{
int old_state;
+ /* Can not do failover during the process of VM's loading VMstate, Or
+ * it will break the secondary VM.
+ */
+ if (vmstate_loading) {
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_RELAUNCH);
+ if (old_state != FAILOVER_STATUS_HANDLING) {
+ error_report("Unknow error while do failover for secondary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+ }
/* It means that VM exit from COLO state */
colo = NULL;
@@ -528,13 +541,22 @@ void *colo_process_incoming_checkpoints(void *opaque)
qemu_mutex_lock_iothread();
qemu_system_reset(VMRESET_SILENT);
+ vmstate_loading = true;
if (qemu_loadvm_state(fb) < 0) {
error_report("COLO: loadvm failed");
+ vmstate_loading = false;
qemu_mutex_unlock_iothread();
goto out;
}
+
+ vmstate_loading = false;
qemu_mutex_unlock_iothread();
+ if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) {
+ failover_set_state(FAILOVER_STATUS_RELAUNCH, FAILOVER_STATUS_NONE);
+ failover_request_active(NULL);
+ goto out;
+ }
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_LOADED);
if (ret < 0) {
goto out;
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 20/34] COLO: Add new command parameter 'forward_nic' 'colo_script' for net
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (18 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 19/34] COLO failover: Don't do failover during loading VM's state zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 21/34] COLO NIC: Init/remove colo nic devices when add/cleanup tap devices zhanghailiang
` (13 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Markus Armbruster, Jason Wang, yunhong.jiang,
eddie.dong, peter.huangpeng, dgilbert, arei.gonglei,
Stefan Hajnoczi, amit.shah, zhanghailiang
The 'forward_nic' should be assigned with network name,
for exmple, 'eth2'. It will be parameter of 'colo_script',
'colo_script' should be assigned with an scirpt path.
We parse these parameter in tap.
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: Eric Blake <eblake@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/colo-nic.h | 23 +++++++++++++++++++++++
include/net/net.h | 2 ++
net/tap.c | 26 +++++++++++++++++++++++---
qapi-schema.json | 8 +++++++-
qemu-options.hx | 7 +++++++
5 files changed, 62 insertions(+), 4 deletions(-)
create mode 100644 include/net/colo-nic.h
diff --git a/include/net/colo-nic.h b/include/net/colo-nic.h
new file mode 100644
index 0000000..3075d97
--- /dev/null
+++ b/include/net/colo-nic.h
@@ -0,0 +1,23 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef COLO_NIC_H
+#define COLO_NIC_H
+
+typedef struct COLONicState {
+ char nicname[128]; /* forward dev */
+ char script[1024]; /* colo script */
+ char ifname[128]; /* e.g. tap name */
+} COLONicState;
+
+#endif
diff --git a/include/net/net.h b/include/net/net.h
index 6a6cbef..f3bf8e6 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -8,6 +8,7 @@
#include "net/queue.h"
#include "migration/vmstate.h"
#include "qapi-types.h"
+#include "net/colo-nic.h"
#define MAX_QUEUE_NUM 1024
@@ -88,6 +89,7 @@ struct NetClientState {
char *model;
char *name;
char info_str[256];
+ COLONicState cns;
unsigned receive_disabled : 1;
NetClientDestructor *destructor;
unsigned int queue_index;
diff --git a/net/tap.c b/net/tap.c
index bd01590..ad99fe3 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -632,6 +632,7 @@ static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
Error *err = NULL;
TAPState *s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
int vhostfd;
+ NetClientState *nc = &(s->nc);
tap_set_sndbuf(s->fd, tap, &err);
if (err) {
@@ -656,6 +657,16 @@ static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
}
}
+ snprintf(nc->cns.ifname, sizeof(nc->cns.ifname), "%s", ifname);
+ if (tap->has_colo_script) {
+ snprintf(nc->cns.script, sizeof(nc->cns.script), "%s",
+ tap->colo_script);
+ }
+ if (tap->has_forward_nic) {
+ snprintf(nc->cns.nicname, sizeof(nc->cns.nicname), "%s",
+ tap->forward_nic);
+ }
+
if (tap->has_vhost ? tap->vhost :
vhostfdname || (tap->has_vhostforce && tap->vhostforce)) {
VhostNetOptions options;
@@ -774,9 +785,10 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
if (tap->has_ifname || tap->has_script || tap->has_downscript ||
tap->has_vnet_hdr || tap->has_helper || tap->has_queues ||
- tap->has_vhostfd) {
+ tap->has_vhostfd || tap->has_colo_script || tap->has_forward_nic) {
error_setg(errp, "ifname=, script=, downscript=, vnet_hdr=, "
"helper=, queues=, and vhostfd= "
+ "colo_script=, and forward_nic= "
"are invalid with fds=");
return -1;
}
@@ -819,9 +831,11 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
}
} else if (tap->has_helper) {
if (tap->has_ifname || tap->has_script || tap->has_downscript ||
- tap->has_vnet_hdr || tap->has_queues || tap->has_vhostfds) {
+ tap->has_vnet_hdr || tap->has_queues || tap->has_vhostfds ||
+ tap->has_colo_script || tap->has_forward_nic) {
error_setg(errp, "ifname=, script=, downscript=, vnet_hdr=, "
- "queues=, and vhostfds= are invalid with helper=");
+ "queues=, and vhostfds=, colo_script=, and "
+ "forward_nic= are invalid with helper=");
return -1;
}
@@ -843,6 +857,12 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
return -1;
}
} else {
+ if (queues > 1 && (tap->has_colo_script || tap->has_forward_nic)) {
+ error_report("queues > 1 is invalid if colo_script or "
+ "forward_nic is specified");
+ return -1;
+ }
+
if (tap->has_vhostfds) {
error_setg(errp, "vhostfds= is invalid if fds= wasn't specified");
return -1;
diff --git a/qapi-schema.json b/qapi-schema.json
index 94d178a..91c4ea5 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -2296,6 +2296,10 @@
#
# @queues: #optional number of queues to be created for multiqueue capable tap
#
+# @forward_nic: #optional the name of host physical forward nic for COLO (Since 2.4)
+#
+# @colo_script: #optional the script file which used by COLO (Since 2.4)
+#
# Since 1.2
##
{ 'struct': 'NetdevTapOptions',
@@ -2312,7 +2316,9 @@
'*vhostfd': 'str',
'*vhostfds': 'str',
'*vhostforce': 'bool',
- '*queues': 'uint32'} }
+ '*queues': 'uint32',
+ '*forward_nic': 'str',
+ '*colo_script': 'str'} }
##
# @NetdevSocketOptions
diff --git a/qemu-options.hx b/qemu-options.hx
index 7b8efbf..dd72c8d 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -1475,6 +1475,9 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
"-netdev tap,id=str[,fd=h][,fds=x:y:...:z][,ifname=name][,script=file][,downscript=dfile]\n"
" [,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off]\n"
" [,vhostfd=h][,vhostfds=x:y:...:z][,vhostforce=on|off][,queues=n]\n"
+#ifdef CONFIG_COLO
+ " [,forward_nic=nicname][,colo_script=scriptfile]\n"
+#endif
" configure a host TAP network backend with ID 'str'\n"
" use network scripts 'file' (default=" DEFAULT_NETWORK_SCRIPT ")\n"
" to configure it and 'dfile' (default=" DEFAULT_NETWORK_DOWN_SCRIPT ")\n"
@@ -1494,6 +1497,10 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
" use 'vhostfd=h' to connect to an already opened vhost net device\n"
" use 'vhostfds=x:y:...:z to connect to multiple already opened vhost net devices\n"
" use 'queues=n' to specify the number of queues to be created for multiqueue TAP\n"
+#ifdef CONFIG_COLO
+ " use 'forward_nic=nicname' to specify the host physical forward nic for QEMU\n"
+ " use 'colo_script=scriptfile' to specify script file when colo is enabled\n"
+#endif
"-netdev bridge,id=str[,br=bridge][,helper=helper]\n"
" configure a host TAP network backend with ID 'str' that is\n"
" connected to a bridge (default=" DEFAULT_BRIDGE_INTERFACE ")\n"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 21/34] COLO NIC: Init/remove colo nic devices when add/cleanup tap devices
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (19 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 20/34] COLO: Add new command parameter 'forward_nic' 'colo_script' for net zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 22/34] tap: Make launch_script() public zhanghailiang
` (12 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Jason Wang, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, Stefan Hajnoczi,
amit.shah, zhanghailiang
When go into COLO mode, we need to some init work for all VM's nics.
Here we use a list to record these nic, and for now we only support
the 'tap' nic backend.
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/colo-nic.h | 5 ++++
net/Makefile.objs | 1 +
net/colo-nic.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
net/net.c | 2 ++
net/tap.c | 12 ++++++----
stubs/migration-colo.c | 9 ++++++++
6 files changed, 86 insertions(+), 5 deletions(-)
create mode 100644 net/colo-nic.c
diff --git a/include/net/colo-nic.h b/include/net/colo-nic.h
index 3075d97..3941b6e 100644
--- a/include/net/colo-nic.h
+++ b/include/net/colo-nic.h
@@ -14,10 +14,15 @@
#ifndef COLO_NIC_H
#define COLO_NIC_H
+#include "migration/colo.h"
+
typedef struct COLONicState {
char nicname[128]; /* forward dev */
char script[1024]; /* colo script */
char ifname[128]; /* e.g. tap name */
} COLONicState;
+void colo_add_nic_devices(COLONicState *cns);
+void colo_remove_nic_devices(COLONicState *cns);
+
#endif
diff --git a/net/Makefile.objs b/net/Makefile.objs
index ec19cb3..73f4a81 100644
--- a/net/Makefile.objs
+++ b/net/Makefile.objs
@@ -13,3 +13,4 @@ common-obj-$(CONFIG_HAIKU) += tap-haiku.o
common-obj-$(CONFIG_SLIRP) += slirp.o
common-obj-$(CONFIG_VDE) += vde.o
common-obj-$(CONFIG_NETMAP) += netmap.o
+common-obj-$(CONFIG_COLO) += colo-nic.o
diff --git a/net/colo-nic.c b/net/colo-nic.c
new file mode 100644
index 0000000..b6a8330
--- /dev/null
+++ b/net/colo-nic.c
@@ -0,0 +1,62 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ *
+ */
+#include "include/migration/migration.h"
+#include "migration/colo.h"
+#include "net/net.h"
+#include "net/colo-nic.h"
+#include "qemu/error-report.h"
+
+typedef struct nic_device {
+ COLONicState *cns;
+ int (*configure)(COLONicState *cns, bool up, int side, int index);
+ QTAILQ_ENTRY(nic_device) next;
+ bool is_up;
+} nic_device;
+
+QTAILQ_HEAD(, nic_device) nic_devices = QTAILQ_HEAD_INITIALIZER(nic_devices);
+
+void colo_add_nic_devices(COLONicState *cns)
+{
+ struct nic_device *nic;
+ NetClientState *nc = container_of(cns, NetClientState, cns);
+
+ if (nc->info->type == NET_CLIENT_OPTIONS_KIND_HUBPORT ||
+ nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
+ return;
+ }
+ QTAILQ_FOREACH(nic, &nic_devices, next) {
+ NetClientState *nic_nc = container_of(nic->cns, NetClientState, cns);
+ if ((nic_nc->peer && nic_nc->peer == nc) ||
+ (nc->peer && nc->peer == nic_nc)) {
+ return;
+ }
+ }
+
+ nic = g_malloc0(sizeof(*nic));
+ nic->configure = NULL;
+ nic->cns = cns;
+
+ QTAILQ_INSERT_TAIL(&nic_devices, nic, next);
+}
+
+void colo_remove_nic_devices(COLONicState *cns)
+{
+ struct nic_device *nic, *next_nic;
+
+ QTAILQ_FOREACH_SAFE(nic, &nic_devices, next, next_nic) {
+ if (nic->cns == cns) {
+ QTAILQ_REMOVE(&nic_devices, nic, next);
+ g_free(nic);
+ }
+ }
+}
diff --git a/net/net.c b/net/net.c
index 6ff7fec..8e9c719 100644
--- a/net/net.c
+++ b/net/net.c
@@ -284,6 +284,7 @@ static void qemu_net_client_setup(NetClientState *nc,
peer->peer = nc;
}
QTAILQ_INSERT_TAIL(&net_clients, nc, next);
+ colo_add_nic_devices(&nc->cns);
nc->incoming_queue = qemu_new_net_queue(nc);
nc->destructor = destructor;
@@ -359,6 +360,7 @@ void *qemu_get_nic_opaque(NetClientState *nc)
static void qemu_cleanup_net_client(NetClientState *nc)
{
QTAILQ_REMOVE(&net_clients, nc, next);
+ colo_remove_nic_devices(&nc->cns);
if (nc->info->cleanup) {
nc->info->cleanup(nc);
diff --git a/net/tap.c b/net/tap.c
index ad99fe3..c2135cd 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -41,6 +41,7 @@
#include "qemu/error-report.h"
#include "net/tap.h"
+#include "net/colo-nic.h"
#include "net/vhost_net.h"
@@ -627,7 +628,8 @@ static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
const char *model, const char *name,
const char *ifname, const char *script,
const char *downscript, const char *vhostfdname,
- int vnet_hdr, int fd, Error **errp)
+ int vnet_hdr, int fd, bool setup_colo,
+ Error **errp)
{
Error *err = NULL;
TAPState *s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
@@ -773,7 +775,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
net_init_tap_one(tap, peer, "tap", name, NULL,
script, downscript,
- vhostfdname, vnet_hdr, fd, &err);
+ vhostfdname, vnet_hdr, fd, true, &err);
if (err) {
error_propagate(errp, err);
return -1;
@@ -823,7 +825,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
net_init_tap_one(tap, peer, "tap", name, ifname,
script, downscript,
tap->has_vhostfds ? vhost_fds[i] : NULL,
- vnet_hdr, fd, &err);
+ vnet_hdr, fd, false, &err);
if (err) {
error_propagate(errp, err);
return -1;
@@ -850,7 +852,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
net_init_tap_one(tap, peer, "bridge", name, ifname,
script, downscript, vhostfdname,
- vnet_hdr, fd, &err);
+ vnet_hdr, fd, false, &err);
if (err) {
error_propagate(errp, err);
close(fd);
@@ -895,7 +897,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
net_init_tap_one(tap, peer, "tap", name, ifname,
i >= 1 ? "no" : script,
i >= 1 ? "no" : downscript,
- vhostfdname, vnet_hdr, fd, &err);
+ vhostfdname, vnet_hdr, fd, i == 0, &err);
if (err) {
error_propagate(errp, err);
close(fd);
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index 2f0b455..91f35e9 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -12,6 +12,7 @@
#include "migration/colo.h"
#include "qmp-commands.h"
+#include "net/colo-nic.h"
bool colo_supported(void)
{
@@ -37,6 +38,14 @@ void *colo_process_incoming_checkpoints(void *opaque)
return NULL;
}
+void colo_add_nic_devices(COLONicState *cns)
+{
+}
+
+void colo_remove_nic_devices(COLONicState *cns)
+{
+}
+
void qmp_colo_lost_heartbeat(Error **errp)
{
error_setg(errp, "COLO is not supported, please rerun configure"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 22/34] tap: Make launch_script() public
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (20 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 21/34] COLO NIC: Init/remove colo nic devices when add/cleanup tap devices zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 23/34] COLO NIC: Implement colo nic device interface configure() zhanghailiang
` (11 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Jason Wang, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, Stefan Hajnoczi,
amit.shah, zhanghailiang
We also change the parameters of launch_script().
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/tap.h | 2 ++
net/tap.c | 31 ++++++++++++++++++-------------
2 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/include/net/tap.h b/include/net/tap.h
index 5da4edc..ac99b31 100644
--- a/include/net/tap.h
+++ b/include/net/tap.h
@@ -38,4 +38,6 @@ int tap_get_fd(NetClientState *nc);
struct vhost_net;
struct vhost_net *tap_get_vhost_net(NetClientState *nc);
+void launch_script(char *const args[], int fd, Error **errp);
+
#endif /* QEMU_NET_TAP_H */
diff --git a/net/tap.c b/net/tap.c
index c2135cd..a715636 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -60,9 +60,6 @@ typedef struct TAPState {
unsigned host_vnet_hdr_len;
} TAPState;
-static void launch_script(const char *setup_script, const char *ifname,
- int fd, Error **errp);
-
static void tap_send(void *opaque);
static void tap_writable(void *opaque);
@@ -305,7 +302,14 @@ static void tap_cleanup(NetClientState *nc)
qemu_purge_queued_packets(nc);
if (s->down_script[0]) {
- launch_script(s->down_script, s->down_script_arg, s->fd, &err);
+ char *args[3];
+ char **parg;
+
+ parg = args;
+ *parg++ = (char *)s->down_script;
+ *parg++ = (char *)s->down_script_arg;
+ *parg = NULL;
+ launch_script(args, s->fd, &err);
if (err) {
error_report_err(err);
}
@@ -382,12 +386,10 @@ static TAPState *net_tap_fd_init(NetClientState *peer,
return s;
}
-static void launch_script(const char *setup_script, const char *ifname,
- int fd, Error **errp)
+void launch_script(char *const args[], int fd, Error **errp)
{
int pid, status;
- char *args[3];
- char **parg;
+ const char *setup_script = args[0];
/* try to launch network script */
pid = fork();
@@ -404,10 +406,6 @@ static void launch_script(const char *setup_script, const char *ifname,
close(i);
}
}
- parg = args;
- *parg++ = (char *)setup_script;
- *parg++ = (char *)ifname;
- *parg = NULL;
execv(setup_script, args);
_exit(1);
} else {
@@ -611,7 +609,14 @@ static int net_tap_init(const NetdevTapOptions *tap, int *vnet_hdr,
if (setup_script &&
setup_script[0] != '\0' &&
strcmp(setup_script, "no") != 0) {
- launch_script(setup_script, ifname, fd, &err);
+ char *args[3];
+ char **parg;
+ parg = args;
+ *parg++ = (char *)setup_script;
+ *parg++ = (char *)ifname;
+ *parg = NULL;
+
+ launch_script(args, fd, &err);
if (err) {
error_propagate(errp, err);
close(fd);
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 23/34] COLO NIC: Implement colo nic device interface configure()
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (21 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 22/34] tap: Make launch_script() public zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 24/34] colo-nic: Handle secondary VM's original net device configure zhanghailiang
` (10 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Jason Wang, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, Stefan Hajnoczi,
amit.shah, zhanghailiang
Implement colo nic device interface configure()
add a script to configure nic devices:
${QEMU_SCRIPT_DIR}/colo-proxy-script.sh
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/tap.h | 17 ++++++++
net/colo-nic.c | 48 ++++++++++++++++++++++-
net/tap.c | 17 --------
scripts/colo-proxy-script.sh | 92 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 156 insertions(+), 18 deletions(-)
create mode 100755 scripts/colo-proxy-script.sh
diff --git a/include/net/tap.h b/include/net/tap.h
index ac99b31..9688765 100644
--- a/include/net/tap.h
+++ b/include/net/tap.h
@@ -29,6 +29,23 @@
#include "qemu-common.h"
#include "qapi-types.h"
#include "standard-headers/linux/virtio_net.h"
+#include "net/net.h"
+#include "net/vhost_net.h"
+
+typedef struct TAPState {
+ NetClientState nc;
+ int fd;
+ char down_script[1024];
+ char down_script_arg[128];
+ uint8_t buf[NET_BUFSIZE];
+ bool read_poll;
+ bool write_poll;
+ bool using_vnet_hdr;
+ bool has_ufo;
+ bool enabled;
+ VHostNetState *vhost_net;
+ unsigned host_vnet_hdr_len;
+} TAPState;
int tap_enable(NetClientState *nc);
int tap_disable(NetClientState *nc);
diff --git a/net/colo-nic.c b/net/colo-nic.c
index b6a8330..4ce08c0 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -15,6 +15,7 @@
#include "net/net.h"
#include "net/colo-nic.h"
#include "qemu/error-report.h"
+#include "net/tap.h"
typedef struct nic_device {
COLONicState *cns;
@@ -23,8 +24,53 @@ typedef struct nic_device {
bool is_up;
} nic_device;
+static int launch_colo_script(COLONicState *cns, bool up, int side, int index)
+{
+ NetClientState *nc = container_of(cns, NetClientState, cns);
+ TAPState *s = DO_UPCAST(TAPState, nc, nc);
+ int i, argc = 6;
+ char *argv[7], index_str[32];
+ char **parg;
+ Error *err = NULL;
+
+ parg = argv;
+ *parg++ = cns->script;
+ *parg++ = (char *)(side == COLO_MODE_SECONDARY ? "secondary" : "primary");
+ *parg++ = (char *)(up ? "install" : "uninstall");
+ *parg++ = cns->nicname;
+ *parg++ = cns->ifname;
+ sprintf(index_str, "%d", index);
+ *parg++ = index_str;
+ *parg = NULL;
+
+ for (i = 0; i < argc; i++) {
+ if (!argv[i][0]) {
+ error_report("Can not get colo_script argument");
+ return -1;
+ }
+ }
+
+ launch_script(argv, s->fd, &err);
+ if (err) {
+ error_report_err(err);
+ return -1;
+ }
+ return 0;
+}
+
QTAILQ_HEAD(, nic_device) nic_devices = QTAILQ_HEAD_INITIALIZER(nic_devices);
+static int colo_nic_configure(COLONicState *cns,
+ bool up, int side, int index)
+{
+ if (!cns && index <= 0) {
+ error_report("Can not parse colo_script or forward_nic");
+ return -1;
+ }
+
+ return launch_colo_script(cns, up, side, index);
+}
+
void colo_add_nic_devices(COLONicState *cns)
{
struct nic_device *nic;
@@ -43,7 +89,7 @@ void colo_add_nic_devices(COLONicState *cns)
}
nic = g_malloc0(sizeof(*nic));
- nic->configure = NULL;
+ nic->configure = colo_nic_configure;
nic->cns = cns;
QTAILQ_INSERT_TAIL(&nic_devices, nic, next);
diff --git a/net/tap.c b/net/tap.c
index a715636..0e3bdbd 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -43,23 +43,6 @@
#include "net/tap.h"
#include "net/colo-nic.h"
-#include "net/vhost_net.h"
-
-typedef struct TAPState {
- NetClientState nc;
- int fd;
- char down_script[1024];
- char down_script_arg[128];
- uint8_t buf[NET_BUFSIZE];
- bool read_poll;
- bool write_poll;
- bool using_vnet_hdr;
- bool has_ufo;
- bool enabled;
- VHostNetState *vhost_net;
- unsigned host_vnet_hdr_len;
-} TAPState;
-
static void tap_send(void *opaque);
static void tap_writable(void *opaque);
diff --git a/scripts/colo-proxy-script.sh b/scripts/colo-proxy-script.sh
new file mode 100755
index 0000000..47c4fb5
--- /dev/null
+++ b/scripts/colo-proxy-script.sh
@@ -0,0 +1,92 @@
+#!/bin/sh
+#usage:
+# colo-proxy-script.sh primary/secondary install/uninstall phy_if virt_if index
+#.e.g:
+# colo-proxy-script.sh primary install eth2 tap0 1
+
+side=$1
+action=$2
+phy_if=$3
+virt_if=$4
+index=$5
+br=br1
+failover_br=br0
+
+script_usage()
+{
+ echo -n "usage: ./colo-proxy-script.sh primary/secondary "
+ echo -e "install/uninstall phy_if virt_if index\n"
+}
+
+primary_install()
+{
+ tc qdisc add dev $virt_if root handle 1: prio
+ tc filter add dev $virt_if parent 1: protocol ip prio 10 u32 match u32 \
+ 0 0 flowid 1:2 action mirred egress mirror dev $phy_if
+ tc filter add dev $virt_if parent 1: protocol arp prio 11 u32 match u32 \
+ 0 0 flowid 1:2 action mirred egress mirror dev $phy_if
+ tc filter add dev $virt_if parent 1: protocol ipv6 prio 12 u32 match u32 \
+ 0 0 flowid 1:2 action mirred egress mirror dev $phy_if
+
+ /usr/local/sbin/iptables -t mangle -I PREROUTING -m physdev --physdev-in \
+ $virt_if -j PMYCOLO --index $index --forward-dev $phy_if
+ /usr/local/sbin/ip6tables -t mangle -I PREROUTING -m physdev --physdev-in \
+ $virt_if -j PMYCOLO --index $index --forward-dev $phy_if
+ /usr/local/sbin/arptables -I INPUT -i $phy_if -j MARK --set-mark $index
+}
+
+primary_uninstall()
+{
+ tc filter del dev $virt_if parent 1: protocol ip prio 10 u32 match u32 \
+ 0 0 flowid 1:2 action mirred egress mirror dev $phy_if
+ tc filter del dev $virt_if parent 1: protocol arp prio 11 u32 match u32 \
+ 0 0 flowid 1:2 action mirred egress mirror dev $phy_if
+ tc filter del dev $virt_if parent 1: protocol ipv6 prio 12 u32 match u32 \
+ 0 0 flowid 1:2 action mirred egress mirror dev $phy_if
+ tc qdisc del dev $virt_if root handle 1: prio
+
+ /usr/local/sbin/iptables -t mangle -D PREROUTING -m physdev --physdev-in \
+ $virt_if -j PMYCOLO --index $index --forward-dev $phy_if
+ /usr/local/sbin/ip6tables -t mangle -D PREROUTING -m physdev --physdev-in \
+ $virt_if -j PMYCOLO --index $index --forward-dev $phy_if
+ /usr/local/sbin/arptables -D INPUT -i $phy_if -j MARK --set-mark $index
+}
+
+secondary_install()
+{
+ brctl addif $br $phy_if
+
+ /usr/local/sbin/iptables -t mangle -I PREROUTING -m physdev --physdev-in \
+ $virt_if -j SECCOLO --index $index
+ /usr/local/sbin/ip6tables -t mangle -I PREROUTING -m physdev --physdev-in \
+ $virt_if -j SECCOLO --index $index
+}
+
+secondary_uninstall()
+{
+ brctl delif $br $phy_if
+ brctl delif $br $virt_if
+ brctl addif $failover_br $virt_if
+
+ /usr/local/sbin/iptables -t mangle -D PREROUTING -m physdev --physdev-in \
+ $virt_if -j SECCOLO --index $index
+ /usr/local/sbin/ip6tables -t mangle -D PREROUTING -m physdev --physdev-in \
+ $virt_if -j SECCOLO --index $index
+}
+
+if [ $# -ne 5 ]; then
+ script_usage
+ exit 1
+fi
+
+if [ "x$side" != "xprimary" ] && [ "x$side" != "xsecondary" ]; then
+ script_usage
+ exit 2
+fi
+
+if [ "x$action" != "xinstall" ] && [ "x$action" != "xuninstall" ]; then
+ script_usage
+ exit 3
+fi
+
+${side}_${action}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 24/34] colo-nic: Handle secondary VM's original net device configure
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (22 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 23/34] COLO NIC: Implement colo nic device interface configure() zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 25/34] COLO NIC: Implement colo nic init/destroy function zhanghailiang
` (9 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, zhanghailiang
For secondary VM, we need to reconfigure its original net devices,
Before go into COLO mode, we detach its original net devices (here is tap)
from its default configure (here is bridge), and
attach the net devices to forward bridge.
When exit from COLO mode, we resume its origianl configure.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
---
include/net/colo-nic.h | 2 ++
net/colo-nic.c | 52 ++++++++++++++++++++++++++++++++++-
net/tap.c | 4 +++
scripts/colo-proxy-script.sh | 65 ++++++++++++++++++++++++++++++++++++++++----
4 files changed, 116 insertions(+), 7 deletions(-)
diff --git a/include/net/colo-nic.h b/include/net/colo-nic.h
index 3941b6e..f1d9c25 100644
--- a/include/net/colo-nic.h
+++ b/include/net/colo-nic.h
@@ -20,6 +20,8 @@ typedef struct COLONicState {
char nicname[128]; /* forward dev */
char script[1024]; /* colo script */
char ifname[128]; /* e.g. tap name */
+ char qemu_ifup[1024]; /* script that setup nic, e.g. /etc/qemu-ifup */
+ char *qemu_ifdown; /* script that cleanup nic e.g. /etc/qemu-ifdown */
} COLONicState;
void colo_add_nic_devices(COLONicState *cns);
diff --git a/net/colo-nic.c b/net/colo-nic.c
index 4ce08c0..4b53f72 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -58,17 +58,67 @@ static int launch_colo_script(COLONicState *cns, bool up, int side, int index)
return 0;
}
+/* For secondary VM, we need to cleanup its original configure
+ * when go into COLO state, when exit from COLO state, we need to
+ * resume its old configure
+*/
+static int handle_old_nic_configure(COLONicState *cns, int fd, bool cleanup)
+{
+ Error *err = NULL;
+ char *args[3];
+ char **parg;
+
+ parg = args;
+ *parg++ = cleanup ? cns->qemu_ifdown : (char *)cns->qemu_ifup;
+ *parg++ = (char *)cns->ifname;
+ *parg = NULL;
+ launch_script(args, fd, &err);
+ if (err) {
+ error_report_err(err);
+ return -1;
+ }
+ return 0;
+}
+
QTAILQ_HEAD(, nic_device) nic_devices = QTAILQ_HEAD_INITIALIZER(nic_devices);
static int colo_nic_configure(COLONicState *cns,
bool up, int side, int index)
{
+ NetClientState *nc = container_of(cns, NetClientState, cns);
+ TAPState *s = DO_UPCAST(TAPState, nc, nc);
+
if (!cns && index <= 0) {
error_report("Can not parse colo_script or forward_nic");
return -1;
}
- return launch_colo_script(cns, up, side, index);
+ switch (side) {
+ case COLO_MODE_PRIMARY:
+ return launch_colo_script(cns, up, side, index);
+ break;
+ case COLO_MODE_SECONDARY:
+ if (!cns->qemu_ifup[0] || !cns->qemu_ifdown || !cns->qemu_ifdown[0]) {
+ error_report("ifup(e.g. /etc/qemu-ifup) and ifdown(e.g."
+ "/etc/qemu-ifdown)script are needed for COLO");
+ return -1;
+ }
+ if (up) {
+ if (handle_old_nic_configure(cns, s->fd, true) < 0) {
+ return -1;
+ }
+ return launch_colo_script(cns, up, side, index);
+ } else {
+ if (launch_colo_script(cns, up, side, index) < 0) {
+ return -1;
+ }
+ return handle_old_nic_configure(cns, s->fd, false);
+ }
+ break;
+ default:
+ break;
+ }
+ return -1;
}
void colo_add_nic_devices(COLONicState *cns)
diff --git a/net/tap.c b/net/tap.c
index 0e3bdbd..ecd057f 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -644,6 +644,10 @@ static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
snprintf(s->down_script, sizeof(s->down_script), "%s", downscript);
snprintf(s->down_script_arg, sizeof(s->down_script_arg),
"%s", ifname);
+ nc->cns.qemu_ifdown = s->down_script;
+ }
+ if (strcmp(script, "no") != 0) {
+ strcpy(nc->cns.qemu_ifup, script);
}
}
diff --git a/scripts/colo-proxy-script.sh b/scripts/colo-proxy-script.sh
index 47c4fb5..03e3d98 100755
--- a/scripts/colo-proxy-script.sh
+++ b/scripts/colo-proxy-script.sh
@@ -9,8 +9,6 @@ action=$2
phy_if=$3
virt_if=$4
index=$5
-br=br1
-failover_br=br0
script_usage()
{
@@ -52,9 +50,66 @@ primary_uninstall()
/usr/local/sbin/arptables -D INPUT -i $phy_if -j MARK --set-mark $index
}
+attach_forward_bridge()
+{
+ if brctl show |grep -q colobr ; then
+ colobr=`brctl show | grep colobr | awk '{print $1}' |tail -1`
+ if brctl show $colobr |grep -qw $phy_if ; then
+ ip link set $colobr up
+ brctl addif $colobr $virt_if
+ return 0
+ fi
+ for (( i=0; i<100; i++))
+ do
+ if [ "$colobr" == "colobr$i" ]; then
+ colobr=colobr$((i+1))
+ break
+ fi
+ done
+ if [ $i -eq 100 ]; then
+ echo "there are to many colobr"
+ colobr=
+ exit 1
+ fi
+ else
+ colobr=colobr0
+ fi
+
+ brctl addbr $colobr
+ ip link set dev $colobr up
+ brctl addif $colobr $phy_if
+ brctl addif $colobr $virt_if
+}
+
+detach_forward_bridge()
+{
+
+ bridges="`brctl show | grep -v -e 'bridge name' -e ^$'\t' |\
+ awk -F'\t' '{print $1}'`"
+ for bridge in $bridges
+ do
+ if brctl show $bridge |grep -qw $virt_if ; then
+ colobr=$bridge
+ fi
+ done
+
+ if [ "X$colobr" == "X" ]; then
+ return
+ fi
+ brctl delif $colobr $virt_if
+ has_slave=`brctl show $colobr |wc -l`
+ if [ $has_slave -ne 2 ]; then
+ return 0
+ fi
+
+ brctl delif $colobr $phy_if
+ ip link set $colobr down
+ brctl delbr $colobr
+}
+
secondary_install()
{
- brctl addif $br $phy_if
+ attach_forward_bridge
/usr/local/sbin/iptables -t mangle -I PREROUTING -m physdev --physdev-in \
$virt_if -j SECCOLO --index $index
@@ -64,9 +119,7 @@ secondary_install()
secondary_uninstall()
{
- brctl delif $br $phy_if
- brctl delif $br $virt_if
- brctl addif $failover_br $virt_if
+ detach_forward_bridge
/usr/local/sbin/iptables -t mangle -D PREROUTING -m physdev --physdev-in \
$virt_if -j SECCOLO --index $index
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 25/34] COLO NIC: Implement colo nic init/destroy function
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (23 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 24/34] colo-nic: Handle secondary VM's original net device configure zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 26/34] COLO NIC: Some init work related with proxy module zhanghailiang
` (8 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Jason Wang, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, Stefan Hajnoczi,
amit.shah, zhanghailiang
When in colo mode, call colo nic init/destroy function.
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/colo-nic.h | 3 +++
migration/colo.c | 15 +++++++++++
net/colo-nic.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 87 insertions(+)
diff --git a/include/net/colo-nic.h b/include/net/colo-nic.h
index f1d9c25..7b8ff57 100644
--- a/include/net/colo-nic.h
+++ b/include/net/colo-nic.h
@@ -27,4 +27,7 @@ typedef struct COLONicState {
void colo_add_nic_devices(COLONicState *cns);
void colo_remove_nic_devices(COLONicState *cns);
+int colo_proxy_init(enum COLOMode mode);
+void colo_proxy_destroy(enum COLOMode mode);
+
#endif
diff --git a/migration/colo.c b/migration/colo.c
index aab7b38..27b41fc 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -16,6 +16,7 @@
#include "qemu/error-report.h"
#include "migration/failover.h"
#include "qapi-event.h"
+#include "net/colo-nic.h"
enum {
COLO_CHECPOINT_READY = 0x46,
@@ -314,6 +315,11 @@ static void *colo_thread(void *opaque)
QEMUFile *colo_control = NULL;
int i, ret;
+ if (colo_proxy_init(COLO_MODE_PRIMARY) != 0) {
+ error_report("Init colo proxy error");
+ goto out;
+ }
+
colo_control = qemu_fopen_socket(qemu_get_fd(s->file), "rb");
if (!colo_control) {
error_report("Open colo_control failed!");
@@ -387,6 +393,8 @@ out:
qemu_bh_schedule(s->cleanup_bh);
qemu_mutex_unlock_iothread();
+ colo_proxy_destroy(COLO_MODE_PRIMARY);
+
return NULL;
}
@@ -451,6 +459,12 @@ void *colo_process_incoming_checkpoints(void *opaque)
colo = qemu_coroutine_self();
assert(colo != NULL);
+ /* configure the network */
+ if (colo_proxy_init(COLO_MODE_SECONDARY) != 0) {
+ error_report("Init colo proxy error\n");
+ goto out;
+ }
+
ctl = qemu_fopen_socket(fd, "wb");
if (!ctl) {
error_report("Can't open incoming channel!");
@@ -617,5 +631,6 @@ out:
loadvm_exit_colo();
+ colo_proxy_destroy(COLO_MODE_SECONDARY);
return NULL;
}
diff --git a/net/colo-nic.c b/net/colo-nic.c
index 4b53f72..5c24169 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -121,6 +121,57 @@ static int colo_nic_configure(COLONicState *cns,
return -1;
}
+static int configure_one_nic(COLONicState *cns,
+ bool up, int side, int index)
+{
+ struct nic_device *nic;
+
+ assert(cns);
+
+ QTAILQ_FOREACH(nic, &nic_devices, next) {
+ if (nic->cns == cns) {
+ if (up == nic->is_up) {
+ return 0;
+ }
+
+ if (!nic->configure || (nic->configure(nic->cns, up, side, index) &&
+ up)) {
+ return -1;
+ }
+ nic->is_up = up;
+ return 0;
+ }
+ }
+
+ return -1;
+}
+
+static int configure_nic(int side, int index)
+{
+ struct nic_device *nic;
+
+ if (QTAILQ_EMPTY(&nic_devices)) {
+ return -1;
+ }
+
+ QTAILQ_FOREACH(nic, &nic_devices, next) {
+ if (configure_one_nic(nic->cns, 1, side, index)) {
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void teardown_nic(int side, int index)
+{
+ struct nic_device *nic;
+
+ QTAILQ_FOREACH(nic, &nic_devices, next) {
+ configure_one_nic(nic->cns, 0, side, index);
+ }
+}
+
void colo_add_nic_devices(COLONicState *cns)
{
struct nic_device *nic;
@@ -151,8 +202,26 @@ void colo_remove_nic_devices(COLONicState *cns)
QTAILQ_FOREACH_SAFE(nic, &nic_devices, next, next_nic) {
if (nic->cns == cns) {
+ configure_one_nic(cns, 0, get_colo_mode(), getpid());
QTAILQ_REMOVE(&nic_devices, nic, next);
g_free(nic);
}
}
}
+
+int colo_proxy_init(enum COLOMode mode)
+{
+ int ret = -1;
+
+ ret = configure_nic(mode, getpid());
+ if (ret != 0) {
+ error_report("excute colo-proxy-script failed");
+ }
+
+ return ret;
+}
+
+void colo_proxy_destroy(enum COLOMode mode)
+{
+ teardown_nic(mode, getpid());
+}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 26/34] COLO NIC: Some init work related with proxy module
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (24 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 25/34] COLO NIC: Implement colo nic init/destroy function zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 27/34] COLO: Handle nfnetlink message from " zhanghailiang
` (7 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Jason Wang, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, Stefan Hajnoczi,
amit.shah, zhanghailiang
Implement communication protocol with proxy module by using
nfnetlink, which requires libnfnetlink libs.
Tell proxy module to do initialization work and moreover ask
kernel to acknowledge the request. It's is necessary for the first
time because Netlink is not a reliable protocol.
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
configure | 22 +++++++-
net/colo-nic.c | 160 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 180 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index a2475db..50f7e05 100755
--- a/configure
+++ b/configure
@@ -2342,7 +2342,25 @@ EOF
rdma="no"
fi
fi
-
+##########################################
+# COLO needs libnfnetlink libraries
+if test "$colo" != "no"; then
+ cat > $TMPC <<EOF
+#include <libnfnetlink/libnfnetlink.h>
+int main(void) { return 0; }
+EOF
+ colo_libs="-lnfnetlink"
+ if compile_prog "" "$colo_libs"; then
+ colo="yes"
+ libs_softmmu="$libs_softmmu $colo_libs"
+ else
+ if test "$colo" = "yes" ; then
+ error_exit "libnfnetlink is required for colo feature." \
+ "Make sure to have the libnfnetlink devel and headers installed."
+ fi
+ colo="no"
+ fi
+fi
##########################################
# VNC TLS/WS detection
if test "$vnc" = "yes" -a "$vnc_tls" != "no" ; then
@@ -2615,7 +2633,7 @@ EOF
if compile_prog "$cfl" "$lib" ; then
:
else
- error_exit "$drv check failed" \
+ rror_exit "$drv check failed" \
"Make sure to have the $drv libs and headers installed."
fi
}
diff --git a/net/colo-nic.c b/net/colo-nic.c
index 5c24169..c8e7734 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -10,6 +10,12 @@
* later. See the COPYING file in the top-level directory.
*
*/
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <linux/netlink.h>
+#include <libnfnetlink/libnfnetlink.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
#include "include/migration/migration.h"
#include "migration/colo.h"
#include "net/net.h"
@@ -17,6 +23,53 @@
#include "qemu/error-report.h"
#include "net/tap.h"
+/* Remove the follow define after proxy is merged into kernel,
+* using #include <libnfnetlink/libnfnetlink.h> instead.
+*/
+#define NFNL_SUBSYS_COLO 12
+
+/* Message Format
+* <---NLMSG_ALIGN(hlen)-----><-------------- NLMSG_ALIGN(len)----------------->
+* +--------------------+- - -+- - - - - - - - - - - - - - +- - - - - - + - - -+
+* | Header | Pad | Netfilter Netlink Header | Attributes | Pad |
+* | struct nlmsghdr | | struct nfgenmsg | | |
+* +--------------------+- - -+- - - - - - - - - - - - - - + - - - - - -+ - - -+
+*/
+
+enum nfnl_colo_msg_types {
+ NFCOLO_KERNEL_NOTIFY, /* Used by proxy module to notify qemu */
+
+ NFCOLO_DO_CHECKPOINT,
+ NFCOLO_DO_FAILOVER,
+ NFCOLO_PROXY_INIT,
+ NFCOLO_PROXY_RESET,
+
+ NFCOLO_MSG_MAX
+};
+
+enum nfnl_colo_kernel_notify_attributes {
+ NFNL_COLO_KERNEL_NOTIFY_UNSPEC,
+ NFNL_COLO_COMPARE_RESULT,
+ __NFNL_COLO_KERNEL_NOTIFY_MAX
+};
+
+#define NFNL_COLO_KERNEL_NOTIFY_MAX (__NFNL_COLO_KERNEL_NOTIFY_MAX - 1)
+
+enum nfnl_colo_attributes {
+ NFNL_COLO_UNSPEC,
+ NFNL_COLO_MODE,
+ __NFNL_COLO_MAX
+};
+#define NFNL_COLO_MAX (__NFNL_COLO_MAX - 1)
+
+struct nfcolo_msg_mode {
+ u_int8_t mode;
+};
+
+struct nfcolo_packet_compare { /* Unused */
+ int32_t different;
+};
+
typedef struct nic_device {
COLONicState *cns;
int (*configure)(COLONicState *cns, bool up, int side, int index);
@@ -24,6 +77,9 @@ typedef struct nic_device {
bool is_up;
} nic_device;
+static struct nfnl_handle *nfnlh;
+static struct nfnl_subsys_handle *nfnlssh;
+
static int launch_colo_script(COLONicState *cns, bool up, int side, int index)
{
NetClientState *nc = container_of(cns, NetClientState, cns);
@@ -209,19 +265,123 @@ void colo_remove_nic_devices(COLONicState *cns)
}
}
+static int colo_proxy_send(enum nfnl_colo_msg_types msg_type,
+ enum COLOMode mode, int flag, void *unused)
+{
+ struct nfcolo_msg_mode params;
+ union {
+ char buf[NFNL_HEADER_LEN
+ + NFA_LENGTH(sizeof(struct nfcolo_msg_mode))];
+ struct nlmsghdr nmh;
+ } u;
+ int ret;
+
+ if (!nfnlssh || !nfnlh) {
+ error_report("nfnlssh and nfnlh are uninited");
+ return -1;
+ }
+ nfnl_fill_hdr(nfnlssh, &u.nmh, 0, AF_UNSPEC, 1,
+ msg_type, NLM_F_REQUEST | flag);
+ params.mode = mode;
+ u.nmh.nlmsg_pid = nfnl_portid(nfnlh);
+ ret = nfnl_addattr_l(&u.nmh, sizeof(u), NFNL_COLO_MODE, ¶ms,
+ sizeof(params));
+ if (ret < 0) {
+ error_report("call nfnl_addattr_l failed");
+ return ret;
+ }
+ ret = nfnl_send(nfnlh, &u.nmh);
+ if (ret < 0) {
+ error_report("call nfnl_send failed");
+ }
+ return ret;
+}
+
+static int check_proxy_ack(void)
+{
+ unsigned char *buf = g_malloc0(2048);
+ struct nlmsghdr *nlmsg;
+ int len;
+ int ret = -1;
+
+ len = nfnl_recv(nfnlh, buf, 2048);
+ if (len <= 0) {
+ error_report("nfnl_recv received nothing");
+ goto err;
+ }
+ nlmsg = (struct nlmsghdr *)buf;
+
+ if (nlmsg->nlmsg_type == NLMSG_ERROR) {
+ struct nlmsgerr *err = (struct nlmsgerr *)NLMSG_DATA(nlmsg);
+
+ if (err->error) {
+ error_report("Received error message:%d", -err->error);
+ goto err;
+ }
+ }
+
+ ret = 0;
+err:
+ g_free(buf);
+ return ret;
+}
+
int colo_proxy_init(enum COLOMode mode)
{
int ret = -1;
+ nfnlh = nfnl_open();
+ if (!nfnlh) {
+ error_report("call nfnl_open failed");
+ return -1;
+ }
+ /* Note:
+ * Here we must ensure that the nl_pid (also nlmsg_pid in nlmsghdr ) equal
+ * to the process ID of VM, becase we use it to identify the VM in proxy
+ * module.
+ */
+ if (nfnl_portid(nfnlh) != getpid()) {
+ error_report("More than one netlink of NETLINK_NETFILTER type exist");
+ return -1;
+ }
+ /* disable netlink sequence tracking by default */
+ nfnl_unset_sequence_tracking(nfnlh);
+ nfnlssh = nfnl_subsys_open(nfnlh, NFNL_SUBSYS_COLO, NFCOLO_MSG_MAX, 0);
+ if (!nfnlssh) {
+ error_report("call nfnl_subsys_open failed");
+ goto err_out;
+ }
+
+ /* Netlink is not a reliable protocol, So it is necessary to request proxy
+ * module to acknowledge in the first time.
+ */
+ ret = colo_proxy_send(NFCOLO_PROXY_INIT, mode, NLM_F_ACK, NULL);
+ if (ret < 0) {
+ goto err_out;
+ }
+
+ ret = check_proxy_ack();
+ if (ret < 0) {
+ goto err_out;
+ }
+
ret = configure_nic(mode, getpid());
if (ret != 0) {
error_report("excute colo-proxy-script failed");
+ goto err_out;
}
+ return 0;
+err_out:
+ nfnl_close(nfnlh);
+ nfnlh = NULL;
return ret;
}
void colo_proxy_destroy(enum COLOMode mode)
{
+ if (nfnlh) {
+ nfnl_close(nfnlh);
+ }
teardown_nic(mode, getpid());
}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 27/34] COLO: Handle nfnetlink message from proxy module
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (25 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 26/34] COLO NIC: Some init work related with proxy module zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 28/34] COLO: Do checkpoint according to the result of packets comparation zhanghailiang
` (6 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Jason Wang, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, Stefan Hajnoczi,
amit.shah, zhanghailiang
Proxy module will send message to qemu through nfnetlink.
Now, the message only contains the result of packets comparation.
We use a global variable 'packet_compare_different' to store the result.
And this variable should be accessed by using atomic related function,
such as 'atomic_set' 'atomic_xchg'.
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
net/colo-nic.c | 42 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/net/colo-nic.c b/net/colo-nic.c
index c8e7734..49380ce 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -22,6 +22,7 @@
#include "net/colo-nic.h"
#include "qemu/error-report.h"
#include "net/tap.h"
+#include "trace.h"
/* Remove the follow define after proxy is merged into kernel,
* using #include <libnfnetlink/libnfnetlink.h> instead.
@@ -79,6 +80,7 @@ typedef struct nic_device {
static struct nfnl_handle *nfnlh;
static struct nfnl_subsys_handle *nfnlssh;
+static int32_t packet_compare_different; /* The result of packet comparing */
static int launch_colo_script(COLONicState *cns, bool up, int side, int index)
{
@@ -297,6 +299,38 @@ static int colo_proxy_send(enum nfnl_colo_msg_types msg_type,
return ret;
}
+static int __colo_rcv_pkt(struct nlmsghdr *nlh, struct nfattr *nfa[],
+ void *data)
+{
+ /* struct nfgenmsg *nfmsg = NLMSG_DATA(nlh); */
+ int32_t result = ntohl(nfnl_get_data(nfa, NFNL_COLO_COMPARE_RESULT,
+ int32_t));
+
+ atomic_set(&packet_compare_different, result);
+ trace_colo_rcv_pkt(result);
+ return 0;
+}
+
+static struct nfnl_callback colo_nic_cb = {
+ .call = &__colo_rcv_pkt,
+ .attr_count = NFNL_COLO_KERNEL_NOTIFY_MAX,
+};
+
+static void colo_proxy_recv(void *opaque)
+{
+ unsigned char *buf = g_malloc0(2048);
+ int len;
+ int ret;
+
+ len = nfnl_recv(nfnlh, buf, 2048);
+ ret = nfnl_handle_packet(nfnlh, (char *)buf, len);
+ if (ret < 0) {/* Notify colo thread the error */
+ atomic_set(&packet_compare_different, -1);
+ error_report("call nfnl_handle_packet failed");
+ }
+ g_free(buf);
+}
+
static int check_proxy_ack(void)
{
unsigned char *buf = g_malloc0(2048);
@@ -352,6 +386,11 @@ int colo_proxy_init(enum COLOMode mode)
goto err_out;
}
+ ret = nfnl_callback_register(nfnlssh, NFCOLO_KERNEL_NOTIFY, &colo_nic_cb);
+ if (ret < 0) {
+ goto err_out;
+ }
+
/* Netlink is not a reliable protocol, So it is necessary to request proxy
* module to acknowledge in the first time.
*/
@@ -371,6 +410,8 @@ int colo_proxy_init(enum COLOMode mode)
goto err_out;
}
+ qemu_set_fd_handler(nfnl_fd(nfnlh), colo_proxy_recv, NULL, NULL);
+
return 0;
err_out:
nfnl_close(nfnlh);
@@ -381,6 +422,7 @@ err_out:
void colo_proxy_destroy(enum COLOMode mode)
{
if (nfnlh) {
+ qemu_set_fd_handler(nfnl_fd(nfnlh), NULL, NULL, NULL);
nfnl_close(nfnlh);
}
teardown_nic(mode, getpid());
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 28/34] COLO: Do checkpoint according to the result of packets comparation
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (26 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 27/34] COLO: Handle nfnetlink message from " zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 29/34] COLO: Improve checkpoint efficiency by do additional periodic checkpoint zhanghailiang
` (5 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, zhanghailiang
Only do checkpoint, when the PVM's and SVM's output net packets are inconsistent,
We also limit the min time between two continuous checkpoint action, to
give VM a change to run.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/colo-nic.h | 2 ++
migration/colo.c | 32 ++++++++++++++++++++++++++++++++
net/colo-nic.c | 5 +++++
3 files changed, 39 insertions(+)
diff --git a/include/net/colo-nic.h b/include/net/colo-nic.h
index 7b8ff57..271ab8b 100644
--- a/include/net/colo-nic.h
+++ b/include/net/colo-nic.h
@@ -30,4 +30,6 @@ void colo_remove_nic_devices(COLONicState *cns);
int colo_proxy_init(enum COLOMode mode);
void colo_proxy_destroy(enum COLOMode mode);
+int colo_proxy_compare(void);
+
#endif
diff --git a/migration/colo.c b/migration/colo.c
index 27b41fc..815f2ab 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -18,6 +18,13 @@
#include "qapi-event.h"
#include "net/colo-nic.h"
+/*
+* We should not do checkpoint one after another without any time interval,
+* Because this will lead continuous 'stop' status for VM.
+* CHECKPOINT_MIN_PERIOD is the min time limit between two checkpoint action.
+*/
+#define CHECKPOINT_MIN_PERIOD 100 /* unit: ms */
+
enum {
COLO_CHECPOINT_READY = 0x46,
@@ -313,6 +320,7 @@ static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
QEMUFile *colo_control = NULL;
+ int64_t current_time, checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
int i, ret;
if (colo_proxy_init(COLO_MODE_PRIMARY) != 0) {
@@ -348,15 +356,39 @@ static void *colo_thread(void *opaque)
trace_colo_vm_state_change("stop", "run");
while (s->state == MIGRATION_STATUS_COLO) {
+ int proxy_checkpoint_req;
+
if (failover_request_is_active()) {
error_report("failover request");
goto out;
}
+ /* wait for a colo checkpoint */
+ proxy_checkpoint_req = colo_proxy_compare();
+ if (proxy_checkpoint_req < 0) {
+ goto out;
+ } else if (!proxy_checkpoint_req) {
+ /*
+ * No checkpoint is needed, wait for 1ms and then
+ * check if we need checkpoint again
+ */
+ g_usleep(1000);
+ continue;
+ } else {
+ int64_t interval;
+
+ current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
+ interval = current_time - checkpoint_time;
+ if (interval < CHECKPOINT_MIN_PERIOD) {
+ /* Limit the min time between two checkpoint */
+ g_usleep((1000*(CHECKPOINT_MIN_PERIOD - interval)));
+ }
+ }
/* start a colo checkpoint */
if (colo_do_checkpoint_transaction(s, colo_control)) {
goto out;
}
+ checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
}
out:
diff --git a/net/colo-nic.c b/net/colo-nic.c
index 49380ce..d2073b8 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -427,3 +427,8 @@ void colo_proxy_destroy(enum COLOMode mode)
}
teardown_nic(mode, getpid());
}
+
+int colo_proxy_compare(void)
+{
+ return atomic_xchg(&packet_compare_different, 0);
+}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 29/34] COLO: Improve checkpoint efficiency by do additional periodic checkpoint
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (27 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 28/34] COLO: Do checkpoint according to the result of packets comparation zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 30/34] COLO: Add colo-set-checkpoint-period command zhanghailiang
` (4 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Yang Hongyang, zhanghailiang
Besides normal checkpoint which according to the result of net packets
comparing, We do additional checkpoint periodically, it will reduce the number
of dirty pages when do one checkpoint, if we don't do checkpoint for a long
time (This is a special case when the net packets is always consistent).
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
migration/colo.c | 29 +++++++++++++++++++++--------
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index 815f2ab..64862dc 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -10,6 +10,7 @@
* later. See the COPYING file in the top-level directory.
*/
+#include "qemu/timer.h"
#include "sysemu/sysemu.h"
#include "migration/colo.h"
#include "trace.h"
@@ -25,6 +26,13 @@
*/
#define CHECKPOINT_MIN_PERIOD 100 /* unit: ms */
+/*
+ * force checkpoint timer: unit ms
+ * this is large because COLO checkpoint will mostly depend on
+ * COLO compare module.
+ */
+#define CHECKPOINT_MAX_PEROID 10000
+
enum {
COLO_CHECPOINT_READY = 0x46,
@@ -366,14 +374,7 @@ static void *colo_thread(void *opaque)
proxy_checkpoint_req = colo_proxy_compare();
if (proxy_checkpoint_req < 0) {
goto out;
- } else if (!proxy_checkpoint_req) {
- /*
- * No checkpoint is needed, wait for 1ms and then
- * check if we need checkpoint again
- */
- g_usleep(1000);
- continue;
- } else {
+ } else if (proxy_checkpoint_req) {
int64_t interval;
current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
@@ -382,8 +383,20 @@ static void *colo_thread(void *opaque)
/* Limit the min time between two checkpoint */
g_usleep((1000*(CHECKPOINT_MIN_PERIOD - interval)));
}
+ goto do_checkpoint;
+ }
+
+ /*
+ * No proxy checkpoint is request, wait for 100ms
+ * and then check if we need checkpoint again.
+ */
+ current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
+ if (current_time - checkpoint_time < CHECKPOINT_MAX_PEROID) {
+ g_usleep(100000);
+ continue;
}
+do_checkpoint:
/* start a colo checkpoint */
if (colo_do_checkpoint_transaction(s, colo_control)) {
goto out;
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 30/34] COLO: Add colo-set-checkpoint-period command
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (28 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 29/34] COLO: Improve checkpoint efficiency by do additional periodic checkpoint zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 31/34] COLO NIC: Implement NIC checkpoint and failover zhanghailiang
` (3 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, Markus Armbruster, yunhong.jiang, eddie.dong,
peter.huangpeng, dgilbert, arei.gonglei, amit.shah,
Luiz Capitulino, zhanghailiang
With this command, we can control the period of checkpoint, if
there is no comparison of net packets.
Cc: Luiz Capitulino <lcapitulino@redhat.com>
Cc: Eric Blake <eblake@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
hmp-commands.hx | 15 +++++++++++++++
hmp.c | 7 +++++++
hmp.h | 1 +
migration/colo.c | 11 ++++++++++-
qapi-schema.json | 13 +++++++++++++
qmp-commands.hx | 22 ++++++++++++++++++++++
stubs/migration-colo.c | 4 ++++
7 files changed, 72 insertions(+), 1 deletion(-)
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 410637f..9164961 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1049,6 +1049,21 @@ Tell COLO that heartbeat is lost, a failover or takeover is needed.
ETEXI
{
+ .name = "colo_set_checkpoint_period",
+ .args_type = "value:i",
+ .params = "value",
+ .help = "set checkpoint period (in ms) for colo. "
+ "Defaults to 100ms",
+ .mhandler.cmd = hmp_colo_set_checkpoint_period,
+ },
+
+STEXI
+@item migrate_set_checkpoint_period @var{value}
+@findex migrate_set_checkpoint_period
+Set checkpoint period to @var{value} (in ms) for colo.
+ETEXI
+
+ {
.name = "client_migrate_info",
.args_type = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
.params = "protocol hostname port tls-port cert-subject",
diff --git a/hmp.c b/hmp.c
index 7f3a8a9..c464fc9 100644
--- a/hmp.c
+++ b/hmp.c
@@ -1280,6 +1280,13 @@ void hmp_colo_lost_heartbeat(Monitor *mon, const QDict *qdict)
hmp_handle_error(mon, &err);
}
+void hmp_colo_set_checkpoint_period(Monitor *mon, const QDict *qdict)
+{
+ int64_t value = qdict_get_int(qdict, "value");
+
+ qmp_colo_set_checkpoint_period(value, NULL);
+}
+
void hmp_set_password(Monitor *mon, const QDict *qdict)
{
const char *protocol = qdict_get_str(qdict, "protocol");
diff --git a/hmp.h b/hmp.h
index c36c99c..d66dc76 100644
--- a/hmp.h
+++ b/hmp.h
@@ -69,6 +69,7 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict);
void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict);
void hmp_client_migrate_info(Monitor *mon, const QDict *qdict);
void hmp_colo_lost_heartbeat(Monitor *mon, const QDict *qdict);
+void hmp_colo_set_checkpoint_period(Monitor *mon, const QDict *qdict);
void hmp_set_password(Monitor *mon, const QDict *qdict);
void hmp_expire_password(Monitor *mon, const QDict *qdict);
void hmp_eject(Monitor *mon, const QDict *qdict);
diff --git a/migration/colo.c b/migration/colo.c
index 64862dc..375088b 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -18,6 +18,7 @@
#include "migration/failover.h"
#include "qapi-event.h"
#include "net/colo-nic.h"
+#include "qmp-commands.h"
/*
* We should not do checkpoint one after another without any time interval,
@@ -71,6 +72,9 @@ enum {
static QEMUBH *colo_bh;
static bool vmstate_loading;
static Coroutine *colo;
+
+int64_t colo_checkpoint_period = CHECKPOINT_MAX_PEROID;
+
/* colo buffer */
#define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
QEMUSizedBuffer *colo_buffer;
@@ -91,6 +95,11 @@ bool loadvm_in_colo_state(void)
return colo != NULL;
}
+void qmp_colo_set_checkpoint_period(int64_t value, Error **errp)
+{
+ colo_checkpoint_period = value;
+}
+
static bool colo_runstate_is_stopped(void)
{
return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
@@ -391,7 +400,7 @@ static void *colo_thread(void *opaque)
* and then check if we need checkpoint again.
*/
current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
- if (current_time - checkpoint_time < CHECKPOINT_MAX_PEROID) {
+ if (current_time - checkpoint_time < colo_checkpoint_period) {
g_usleep(100000);
continue;
}
diff --git a/qapi-schema.json b/qapi-schema.json
index 91c4ea5..2bb8778 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -691,6 +691,19 @@
{ 'command': 'colo-lost-heartbeat' }
##
+# @colo-set-checkpoint-period
+#
+# Set colo checkpoint period
+#
+# @value: period of colo checkpoint in ms
+#
+# Returns: nothing on success
+#
+# Since: 2.4
+##
+{ 'command': 'colo-set-checkpoint-period', 'data': {'value': 'int'} }
+
+##
# @MouseInfo:
#
# Information about a mouse device.
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 443ef8e..3462ec8 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -800,6 +800,28 @@ Example:
EQMP
{
+ .name = "colo-set-checkpoint-period",
+ .args_type = "value:i",
+ .mhandler.cmd_new = qmp_marshal_input_colo_set_checkpoint_period,
+ },
+
+SQMP
+colo-set-checkpoint-period
+--------------------------
+
+set checkpoint period
+
+Arguments:
+- "value": checkpoint period
+
+Example:
+
+-> { "execute": "colo-set-checkpoint-period", "arguments": { "value": "1000" } }
+<- { "return": {} }
+
+EQMP
+
+ {
.name = "client_migrate_info",
.args_type = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
.params = "protocol hostname port tls-port cert-subject",
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index 91f35e9..c648935 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -52,3 +52,7 @@ void qmp_colo_lost_heartbeat(Error **errp)
" with --enable-colo option in order to support"
" COLO feature");
}
+
+void qmp_colo_set_checkpoint_period(int64_t value, Error **errp)
+{
+}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 31/34] COLO NIC: Implement NIC checkpoint and failover
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (29 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 30/34] COLO: Add colo-set-checkpoint-period command zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 32/34] COLO: Disable qdev hotplug when VM is in COLO mode zhanghailiang
` (2 subsequent siblings)
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, zhanghailiang
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/net/colo-nic.h | 2 ++
migration/colo.c | 21 ++++++++++++++++++---
net/colo-nic.c | 23 +++++++++++++++++++++++
3 files changed, 43 insertions(+), 3 deletions(-)
diff --git a/include/net/colo-nic.h b/include/net/colo-nic.h
index 271ab8b..a9960f1 100644
--- a/include/net/colo-nic.h
+++ b/include/net/colo-nic.h
@@ -31,5 +31,7 @@ int colo_proxy_init(enum COLOMode mode);
void colo_proxy_destroy(enum COLOMode mode);
int colo_proxy_compare(void);
+int colo_proxy_failover(void);
+int colo_proxy_checkpoint(enum COLOMode mode);
#endif
diff --git a/migration/colo.c b/migration/colo.c
index 375088b..3dc7eee 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -129,6 +129,11 @@ static void secondary_vm_do_failover(void)
}
}
+ if (colo_proxy_failover() != 0) {
+ error_report("colo proxy failed to do failover");
+ }
+ colo_proxy_destroy(COLO_MODE_SECONDARY);
+
/* It means that VM exit from COLO state */
colo = NULL;
@@ -160,6 +165,8 @@ static void primary_vm_do_failover(void)
vm_stop_force_state(RUN_STATE_COLO);
}
+ colo_proxy_destroy(COLO_MODE_PRIMARY);
+
if (s->state != MIGRATION_STATUS_FAILED) {
migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
}
@@ -288,6 +295,11 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
qemu_fflush(trans);
+ ret = colo_proxy_checkpoint(COLO_MODE_PRIMARY);
+ if (ret < 0) {
+ goto out;
+ }
+
ret = colo_ctl_put(s->file, COLO_CHECKPOINT_SEND);
if (ret < 0) {
goto out;
@@ -447,8 +459,6 @@ out:
qemu_bh_schedule(s->cleanup_bh);
qemu_mutex_unlock_iothread();
- colo_proxy_destroy(COLO_MODE_PRIMARY);
-
return NULL;
}
@@ -575,6 +585,11 @@ void *colo_process_incoming_checkpoints(void *opaque)
goto out;
}
+ ret = colo_proxy_checkpoint(COLO_MODE_SECONDARY);
+ if (ret < 0) {
+ goto out;
+ }
+
ret = colo_ctl_get(f, COLO_CHECKPOINT_SEND);
if (ret < 0) {
goto out;
@@ -663,6 +678,7 @@ out:
* just kill Secondary VM
*/
error_report("SVM is going to exit in default!");
+ colo_proxy_destroy(COLO_MODE_SECONDARY);
exit(1);
} else {
/* if we went here, means Primary VM may dead, we are doing failover */
@@ -685,6 +701,5 @@ out:
loadvm_exit_colo();
- colo_proxy_destroy(COLO_MODE_SECONDARY);
return NULL;
}
diff --git a/net/colo-nic.c b/net/colo-nic.c
index d2073b8..62bd35a 100644
--- a/net/colo-nic.c
+++ b/net/colo-nic.c
@@ -428,6 +428,29 @@ void colo_proxy_destroy(enum COLOMode mode)
teardown_nic(mode, getpid());
}
+/*
+* Note: Weird, Only the VM in slave side need to do failover work !!!
+*/
+int colo_proxy_failover(void)
+{
+ if (colo_proxy_send(NFCOLO_DO_FAILOVER, COLO_MODE_SECONDARY, 0, NULL) < 0) {
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+* Note: Only the VM in master side need to do checkpoint
+*/
+int colo_proxy_checkpoint(enum COLOMode mode)
+{
+ if (colo_proxy_send(NFCOLO_DO_CHECKPOINT, mode, 0, NULL) < 0) {
+ return -1;
+ }
+ return 0;
+}
+
int colo_proxy_compare(void)
{
return atomic_xchg(&packet_compare_different, 0);
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 32/34] COLO: Disable qdev hotplug when VM is in COLO mode
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (30 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 31/34] COLO NIC: Implement NIC checkpoint and failover zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 33/34] COLO: Implement shutdown checkpoint zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 34/34] COLO: Add block replication into colo process zhanghailiang
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Yang Hongyang, zhanghailiang
COLO do not support qdev hotplug migration, disable it.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
---
migration/colo.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/migration/colo.c b/migration/colo.c
index 3dc7eee..f5bad95 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -10,6 +10,7 @@
* later. See the COPYING file in the top-level directory.
*/
+#include "hw/qdev-core.h"
#include "qemu/timer.h"
#include "sysemu/sysemu.h"
#include "migration/colo.h"
@@ -348,6 +349,7 @@ out:
static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
+ int dev_hotplug = qdev_hotplug;
QEMUFile *colo_control = NULL;
int64_t current_time, checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
int i, ret;
@@ -363,6 +365,8 @@ static void *colo_thread(void *opaque)
goto out;
}
+ qdev_hotplug = 0;
+
/*
* Wait for Secondary finish loading vm states and enter COLO
* restore.
@@ -459,6 +463,8 @@ out:
qemu_bh_schedule(s->cleanup_bh);
qemu_mutex_unlock_iothread();
+ qdev_hotplug = dev_hotplug;
+
return NULL;
}
@@ -516,10 +522,13 @@ void *colo_process_incoming_checkpoints(void *opaque)
struct colo_incoming *colo_in = opaque;
QEMUFile *f = colo_in->file;
int fd = qemu_get_fd(f);
+ int dev_hotplug = qdev_hotplug;
QEMUFile *ctl = NULL, *fb = NULL;
int i, ret;
uint64_t total_size;
+ qdev_hotplug = 0;
+
colo = qemu_coroutine_self();
assert(colo != NULL);
@@ -701,5 +710,7 @@ out:
loadvm_exit_colo();
+ qdev_hotplug = dev_hotplug;
+
return NULL;
}
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 33/34] COLO: Implement shutdown checkpoint
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (31 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 32/34] COLO: Disable qdev hotplug when VM is in COLO mode zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 34/34] COLO: Add block replication into colo process zhanghailiang
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, Lai Jiangshan, arei.gonglei, amit.shah, Paolo Bonzini,
zhanghailiang
For Secondary VM, we forbid it shutdown directly when in COLO mode,
FOR Primary VM's shutdown, we should do some work to ensure the consistent action
between PVM and SVM.
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
include/sysemu/sysemu.h | 3 +++
migration/colo.c | 32 +++++++++++++++++++++++++++++++-
vl.c | 26 ++++++++++++++++++++++++--
3 files changed, 58 insertions(+), 3 deletions(-)
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 44570d1..ab026ab 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -52,6 +52,8 @@ typedef enum WakeupReason {
QEMU_WAKEUP_REASON_OTHER,
} WakeupReason;
+extern int colo_shutdown_requested;
+
void qemu_system_reset_request(void);
void qemu_system_suspend_request(void);
void qemu_register_suspend_notifier(Notifier *notifier);
@@ -59,6 +61,7 @@ void qemu_system_wakeup_request(WakeupReason reason);
void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
void qemu_register_wakeup_notifier(Notifier *notifier);
void qemu_system_shutdown_request(void);
+void qemu_system_shutdown_request_core(void);
void qemu_system_powerdown_request(void);
void qemu_register_powerdown_notifier(Notifier *notifier);
void qemu_system_debug_request(void);
diff --git a/migration/colo.c b/migration/colo.c
index f5bad95..d6f2c73 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -68,6 +68,8 @@ enum {
COLO_CHECKPOINT_SEND,
COLO_CHECKPOINT_RECEIVED,
COLO_CHECKPOINT_LOADED,
+
+ COLO_GUEST_SHUTDOWN
};
static QEMUBH *colo_bh;
@@ -247,7 +249,7 @@ static int colo_ctl_get(QEMUFile *f, uint64_t require)
static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
{
- int ret;
+ int colo_shutdown, ret;
size_t size;
QEMUFile *trans = NULL;
@@ -274,6 +276,7 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
}
/* suspend and save vm state to colo buffer */
qemu_mutex_lock_iothread();
+ colo_shutdown = colo_shutdown_requested;
vm_stop_force_state(RUN_STATE_COLO);
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("run", "stop");
@@ -289,6 +292,7 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
/* Disable block migration */
s->params.blk = 0;
s->params.shared = 0;
+ qemu_savevm_state_header(trans);
qemu_savevm_state_begin(trans, &s->params);
qemu_mutex_lock_iothread();
qemu_savevm_state_complete(trans);
@@ -331,6 +335,16 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
}
trace_colo_receive_message("COLO_CHECKPOINT_LOADED");
+ if (colo_shutdown) {
+ colo_ctl_put(s->file, COLO_GUEST_SHUTDOWN);
+ qemu_fflush(s->file);
+ colo_shutdown_requested = 0;
+ qemu_system_shutdown_request_core();
+ while (1) {
+ ;
+ }
+ }
+
ret = 0;
/* resume master */
qemu_mutex_lock_iothread();
@@ -395,6 +409,10 @@ static void *colo_thread(void *opaque)
error_report("failover request");
goto out;
}
+
+ if (colo_shutdown_requested) {
+ goto do_checkpoint;
+ }
/* wait for a colo checkpoint */
proxy_checkpoint_req = colo_proxy_compare();
if (proxy_checkpoint_req < 0) {
@@ -512,6 +530,18 @@ static int colo_wait_handle_cmd(QEMUFile *f, int *checkpoint_request)
case COLO_CHECKPOINT_NEW:
*checkpoint_request = 1;
return 0;
+ case COLO_GUEST_SHUTDOWN:
+ qemu_mutex_lock_iothread();
+ vm_stop_force_state(RUN_STATE_COLO);
+ qemu_system_shutdown_request_core();
+ qemu_mutex_unlock_iothread();
+ trace_colo_receive_message("COLO_GUEST_SHUTDOWN");
+ /* the main thread will exit and termiante the whole
+ * process, do we need some cleanup?
+ */
+ for (;;) {
+ ;
+ }
default:
return -1;
}
diff --git a/vl.c b/vl.c
index dece65b..a21a5b2 100644
--- a/vl.c
+++ b/vl.c
@@ -1636,6 +1636,8 @@ static NotifierList wakeup_notifiers =
NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
static uint32_t wakeup_reason_mask = ~(1 << QEMU_WAKEUP_REASON_NONE);
+int colo_shutdown_requested;
+
int qemu_shutdown_requested_get(void)
{
return shutdown_requested;
@@ -1752,6 +1754,10 @@ void qemu_system_reset(bool report)
void qemu_system_reset_request(void)
{
if (no_reboot) {
+ qemu_system_shutdown_request();
+ if (!shutdown_requested) {/* colo handle it ? */
+ return;
+ }
shutdown_requested = 1;
} else {
reset_requested = 1;
@@ -1820,13 +1826,29 @@ void qemu_system_killed(int signal, pid_t pid)
qemu_system_shutdown_request();
}
-void qemu_system_shutdown_request(void)
+void qemu_system_shutdown_request_core(void)
{
- trace_qemu_system_shutdown_request();
shutdown_requested = 1;
qemu_notify_event();
}
+void qemu_system_shutdown_request(void)
+{
+ trace_qemu_system_shutdown_request();
+ /*
+ * if in colo mode, we need do some significant work before respond to the
+ * shutdown request.
+ */
+ if (loadvm_in_colo_state()) {
+ return ; /* primary's responsibility */
+ }
+ if (migrate_in_colo_state()) {
+ colo_shutdown_requested = 1;
+ return;
+ }
+ qemu_system_shutdown_request_core();
+}
+
static void qemu_system_powerdown(void)
{
qapi_event_send_powerdown(&error_abort);
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread* [Qemu-devel] [PATCH COLO-Frame v7 34/34] COLO: Add block replication into colo process
2015-07-09 3:16 [Qemu-devel] [PATCH COLO-Frame v7 00/34] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT) zhanghailiang
` (32 preceding siblings ...)
2015-07-09 3:16 ` [Qemu-devel] [PATCH COLO-Frame v7 33/34] COLO: Implement shutdown checkpoint zhanghailiang
@ 2015-07-09 3:16 ` zhanghailiang
33 siblings, 0 replies; 37+ messages in thread
From: zhanghailiang @ 2015-07-09 3:16 UTC (permalink / raw)
To: qemu-devel
Cc: lizhijian, quintela, yunhong.jiang, eddie.dong, peter.huangpeng,
dgilbert, arei.gonglei, amit.shah, Yang Hongyang, zhanghailiang
Make sure master start block replication after slave's block replication started.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
---
migration/colo.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++---
trace-events | 2 ++
2 files changed, 70 insertions(+), 3 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index d6f2c73..067102f 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -20,6 +20,7 @@
#include "qapi-event.h"
#include "net/colo-nic.h"
#include "qmp-commands.h"
+#include "block/block_int.h"
/*
* We should not do checkpoint one after another without any time interval,
@@ -119,6 +120,8 @@ static bool colo_runstate_is_stopped(void)
static void secondary_vm_do_failover(void)
{
int old_state;
+ Error *local_err = NULL;
+
/* Can not do failover during the process of VM's loading VMstate, Or
* it will break the secondary VM.
*/
@@ -137,6 +140,12 @@ static void secondary_vm_do_failover(void)
}
colo_proxy_destroy(COLO_MODE_SECONDARY);
+ bdrv_stop_replication_all(true, &local_err);
+ if (local_err) {
+ error_report_err(local_err);
+ }
+ trace_colo_stop_block_replication("failover");
+
/* It means that VM exit from COLO state */
colo = NULL;
@@ -163,6 +172,7 @@ static void primary_vm_do_failover(void)
{
MigrationState *s = migrate_get_current();
int old_state;
+ Error *local_err = NULL;
if (!colo_runstate_is_stopped()) {
vm_stop_force_state(RUN_STATE_COLO);
@@ -174,6 +184,12 @@ static void primary_vm_do_failover(void)
migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
}
+ bdrv_stop_replication_all(true, &local_err);
+ if (local_err) {
+ error_report_err(local_err);
+ }
+ trace_colo_stop_block_replication("failover");
+
vm_start();
old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
@@ -252,6 +268,7 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
int colo_shutdown, ret;
size_t size;
QEMUFile *trans = NULL;
+ Error *local_err = NULL;
ret = colo_ctl_put(s->file, COLO_CHECKPOINT_NEW);
if (ret < 0) {
@@ -305,6 +322,16 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
goto out;
}
+ /* we call this api although this may do nothing on primary side */
+ qemu_mutex_lock_iothread();
+ bdrv_do_checkpoint_all(&local_err);
+ qemu_mutex_unlock_iothread();
+ if (local_err) {
+ error_report_err(local_err);
+ ret = -1;
+ goto out;
+ }
+
ret = colo_ctl_put(s->file, COLO_CHECKPOINT_SEND);
if (ret < 0) {
goto out;
@@ -336,6 +363,10 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
trace_colo_receive_message("COLO_CHECKPOINT_LOADED");
if (colo_shutdown) {
+ qemu_mutex_lock_iothread();
+ bdrv_stop_replication_all(false, NULL);
+ trace_colo_stop_block_replication("shutdown");
+ qemu_mutex_unlock_iothread();
colo_ctl_put(s->file, COLO_GUEST_SHUTDOWN);
qemu_fflush(s->file);
colo_shutdown_requested = 0;
@@ -367,6 +398,7 @@ static void *colo_thread(void *opaque)
QEMUFile *colo_control = NULL;
int64_t current_time, checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
int i, ret;
+ Error *local_err = NULL;
if (colo_proxy_init(COLO_MODE_PRIMARY) != 0) {
error_report("Init colo proxy error");
@@ -398,6 +430,12 @@ static void *colo_thread(void *opaque)
}
qemu_mutex_lock_iothread();
+ /* start block replication */
+ bdrv_start_replication_all(REPLICATION_MODE_PRIMARY, &local_err);
+ if (local_err) {
+ goto out;
+ }
+ trace_colo_start_block_replication();
vm_start();
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "run");
@@ -448,7 +486,11 @@ do_checkpoint:
}
out:
- error_report("colo: some error happens in colo_thread");
+ if (local_err) {
+ error_report_err(local_err);
+ } else {
+ error_report("colo: some error happens in colo_thread");
+ }
qapi_event_send_colo_exit("primary", true, "unknown", NULL);;
/* Give users time (2s) to get involved in this verdict */
for (i = 0; i < 10; i++) {
@@ -533,6 +575,8 @@ static int colo_wait_handle_cmd(QEMUFile *f, int *checkpoint_request)
case COLO_GUEST_SHUTDOWN:
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
+ bdrv_stop_replication_all(false, NULL);
+ trace_colo_stop_block_replication("shutdown");
qemu_system_shutdown_request_core();
qemu_mutex_unlock_iothread();
trace_colo_receive_message("COLO_GUEST_SHUTDOWN");
@@ -556,6 +600,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
QEMUFile *ctl = NULL, *fb = NULL;
int i, ret;
uint64_t total_size;
+ Error *local_err = NULL;
qdev_hotplug = 0;
@@ -585,6 +630,15 @@ void *colo_process_incoming_checkpoints(void *opaque)
goto out;
}
+ qemu_mutex_lock_iothread();
+ /* start block replication */
+ bdrv_start_replication_all(REPLICATION_MODE_SECONDARY, &local_err);
+ if (local_err) {
+ goto out;
+ }
+ qemu_mutex_unlock_iothread();
+ trace_colo_start_block_replication();
+
ret = colo_ctl_put(ctl, COLO_CHECPOINT_READY);
if (ret < 0) {
goto out;
@@ -671,8 +725,15 @@ void *colo_process_incoming_checkpoints(void *opaque)
goto out;
}
- vmstate_loading = false;
+ /* discard colo disk buffer */
+ bdrv_do_checkpoint_all(&local_err);
qemu_mutex_unlock_iothread();
+ if (local_err) {
+ vmstate_loading = false;
+ goto out;
+ }
+
+ vmstate_loading = false;
if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) {
failover_set_state(FAILOVER_STATUS_RELAUNCH, FAILOVER_STATUS_NONE);
@@ -695,7 +756,11 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
out:
- error_report("Detect some error or get a failover request");
+ if (local_err) {
+ error_report_err(local_err);
+ } else {
+ error_report("Detect some error or get a failover request");
+ }
/*
* Here, we raise a qmp event to the user,
* It can help user to know what happens, and help deciding whether to
diff --git a/trace-events b/trace-events
index 66bcd08..914eb90 100644
--- a/trace-events
+++ b/trace-events
@@ -1477,6 +1477,8 @@ colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
colo_receive_message(const char *msg) "Receive '%s'"
failover_set_state(int new_state) "new state %d"
colo_rcv_pkt(int result) "Result of net packets comparing is different: %d"
+colo_start_block_replication(void) "Block replication is started"
+colo_stop_block_replication(const char *reason) "Block replication is stopped(reason: '%s')"
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
--
1.7.12.4
^ permalink raw reply related [flat|nested] 37+ messages in thread