From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:43920) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1ZX3L3-0005TW-IJ for qemu-devel@nongnu.org; Wed, 02 Sep 2015 04:24:14 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1ZX3L2-00011d-7j for qemu-devel@nongnu.org; Wed, 02 Sep 2015 04:24:13 -0400 Received: from szxga02-in.huawei.com ([119.145.14.65]:40971) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1ZX3L0-0000xm-Rq for qemu-devel@nongnu.org; Wed, 02 Sep 2015 04:24:12 -0400 From: zhanghailiang Date: Wed, 2 Sep 2015 16:22:56 +0800 Message-ID: <1441182199-8328-10-git-send-email-zhang.zhanghailiang@huawei.com> In-Reply-To: <1441182199-8328-1-git-send-email-zhang.zhanghailiang@huawei.com> References: <1441182199-8328-1-git-send-email-zhang.zhanghailiang@huawei.com> MIME-Version: 1.0 Content-Type: text/plain Subject: [Qemu-devel] [PATCH COLO-Frame v9 09/32] COLO: Implement colo checkpoint protocol List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: qemu-devel@nongnu.org Cc: lizhijian@cn.fujitsu.com, quintela@redhat.com, yunhong.jiang@intel.com, eddie.dong@intel.com, peter.huangpeng@huawei.com, dgilbert@redhat.com, arei.gonglei@huawei.com, stefanha@redhat.com, amit.shah@redhat.com, yanghy@cn.fujitsu.com, zhanghailiang We need communications protocol of user-defined to control the checkpoint process. The new checkpoint request is started by Primary VM, and the interactive process like below: Checkpoint synchronizing points, Primary Secondary 'checkpoint-request' @ -----------------------------> Suspend (In hybrid mode) 'checkpoint-reply' <------------------------------ @ Suspend&Save state 'vmstate-send' @ -----------------------------> Send state Receive state 'vmstate-received' <------------------------------ @ Release packets Load state 'vmstate-load' <------------------------------ @ Resume Resume (In hybrid mode) Start Comparing (In hybrid mode) NOTE: 1) '@' who sends the message 2) Every sync-point is synchronized by two sides with only one handshake(single direction) for low-latency. If more strict synchronization is required, a opposite direction sync-point should be added. 3) Since sync-points are single direction, the remote side may go forward a lot when this side just receives the sync-point. 4) For now, we only support 'periodic' checkpoint, for which the Secondary VM is not running, later we will support 'hybrid' mode. Signed-off-by: zhanghailiang Signed-off-by: Yang Hongyang Signed-off-by: Li Zhijian Signed-off-by: Gonglei --- migration/colo.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- qapi-schema.json | 26 ++++++++ trace-events | 3 +- 3 files changed, 218 insertions(+), 3 deletions(-) diff --git a/migration/colo.c b/migration/colo.c index 5f4fb20..526ee85 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -10,10 +10,12 @@ * later. See the COPYING file in the top-level directory. */ +#include #include "sysemu/sysemu.h" #include "migration/colo.h" #include "trace.h" #include "qemu/error-report.h" +#include "qemu/sockets.h" static QEMUBH *colo_bh; @@ -36,6 +38,103 @@ bool migration_incoming_in_colo_state(void) return (mis && (mis->state == MIGRATION_STATUS_COLO)); } +/* colo checkpoint control helper */ +static int colo_ctl_put(QEMUFile *f, uint32_t cmd, uint64_t value) +{ + int ret = 0; + + qemu_put_be32(f, cmd); + qemu_put_be64(f, value); + qemu_fflush(f); + + ret = qemu_file_get_error(f); + trace_colo_ctl_put(COLOCmd_lookup[cmd]); + + return ret; +} + +static int colo_ctl_get_cmd(QEMUFile *f, uint32_t *cmd) +{ + int ret = 0; + + *cmd = qemu_get_be32(f); + ret = qemu_file_get_error(f); + if (ret < 0) { + return ret; + } + if (*cmd >= COLO_CMD_MAX) { + error_report("Invalid colo command, get cmd:%d", *cmd); + return -EINVAL; + } + trace_colo_ctl_get(COLOCmd_lookup[*cmd]); + + return 0; +} + +static int colo_ctl_get(QEMUFile *f, uint32_t require) +{ + int ret; + uint32_t cmd; + uint64_t value; + + ret = colo_ctl_get_cmd(f, &cmd); + if (ret < 0) { + return ret; + } + if (cmd != require) { + error_report("Unexpect colo command, expect:%d, but get cmd:%d", + require, cmd); + return -EINVAL; + } + + value = qemu_get_be64(f); + ret = qemu_file_get_error(f); + if (ret < 0) { + return ret; + } + + return value; +} + +static int colo_do_checkpoint_transaction(MigrationState *s) +{ + int ret; + + ret = colo_ctl_put(s->to_dst_file, COLO_CMD_CHECKPOINT_REQUEST, 0); + if (ret < 0) { + goto out; + } + + ret = colo_ctl_get(s->from_dst_file, COLO_CMD_CHECKPOINT_REPLY); + if (ret < 0) { + goto out; + } + + /* TODO: suspend and save vm state to colo buffer */ + + ret = colo_ctl_put(s->to_dst_file, COLO_CMD_VMSTATE_SEND, 0); + if (ret < 0) { + goto out; + } + + /* TODO: send vmstate to Secondary */ + + ret = colo_ctl_get(s->from_dst_file, COLO_CMD_VMSTATE_RECEIVED); + if (ret < 0) { + goto out; + } + + ret = colo_ctl_get(s->from_dst_file, COLO_CMD_VMSTATE_LOADED); + if (ret < 0) { + goto out; + } + + /* TODO: resume Primary */ + +out: + return ret; +} + static void *colo_thread(void *opaque) { MigrationState *s = opaque; @@ -54,12 +153,27 @@ static void *colo_thread(void *opaque) goto out; } + /* + * Wait for Secondary finish loading vm states and enter COLO + * restore. + */ + ret = colo_ctl_get(s->from_dst_file, COLO_CMD_CHECKPOINT_READY); + if (ret < 0) { + goto out; + } + qemu_mutex_lock_iothread(); vm_start(); qemu_mutex_unlock_iothread(); trace_colo_vm_state_change("stop", "run"); - /*TODO: COLO checkpoint savevm loop*/ + while (s->state == MIGRATION_STATUS_COLO) { + /* start a colo checkpoint */ + ret = colo_do_checkpoint_transaction(s); + if (ret < 0) { + goto out; + } + } out: if (ret < 0) { @@ -105,6 +219,39 @@ void colo_init_checkpointer(MigrationState *s) qemu_bh_schedule(colo_bh); } +/* + * return: + * 0: start a checkpoint + * -1: some error happened, exit colo restore + */ +static int colo_wait_handle_cmd(QEMUFile *f, int *checkpoint_request) +{ + int ret; + uint32_t cmd; + uint64_t value; + + ret = colo_ctl_get_cmd(f, &cmd); + if (ret < 0) { + /* do failover ? */ + return ret; + } + /* Fix me: this value should be 0, which is not so good, + * should be used for checking ? + */ + value = qemu_get_be64(f); + if (value != 0) { + return -EINVAL; + } + + switch (cmd) { + case COLO_CMD_CHECKPOINT_REQUEST: + *checkpoint_request = 1; + return 0; + default: + return -EINVAL; + } +} + void *colo_process_incoming_thread(void *opaque) { MigrationIncomingState *mis = opaque; @@ -124,7 +271,48 @@ void *colo_process_incoming_thread(void *opaque) error_report("Can't open incoming channel!"); goto out; } - /* TODO: COLO checkpoint restore loop */ + + ret = colo_ctl_put(mis->to_src_file, COLO_CMD_CHECKPOINT_READY, 0); + if (ret < 0) { + goto out; + } + + while (mis->state == MIGRATION_STATUS_COLO) { + int request = 0; + int ret = colo_wait_handle_cmd(mis->from_src_file, &request); + + if (ret < 0) { + break; + } else { + if (!request) { + continue; + } + } + + ret = colo_ctl_put(mis->to_src_file, COLO_CMD_CHECKPOINT_REPLY, 0); + if (ret < 0) { + goto out; + } + + ret = colo_ctl_get(mis->from_src_file, COLO_CMD_VMSTATE_SEND); + if (ret < 0) { + goto out; + } + + /* TODO: read migration data into colo buffer */ + + ret = colo_ctl_put(mis->to_src_file, COLO_CMD_VMSTATE_RECEIVED, 0); + if (ret < 0) { + goto out; + } + + /* TODO: load vm state */ + + ret = colo_ctl_put(mis->to_src_file, COLO_CMD_VMSTATE_LOADED, 0); + if (ret < 0) { + goto out; + } +} out: if (ret < 0) { diff --git a/qapi-schema.json b/qapi-schema.json index 6dd5c7c..0d3d6e2 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -664,6 +664,32 @@ '*tls-port': 'int', '*cert-subject': 'str' } } ## +# @COLOCmd +# +# The colo command +# +# @invalid: unknown command +# +# @checkpoint-ready: SVM is ready for checkpointing +# +# @checkpoint-request: PVM tells SVM to prepare for new checkpointing +# +# @checkpoint-reply: SVM gets PVM's checkpoint request +# +# @vmstate-send: VM's state will be sent by PVM. +# +# @vmstate-received: VM's state has been received by SVM +# +# @vmstate-loaded: VM's state has been loaded by SVM +# +# Since: 2.5 +## +{ 'enum': 'COLOCmd', + 'data': [ 'invalid', 'checkpoint-ready', 'checkpoint-request', + 'checkpoint-reply', 'vmstate-send', 'vmstate-size', + 'vmstate-received', 'vmstate-loaded', 'guest-shutdown', + 'ram-steal'] } + # @MouseInfo: # # Information about a mouse device. diff --git a/trace-events b/trace-events index 352e9c3..7fdf723 100644 --- a/trace-events +++ b/trace-events @@ -1474,7 +1474,8 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) "" # migration/colo.c colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'" -colo_receive_message(const char *msg) "Receive '%s'" +colo_ctl_put(const char *msg) "Send '%s'" +colo_ctl_get(const char *msg) "Receive '%s'" # kvm-all.c kvm_ioctl(int type, void *arg) "type 0x%x, arg %p" -- 1.8.3.1