[Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Lei Li]

This patch series tries to introduce a mechanism using side
channel pipe for RAM via SCM_RIGHTS with unix domain socket
protocol migration.

This side channel is used for the page flipping by vmsplice,
which is the internal mechanism for localhost migration that
we are trying to add to QEMU. The backgroud info and previous
patch series for reference,

Localhost migration
http://lists.nongnu.org/archive/html/qemu-devel/2013-08/msg02916.html

migration: Introduce side channel for RAM
http://lists.gnu.org/archive/html/qemu-devel/2013-09/msg04043.html

I have picked patches from the localhost migration series and rebased 
it on the series of side channel, now it is a complete series that
passed the basic test.

Please let me know if there is anything needs to be fixed or improved.
Your suggestions and comments are very welcome, and thanks for Paolo
for his review and useful suggestions.


Changes since V1:
  Address suggestions from Paolo Bonzini including:

    - Use Unix socket QEMUFile as basis of code and adjust the way
      of overriding RDMA hooks.

    - Involve the vmsplice for page flipping.

    - Add new RunState RUN_STATE_FLIPPING_MIGRATE and add it to
      runstate_needs_reset() for the adjustment of the current
      migration process with page flipping.



Lei Li (17):
  rename is_active to is_block_active
  QAPI: introduce magration capability unix_page_flipping
  migration: add migrate_unix_page_flipping()
  qmp-command.hx: add missing docs for migration capabilites
  migration-local: add QEMUFileLocal with socket based QEMUFile
  migration-local: introduce qemu_fopen_socket_local()
  migration-local: add send_pipefd()
  migration-local: add recv_pipefd()
  migration-local: override before_ram_iterate to send pipefd
  savevm: adjust ram_control_save_page with page flipping
  migration-local: override save_page for page transmit
  migration-local: override hook_ram_load 
  migration-unix: replace qemu_fopen_socket with qemu_fopen_socket_local
  add new RanState RAN_STATE_FLIPPING_MIGRATE
  migration-unix: page flipping support on unix outgoing
  migration: adjust migration_thread() process for unix_page_flipping
  hmp: better fomat for info migrate_capabilities

 Makefile.target               |   1 +
 block-migration.c             |   2 +-
 migration-local.c             | 512 ++++++++++++++++++++++++++++++++++++++++++
 hmp.c                         |   5 +-
 include/migration/migration.h |   3 +
 include/migration/qemu-file.h |   2 +
 include/migration/vmstate.h   |   2 +-
 migration-unix.c              |  27 ++-
 migration.c                   |  18 +-
 qapi-schema.json              |  18 +-
 qmp-commands.hx               |   8 +
 savevm.c                      |  21 +-
 vl.c                          |  12 +-
 13 files changed, 624 insertions(+), 27 deletions(-)
 create mode 100644 migration-local.c

[Qemu-devel] [PATCH 01/17] rename is_active to is_block_active

[Lei Li]

is_active is used to identify block migration, rename to
is_block_active to make it more clear.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 block-migration.c           |    2 +-
 include/migration/vmstate.h |    2 +-
 savevm.c                    |   16 ++++++++--------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/block-migration.c b/block-migration.c
index daf9ec1..b637695 100644
--- a/block-migration.c
+++ b/block-migration.c
@@ -834,7 +834,7 @@ SaveVMHandlers savevm_block_handlers = {
     .save_live_pending = block_save_pending,
     .load_state = block_load,
     .cancel = block_migration_cancel,
-    .is_active = block_is_active,
+    .is_block_active = block_is_active,
 };
 
 void blk_mig_init(void)
diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 9d09e60..c634d65 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -42,7 +42,7 @@ typedef struct SaveVMHandlers {
     int (*save_live_complete)(QEMUFile *f, void *opaque);
 
     /* This runs both outside and inside the iothread lock.  */
-    bool (*is_active)(void *opaque);
+    bool (*is_block_active)(void *opaque);
 
     /* This runs outside the iothread lock in the migration case, and
      * within the lock in the savevm case.  The callback had better only
diff --git a/savevm.c b/savevm.c
index 2f631d4..56b8643 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1867,8 +1867,8 @@ void qemu_savevm_state_begin(QEMUFile *f,
         if (!se->ops || !se->ops->save_live_setup) {
             continue;
         }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
+        if (se->ops && se->ops->is_block_active) {
+            if (!se->ops->is_block_active(se->opaque)) {
                 continue;
             }
         }
@@ -1907,8 +1907,8 @@ int qemu_savevm_state_iterate(QEMUFile *f)
         if (!se->ops || !se->ops->save_live_iterate) {
             continue;
         }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
+        if (se->ops && se->ops->is_block_active) {
+            if (!se->ops->is_block_active(se->opaque)) {
                 continue;
             }
         }
@@ -1948,8 +1948,8 @@ void qemu_savevm_state_complete(QEMUFile *f)
         if (!se->ops || !se->ops->save_live_complete) {
             continue;
         }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
+        if (se->ops && se->ops->is_block_active) {
+            if (!se->ops->is_block_active(se->opaque)) {
                 continue;
             }
         }
@@ -2002,8 +2002,8 @@ uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
         if (!se->ops || !se->ops->save_live_pending) {
             continue;
         }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
+        if (se->ops && se->ops->is_block_active) {
+            if (!se->ops->is_block_active(se->opaque)) {
                 continue;
             }
         }
-- 
1.7.7.6

[Qemu-devel] [PATCH 02/17] QAPI: introduce magration capability unix_page_flipping

[Lei Li]

Introduce unix_page_flipping to MigrationCapability for localhost
migration.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 qapi-schema.json |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/qapi-schema.json b/qapi-schema.json
index 60f3fd1..523a5b2 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -661,10 +661,16 @@
 # @auto-converge: If enabled, QEMU will automatically throttle down the guest
 #          to speed up convergence of RAM migration. (since 1.6)
 #
+# @unix-page-flipping: If enabled, QEMU will support localhost migration. This
+#          feature allows live upgrade of a running QEMU instance by doing localhost
+#          migration with page flipping. It requires the source and destination
+#          are both on localhost. Disabled by default. (since 1.7)
+#
 # Since: 1.2
 ##
 { 'enum': 'MigrationCapability',
-  'data': ['xbzrle', 'x-rdma-pin-all', 'auto-converge', 'zero-blocks'] }
+  'data': ['xbzrle', 'x-rdma-pin-all', 'auto-converge', 'zero-blocks',
+           'unix-page-flipping'] }
 
 ##
 # @MigrationCapabilityStatus
-- 
1.7.7.6

[Qemu-devel] [PATCH 03/17] migration: add migrate_unix_page_flipping()

[Lei Li]

Add migrate_unix_page_flipping() to check if
MIGRATION_CAPABILITY_UNIX_PAGE_FLIPPING is enabled.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 include/migration/migration.h |    3 +++
 migration.c                   |    9 +++++++++
 2 files changed, 12 insertions(+), 0 deletions(-)

diff --git a/include/migration/migration.h b/include/migration/migration.h
index 140e6b4..7e5d01a 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -131,10 +131,13 @@ void migrate_add_blocker(Error *reason);
 void migrate_del_blocker(Error *reason);
 
 bool migrate_rdma_pin_all(void);
+
 bool migrate_zero_blocks(void);
 
 bool migrate_auto_converge(void);
 
+bool migrate_unix_page_flipping(void);
+
 int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
                          uint8_t *dst, int dlen);
 int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen);
diff --git a/migration.c b/migration.c
index 2b1ab20..4ac466b 100644
--- a/migration.c
+++ b/migration.c
@@ -541,6 +541,15 @@ int64_t migrate_xbzrle_cache_size(void)
     return s->xbzrle_cache_size;
 }
 
+bool migrate_unix_page_flipping(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_UNIX_PAGE_FLIPPING];
+}
+
 /* migration thread support */
 
 static void *migration_thread(void *opaque)
-- 
1.7.7.6

[Qemu-devel] [PATCH 04/17] qmp-command.hx: add missing docs for migration capabilites

[Lei Li]

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 qmp-commands.hx |    8 ++++++++
 1 files changed, 8 insertions(+), 0 deletions(-)

diff --git a/qmp-commands.hx b/qmp-commands.hx
index fba15cd..650a3a8 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -2898,6 +2898,10 @@ migrate-set-capabilities
 Enable/Disable migration capabilities
 
 - "xbzrle": XBZRLE support
+- "x-rdma-pin-all": RDMA support
+- "zero-blocks": zero-blocks support
+- "auto-converge": Auto converge support
+- "unix-page-flipping": Page flipping support
 
 Arguments:
 
@@ -2922,6 +2926,10 @@ Query current migration capabilities
 
 - "capabilities": migration capabilities state
          - "xbzrle" : XBZRLE state (json-bool)
+         - "x-rdma-pin-all": RDMA state (json-bool)
+         - "zero-blocks": zero-blocks state (json-bool)
+         - "auto-converge": Auto converge state (json-bool)
+         - "unix-page-flipping": Page flipping state (json-bool)
 
 Arguments:
 
-- 
1.7.7.6

[Qemu-devel] [PATCH 05/17] migration-local: add QEMUFileLocal with socket based QEMUFile

[Lei Li]

This patch adds QEMUFileLocal with copy of socket based QEMUFile, will
be used as the basis code for Unix socket protocol migration and page
flipping migration.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 Makefile.target   |    1 +
 migration-local.c |  121 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+), 0 deletions(-)
 create mode 100644 migration-local.c

diff --git a/Makefile.target b/Makefile.target
index af6ac7e..aa09960 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -117,6 +117,7 @@ obj-$(CONFIG_KVM) += kvm-all.o
 obj-y += memory.o savevm.o cputlb.o
 obj-y += memory_mapping.o
 obj-y += dump.o
+obj-y += migration-local.o
 LIBS+=$(libs_softmmu)
 
 # xen support
diff --git a/migration-local.c b/migration-local.c
new file mode 100644
index 0000000..8b9e10e
--- /dev/null
+++ b/migration-local.c
@@ -0,0 +1,121 @@
+/*
+ * QEMU localhost migration with page flipping
+ *
+ * Copyright IBM, Corp. 2013
+ *
+ * Authors:
+ *   Lei Li   <lilei@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "config-host.h"
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "exec/cpu-common.h"
+#include "config.h"
+#include "exec/cpu-all.h"
+#include "exec/memory.h"
+#include "exec/memory-internal.h"
+#include "monitor/monitor.h"
+#include "migration/qemu-file.h"
+#include "qemu/iov.h"
+#include "sysemu/arch_init.h"
+#include "sysemu/sysemu.h"
+#include "block/block.h"
+#include "qemu/sockets.h"
+#include "migration/block.h"
+#include "qemu/thread.h"
+#include "qmp-commands.h"
+#include "trace.h"
+#include "qemu/osdep.h"
+
+//#define DEBUG_MIGRATION_LOCAL
+
+#ifdef DEBUG_MIGRATION_LOCAL
+#define DPRINTF(fmt, ...) \
+    do { printf("migration-local: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+
+typedef struct QEMUFileLocal {
+    QEMUFile *file;
+    int sockfd;
+    int pipefd[2];
+    bool unix_page_flipping;
+} QEMUFileLocal;
+
+static int qemu_local_get_sockfd(void *opaque)
+{
+    QEMUFileLocal *s = opaque;
+
+    return s->sockfd;
+}
+
+static int qemu_local_get_buffer(void *opaque, uint8_t *buf,
+                                 int64_t pos, int size)
+{
+    QEMUFileLocal *s = opaque;
+    ssize_t len;
+
+    for (;;) {
+        len = qemu_recv(s->sockfd, buf, size, 0);
+        if (len != -1) {
+            break;
+        }
+
+        if (socket_error() == EAGAIN) {
+            yield_until_fd_readable(s->sockfd);
+        } else if (socket_error() != EINTR) {
+            break;
+        }
+    }
+
+    if (len == -1) {
+        len = -socket_error();
+    }
+
+    return len;
+}
+
+static ssize_t qemu_local_writev_buffer(void *opaque, struct iovec *iov,
+                                        int iovcnt, int64_t pos)
+{
+    QEMUFileLocal *s = opaque;
+    ssize_t len;
+    ssize_t size = iov_size(iov, iovcnt);
+
+    len = iov_send(s->sockfd, iov, iovcnt, 0, size);
+    if (len < size) {
+        len = -socket_error();
+    }
+
+    return len;
+}
+
+static int qemu_local_close(void *opaque)
+{
+    QEMUFileLocal *s = opaque;
+
+    closesocket(s->sockfd);
+    g_free(s);
+
+    return 0;
+}
+
+static const QEMUFileOps pipe_read_ops = {
+    .get_fd        = qemu_local_get_sockfd,
+    .get_buffer    = qemu_local_get_buffer,
+    .close         = qemu_local_close,
+};
+
+static const QEMUFileOps pipe_write_ops = {
+    .get_fd             = qemu_local_get_sockfd,
+    .writev_buffer      = qemu_local_writev_buffer,
+    .close              = qemu_local_close,
+};
-- 
1.7.7.6

[Qemu-devel] [PATCH 06/17] migration-local: introduce qemu_fopen_socket_local()

[Lei Li]

Add qemu_fopen_socket_local() to open QEMUFileLocal introduced
earlier. It will create a pipe in write mode if unix_page_flipping
is enabled, adjust qemu_local_close() to close pipe as well.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 include/migration/qemu-file.h |    2 +
 migration-local.c             |   46 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 0 deletions(-)

diff --git a/include/migration/qemu-file.h b/include/migration/qemu-file.h
index 0f757fb..f9b104a 100644
--- a/include/migration/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -99,6 +99,8 @@ QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fdopen(int fd, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
+QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode);
+
 int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 int64_t qemu_ftell(QEMUFile *f);
diff --git a/migration-local.c b/migration-local.c
index 8b9e10e..6214ff9 100644
--- a/migration-local.c
+++ b/migration-local.c
@@ -103,6 +103,12 @@ static int qemu_local_close(void *opaque)
     QEMUFileLocal *s = opaque;
 
     closesocket(s->sockfd);
+
+    if (s->unix_page_flipping) {
+        close(s->pipefd[0]);
+        close(s->pipefd[1]);
+    }
+
     g_free(s);
 
     return 0;
@@ -119,3 +125,43 @@ static const QEMUFileOps pipe_write_ops = {
     .writev_buffer      = qemu_local_writev_buffer,
     .close              = qemu_local_close,
 };
+
+QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode)
+{
+    QEMUFileLocal *s;
+    int pipefd[2];
+
+    if (qemu_file_mode_is_not_valid(mode)) {
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUFileLocal));
+    s->sockfd = sockfd;
+
+    if (migrate_unix_page_flipping()) {
+        s->unix_page_flipping = 1;
+    }
+
+    if (mode[0] == 'w') {
+        if (s->unix_page_flipping) {
+            if (pipe(pipefd) < 0) {
+                fprintf(stderr, "failed to create pipe\n");
+                goto fail;
+            }
+
+            s->pipefd[0] = pipefd[0];
+            s->pipefd[1] = pipefd[1];
+        }
+
+        qemu_set_block(s->sockfd);
+        s->file = qemu_fopen_ops(s, &pipe_write_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &pipe_read_ops);
+    }
+
+    return s->file;
+
+fail:
+    g_free(s);
+    return NULL;
+}
-- 
1.7.7.6

[Qemu-devel] [PATCH 07/17] migration-local: add send_pipefd()

[Lei Li]

This patch adds send_pipefd() to pass the pipe file descriptor
to destination process.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-local.c |   53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 53 insertions(+), 0 deletions(-)

diff --git a/migration-local.c b/migration-local.c
index 6214ff9..f2871d8 100644
--- a/migration-local.c
+++ b/migration-local.c
@@ -165,3 +165,56 @@ fail:
     g_free(s);
     return NULL;
 }
+
+
+/*
+ * Pass a pipe file descriptor to another process.
+ *
+ * Return negative value If pipefd < 0. Return 0 on
+ * success.
+ *
+ */
+static int send_pipefd(int sockfd, int pipefd)
+{
+    struct msghdr msg;
+    struct iovec iov[1];
+    ssize_t ret;
+
+    union {
+      struct cmsghdr cm;
+      char control[CMSG_SPACE(sizeof(int))];
+    } control_un;
+    struct cmsghdr *cmptr;
+    char req[1] = { 0x01 };
+
+    if (pipefd < 0) {
+        msg.msg_control = NULL;
+        msg.msg_controllen = 0;
+        /* Negative status means error */
+        req[0] = pipefd;
+    } else {
+        msg.msg_control = control_un.control;
+        msg.msg_controllen = sizeof(control_un.control);
+
+        cmptr = CMSG_FIRSTHDR(&msg);
+        cmptr->cmsg_len = CMSG_LEN(sizeof(int));
+        cmptr->cmsg_level = SOL_SOCKET;
+        cmptr->cmsg_type = SCM_RIGHTS;
+        *((int *) CMSG_DATA(cmptr)) = pipefd;
+
+        msg.msg_name = NULL;
+        msg.msg_namelen = 0;
+
+        iov[0].iov_base = req;
+        iov[0].iov_len = sizeof(req);
+        msg.msg_iov = iov;
+        msg.msg_iovlen = 1;
+    }
+
+    ret = sendmsg(sockfd, &msg, 0);
+    if (ret <= 0) {
+        DPRINTF("sendmsg error: %s\n", strerror(errno));
+    }
+
+    return ret;
+}
-- 
1.7.7.6

[Qemu-devel] [PATCH 08/17] migration-local: add recv_pipefd()

[Lei Li]

This patch adds recv_pipefd() to receive the pipe file descriptor
that passed by source process.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-local.c |   64 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 64 insertions(+), 0 deletions(-)

diff --git a/migration-local.c b/migration-local.c
index f2871d8..ed0ae6c 100644
--- a/migration-local.c
+++ b/migration-local.c
@@ -218,3 +218,67 @@ static int send_pipefd(int sockfd, int pipefd)
 
     return ret;
 }
+
+/*
+ * Receive a pipe file descriptor from a source process
+ * via unix socket.
+ *
+ * Return negative value if there has been an recvmsg error or
+ * no fd to be received. Return 0 if the connection closed by
+ * source. Return file descriptor on success.
+ *
+ */
+static int recv_pipefd(int sockfd)
+{
+    struct msghdr msg;
+    struct iovec iov[1];
+    ssize_t n;
+    int pipefd = -1;
+    char req[1];
+
+    union {
+      struct cmsghdr cm;
+      char control[CMSG_SPACE(sizeof(int))];
+    } control_un;
+    struct cmsghdr *cmptr;
+
+    msg.msg_control = control_un.control;
+    msg.msg_controllen = sizeof(control_un.control);
+
+    msg.msg_name = NULL;
+    msg.msg_namelen = 0;
+
+    iov[0].iov_base = req;
+    iov[0].iov_len = sizeof(req);
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+
+    if ( (n = recvmsg(sockfd, &msg, 0)) <= 0) {
+        fprintf(stderr, "recvmsg error: %s\n", strerror(errno));
+        return n;
+    }
+
+    /* req 0x01 means there is a file descriptor to receive */
+    if (req[0] != 0x01) {
+        return pipefd;
+    }
+
+    if ( (cmptr = CMSG_FIRSTHDR(&msg)) != NULL &&
+        cmptr->cmsg_len == CMSG_LEN(sizeof(int))) {
+        if (cmptr->cmsg_level != SOL_SOCKET) {
+            DPRINTF(stderr, "control level != SOL_SOCKET\n");
+            return pipefd;
+        } else if (cmptr->cmsg_type != SCM_RIGHTS) {
+            DPRINTF(stderr, "control type != SCM_RIGHTS\n");
+            return pipefd;
+        }
+        /* The pipe file descriptor to be received */
+        pipefd = *((int *) CMSG_DATA(cmptr));
+        DPRINTF("pipefd received successfully: %d\n", pipefd);
+    } else {
+        /* Descriptor was not passed */
+        DPRINTF(stderr, "pipefd was not passed\n");
+    }
+
+    return pipefd;
+}
-- 
1.7.7.6

[Qemu-devel] [PATCH 09/17] migration-local: override before_ram_iterate to send pipefd

[Lei Li]

Override befor_ram_iterate to send pipefd. It will write the
RAM_SAVE_FLAG_HOOK flags which will trigger the load hook to
receive it.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-local.c |   26 ++++++++++++++++++++++++++
 1 files changed, 26 insertions(+), 0 deletions(-)

diff --git a/migration-local.c b/migration-local.c
index ed0ae6c..92c661c 100644
--- a/migration-local.c
+++ b/migration-local.c
@@ -114,6 +114,31 @@ static int qemu_local_close(void *opaque)
     return 0;
 }
 
+static int send_pipefd(int sockfd, int pipefd);
+
+static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
+                                  uint64_t flags)
+{
+    QEMUFileLocal *s = opaque;
+    int ret;
+
+    if (s->unix_page_flipping) {
+        /* Avoid sending pipe fd again in ram_save_complete() stage */
+        if (flags != RAM_CONTROL_FINISH) {
+            qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
+            qemu_fflush(f);
+            ret = send_pipefd(s->sockfd, s->pipefd[0]);
+            if (ret < 0) {
+                fprintf(stderr, "failed to pass pipe\n");
+                return ret;
+            }
+            DPRINTF("pipe fd was sent\n");
+        }
+    }
+
+    return 0;
+}
+
 static const QEMUFileOps pipe_read_ops = {
     .get_fd        = qemu_local_get_sockfd,
     .get_buffer    = qemu_local_get_buffer,
@@ -124,6 +149,7 @@ static const QEMUFileOps pipe_write_ops = {
     .get_fd             = qemu_local_get_sockfd,
     .writev_buffer      = qemu_local_writev_buffer,
     .close              = qemu_local_close,
+    .before_ram_iterate = qemu_local_send_pipefd,
 };
 
 QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode)
-- 
1.7.7.6

[Qemu-devel] [PATCH 10/17] migration-local: override save_page for page transmit

[Lei Li]

This patch implements save_page callback for the outside
of page flipping. It will write the address of the page
on the Unix socket and flip the page data on pipe by
vmsplice(). Every page address would have a header flag
RAM_SAVE_FLAG_HOOK.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-local.c |   54 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 54 insertions(+), 0 deletions(-)

diff --git a/migration-local.c b/migration-local.c
index 92c661c..7ccec30 100644
--- a/migration-local.c
+++ b/migration-local.c
@@ -139,6 +139,59 @@ static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
     return 0;
 }
 
+static size_t qemu_local_save_ram(QEMUFile *f, void *opaque,
+                                  ram_addr_t block_offset, ram_addr_t offset,
+                                  size_t size, int *bytes_sent)
+{
+    QEMUFileLocal *s = opaque;
+    ram_addr_t current_addr = block_offset + offset;
+    void *ram_addr;
+    ssize_t ret;
+
+    if (s->unix_page_flipping) {
+        qemu_fflush(s->file);
+        qemu_put_be64(s->file, RAM_SAVE_FLAG_HOOK);
+
+        /* Write page address to unix socket */
+        qemu_put_be64(s->file, current_addr);
+
+        ram_addr = qemu_get_ram_ptr(current_addr);
+
+        /* vmsplice page data to pipe */
+        struct iovec iov = {
+            .iov_base = ram_addr,
+            .iov_len  = size,
+        };
+
+        /*
+         * The flag SPLICE_F_MOVE is introduced in kernel for the page
+         * flipping feature in QEMU, which will movie pages rather than
+         * copying, previously unused.
+         *
+         * If a move is not possible the kernel will transparently falls
+         * back to copying data.
+         *
+         * For older kernels the SPLICE_F_MOVE would be ignored and a copy
+         * would occur.
+         */
+        ret = vmsplice(s->pipefd[1], &iov, 1, SPLICE_F_GIFT | SPLICE_F_MOVE);
+        if (ret == -1) {
+            if (errno != EAGAIN && errno != EINTR) {
+                fprintf(stderr, "vmsplice save error: %s\n", strerror(errno));
+                return ret;
+            }
+        } else {
+            if (bytes_sent) {
+                *bytes_sent = 1;
+            }
+            DPRINTF("block_offset: %lu, offset: %lu\n", block_offset, offset);
+            return 0;
+        }
+    }
+
+    return RAM_SAVE_CONTROL_NOT_SUPP;
+}
+
 static const QEMUFileOps pipe_read_ops = {
     .get_fd        = qemu_local_get_sockfd,
     .get_buffer    = qemu_local_get_buffer,
@@ -150,6 +203,7 @@ static const QEMUFileOps pipe_write_ops = {
     .writev_buffer      = qemu_local_writev_buffer,
     .close              = qemu_local_close,
     .before_ram_iterate = qemu_local_send_pipefd,
+    .save_page          = qemu_local_save_ram
 };
 
 QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode)
-- 
1.7.7.6

[Qemu-devel] [PATCH 11/17] savevm: adjust ram_control_save_page for page flipping

[Lei Li]

As callback save_page will always be opened by
qemu_fopen_socket_local(), and without unix_page_flipping
it will return RAM_SAVE_CONTROL_NOT_SUPP, it leads to a
wrong qemu_file_set_error() based on the current logic.
So this patch adds RAM_SAVE_CONTROL_NOT_SUPP to the check.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 savevm.c |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/savevm.c b/savevm.c
index 56b8643..b102275 100644
--- a/savevm.c
+++ b/savevm.c
@@ -668,7 +668,8 @@ size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset,
         int ret = f->ops->save_page(f, f->opaque, block_offset,
                                     offset, size, bytes_sent);
 
-        if (ret != RAM_SAVE_CONTROL_DELAYED) {
+        if (ret != RAM_SAVE_CONTROL_DELAYED &&
+            ret != RAM_SAVE_CONTROL_NOT_SUPP) {
             if (bytes_sent && *bytes_sent > 0) {
                 qemu_update_position(f, *bytes_sent);
             } else if (ret < 0) {
-- 
1.7.7.6

[Qemu-devel] [PATCH 12/17] migration-local: override hook_ram_load

[Lei Li]

Override hook_ram_load to receive the pipe file descriptor
passed by source process and page address which will be
extracted to vmsplice the page data from pipe.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-local.c |   85 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 85 insertions(+), 0 deletions(-)

diff --git a/migration-local.c b/migration-local.c
index 7ccec30..083a24f 100644
--- a/migration-local.c
+++ b/migration-local.c
@@ -32,6 +32,8 @@
 #include "trace.h"
 #include "qemu/osdep.h"
 
+#define BEFORE_PIPE_FD 199
+
 //#define DEBUG_MIGRATION_LOCAL
 
 #ifdef DEBUG_MIGRATION_LOCAL
@@ -50,6 +52,8 @@ typedef struct QEMUFileLocal {
     bool unix_page_flipping;
 } QEMUFileLocal;
 
+static bool pipefd_passed;
+
 static int qemu_local_get_sockfd(void *opaque)
 {
     QEMUFileLocal *s = opaque;
@@ -64,6 +68,22 @@ static int qemu_local_get_buffer(void *opaque, uint8_t *buf,
     ssize_t len;
 
     for (;;) {
+        /*
+         * FIX ME: BEFORE_PIPE_FD is hard-coded and checked temporarily here
+         * because that the control message of passed pipe file descriptor
+         * might be 'eaten' to stream file by qemu_recv(), which would lead
+         * to the failure of recv_pipefd(), as it should stay in the socket
+         * and received by the real receiver recvmsg().
+         *
+         * Although this message is followed by the first load_hook flags
+         * RAM_SAVE_FLAG_HOOK, the incoming side is hardly to avoid this
+         * as it would fill it into the stream file before any check action
+         * taken. Need to find a way out to fix this.
+         */
+        if (size > BEFORE_PIPE_FD && !pipefd_passed) {
+            size = BEFORE_PIPE_FD;
+        }
+
         len = qemu_recv(s->sockfd, buf, size, 0);
         if (len != -1) {
             break;
@@ -115,6 +135,7 @@ static int qemu_local_close(void *opaque)
 }
 
 static int send_pipefd(int sockfd, int pipefd);
+static int recv_pipefd(int sockfd);
 
 static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
                                   uint64_t flags)
@@ -192,10 +213,74 @@ static size_t qemu_local_save_ram(QEMUFile *f, void *opaque,
     return RAM_SAVE_CONTROL_NOT_SUPP;
 }
 
+static int qemu_local_ram_load(QEMUFile *f, void *opaque,
+                               uint64_t flags)
+{
+    QEMUFileLocal *s = opaque;
+    ram_addr_t addr;
+    struct iovec iov;
+    ssize_t ret;
+
+    /* Receive the pipe file descripter passed from source process */
+    if (!pipefd_passed) {
+        s->pipefd[0] = recv_pipefd(s->sockfd);
+        if (s->pipefd[0] <= 0) {
+            fprintf(stderr, "failed to receive pipe fd: %d\n", s->pipefd[0]);
+        } else {
+            pipefd_passed = 1;
+            DPRINTF(stderr, "succeed\n");
+        }
+
+        return s->pipefd[0];
+    }
+
+    if (pipefd_passed) {
+        void *host;
+
+        /*
+         * Extract the page address from the 8-byte record and
+         * read the page data from the pipe.
+         */
+        addr = qemu_get_be64(s->file);
+        host = qemu_get_ram_ptr(addr);
+
+        iov.iov_base = host;
+        iov.iov_len = TARGET_PAGE_SIZE;
+
+        /* The flag SPLICE_F_MOVE is introduced in kernel for the page
+         * flipping feature in QEMU, which will movie pages rather than
+         * copying, previously unused.
+         *
+         * If a move is not possible the kernel will transparently falls
+         * back to copying data.
+         *
+         * For older kernels the SPLICE_F_MOVE would be ignored and a copy
+         * would occur.
+         */
+        ret = vmsplice(s->pipefd[0], &iov, 1, SPLICE_F_MOVE);
+        if (ret == -1) {
+            if (errno != EAGAIN && errno != EINTR) {
+                fprintf(stderr, "vmsplice() load error: %s", strerror(errno));
+                return ret;
+            }
+            DPRINTF("vmsplice load error\n");
+        } else if (ret == 0) {
+            DPRINTF(stderr, "load_page: zero read\n");
+        }
+
+        DPRINTF("vmsplice (read): %zu\n", ret);
+        return ret;
+    }
+
+    return 0;
+}
+
+
 static const QEMUFileOps pipe_read_ops = {
     .get_fd        = qemu_local_get_sockfd,
     .get_buffer    = qemu_local_get_buffer,
     .close         = qemu_local_close,
+    .hook_ram_load = qemu_local_ram_load
 };
 
 static const QEMUFileOps pipe_write_ops = {
-- 
1.7.7.6

[Qemu-devel] [PATCH 13/17] migration-unix: replace qemu_fopen_socket with qemu_fopen_socket_local

[Lei Li]

Relace qemu_fopen_socket with qemu_fopen_socket_local in Unix
protocol migration.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-unix.c |   18 ++++++++++++++----
 1 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/migration-unix.c b/migration-unix.c
index 651fc5b..d3a151a 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -37,12 +37,22 @@ static void unix_wait_for_connect(int fd, void *opaque)
     if (fd < 0) {
         DPRINTF("migrate connect error\n");
         s->file = NULL;
-        migrate_fd_error(s);
+        goto fails;
     } else {
         DPRINTF("migrate connect success\n");
-        s->file = qemu_fopen_socket(fd, "wb");
+
+        s->file = qemu_fopen_socket_local(fd, "wb");
+        if (s->file == NULL) {
+            DPRINTF("failed to open local socket\n");
+            goto fail;
+        }
+
         migrate_fd_connect(s);
+        return;
     }
+
+fail:
+    migrate_fd_error(s);
 }
 
 void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp)
@@ -71,9 +81,9 @@ static void unix_accept_incoming_migration(void *opaque)
         goto out;
     }
 
-    f = qemu_fopen_socket(c, "rb");
+    f = qemu_fopen_socket_local(c, "rb");
     if (f == NULL) {
-        fprintf(stderr, "could not qemu_fopen socket\n");
+        fprintf(stderr, "could not qemu_fopen socket local\n");
         goto out;
     }
 
-- 
1.7.7.6

[Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Lei Li]

Introduce new RanState RAN_STATE_FLIPPING_MIGRATE and
add it to runstate_needs_reset().

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 qapi-schema.json |   11 +++++++----
 vl.c             |   12 +++++++++++-
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/qapi-schema.json b/qapi-schema.json
index 523a5b2..8178d0c 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -176,12 +176,15 @@
 # @watchdog: the watchdog action is configured to pause and has been triggered
 #
 # @guest-panicked: guest has been panicked as a result of guest OS panic
+#
+# @flipping-migrate: guest is paused to start unix_page_flipping migration
+# process
 ##
 { 'enum': 'RunState',
-  'data': [ 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused',
-            'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm',
-            'running', 'save-vm', 'shutdown', 'suspended', 'watchdog',
-            'guest-panicked' ] }
+  'data': [ 'debug', 'flipping-migrate', 'inmigrate', 'internal-error',
+            'io-error', 'paused', 'postmigrate', 'prelaunch', 'finish-migrate',
+            'restore-vm', 'running', 'save-vm', 'shutdown', 'suspended',
+            'watchdog', 'guest-panicked' ] }
 
 ##
 # @SnapshotInfo
diff --git a/vl.c b/vl.c
index b42ac67..fcdf981 100644
--- a/vl.c
+++ b/vl.c
@@ -601,6 +601,7 @@ static const RunStateTransition runstate_transitions_def[] = {
 
     { RUN_STATE_PAUSED, RUN_STATE_RUNNING },
     { RUN_STATE_PAUSED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_PAUSED, RUN_STATE_FLIPPING_MIGRATE },
 
     { RUN_STATE_POSTMIGRATE, RUN_STATE_RUNNING },
     { RUN_STATE_POSTMIGRATE, RUN_STATE_FINISH_MIGRATE },
@@ -624,23 +625,31 @@ static const RunStateTransition runstate_transitions_def[] = {
     { RUN_STATE_RUNNING, RUN_STATE_SHUTDOWN },
     { RUN_STATE_RUNNING, RUN_STATE_WATCHDOG },
     { RUN_STATE_RUNNING, RUN_STATE_GUEST_PANICKED },
+    { RUN_STATE_RUNNING, RUN_STATE_FLIPPING_MIGRATE },
 
     { RUN_STATE_SAVE_VM, RUN_STATE_RUNNING },
 
     { RUN_STATE_SHUTDOWN, RUN_STATE_PAUSED },
     { RUN_STATE_SHUTDOWN, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_SHUTDOWN, RUN_STATE_FLIPPING_MIGRATE },
 
     { RUN_STATE_DEBUG, RUN_STATE_SUSPENDED },
     { RUN_STATE_RUNNING, RUN_STATE_SUSPENDED },
     { RUN_STATE_SUSPENDED, RUN_STATE_RUNNING },
     { RUN_STATE_SUSPENDED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_SUSPENDED, RUN_STATE_FLIPPING_MIGRATE },
 
     { RUN_STATE_WATCHDOG, RUN_STATE_RUNNING },
     { RUN_STATE_WATCHDOG, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_WATCHDOG, RUN_STATE_FLIPPING_MIGRATE },
 
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_PAUSED },
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_FINISH_MIGRATE },
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_DEBUG },
+    { RUN_STATE_GUEST_PANICKED, RUN_STATE_FLIPPING_MIGRATE },
+
+    { RUN_STATE_FLIPPING_MIGRATE, RUN_STATE_RUNNING },
+    { RUN_STATE_FLIPPING_MIGRATE, RUN_STATE_POSTMIGRATE },
 
     { RUN_STATE_MAX, RUN_STATE_MAX },
 };
@@ -687,7 +696,8 @@ bool runstate_needs_reset(void)
 {
     return runstate_check(RUN_STATE_INTERNAL_ERROR) ||
         runstate_check(RUN_STATE_SHUTDOWN) ||
-        runstate_check(RUN_STATE_GUEST_PANICKED);
+        runstate_check(RUN_STATE_GUEST_PANICKED) ||
+        runstate_check(RUN_STATE_FLIPPING_MIGRATE);
 }
 
 StatusInfo *qmp_query_status(Error **errp)
-- 
1.7.7.6

[Qemu-devel] [PATCH 15/17] migration-unix: page flipping support on unix outgoing

[Lei Li]

Add page flipping support on unix outgoing part by stopping
VM with the new RunState RUN_STATE_FLIPPING_MIGRATE before
invoking migration if unix_page_flipping enabled.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration-unix.c |   13 ++++++++++++-
 1 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/migration-unix.c b/migration-unix.c
index d3a151a..f2bfd38 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -19,6 +19,7 @@
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "block/block.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG_MIGRATION_UNIX
 
@@ -33,11 +34,12 @@
 static void unix_wait_for_connect(int fd, void *opaque)
 {
     MigrationState *s = opaque;
+    int ret;
 
     if (fd < 0) {
         DPRINTF("migrate connect error\n");
         s->file = NULL;
-        goto fails;
+        goto fail;
     } else {
         DPRINTF("migrate connect success\n");
 
@@ -47,6 +49,15 @@ static void unix_wait_for_connect(int fd, void *opaque)
             goto fail;
         }
 
+        /* Stop VM before invoking migration if unix_page_flipping enabled */
+        if (migrate_unix_page_flipping()) {
+            ret = vm_stop_force_state(RUN_STATE_FLIPPING_MIGRATE);
+            if (ret < 0) {
+                DPRINTF("failed to stop VM\n");
+                goto fail;
+            }
+        }
+
         migrate_fd_connect(s);
         return;
     }
-- 
1.7.7.6

[Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration.c |   10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/migration.c b/migration.c
index 4ac466b..568b73a 100644
--- a/migration.c
+++ b/migration.c
@@ -579,10 +579,11 @@ static void *migration_thread(void *opaque)
             pending_size = qemu_savevm_state_pending(s->file, max_size);
             DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
                     pending_size, max_size);
-            if (pending_size && pending_size >= max_size) {
+            if (pending_size && pending_size >= max_size &&
+                !migrate_unix_page_flipping()) {
                 qemu_savevm_state_iterate(s->file);
             } else {
-                int ret;
+                int ret = 0;
 
                 DPRINTF("done iterating\n");
                 qemu_mutex_lock_iothread();
@@ -590,7 +591,10 @@ static void *migration_thread(void *opaque)
                 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
                 old_vm_running = runstate_is_running();
 
-                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                if (!runstate_needs_reset()) {
+                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                }
+
                 if (ret >= 0) {
                     qemu_file_set_rate_limit(s->file, INT_MAX);
                     qemu_savevm_state_complete(s->file);
-- 
1.7.7.6

[Qemu-devel] [PATCH 17/17] hmp: better format for info migrate_capabilities

[Lei Li]

As there might be more capabilities introduced, better to display
it in lines.

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 hmp.c |    5 ++---
 1 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/hmp.c b/hmp.c
index 32ee285..dcfa2f9 100644
--- a/hmp.c
+++ b/hmp.c
@@ -226,13 +226,12 @@ void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict)
     caps = qmp_query_migrate_capabilities(NULL);
 
     if (caps) {
-        monitor_printf(mon, "capabilities: ");
+        monitor_printf(mon, "Capabilities:\n");
         for (cap = caps; cap; cap = cap->next) {
-            monitor_printf(mon, "%s: %s ",
+            monitor_printf(mon, "%s: %s\n",
                            MigrationCapability_lookup[cap->value->capability],
                            cap->value->state ? "on" : "off");
         }
-        monitor_printf(mon, "\n");
     }
 
     qapi_free_MigrationCapabilityStatusList(caps);
-- 
1.7.7.6

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Eric Blake]

[-- Attachment #1: Type: text/plain, Size: 2186 bytes --]

On 10/22/2013 04:25 AM, Lei Li wrote:
> Introduce new RanState RAN_STATE_FLIPPING_MIGRATE and
> add it to runstate_needs_reset().
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  qapi-schema.json |   11 +++++++----
>  vl.c             |   12 +++++++++++-
>  2 files changed, 18 insertions(+), 5 deletions(-)
> 
> diff --git a/qapi-schema.json b/qapi-schema.json
> index 523a5b2..8178d0c 100644
> --- a/qapi-schema.json
> +++ b/qapi-schema.json
> @@ -176,12 +176,15 @@
>  # @watchdog: the watchdog action is configured to pause and has been triggered
>  #
>  # @guest-panicked: guest has been panicked as a result of guest OS panic
> +#
> +# @flipping-migrate: guest is paused to start unix_page_flipping migration
> +# process

We probably ought to enhance the docs to mention '(since 1.8)' for this
field (and likewise for other enum values added after the original
introduction of the enum).

Last time we added a new user-visible runstate, it broke migration with
older libvirt versions that weren't prepared to see the new state (hmm,
I need to check if libvirt has fixed that in the meantime; adding a
cc...).  Paolo's advice at the time was that it is okay to require a new
libvirt when using a new qemu, and that libvirt should be taught to
treat all unknown RunState as if they were 'running'; although for this
particular addition it might be nicer to have libvirt lump 'inmigrate'
and 'flipping-migrate' to the same usage.

>  ##
>  { 'enum': 'RunState',
> -  'data': [ 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused',
> -            'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm',
> -            'running', 'save-vm', 'shutdown', 'suspended', 'watchdog',
> -            'guest-panicked' ] }
> +  'data': [ 'debug', 'flipping-migrate', 'inmigrate', 'internal-error',
> +            'io-error', 'paused', 'postmigrate', 'prelaunch', 'finish-migrate',
> +            'restore-vm', 'running', 'save-vm', 'shutdown', 'suspended',
> +            'watchdog', 'guest-panicked' ] }
>  


-- 
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 621 bytes --]

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Lei Li]

On 10/22/2013 11:51 AM, Eric Blake wrote:
> On 10/22/2013 04:25 AM, Lei Li wrote:
>> Introduce new RanState RAN_STATE_FLIPPING_MIGRATE and
>> add it to runstate_needs_reset().
>>
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   qapi-schema.json |   11 +++++++----
>>   vl.c             |   12 +++++++++++-
>>   2 files changed, 18 insertions(+), 5 deletions(-)
>>
>> diff --git a/qapi-schema.json b/qapi-schema.json
>> index 523a5b2..8178d0c 100644
>> --- a/qapi-schema.json
>> +++ b/qapi-schema.json
>> @@ -176,12 +176,15 @@
>>   # @watchdog: the watchdog action is configured to pause and has been triggered
>>   #
>>   # @guest-panicked: guest has been panicked as a result of guest OS panic
>> +#
>> +# @flipping-migrate: guest is paused to start unix_page_flipping migration
>> +# process
> We probably ought to enhance the docs to mention '(since 1.8)' for this
> field (and likewise for other enum values added after the original
> introduction of the enum).

Hi Eric,

Sure, will do.

BTW, I was hoping this feature could be accepted and merged to QEMU 1.7
release.

>
> Last time we added a new user-visible runstate, it broke migration with
> older libvirt versions that weren't prepared to see the new state (hmm,
> I need to check if libvirt has fixed that in the meantime; adding a
> cc...).  Paolo's advice at the time was that it is okay to require a new
> libvirt when using a new qemu, and that libvirt should be taught to
> treat all unknown RunState as if they were 'running'; although for this
> particular addition it might be nicer to have libvirt lump 'inmigrate'
> and 'flipping-migrate' to the same usage.

I am not sure whether these two runstate could be lumped with same usage
in libvirt. Whatever, looks like 'inmigrate' has a transition from 'prelaunch'
to it in QEMU, which reminds me that it may need to add another transition
from 'prelaunch' to 'flipping-migrate' too.

>
>>   ##
>>   { 'enum': 'RunState',
>> -  'data': [ 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused',
>> -            'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm',
>> -            'running', 'save-vm', 'shutdown', 'suspended', 'watchdog',
>> -            'guest-panicked' ] }
>> +  'data': [ 'debug', 'flipping-migrate', 'inmigrate', 'internal-error',
>> +            'io-error', 'paused', 'postmigrate', 'prelaunch', 'finish-migrate',
>> +            'restore-vm', 'running', 'save-vm', 'shutdown', 'suspended',
>> +            'watchdog', 'guest-panicked' ] }
>>   
>


-- 
Lei

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Eric Blake]

[-- Attachment #1: Type: text/plain, Size: 879 bytes --]

On 10/22/2013 07:28 AM, Lei Li wrote:

>>>   # @guest-panicked: guest has been panicked as a result of guest OS
>>> panic
>>> +#
>>> +# @flipping-migrate: guest is paused to start unix_page_flipping
>>> migration
>>> +# process
>> We probably ought to enhance the docs to mention '(since 1.8)' for this
>> field (and likewise for other enum values added after the original
>> introduction of the enum).
> 
> Hi Eric,
> 
> Sure, will do.
> 
> BTW, I was hoping this feature could be accepted and merged to QEMU 1.7
> release.

http://wiki.qemu.org/Planning/1.7
Soft freeze has already happened, so it's up to the maintainers whether
there is still time to be adding this feature in 1.7 - but yes, that
would affect the tag you list in your docs.

-- 
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 621 bytes --]

Re: [Qemu-devel] [PATCH 01/17] rename is_active to is_block_active

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> is_active is used to identify block migration, rename to
> is_block_active to make it more clear.

No, is_active is used to identify whether a set of SaveVMHandlers is
active.  The default is true, so only block migration is using it.  But
we could use it in the future for other features (probably using
migration capabilities instead of a flag as is the case for block).

Paolo

> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  block-migration.c           |    2 +-
>  include/migration/vmstate.h |    2 +-
>  savevm.c                    |   16 ++++++++--------
>  3 files changed, 10 insertions(+), 10 deletions(-)
> 
> diff --git a/block-migration.c b/block-migration.c
> index daf9ec1..b637695 100644
> --- a/block-migration.c
> +++ b/block-migration.c
> @@ -834,7 +834,7 @@ SaveVMHandlers savevm_block_handlers = {
>      .save_live_pending = block_save_pending,
>      .load_state = block_load,
>      .cancel = block_migration_cancel,
> -    .is_active = block_is_active,
> +    .is_block_active = block_is_active,
>  };
>  
>  void blk_mig_init(void)
> diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
> index 9d09e60..c634d65 100644
> --- a/include/migration/vmstate.h
> +++ b/include/migration/vmstate.h
> @@ -42,7 +42,7 @@ typedef struct SaveVMHandlers {
>      int (*save_live_complete)(QEMUFile *f, void *opaque);
>  
>      /* This runs both outside and inside the iothread lock.  */
> -    bool (*is_active)(void *opaque);
> +    bool (*is_block_active)(void *opaque);
>  
>      /* This runs outside the iothread lock in the migration case, and
>       * within the lock in the savevm case.  The callback had better only
> diff --git a/savevm.c b/savevm.c
> index 2f631d4..56b8643 100644
> --- a/savevm.c
> +++ b/savevm.c
> @@ -1867,8 +1867,8 @@ void qemu_savevm_state_begin(QEMUFile *f,
>          if (!se->ops || !se->ops->save_live_setup) {
>              continue;
>          }
> -        if (se->ops && se->ops->is_active) {
> -            if (!se->ops->is_active(se->opaque)) {
> +        if (se->ops && se->ops->is_block_active) {
> +            if (!se->ops->is_block_active(se->opaque)) {
>                  continue;
>              }
>          }
> @@ -1907,8 +1907,8 @@ int qemu_savevm_state_iterate(QEMUFile *f)
>          if (!se->ops || !se->ops->save_live_iterate) {
>              continue;
>          }
> -        if (se->ops && se->ops->is_active) {
> -            if (!se->ops->is_active(se->opaque)) {
> +        if (se->ops && se->ops->is_block_active) {
> +            if (!se->ops->is_block_active(se->opaque)) {
>                  continue;
>              }
>          }
> @@ -1948,8 +1948,8 @@ void qemu_savevm_state_complete(QEMUFile *f)
>          if (!se->ops || !se->ops->save_live_complete) {
>              continue;
>          }
> -        if (se->ops && se->ops->is_active) {
> -            if (!se->ops->is_active(se->opaque)) {
> +        if (se->ops && se->ops->is_block_active) {
> +            if (!se->ops->is_block_active(se->opaque)) {
>                  continue;
>              }
>          }
> @@ -2002,8 +2002,8 @@ uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
>          if (!se->ops || !se->ops->save_live_pending) {
>              continue;
>          }
> -        if (se->ops && se->ops->is_active) {
> -            if (!se->ops->is_active(se->opaque)) {
> +        if (se->ops && se->ops->is_block_active) {
> +            if (!se->ops->is_block_active(se->opaque)) {
>                  continue;
>              }
>          }
> 

Re: [Qemu-devel] [PATCH 02/17] QAPI: introduce magration capability unix_page_flipping

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> +# @unix-page-flipping: If enabled, QEMU will support localhost migration. This
> +#          feature allows live upgrade of a running QEMU instance by doing localhost
> +#          migration with page flipping. It requires the source and destination
> +#          are both on localhost. Disabled by default. (since 1.7)
> +#

If enabled, QEMU can optimize migration when the destination is a QEMU
process that runs on the same host as the source (as is the case for
live upgrade).  If the migration transport is a Unix socket, QEMU will
flip RAM pages directly to the destination, so that memory is only
allocated twice for the source and destination processes. Disabled by
default. (since 1.8)

Paolo

Re: [Qemu-devel] [PATCH 03/17] migration: add migrate_unix_page_flipping()

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Add migrate_unix_page_flipping() to check if
> MIGRATION_CAPABILITY_UNIX_PAGE_FLIPPING is enabled.
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  include/migration/migration.h |    3 +++
>  migration.c                   |    9 +++++++++
>  2 files changed, 12 insertions(+), 0 deletions(-)
> 
> diff --git a/include/migration/migration.h b/include/migration/migration.h
> index 140e6b4..7e5d01a 100644
> --- a/include/migration/migration.h
> +++ b/include/migration/migration.h
> @@ -131,10 +131,13 @@ void migrate_add_blocker(Error *reason);
>  void migrate_del_blocker(Error *reason);
>  
>  bool migrate_rdma_pin_all(void);
> +
>  bool migrate_zero_blocks(void);
>  
>  bool migrate_auto_converge(void);
>  
> +bool migrate_unix_page_flipping(void);
> +
>  int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
>                           uint8_t *dst, int dlen);
>  int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen);
> diff --git a/migration.c b/migration.c
> index 2b1ab20..4ac466b 100644
> --- a/migration.c
> +++ b/migration.c
> @@ -541,6 +541,15 @@ int64_t migrate_xbzrle_cache_size(void)
>      return s->xbzrle_cache_size;
>  }
>  
> +bool migrate_unix_page_flipping(void)
> +{
> +    MigrationState *s;
> +
> +    s = migrate_get_current();
> +
> +    return s->enabled_capabilities[MIGRATION_CAPABILITY_UNIX_PAGE_FLIPPING];
> +}
> +
>  /* migration thread support */
>  
>  static void *migration_thread(void *opaque)
> 

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

Re: [Qemu-devel] [PATCH 04/17] qmp-command.hx: add missing docs for migration capabilites

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  qmp-commands.hx |    8 ++++++++
>  1 files changed, 8 insertions(+), 0 deletions(-)
> 
> diff --git a/qmp-commands.hx b/qmp-commands.hx
> index fba15cd..650a3a8 100644
> --- a/qmp-commands.hx
> +++ b/qmp-commands.hx
> @@ -2898,6 +2898,10 @@ migrate-set-capabilities
>  Enable/Disable migration capabilities
>  
>  - "xbzrle": XBZRLE support
> +- "x-rdma-pin-all": RDMA support

Pin all pages during RDMA support.

> +- "zero-blocks": zero-blocks support

Compress zero blocks during block migration.

> +- "auto-converge": Auto converge support

Block VCPU to help convergence of migration

> +- "unix-page-flipping": Page flipping support

Page flipping for live QEMU upgrade

>  Arguments:
>  
> @@ -2922,6 +2926,10 @@ Query current migration capabilities
>  
>  - "capabilities": migration capabilities state
>           - "xbzrle" : XBZRLE state (json-bool)
> +         - "x-rdma-pin-all": RDMA state (json-bool)
> +         - "zero-blocks": zero-blocks state (json-bool)
> +         - "auto-converge": Auto converge state (json-bool)
> +         - "unix-page-flipping": Page flipping state (json-bool)
>  
>  Arguments:
>  
> 

Please separate page flipping in a separate patch and send it for 1.7.
Once you do that, patches 2/3/4 can be merged.

Paolo

Re: [Qemu-devel] [PATCH 12/17] migration-local: override hook_ram_load

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Override hook_ram_load to receive the pipe file descriptor
> passed by source process and page address which will be
> extracted to vmsplice the page data from pipe.
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  migration-local.c |   85 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 85 insertions(+), 0 deletions(-)
> 
> diff --git a/migration-local.c b/migration-local.c
> index 7ccec30..083a24f 100644
> --- a/migration-local.c
> +++ b/migration-local.c
> @@ -32,6 +32,8 @@
>  #include "trace.h"
>  #include "qemu/osdep.h"
>  
> +#define BEFORE_PIPE_FD 199
> +
>  //#define DEBUG_MIGRATION_LOCAL
>  
>  #ifdef DEBUG_MIGRATION_LOCAL
> @@ -50,6 +52,8 @@ typedef struct QEMUFileLocal {
>      bool unix_page_flipping;
>  } QEMUFileLocal;
>  
> +static bool pipefd_passed;
> +
>  static int qemu_local_get_sockfd(void *opaque)
>  {
>      QEMUFileLocal *s = opaque;
> @@ -64,6 +68,22 @@ static int qemu_local_get_buffer(void *opaque, uint8_t *buf,
>      ssize_t len;
>  
>      for (;;) {
> +        /*
> +         * FIX ME: BEFORE_PIPE_FD is hard-coded and checked temporarily here
> +         * because that the control message of passed pipe file descriptor
> +         * might be 'eaten' to stream file by qemu_recv(), which would lead
> +         * to the failure of recv_pipefd(), as it should stay in the socket
> +         * and received by the real receiver recvmsg().
> +         *
> +         * Although this message is followed by the first load_hook flags
> +         * RAM_SAVE_FLAG_HOOK, the incoming side is hardly to avoid this
> +         * as it would fill it into the stream file before any check action
> +         * taken. Need to find a way out to fix this.
> +         */
> +        if (size > BEFORE_PIPE_FD && !pipefd_passed) {
> +            size = BEFORE_PIPE_FD;
> +        }

I think you can simply use recvmsg always, or at least until !pipefd_passed.

>          len = qemu_recv(s->sockfd, buf, size, 0);
>          if (len != -1) {
>              break;
> @@ -115,6 +135,7 @@ static int qemu_local_close(void *opaque)
>  }
>  
>  static int send_pipefd(int sockfd, int pipefd);
> +static int recv_pipefd(int sockfd);
>  
>  static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
>                                    uint64_t flags)
> @@ -192,10 +213,74 @@ static size_t qemu_local_save_ram(QEMUFile *f, void *opaque,
>      return RAM_SAVE_CONTROL_NOT_SUPP;
>  }
>  
> +static int qemu_local_ram_load(QEMUFile *f, void *opaque,
> +                               uint64_t flags)
> +{
> +    QEMUFileLocal *s = opaque;
> +    ram_addr_t addr;
> +    struct iovec iov;
> +    ssize_t ret;
> +
> +    /* Receive the pipe file descripter passed from source process */
> +    if (!pipefd_passed) {
> +        s->pipefd[0] = recv_pipefd(s->sockfd);
> +        if (s->pipefd[0] <= 0) {
> +            fprintf(stderr, "failed to receive pipe fd: %d\n", s->pipefd[0]);
> +        } else {
> +            pipefd_passed = 1;
> +            DPRINTF(stderr, "succeed\n");
> +        }
> +
> +        return s->pipefd[0];
> +    }
> +
> +    if (pipefd_passed) {
> +        void *host;
> +
> +        /*
> +         * Extract the page address from the 8-byte record and
> +         * read the page data from the pipe.
> +         */
> +        addr = qemu_get_be64(s->file);
> +        host = qemu_get_ram_ptr(addr);
> +
> +        iov.iov_base = host;
> +        iov.iov_len = TARGET_PAGE_SIZE;
> +
> +        /* The flag SPLICE_F_MOVE is introduced in kernel for the page
> +         * flipping feature in QEMU, which will movie pages rather than
> +         * copying, previously unused.
> +         *
> +         * If a move is not possible the kernel will transparently falls
> +         * back to copying data.
> +         *
> +         * For older kernels the SPLICE_F_MOVE would be ignored and a copy
> +         * would occur.
> +         */
> +        ret = vmsplice(s->pipefd[0], &iov, 1, SPLICE_F_MOVE);
> +        if (ret == -1) {
> +            if (errno != EAGAIN && errno != EINTR) {
> +                fprintf(stderr, "vmsplice() load error: %s", strerror(errno));
> +                return ret;
> +            }
> +            DPRINTF("vmsplice load error\n");
> +        } else if (ret == 0) {
> +            DPRINTF(stderr, "load_page: zero read\n");
> +        }
> +
> +        DPRINTF("vmsplice (read): %zu\n", ret);
> +        return ret;
> +    }
> +
> +    return 0;
> +}
> +
> +
>  static const QEMUFileOps pipe_read_ops = {
>      .get_fd        = qemu_local_get_sockfd,
>      .get_buffer    = qemu_local_get_buffer,
>      .close         = qemu_local_close,
> +    .hook_ram_load = qemu_local_ram_load
>  };
>  
>  static const QEMUFileOps pipe_write_ops = {
> -- 1.7.7.6
> 

Otherwise looks good,

Paolo

Re: [Qemu-devel] [PATCH 09/17] migration-local: override before_ram_iterate to send pipefd

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Override befor_ram_iterate to send pipefd. It will write the
> RAM_SAVE_FLAG_HOOK flags which will trigger the load hook to
> receive it.
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  migration-local.c |   26 ++++++++++++++++++++++++++
>  1 files changed, 26 insertions(+), 0 deletions(-)
> 
> diff --git a/migration-local.c b/migration-local.c
> index ed0ae6c..92c661c 100644
> --- a/migration-local.c
> +++ b/migration-local.c
> @@ -114,6 +114,31 @@ static int qemu_local_close(void *opaque)
>      return 0;
>  }
>  
> +static int send_pipefd(int sockfd, int pipefd);
> +
> +static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
> +                                  uint64_t flags)
> +{
> +    QEMUFileLocal *s = opaque;
> +    int ret;
> +
> +    if (s->unix_page_flipping) {
> +        /* Avoid sending pipe fd again in ram_save_complete() stage */
> +        if (flags != RAM_CONTROL_FINISH) {

Why not "flags == RAM_CONTROL_SETUP"?

> +            qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
> +            qemu_fflush(f);
> +            ret = send_pipefd(s->sockfd, s->pipefd[0]);
> +            if (ret < 0) {
> +                fprintf(stderr, "failed to pass pipe\n");
> +                return ret;
> +            }
> +            DPRINTF("pipe fd was sent\n");
> +        }
> +    }
> +
> +    return 0;
> +}
> +
>  static const QEMUFileOps pipe_read_ops = {
>      .get_fd        = qemu_local_get_sockfd,
>      .get_buffer    = qemu_local_get_buffer,
> @@ -124,6 +149,7 @@ static const QEMUFileOps pipe_write_ops = {
>      .get_fd             = qemu_local_get_sockfd,
>      .writev_buffer      = qemu_local_writev_buffer,
>      .close              = qemu_local_close,
> +    .before_ram_iterate = qemu_local_send_pipefd,
>  };
>  
>  QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode)
> 

Otherwise looks good.

Paolo

Re: [Qemu-devel] [PATCH 11/17] savevm: adjust ram_control_save_page for page flipping

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> As callback save_page will always be opened by
> qemu_fopen_socket_local(), and without unix_page_flipping
> it will return RAM_SAVE_CONTROL_NOT_SUPP, it leads to a
> wrong qemu_file_set_error() based on the current logic.
> So this patch adds RAM_SAVE_CONTROL_NOT_SUPP to the check.
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  savevm.c |    3 ++-
>  1 files changed, 2 insertions(+), 1 deletions(-)
> 
> diff --git a/savevm.c b/savevm.c
> index 56b8643..b102275 100644
> --- a/savevm.c
> +++ b/savevm.c
> @@ -668,7 +668,8 @@ size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset,
>          int ret = f->ops->save_page(f, f->opaque, block_offset,
>                                      offset, size, bytes_sent);
>  
> -        if (ret != RAM_SAVE_CONTROL_DELAYED) {
> +        if (ret != RAM_SAVE_CONTROL_DELAYED &&
> +            ret != RAM_SAVE_CONTROL_NOT_SUPP) {
>              if (bytes_sent && *bytes_sent > 0) {
>                  qemu_update_position(f, *bytes_sent);
>              } else if (ret < 0) {
> 

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

Re: [Qemu-devel] [PATCH 13/17] migration-unix: replace qemu_fopen_socket with qemu_fopen_socket_local

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Relace qemu_fopen_socket with qemu_fopen_socket_local in Unix
> protocol migration.
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  migration-unix.c |   18 ++++++++++++++----
>  1 files changed, 14 insertions(+), 4 deletions(-)
> 
> diff --git a/migration-unix.c b/migration-unix.c
> index 651fc5b..d3a151a 100644
> --- a/migration-unix.c
> +++ b/migration-unix.c
> @@ -37,12 +37,22 @@ static void unix_wait_for_connect(int fd, void *opaque)
>      if (fd < 0) {
>          DPRINTF("migrate connect error\n");
>          s->file = NULL;
> -        migrate_fd_error(s);
> +        goto fails;
>      } else {
>          DPRINTF("migrate connect success\n");
> -        s->file = qemu_fopen_socket(fd, "wb");
> +
> +        s->file = qemu_fopen_socket_local(fd, "wb");
> +        if (s->file == NULL) {
> +            DPRINTF("failed to open local socket\n");

"failed to open Unix socket"

> +            goto fail;
> +        }
> +
>          migrate_fd_connect(s);
> +        return;
>      }
> +
> +fail:
> +    migrate_fd_error(s);
>  }
>  
>  void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp)
> @@ -71,9 +81,9 @@ static void unix_accept_incoming_migration(void *opaque)
>          goto out;
>      }
>  
> -    f = qemu_fopen_socket(c, "rb");
> +    f = qemu_fopen_socket_local(c, "rb");
>      if (f == NULL) {
> -        fprintf(stderr, "could not qemu_fopen socket\n");
> +        fprintf(stderr, "could not qemu_fopen socket local\n");

"failed to open Unix socket"

>          goto out;
>      }
>  
> 

Otherwise ok.

Paolo

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Paolo Bonzini]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

Il 22/10/2013 09:10, Eric Blake ha scritto:
> http://wiki.qemu.org/Planning/1.7 Soft freeze has already
> happened, so it's up to the maintainers whether there is still time
> to be adding this feature in 1.7 - but yes, that would affect the
> tag you list in your docs.

No, this is a 1.8 feature at this point.

Paolo
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.22 (GNU/Linux)
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iQIcBAEBAgAGBQJSaSqeAAoJEBvWZb6bTYby7sAP+gPF35M0+8YW/ujlDjXqezYw
gjtpSjKJSLx5mF4l9cA9SrQQFSkikfT/5QUZnMfjCXzWobH2y07FdUIfYR+Y5vB0
Op7Ywf01qLOgX1/qZaikZVDLYmduir1qk6Dxp85epGoVJpCC9natLkT3bPIAkB4S
Mvl2oDEnOQD5b5W5gVe80UvLIMOYW1CtJD87u6OiOm4pMVtpOp8xz02ToRO6xSSx
/PjMYY07kWlOGrMkPw/BUX92crxEfzarzhd4OrIfVhEr5l0H9axY2WJKxdMsXREI
TRL77GhqFHKz8O8uAh0G5Y1MKoX3/6mxIA1qSoMyTlDI4+qWVZwt5pJjmJjiGW58
HmsigFdBfvPqDwLCrnIrQd1aRS5oHzL++AncctlbXqayDo4IT+0lM29RBHM1obwO
GlxePNzbiKljvAyJfvIoJlUkfFvXHLZ1a94+AWXiZCcb+gRUcMw7hxmnPmyMr92X
bYkY/ZEtd8FLj+4weFAwWnrAosyp0UC4qF9y9zk9WXjUqz52KPWwVP3M10ElxjQs
XBHGmlf3+eQu1mPYf1yWO57nwHSNisnCktQ9aSiupkYJl2rBwu/qizSyfRlEV+uW
pJ++Uia29527JcYtVVM2u0FcaM5i+ZWVcD/4KVy6lFFQBK7Rt9wmhcAgfd06ynkV
1G+niTsF24mzfoVxcvPD
=BgUk
-----END PGP SIGNATURE-----

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Introduce new RanState RAN_STATE_FLIPPING_MIGRATE and
> add it to runstate_needs_reset().

I am not sure about the name; for one thing, the new state would apply
also to postcopy migration.

But the code looks ok.

Paolo

> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  qapi-schema.json |   11 +++++++----
>  vl.c             |   12 +++++++++++-
>  2 files changed, 18 insertions(+), 5 deletions(-)
> 
> diff --git a/qapi-schema.json b/qapi-schema.json
> index 523a5b2..8178d0c 100644
> --- a/qapi-schema.json
> +++ b/qapi-schema.json
> @@ -176,12 +176,15 @@
>  # @watchdog: the watchdog action is configured to pause and has been triggered
>  #
>  # @guest-panicked: guest has been panicked as a result of guest OS panic
> +#
> +# @flipping-migrate: guest is paused to start unix_page_flipping migration
> +# process
>  ##
>  { 'enum': 'RunState',
> -  'data': [ 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused',
> -            'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm',
> -            'running', 'save-vm', 'shutdown', 'suspended', 'watchdog',
> -            'guest-panicked' ] }
> +  'data': [ 'debug', 'flipping-migrate', 'inmigrate', 'internal-error',
> +            'io-error', 'paused', 'postmigrate', 'prelaunch', 'finish-migrate',
> +            'restore-vm', 'running', 'save-vm', 'shutdown', 'suspended',
> +            'watchdog', 'guest-panicked' ] }
>  
>  ##
>  # @SnapshotInfo
> diff --git a/vl.c b/vl.c
> index b42ac67..fcdf981 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -601,6 +601,7 @@ static const RunStateTransition runstate_transitions_def[] = {
>  
>      { RUN_STATE_PAUSED, RUN_STATE_RUNNING },
>      { RUN_STATE_PAUSED, RUN_STATE_FINISH_MIGRATE },
> +    { RUN_STATE_PAUSED, RUN_STATE_FLIPPING_MIGRATE },
>  
>      { RUN_STATE_POSTMIGRATE, RUN_STATE_RUNNING },
>      { RUN_STATE_POSTMIGRATE, RUN_STATE_FINISH_MIGRATE },
> @@ -624,23 +625,31 @@ static const RunStateTransition runstate_transitions_def[] = {
>      { RUN_STATE_RUNNING, RUN_STATE_SHUTDOWN },
>      { RUN_STATE_RUNNING, RUN_STATE_WATCHDOG },
>      { RUN_STATE_RUNNING, RUN_STATE_GUEST_PANICKED },
> +    { RUN_STATE_RUNNING, RUN_STATE_FLIPPING_MIGRATE },
>  
>      { RUN_STATE_SAVE_VM, RUN_STATE_RUNNING },
>  
>      { RUN_STATE_SHUTDOWN, RUN_STATE_PAUSED },
>      { RUN_STATE_SHUTDOWN, RUN_STATE_FINISH_MIGRATE },
> +    { RUN_STATE_SHUTDOWN, RUN_STATE_FLIPPING_MIGRATE },
>  
>      { RUN_STATE_DEBUG, RUN_STATE_SUSPENDED },
>      { RUN_STATE_RUNNING, RUN_STATE_SUSPENDED },
>      { RUN_STATE_SUSPENDED, RUN_STATE_RUNNING },
>      { RUN_STATE_SUSPENDED, RUN_STATE_FINISH_MIGRATE },
> +    { RUN_STATE_SUSPENDED, RUN_STATE_FLIPPING_MIGRATE },
>  
>      { RUN_STATE_WATCHDOG, RUN_STATE_RUNNING },
>      { RUN_STATE_WATCHDOG, RUN_STATE_FINISH_MIGRATE },
> +    { RUN_STATE_WATCHDOG, RUN_STATE_FLIPPING_MIGRATE },
>  
>      { RUN_STATE_GUEST_PANICKED, RUN_STATE_PAUSED },
>      { RUN_STATE_GUEST_PANICKED, RUN_STATE_FINISH_MIGRATE },
>      { RUN_STATE_GUEST_PANICKED, RUN_STATE_DEBUG },
> +    { RUN_STATE_GUEST_PANICKED, RUN_STATE_FLIPPING_MIGRATE },
> +
> +    { RUN_STATE_FLIPPING_MIGRATE, RUN_STATE_RUNNING },
> +    { RUN_STATE_FLIPPING_MIGRATE, RUN_STATE_POSTMIGRATE },
>  
>      { RUN_STATE_MAX, RUN_STATE_MAX },
>  };
> @@ -687,7 +696,8 @@ bool runstate_needs_reset(void)
>  {
>      return runstate_check(RUN_STATE_INTERNAL_ERROR) ||
>          runstate_check(RUN_STATE_SHUTDOWN) ||
> -        runstate_check(RUN_STATE_GUEST_PANICKED);
> +        runstate_check(RUN_STATE_GUEST_PANICKED) ||
> +        runstate_check(RUN_STATE_FLIPPING_MIGRATE);
>  }
>  
>  StatusInfo *qmp_query_status(Error **errp)
> 

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  migration.c |   10 +++++++---
>  1 files changed, 7 insertions(+), 3 deletions(-)
> 
> diff --git a/migration.c b/migration.c
> index 4ac466b..568b73a 100644
> --- a/migration.c
> +++ b/migration.c
> @@ -579,10 +579,11 @@ static void *migration_thread(void *opaque)
>              pending_size = qemu_savevm_state_pending(s->file, max_size);
>              DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
>                      pending_size, max_size);
> -            if (pending_size && pending_size >= max_size) {
> +            if (pending_size && pending_size >= max_size &&
> +                !migrate_unix_page_flipping()) {

This is a bit ugly but I understand the need.  Perhaps "&&
!runstate_needs_reset()" like below?

Paolo

>                  qemu_savevm_state_iterate(s->file);
>              } else {
> -                int ret;
> +                int ret = 0;
>  
>                  DPRINTF("done iterating\n");
>                  qemu_mutex_lock_iothread();
> @@ -590,7 +591,10 @@ static void *migration_thread(void *opaque)
>                  qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
>                  old_vm_running = runstate_is_running();
>  
> -                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
> +                if (!runstate_needs_reset()) {
> +                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
> +                }
> +
>                  if (ret >= 0) {
>                      qemu_file_set_rate_limit(s->file, INT_MAX);
>                      qemu_savevm_state_complete(s->file);
> 

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Paolo Bonzini]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

Il 22/10/2013 04:51, Eric Blake ha scritto:
> 
> Last time we added a new user-visible runstate, it broke migration
> with older libvirt versions that weren't prepared to see the new
> state (hmm, I need to check if libvirt has fixed that in the
> meantime; adding a cc...).  Paolo's advice at the time was that it
> is okay to require a new libvirt when using a new qemu, and that
> libvirt should be taught to treat all unknown RunState as if they
> were 'running'; although for this particular addition it might be
> nicer to have libvirt lump 'inmigrate' and 'flipping-migrate' to
> the same usage.

This is not a problem in this case, because the new runstate will only
appear after enabling a new migration capability.  Only new libvirt
will be able to enable the capability and thus reach the new runstate.

Poalo
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.22 (GNU/Linux)
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iQIcBAEBAgAGBQJSaSu2AAoJEBvWZb6bTYbycoUP/RDsMA/62ukPU1stvKhoOPLK
+Zi9OOSd8tIAAkHD0rDS5pQePklZ2AqKmOpJujRHhavGA/kKcMhL+79dK22UJSWn
fQPgxiivylZcFWbhF97H9eW27Vmw5la1Ob2onq2g+KY/AvL5yrEpBrN60XipHtrd
uPM2yWlERSlKcQj0nrTv5Oz3p2k9jWQrGzWX16cC+nFAaR9K/D+afOyl/3ZvixKw
ekiIIxStoJQGCogU/rJp1nJcenJfKSOofNKPU8xKkpexK4pw+W1u9GZMaAgzXLRO
Q6p3R6raIRT42Hf820ASHsWeHYak2gYmZ+5FaKGPyjAL/iwKyr4FRp7uGXobN9sD
ziMX5gaXreVdBh8CIOqDrOGa7BNoEkId2sJrrjwmmD418uZNOGK/xdcPa+mFlc8Q
GkJ4y844OckkyLhqQnuVtCVtxGlNCU3Y/XHfTKELdTm1m1jdTdoNnBGFXD1IyTxU
9yPYfQRAeRy7yPEXqQw0qc7ZUj/fhqPPzijwpPrvmFvzAouUZjlpE6QoriiBGzRx
Csq+uwpJ2bsWOC5BGvxhrhNE4F9yhWGkotxod6gvULkOLkm/4Cg5/8aNSeYzInsm
UtJ5+oZD6jzOVW5brkFosoUvWzr6vJEvwoh7DVz5sVkAH/u93WR8jlG+NsTw5fTY
+hthfn5l153w5JDikLyV
=q1Ww
-----END PGP SIGNATURE-----

Re: [Qemu-devel] [PATCH 17/17] hmp: better format for info migrate_capabilities

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> As there might be more capabilities introduced, better to display
> it in lines.
> 
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  hmp.c |    5 ++---
>  1 files changed, 2 insertions(+), 3 deletions(-)
> 
> diff --git a/hmp.c b/hmp.c
> index 32ee285..dcfa2f9 100644
> --- a/hmp.c
> +++ b/hmp.c
> @@ -226,13 +226,12 @@ void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict)
>      caps = qmp_query_migrate_capabilities(NULL);
>  
>      if (caps) {
> -        monitor_printf(mon, "capabilities: ");
> +        monitor_printf(mon, "Capabilities:\n");
>          for (cap = caps; cap; cap = cap->next) {
> -            monitor_printf(mon, "%s: %s ",
> +            monitor_printf(mon, "%s: %s\n",
>                             MigrationCapability_lookup[cap->value->capability],
>                             cap->value->state ? "on" : "off");
>          }
> -        monitor_printf(mon, "\n");
>      }
>  
>      qapi_free_MigrationCapabilityStatusList(caps);
> 

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

Re: [Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Paolo Bonzini]

Il 22/10/2013 04:25, Lei Li ha scritto:
> This patch series tries to introduce a mechanism using side
> channel pipe for RAM via SCM_RIGHTS with unix domain socket
> protocol migration.
> 
> This side channel is used for the page flipping by vmsplice,
> which is the internal mechanism for localhost migration that
> we are trying to add to QEMU. The backgroud info and previous
> patch series for reference,
> 
> Localhost migration
> http://lists.nongnu.org/archive/html/qemu-devel/2013-08/msg02916.html
> 
> migration: Introduce side channel for RAM
> http://lists.gnu.org/archive/html/qemu-devel/2013-09/msg04043.html
> 
> I have picked patches from the localhost migration series and rebased 
> it on the series of side channel, now it is a complete series that
> passed the basic test.
> 
> Please let me know if there is anything needs to be fixed or improved.
> Your suggestions and comments are very welcome, and thanks for Paolo
> for his review and useful suggestions.

Thanks to you for listening. :)

What is performance like?  How much downtime do you get as the guest
size increases?

Paolo

Re: [Qemu-devel] [PATCH 01/17] rename is_active to is_block_active

[Lei Li]

On 10/24/2013 09:46 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> is_active is used to identify block migration, rename to
>> is_block_active to make it more clear.
> No, is_active is used to identify whether a set of SaveVMHandlers is
> active.  The default is true, so only block migration is using it.  But
> we could use it in the future for other features (probably using
> migration capabilities instead of a flag as is the case for block).

It updates my knowledge.
Thanks for your clarifying!

>
> Paolo
>
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   block-migration.c           |    2 +-
>>   include/migration/vmstate.h |    2 +-
>>   savevm.c                    |   16 ++++++++--------
>>   3 files changed, 10 insertions(+), 10 deletions(-)
>>
>> diff --git a/block-migration.c b/block-migration.c
>> index daf9ec1..b637695 100644
>> --- a/block-migration.c
>> +++ b/block-migration.c
>> @@ -834,7 +834,7 @@ SaveVMHandlers savevm_block_handlers = {
>>       .save_live_pending = block_save_pending,
>>       .load_state = block_load,
>>       .cancel = block_migration_cancel,
>> -    .is_active = block_is_active,
>> +    .is_block_active = block_is_active,
>>   };
>>   
>>   void blk_mig_init(void)
>> diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
>> index 9d09e60..c634d65 100644
>> --- a/include/migration/vmstate.h
>> +++ b/include/migration/vmstate.h
>> @@ -42,7 +42,7 @@ typedef struct SaveVMHandlers {
>>       int (*save_live_complete)(QEMUFile *f, void *opaque);
>>   
>>       /* This runs both outside and inside the iothread lock.  */
>> -    bool (*is_active)(void *opaque);
>> +    bool (*is_block_active)(void *opaque);
>>   
>>       /* This runs outside the iothread lock in the migration case, and
>>        * within the lock in the savevm case.  The callback had better only
>> diff --git a/savevm.c b/savevm.c
>> index 2f631d4..56b8643 100644
>> --- a/savevm.c
>> +++ b/savevm.c
>> @@ -1867,8 +1867,8 @@ void qemu_savevm_state_begin(QEMUFile *f,
>>           if (!se->ops || !se->ops->save_live_setup) {
>>               continue;
>>           }
>> -        if (se->ops && se->ops->is_active) {
>> -            if (!se->ops->is_active(se->opaque)) {
>> +        if (se->ops && se->ops->is_block_active) {
>> +            if (!se->ops->is_block_active(se->opaque)) {
>>                   continue;
>>               }
>>           }
>> @@ -1907,8 +1907,8 @@ int qemu_savevm_state_iterate(QEMUFile *f)
>>           if (!se->ops || !se->ops->save_live_iterate) {
>>               continue;
>>           }
>> -        if (se->ops && se->ops->is_active) {
>> -            if (!se->ops->is_active(se->opaque)) {
>> +        if (se->ops && se->ops->is_block_active) {
>> +            if (!se->ops->is_block_active(se->opaque)) {
>>                   continue;
>>               }
>>           }
>> @@ -1948,8 +1948,8 @@ void qemu_savevm_state_complete(QEMUFile *f)
>>           if (!se->ops || !se->ops->save_live_complete) {
>>               continue;
>>           }
>> -        if (se->ops && se->ops->is_active) {
>> -            if (!se->ops->is_active(se->opaque)) {
>> +        if (se->ops && se->ops->is_block_active) {
>> +            if (!se->ops->is_block_active(se->opaque)) {
>>                   continue;
>>               }
>>           }
>> @@ -2002,8 +2002,8 @@ uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
>>           if (!se->ops || !se->ops->save_live_pending) {
>>               continue;
>>           }
>> -        if (se->ops && se->ops->is_active) {
>> -            if (!se->ops->is_active(se->opaque)) {
>> +        if (se->ops && se->ops->is_block_active) {
>> +            if (!se->ops->is_block_active(se->opaque)) {
>>                   continue;
>>               }
>>           }
>>
>


-- 
Lei

Re: [Qemu-devel] [PATCH 02/17] QAPI: introduce magration capability unix_page_flipping

[Lei Li]

On 10/24/2013 09:52 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> +# @unix-page-flipping: If enabled, QEMU will support localhost migration. This
>> +#          feature allows live upgrade of a running QEMU instance by doing localhost
>> +#          migration with page flipping. It requires the source and destination
>> +#          are both on localhost. Disabled by default. (since 1.7)
>> +#
> If enabled, QEMU can optimize migration when the destination is a QEMU
> process that runs on the same host as the source (as is the case for
> live upgrade).  If the migration transport is a Unix socket, QEMU will
> flip RAM pages directly to the destination, so that memory is only
> allocated twice for the source and destination processes. Disabled by
> default. (since 1.8)

I will update with this in next version, thanks.

> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 04/17] qmp-command.hx: add missing docs for migration capabilites

[Lei Li]

On 10/24/2013 09:57 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   qmp-commands.hx |    8 ++++++++
>>   1 files changed, 8 insertions(+), 0 deletions(-)
>>
>> diff --git a/qmp-commands.hx b/qmp-commands.hx
>> index fba15cd..650a3a8 100644
>> --- a/qmp-commands.hx
>> +++ b/qmp-commands.hx
>> @@ -2898,6 +2898,10 @@ migrate-set-capabilities
>>   Enable/Disable migration capabilities
>>   
>>   - "xbzrle": XBZRLE support
>> +- "x-rdma-pin-all": RDMA support
> Pin all pages during RDMA support.
>
>> +- "zero-blocks": zero-blocks support
> Compress zero blocks during block migration.
>
>> +- "auto-converge": Auto converge support
> Block VCPU to help convergence of migration
>
>> +- "unix-page-flipping": Page flipping support
> Page flipping for live QEMU upgrade
>
>>   Arguments:
>>   
>> @@ -2922,6 +2926,10 @@ Query current migration capabilities
>>   
>>   - "capabilities": migration capabilities state
>>            - "xbzrle" : XBZRLE state (json-bool)
>> +         - "x-rdma-pin-all": RDMA state (json-bool)
>> +         - "zero-blocks": zero-blocks state (json-bool)
>> +         - "auto-converge": Auto converge state (json-bool)
>> +         - "unix-page-flipping": Page flipping state (json-bool)
>>   
>>   Arguments:
>>   
>>
> Please separate page flipping in a separate patch and send it for 1.7.
> Once you do that, patches 2/3/4 can be merged.

Sure, I will send it with your comments later.

>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 09/17] migration-local: override before_ram_iterate to send pipefd

[Lei Li]

On 10/24/2013 10:07 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> Override befor_ram_iterate to send pipefd. It will write the
>> RAM_SAVE_FLAG_HOOK flags which will trigger the load hook to
>> receive it.
>>
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   migration-local.c |   26 ++++++++++++++++++++++++++
>>   1 files changed, 26 insertions(+), 0 deletions(-)
>>
>> diff --git a/migration-local.c b/migration-local.c
>> index ed0ae6c..92c661c 100644
>> --- a/migration-local.c
>> +++ b/migration-local.c
>> @@ -114,6 +114,31 @@ static int qemu_local_close(void *opaque)
>>       return 0;
>>   }
>>   
>> +static int send_pipefd(int sockfd, int pipefd);
>> +
>> +static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
>> +                                  uint64_t flags)
>> +{
>> +    QEMUFileLocal *s = opaque;
>> +    int ret;
>> +
>> +    if (s->unix_page_flipping) {
>> +        /* Avoid sending pipe fd again in ram_save_complete() stage */
>> +        if (flags != RAM_CONTROL_FINISH) {
> Why not "flags == RAM_CONTROL_SETUP"?

As now it actually has two stages SETUP and FINISH,
'flags == RAM_CONTROL_SETUP' is more straight forward.

Thanks!

>
>> +            qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
>> +            qemu_fflush(f);
>> +            ret = send_pipefd(s->sockfd, s->pipefd[0]);
>> +            if (ret < 0) {
>> +                fprintf(stderr, "failed to pass pipe\n");
>> +                return ret;
>> +            }
>> +            DPRINTF("pipe fd was sent\n");
>> +        }
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>>   static const QEMUFileOps pipe_read_ops = {
>>       .get_fd        = qemu_local_get_sockfd,
>>       .get_buffer    = qemu_local_get_buffer,
>> @@ -124,6 +149,7 @@ static const QEMUFileOps pipe_write_ops = {
>>       .get_fd             = qemu_local_get_sockfd,
>>       .writev_buffer      = qemu_local_writev_buffer,
>>       .close              = qemu_local_close,
>> +    .before_ram_iterate = qemu_local_send_pipefd,
>>   };
>>   
>>   QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode)
>>
> Otherwise looks good.
>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 13/17] migration-unix: replace qemu_fopen_socket with qemu_fopen_socket_local

[Lei Li]

On 10/24/2013 10:10 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> Relace qemu_fopen_socket with qemu_fopen_socket_local in Unix
>> protocol migration.
>>
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   migration-unix.c |   18 ++++++++++++++----
>>   1 files changed, 14 insertions(+), 4 deletions(-)
>>
>> diff --git a/migration-unix.c b/migration-unix.c
>> index 651fc5b..d3a151a 100644
>> --- a/migration-unix.c
>> +++ b/migration-unix.c
>> @@ -37,12 +37,22 @@ static void unix_wait_for_connect(int fd, void *opaque)
>>       if (fd < 0) {
>>           DPRINTF("migrate connect error\n");
>>           s->file = NULL;
>> -        migrate_fd_error(s);
>> +        goto fails;
>>       } else {
>>           DPRINTF("migrate connect success\n");
>> -        s->file = qemu_fopen_socket(fd, "wb");
>> +
>> +        s->file = qemu_fopen_socket_local(fd, "wb");
>> +        if (s->file == NULL) {
>> +            DPRINTF("failed to open local socket\n");
> "failed to open Unix socket"

OK.

>
>> +            goto fail;
>> +        }
>> +
>>           migrate_fd_connect(s);
>> +        return;
>>       }
>> +
>> +fail:
>> +    migrate_fd_error(s);
>>   }
>>   
>>   void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp)
>> @@ -71,9 +81,9 @@ static void unix_accept_incoming_migration(void *opaque)
>>           goto out;
>>       }
>>   
>> -    f = qemu_fopen_socket(c, "rb");
>> +    f = qemu_fopen_socket_local(c, "rb");
>>       if (f == NULL) {
>> -        fprintf(stderr, "could not qemu_fopen socket\n");
>> +        fprintf(stderr, "could not qemu_fopen socket local\n");
> "failed to open Unix socket"

OK, thanks.

>
>>           goto out;
>>       }
>>   
>>
> Otherwise ok.
>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Lei Li]

On 10/24/2013 10:13 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> Introduce new RanState RAN_STATE_FLIPPING_MIGRATE and
>> add it to runstate_needs_reset().
> I am not sure about the name; for one thing, the new state would apply
> also to postcopy migration.

About the name, how about 'live-upgrade'?

OK, I'll add the transition between postcopy and this new state.
And should it also apply from 'prelaunch' to 'flipping-migrate' too?


>
> But the code looks ok.
>
> Paolo
>
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   qapi-schema.json |   11 +++++++----
>>   vl.c             |   12 +++++++++++-
>>   2 files changed, 18 insertions(+), 5 deletions(-)
>>
>> diff --git a/qapi-schema.json b/qapi-schema.json
>> index 523a5b2..8178d0c 100644
>> --- a/qapi-schema.json
>> +++ b/qapi-schema.json
>> @@ -176,12 +176,15 @@
>>   # @watchdog: the watchdog action is configured to pause and has been triggered
>>   #
>>   # @guest-panicked: guest has been panicked as a result of guest OS panic
>> +#
>> +# @flipping-migrate: guest is paused to start unix_page_flipping migration
>> +# process
>>   ##
>>   { 'enum': 'RunState',
>> -  'data': [ 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused',
>> -            'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm',
>> -            'running', 'save-vm', 'shutdown', 'suspended', 'watchdog',
>> -            'guest-panicked' ] }
>> +  'data': [ 'debug', 'flipping-migrate', 'inmigrate', 'internal-error',
>> +            'io-error', 'paused', 'postmigrate', 'prelaunch', 'finish-migrate',
>> +            'restore-vm', 'running', 'save-vm', 'shutdown', 'suspended',
>> +            'watchdog', 'guest-panicked' ] }
>>   
>>   ##
>>   # @SnapshotInfo
>> diff --git a/vl.c b/vl.c
>> index b42ac67..fcdf981 100644
>> --- a/vl.c
>> +++ b/vl.c
>> @@ -601,6 +601,7 @@ static const RunStateTransition runstate_transitions_def[] = {
>>   
>>       { RUN_STATE_PAUSED, RUN_STATE_RUNNING },
>>       { RUN_STATE_PAUSED, RUN_STATE_FINISH_MIGRATE },
>> +    { RUN_STATE_PAUSED, RUN_STATE_FLIPPING_MIGRATE },
>>   
>>       { RUN_STATE_POSTMIGRATE, RUN_STATE_RUNNING },
>>       { RUN_STATE_POSTMIGRATE, RUN_STATE_FINISH_MIGRATE },
>> @@ -624,23 +625,31 @@ static const RunStateTransition runstate_transitions_def[] = {
>>       { RUN_STATE_RUNNING, RUN_STATE_SHUTDOWN },
>>       { RUN_STATE_RUNNING, RUN_STATE_WATCHDOG },
>>       { RUN_STATE_RUNNING, RUN_STATE_GUEST_PANICKED },
>> +    { RUN_STATE_RUNNING, RUN_STATE_FLIPPING_MIGRATE },
>>   
>>       { RUN_STATE_SAVE_VM, RUN_STATE_RUNNING },
>>   
>>       { RUN_STATE_SHUTDOWN, RUN_STATE_PAUSED },
>>       { RUN_STATE_SHUTDOWN, RUN_STATE_FINISH_MIGRATE },
>> +    { RUN_STATE_SHUTDOWN, RUN_STATE_FLIPPING_MIGRATE },
>>   
>>       { RUN_STATE_DEBUG, RUN_STATE_SUSPENDED },
>>       { RUN_STATE_RUNNING, RUN_STATE_SUSPENDED },
>>       { RUN_STATE_SUSPENDED, RUN_STATE_RUNNING },
>>       { RUN_STATE_SUSPENDED, RUN_STATE_FINISH_MIGRATE },
>> +    { RUN_STATE_SUSPENDED, RUN_STATE_FLIPPING_MIGRATE },
>>   
>>       { RUN_STATE_WATCHDOG, RUN_STATE_RUNNING },
>>       { RUN_STATE_WATCHDOG, RUN_STATE_FINISH_MIGRATE },
>> +    { RUN_STATE_WATCHDOG, RUN_STATE_FLIPPING_MIGRATE },
>>   
>>       { RUN_STATE_GUEST_PANICKED, RUN_STATE_PAUSED },
>>       { RUN_STATE_GUEST_PANICKED, RUN_STATE_FINISH_MIGRATE },
>>       { RUN_STATE_GUEST_PANICKED, RUN_STATE_DEBUG },
>> +    { RUN_STATE_GUEST_PANICKED, RUN_STATE_FLIPPING_MIGRATE },
>> +
>> +    { RUN_STATE_FLIPPING_MIGRATE, RUN_STATE_RUNNING },
>> +    { RUN_STATE_FLIPPING_MIGRATE, RUN_STATE_POSTMIGRATE },
>>   
>>       { RUN_STATE_MAX, RUN_STATE_MAX },
>>   };
>> @@ -687,7 +696,8 @@ bool runstate_needs_reset(void)
>>   {
>>       return runstate_check(RUN_STATE_INTERNAL_ERROR) ||
>>           runstate_check(RUN_STATE_SHUTDOWN) ||
>> -        runstate_check(RUN_STATE_GUEST_PANICKED);
>> +        runstate_check(RUN_STATE_GUEST_PANICKED) ||
>> +        runstate_check(RUN_STATE_FLIPPING_MIGRATE);
>>   }
>>   
>>   StatusInfo *qmp_query_status(Error **errp)
>>
>


-- 
Lei

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

On 10/24/2013 10:15 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   migration.c |   10 +++++++---
>>   1 files changed, 7 insertions(+), 3 deletions(-)
>>
>> diff --git a/migration.c b/migration.c
>> index 4ac466b..568b73a 100644
>> --- a/migration.c
>> +++ b/migration.c
>> @@ -579,10 +579,11 @@ static void *migration_thread(void *opaque)
>>               pending_size = qemu_savevm_state_pending(s->file, max_size);
>>               DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
>>                       pending_size, max_size);
>> -            if (pending_size && pending_size >= max_size) {
>> +            if (pending_size && pending_size >= max_size &&
>> +                !migrate_unix_page_flipping()) {
> This is a bit ugly but I understand the need.  Perhaps "&&
> !runstate_needs_reset()" like below?

'&& !runstate_needs_reset()' is fine, thanks.

>
> Paolo
>
>>                   qemu_savevm_state_iterate(s->file);
>>               } else {
>> -                int ret;
>> +                int ret = 0;
>>   
>>                   DPRINTF("done iterating\n");
>>                   qemu_mutex_lock_iothread();
>> @@ -590,7 +591,10 @@ static void *migration_thread(void *opaque)
>>                   qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
>>                   old_vm_running = runstate_is_running();
>>   
>> -                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
>> +                if (!runstate_needs_reset()) {
>> +                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
>> +                }
>> +
>>                   if (ret >= 0) {
>>                       qemu_file_set_rate_limit(s->file, INT_MAX);
>>                       qemu_savevm_state_complete(s->file);
>>
>


-- 
Lei

Re: [Qemu-devel] [PATCH 09/17] migration-local: override before_ram_iterate to send pipefd

[Lei Li]

On 10/24/2013 10:07 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> Override befor_ram_iterate to send pipefd. It will write the
>> RAM_SAVE_FLAG_HOOK flags which will trigger the load hook to
>> receive it.
>>
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   migration-local.c |   26 ++++++++++++++++++++++++++
>>   1 files changed, 26 insertions(+), 0 deletions(-)
>>
>> diff --git a/migration-local.c b/migration-local.c
>> index ed0ae6c..92c661c 100644
>> --- a/migration-local.c
>> +++ b/migration-local.c
>> @@ -114,6 +114,31 @@ static int qemu_local_close(void *opaque)
>>       return 0;
>>   }
>>   
>> +static int send_pipefd(int sockfd, int pipefd);
>> +
>> +static int qemu_local_send_pipefd(QEMUFile *f, void *opaque,
>> +                                  uint64_t flags)
>> +{
>> +    QEMUFileLocal *s = opaque;
>> +    int ret;
>> +
>> +    if (s->unix_page_flipping) {
>> +        /* Avoid sending pipe fd again in ram_save_complete() stage */
>> +        if (flags != RAM_CONTROL_FINISH) {
> Why not "flags == RAM_CONTROL_SETUP"?
>
>> +            qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
>> +            qemu_fflush(f);
>> +            ret = send_pipefd(s->sockfd, s->pipefd[0]);
>> +            if (ret < 0) {
>> +                fprintf(stderr, "failed to pass pipe\n");
>> +                return ret;
>> +            }
>> +            DPRINTF("pipe fd was sent\n");
>> +        }
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>>   static const QEMUFileOps pipe_read_ops = {
>>       .get_fd        = qemu_local_get_sockfd,
>>       .get_buffer    = qemu_local_get_buffer,
>> @@ -124,6 +149,7 @@ static const QEMUFileOps pipe_write_ops = {
>>       .get_fd             = qemu_local_get_sockfd,
>>       .writev_buffer      = qemu_local_writev_buffer,
>>       .close              = qemu_local_close,
>> +    .before_ram_iterate = qemu_local_send_pipefd,
>>   };
>>   
>>   QEMUFile *qemu_fopen_socket_local(int sockfd, const char *mode)
>>
> Otherwise looks good.

Just want to confirm, normally, should I take these 'otherwise looks good/ok'
as a 'Reviewed-by' from you If the other comment is fixed in the update version?

>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Lei Li]

On 10/24/2013 10:17 PM, Paolo Bonzini wrote:
> Il 22/10/2013 04:25, Lei Li ha scritto:
>> This patch series tries to introduce a mechanism using side
>> channel pipe for RAM via SCM_RIGHTS with unix domain socket
>> protocol migration.
>>
>> This side channel is used for the page flipping by vmsplice,
>> which is the internal mechanism for localhost migration that
>> we are trying to add to QEMU. The backgroud info and previous
>> patch series for reference,
>>
>> Localhost migration
>> http://lists.nongnu.org/archive/html/qemu-devel/2013-08/msg02916.html
>>
>> migration: Introduce side channel for RAM
>> http://lists.gnu.org/archive/html/qemu-devel/2013-09/msg04043.html
>>
>> I have picked patches from the localhost migration series and rebased
>> it on the series of side channel, now it is a complete series that
>> passed the basic test.
>>
>> Please let me know if there is anything needs to be fixed or improved.
>> Your suggestions and comments are very welcome, and thanks for Paolo
>> for his review and useful suggestions.
> Thanks to you for listening. :)
>
> What is performance like?  How much downtime do you get as the guest
> size increases?

Hi Paolo,

Thanks very much for your continued reviews, as well as those nice suggestions
and comments!

For the performance, we have not yet benchmarked it from QEMU layer, as now it's
just an initial implementation, and Robert Jennings has been worked on the
improvement of performance on kernel side.

Right now just has inaccurate numbers without the new vmsplice, which based on
the result from info migrate, as the guest ram size increases, although the
'total time' is number of times less compared with the current live migration,
but the 'downtime' performs badly.

For a 1GB ram guest,

total time: 702 milliseconds
downtime: 692 milliseconds

And when the ram size of guest increasesexponentially, those numbers are
proportional to it.
  
I will make a list of the performance with the new vmsplice later, I am sure
it'd be much better than this at least.

> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 09/17] migration-local: override before_ram_iterate to send pipefd

[Paolo Bonzini]

Il 25/10/2013 05:38, Lei Li ha scritto:
> 
> Just want to confirm, normally, should I take these 'otherwise looks
> good/ok'
> as a 'Reviewed-by' from you If the other comment is fixed in the update
> version?

Depends on how much the patch changes... right now I'm still expecting
some changes so I didn't really look much at the patch and didn't test
it.  I prefer to take a more complete look at v3 before giving a
"formal" Reviewed-by.

Paolo

Re: [Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Paolo Bonzini]

Il 25/10/2013 06:58, Lei Li ha scritto:
> Right now just has inaccurate numbers without the new vmsplice, which
> based on
> the result from info migrate, as the guest ram size increases, although the
> 'total time' is number of times less compared with the current live
> migration, but the 'downtime' performs badly.

Of course.
> 
> For a 1GB ram guest,
> 
> total time: 702 milliseconds
> downtime: 692 milliseconds
> 
> And when the ram size of guest increasesexponentially, those numbers are
> proportional to it.
>  
> I will make a list of the performance with the new vmsplice later, I am
> sure it'd be much better than this at least.

Yes, please.  Is the memory usage is still 2x without vmsplice?

I think you have a nice proof of concept, but on the other hand this
probably needs to be coupled with some kind of postcopy live migration,
that is:

* the source starts sending data

* but the destination starts running immediately

* if the machine needs a page that is missing, the destination asks the
source to send it

* as soon as it arrives, the destination can restart

Using postcopy is problematic for reliability: if the destination fails,
the virtual machine is lost because the source doesn't have the latest
content of memory.  However, this is a much, much smaller problem for
live QEMU upgrade where the network cannot fail.

If you do this, you can achieve pretty much instantaneous live upgrade,
well within your original 200 ms goals.  But the flipping code with
vmsplice should be needed anyway to avoid doubling memory usage, and
it's looking pretty good in this version already!  I'm relieved that the
RDMA code was designed right!

Paolo

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Paolo Bonzini]

Il 25/10/2013 05:30, Lei Li ha scritto:
>>>
>> I am not sure about the name; for one thing, the new state would apply
>> also to postcopy migration.
> 
> About the name, how about 'live-upgrade'?
> 
> OK, I'll add the transition between postcopy and this new state.

Note I didn't mean "postmigrate".

For a description of postcopy, see my answer to the cover letter ("patch
0").  The new state means "somebody else has newer contents of the
memory".  Perhaps "stale"?

> And should it also apply from 'prelaunch' to 'flipping-migrate' too?

Yes, it should.  Good catch!

Paolo

Re: [Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Anthony Liguori]

[-- Attachment #1: Type: text/plain, Size: 1915 bytes --]

On Oct 25, 2013 8:30 AM, "Paolo Bonzini" <pbonzini@redhat.com> wrote:
>
> Il 25/10/2013 06:58, Lei Li ha scritto:
> > Right now just has inaccurate numbers without the new vmsplice, which
> > based on
> > the result from info migrate, as the guest ram size increases, although
the
> > 'total time' is number of times less compared with the current live
> > migration, but the 'downtime' performs badly.
>
> Of course.
> >
> > For a 1GB ram guest,
> >
> > total time: 702 milliseconds
> > downtime: 692 milliseconds
> >
> > And when the ram size of guest increasesexponentially, those numbers are
> > proportional to it.
> >
> > I will make a list of the performance with the new vmsplice later, I am
> > sure it'd be much better than this at least.
>
> Yes, please.  Is the memory usage is still 2x without vmsplice?
>
> I think you have a nice proof of concept, but on the other hand this
> probably needs to be coupled with some kind of postcopy live migration,
> that is:
>
> * the source starts sending data
>
> * but the destination starts running immediately
>
> * if the machine needs a page that is missing, the destination asks the
> source to send it
>
> * as soon as it arrives, the destination can restart
>
> Using postcopy is problematic for reliability: if the destination fails,
> the virtual machine is lost because the source doesn't have the latest
> content of memory.  However, this is a much, much smaller problem for
> live QEMU upgrade where the network cannot fail.
>
> If you do this, you can achieve pretty much instantaneous live upgrade,
> well within your original 200 ms goals.

This is actually a very nice justification for post copy.

Regards,

Anthony Liguori

But the flipping code with
> vmsplice should be needed anyway to avoid doubling memory usage, and
> it's looking pretty good in this version already!  I'm relieved that the
> RDMA code was designed right!
>
> Paolo
>
>

[-- Attachment #2: Type: text/html, Size: 2553 bytes --]

Re: [Qemu-devel] [PATCH 09/17] migration-local: override before_ram_iterate to send pipefd

[Lei Li]

On 10/25/2013 03:23 PM, Paolo Bonzini wrote:
> Il 25/10/2013 05:38, Lei Li ha scritto:
>> Just want to confirm, normally, should I take these 'otherwise looks
>> good/ok'
>> as a 'Reviewed-by' from you If the other comment is fixed in the update
>> version?
> Depends on how much the patch changes... right now I'm still expecting
> some changes so I didn't really look much at the patch and didn't test
> it.  I prefer to take a more complete look at v3 before giving a
> "formal" Reviewed-by.

I see, thanks for your explanation.

>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 14/17] add new RanState RAN_STATE_FLIPPING_MIGRATE

[Lei Li]

On 10/25/2013 03:31 PM, Paolo Bonzini wrote:
> Il 25/10/2013 05:30, Lei Li ha scritto:
>>> I am not sure about the name; for one thing, the new state would apply
>>> also to postcopy migration.
>> About the name, how about 'live-upgrade'?
>>
>> OK, I'll add the transition between postcopy and this new state.
> Note I didn't mean "postmigrate".
>
> For a description of postcopy, see my answer to the cover letter ("patch

Yes, I've realized that I misunderstood it...

> 0").  The new state means "somebody else has newer contents of the
> memory".  Perhaps "stale"?
>> And should it also apply from 'prelaunch' to 'flipping-migrate' too?
> Yes, it should.  Good catch!
>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Lei Li]

On 10/25/2013 03:30 PM, Paolo Bonzini wrote:
> Il 25/10/2013 06:58, Lei Li ha scritto:
>> Right now just has inaccurate numbers without the new vmsplice, which
>> based on
>> the result from info migrate, as the guest ram size increases, although the
>> 'total time' is number of times less compared with the current live
>> migration, but the 'downtime' performs badly.
> Of course.
>> For a 1GB ram guest,
>>
>> total time: 702 milliseconds
>> downtime: 692 milliseconds
>>
>> And when the ram size of guest increasesexponentially, those numbers are
>> proportional to it.
>>   
>> I will make a list of the performance with the new vmsplice later, I am
>> sure it'd be much better than this at least.
> Yes, please.  Is the memory usage is still 2x without vmsplice?
>
> I think you have a nice proof of concept, but on the other hand this
> probably needs to be coupled with some kind of postcopy live migration,
> that is:
>
> * the source starts sending data
>
> * but the destination starts running immediately
>
> * if the machine needs a page that is missing, the destination asks the
> source to send it
>
> * as soon as it arrives, the destination can restart
>
> Using postcopy is problematic for reliability: if the destination fails,
> the virtual machine is lost because the source doesn't have the latest
> content of memory.  However, this is a much, much smaller problem for
> live QEMU upgrade where the network cannot fail.
>
> If you do this, you can achieve pretty much instantaneous live upgrade,
> well within your original 200 ms goals.  But the flipping code with
> vmsplice should be needed anyway to avoid doubling memory usage, and

Yes, I have read the postcopy migration patches, it does perform very
good on downtime, as just send the vmstates then switch the execution
to destination host. And as you pointed out, it can not avoid
doubling memory usage.

The numbers list above are based on the old vmsplice as I have not yet
worked on the benchmark for performance, it actually copys data rather
than moving. As the feedback for this version is positive, now I am
trying to get a real result out with the new vmsplice.

BTW, kernel side is looking for huge page solution for the improvement of
performance.

The recently patches from kernel as link,

http://article.gmane.org/gmane.linux.kernel/1574277

> it's looking pretty good in this version already!  I'm relieved that the
> RDMA code was designed right!

I am happy with it too. :)
Those RDMA hooks really make thingsmore flexible!


> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 0/17 v2] Localhost migration with side channel for ram

[Lei Li]

On 10/25/2013 08:24 PM, Lei Li wrote:
> On 10/25/2013 03:30 PM, Paolo Bonzini wrote:
>> Il 25/10/2013 06:58, Lei Li ha scritto:
>>> Right now just has inaccurate numbers without the new vmsplice, which
>>> based on
>>> the result from info migrate, as the guest ram size increases, 
>>> although the
>>> 'total time' is number of times less compared with the current live
>>> migration, but the 'downtime' performs badly.
>> Of course.
>>> For a 1GB ram guest,
>>>
>>> total time: 702 milliseconds
>>> downtime: 692 milliseconds
>>>
>>> And when the ram size of guest increasesexponentially, those numbers 
>>> are
>>> proportional to it.
>>>   I will make a list of the performance with the new vmsplice later, 
>>> I am
>>> sure it'd be much better than this at least.
>> Yes, please.  Is the memory usage is still 2x without vmsplice?
>>
>> I think you have a nice proof of concept, but on the other hand this
>> probably needs to be coupled with some kind of postcopy live migration,
>> that is:
>>
>> * the source starts sending data
>>
>> * but the destination starts running immediately
>>
>> * if the machine needs a page that is missing, the destination asks the
>> source to send it
>>
>> * as soon as it arrives, the destination can restart
>>
>> Using postcopy is problematic for reliability: if the destination fails,
>> the virtual machine is lost because the source doesn't have the latest
>> content of memory.  However, this is a much, much smaller problem for
>> live QEMU upgrade where the network cannot fail.
>>
>> If you do this, you can achieve pretty much instantaneous live upgrade,
>> well within your original 200 ms goals.  But the flipping code with
>> vmsplice should be needed anyway to avoid doubling memory usage, and
>
> Yes, I have read the postcopy migration patches, it does perform very
> good on downtime, as just send the vmstates then switch the execution
> to destination host. And as you pointed out, it can not avoid
> doubling memory usage.
>
> The numbers list above are based on the old vmsplice as I have not yet
> worked on the benchmark for performance, it actually copys data rather
> than moving. As the feedback for this version is positive, now I am
> trying to get a real result out with the new vmsplice.
>
> BTW, kernel side is looking for huge page solution for the improvement of
> performance.
>
> The recently patches from kernel as link,
>
> http://article.gmane.org/gmane.linux.kernel/1574277

Hi Paolo,

I have been working on the benchmark of the performance, I am afraid that it may take
a bit more time as there has some problems on the new vmsplice which kernel side is
working on right now.

I will post a v3 of the series with your comments in previous version fixed soon.

>
>> it's looking pretty good in this version already!  I'm relieved that the
>> RDMA code was designed right!
>
> I am happy with it too. :)
> Those RDMA hooks really make thingsmore flexible!
>
>
>> Paolo
>>
>
>


-- 
Lei

[Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration.c |   10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/migration.c b/migration.c
index 4ac466b..0f98ac1 100644
--- a/migration.c
+++ b/migration.c
@@ -579,10 +579,11 @@ static void *migration_thread(void *opaque)
             pending_size = qemu_savevm_state_pending(s->file, max_size);
             DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
                     pending_size, max_size);
-            if (pending_size && pending_size >= max_size) {
+            if (pending_size && pending_size >= max_size &&
+                !runstate_needs_reset()) {
                 qemu_savevm_state_iterate(s->file);
             } else {
-                int ret;
+                int ret = 0;
 
                 DPRINTF("done iterating\n");
                 qemu_mutex_lock_iothread();
@@ -590,7 +591,10 @@ static void *migration_thread(void *opaque)
                 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
                 old_vm_running = runstate_is_running();
 
-                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                if (!runstate_needs_reset()) {
+                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                }
+
                 if (ret >= 0) {
                     qemu_file_set_rate_limit(s->file, INT_MAX);
                     qemu_savevm_state_complete(s->file);
-- 
1.7.7.6

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Paolo Bonzini]

Il 21/11/2013 10:11, Lei Li ha scritto:
> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
> ---
>  migration.c |   10 +++++++---
>  1 files changed, 7 insertions(+), 3 deletions(-)
> 
> diff --git a/migration.c b/migration.c
> index 4ac466b..0f98ac1 100644
> --- a/migration.c
> +++ b/migration.c
> @@ -579,10 +579,11 @@ static void *migration_thread(void *opaque)
>              pending_size = qemu_savevm_state_pending(s->file, max_size);
>              DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
>                      pending_size, max_size);
> -            if (pending_size && pending_size >= max_size) {
> +            if (pending_size && pending_size >= max_size &&
> +                !runstate_needs_reset()) {
>                  qemu_savevm_state_iterate(s->file);

I'm not sure why you need this.

>              } else {
> -                int ret;
> +                int ret = 0;
>  
>                  DPRINTF("done iterating\n");
>                  qemu_mutex_lock_iothread();
> @@ -590,7 +591,10 @@ static void *migration_thread(void *opaque)
>                  qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
>                  old_vm_running = runstate_is_running();
>  
> -                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
> +                if (!runstate_needs_reset()) {
> +                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
> +                }

This however is okay.

Paolo

>                  if (ret >= 0) {
>                      qemu_file_set_rate_limit(s->file, INT_MAX);
>                      qemu_savevm_state_complete(s->file);
> 

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

On 11/26/2013 07:32 PM, Paolo Bonzini wrote:
> Il 21/11/2013 10:11, Lei Li ha scritto:
>> Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
>> ---
>>   migration.c |   10 +++++++---
>>   1 files changed, 7 insertions(+), 3 deletions(-)
>>
>> diff --git a/migration.c b/migration.c
>> index 4ac466b..0f98ac1 100644
>> --- a/migration.c
>> +++ b/migration.c
>> @@ -579,10 +579,11 @@ static void *migration_thread(void *opaque)
>>               pending_size = qemu_savevm_state_pending(s->file, max_size);
>>               DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
>>                       pending_size, max_size);
>> -            if (pending_size && pending_size >= max_size) {
>> +            if (pending_size && pending_size >= max_size &&
>> +                !runstate_needs_reset()) {
>>                   qemu_savevm_state_iterate(s->file);
> I'm not sure why you need this.

The adjustment here is to avoid the iteration stage for page flipping.
Because pending_size = ram_save_remaining() * TARGET_PAGE_SIZE which is
not 0 and pending_size > max_size (0) at start.

In the previous version it was like this:

if (pending_size && pending_size >= max_size &&
     !migrate_unix_page_flipping()) {

And you said 'This is a bit ugly but I understand the need. Perhaps "&&
!runstate_needs_reset()" like below?' :)

>
>>               } else {
>> -                int ret;
>> +                int ret = 0;
>>   
>>                   DPRINTF("done iterating\n");
>>                   qemu_mutex_lock_iothread();
>> @@ -590,7 +591,10 @@ static void *migration_thread(void *opaque)
>>                   qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
>>                   old_vm_running = runstate_is_running();
>>   
>> -                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
>> +                if (!runstate_needs_reset()) {
>> +                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
>> +                }
> This however is okay.
>
> Paolo
>
>>                   if (ret >= 0) {
>>                       qemu_file_set_rate_limit(s->file, INT_MAX);
>>                       qemu_savevm_state_complete(s->file);
>>
>


-- 
Lei

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Paolo Bonzini]

Il 26/11/2013 13:03, Lei Li ha scritto:
>>>
>>> +            if (pending_size && pending_size >= max_size &&
>>> +                !runstate_needs_reset()) {
>>>                   qemu_savevm_state_iterate(s->file);
>> I'm not sure why you need this.
> 
> The adjustment here is to avoid the iteration stage for page flipping.
> Because pending_size = ram_save_remaining() * TARGET_PAGE_SIZE which is
> not 0 and pending_size > max_size (0) at start.

It's still not clear to me that avoiding the iteration stage is
necessary.  I think it's just an optimization to avoid scanning the
bitmap, but:

(1) Juan's bitmap optimization will make this mostly unnecessary

(2) getting good downtime from page flipping will require postcopy anyway.

> And you said 'This is a bit ugly but I understand the need. Perhaps "&&
> !runstate_needs_reset()" like below?' :)

Oops.  I might have said this before thinking about postcopy and/or
before seeing the benchmark results from Juan's patches.  If this part
of the patch is just an optimization, I'd rather leave it out for now.

Thanks for putting up with me. :)

Paolo

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

On 11/26/2013 08:54 PM, Paolo Bonzini wrote:
> Il 26/11/2013 13:03, Lei Li ha scritto:
>>>> +            if (pending_size && pending_size >= max_size &&
>>>> +                !runstate_needs_reset()) {
>>>>                    qemu_savevm_state_iterate(s->file);
>>> I'm not sure why you need this.
>> The adjustment here is to avoid the iteration stage for page flipping.
>> Because pending_size = ram_save_remaining() * TARGET_PAGE_SIZE which is
>> not 0 and pending_size > max_size (0) at start.
> It's still not clear to me that avoiding the iteration stage is

The purpose of it is not just for optimization, but to avoid the
iteration for better alignment.

The current flow of page flipping basically has two stages:

1) ram_save_setup stage, it will send all the bytes in this stages
    to destination, and send_pipefd by ram_control_before_iterate
    at the end of it.
2) ram_save_complete, it will start to transmit the ram page
    in ram_save_block, and send the device state after that.

So it needs to adjust the current migration process to avoid
the iteration stage.

> necessary.  I think it's just an optimization to avoid scanning the
> bitmap, but:
>
> (1) Juan's bitmap optimization will make this mostly unnecessary
>
> (2) getting good downtime from page flipping will require postcopy anyway.
>
>> And you said 'This is a bit ugly but I understand the need. Perhaps "&&
>> !runstate_needs_reset()" like below?' :)
> Oops.  I might have said this before thinking about postcopy and/or
> before seeing the benchmark results from Juan's patches.  If this part
> of the patch is just an optimization, I'd rather leave it out for now.

I am afraid that page flipping can not proceed correctly without this..

>
> Thanks for putting up with me. :)
>
> Paolo
>


-- 
Lei

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Paolo Bonzini]

Il 26/11/2013 14:53, Lei Li ha scritto:
> 1) ram_save_setup stage, it will send all the bytes in this stages
>    to destination, and send_pipefd by ram_control_before_iterate
>    at the end of it.

ram_save_setup runs doesn't send anything from guest RAM.  It sends the
lengths of the various blocks.  As you said, at the end of
ram_save_setup you send the pipefd.

ram_save_iterate runs before ram_save_complete.  ram_save_iterate and
ram_save_complete write data with exactly the same format.  Both of them
can use ram_save_page

It should not matter if some pages are sent as part of ram_save_iterate
and others as part of ram_save_complete.

One possibility is that you are hitting a bug due to the way you ignore
the "0x01" byte that send_pipefd places on the socket.

>> Oops.  I might have said this before thinking about postcopy and/or
>> before seeing the benchmark results from Juan's patches.  If this part
>> of the patch is just an optimization, I'd rather leave it out for now.
> 
> I am afraid that page flipping can not proceed correctly without this..

I really would like to understand why, because it really shouldn't (this
shouldn't be a place where you need a hook).

Paolo

Re: [Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

On 11/26/2013 10:11 PM, Paolo Bonzini wrote:
> Il 26/11/2013 14:53, Lei Li ha scritto:
>> 1) ram_save_setup stage, it will send all the bytes in this stages
>>     to destination, and send_pipefd by ram_control_before_iterate
>>     at the end of it.
> ram_save_setup runs doesn't send anything from guest RAM.  It sends the
> lengths of the various blocks.  As you said, at the end of
> ram_save_setup you send the pipefd.
>
> ram_save_iterate runs before ram_save_complete.  ram_save_iterate and
> ram_save_complete write data with exactly the same format.  Both of them
> can use ram_save_page
>
> It should not matter if some pages are sent as part of ram_save_iterate
> and others as part of ram_save_complete.
>
> One possibility is that you are hitting a bug due to the way you ignore
> the "0x01" byte that send_pipefd places on the socket.
>
>>> Oops.  I might have said this before thinking about postcopy and/or
>>> before seeing the benchmark results from Juan's patches.  If this part
>>> of the patch is just an optimization, I'd rather leave it out for now.
>> I am afraid that page flipping can not proceed correctly without this..
> I really would like to understand why, because it really shouldn't (this
> shouldn't be a place where you need a hook).

Hi Paolo,

Sorry for the late reply.

Yes, you are right!!  I just have a try with this adjustment removed, it
works well...

I remembered that it can not proceed correctly when debugging in previous
version without this as in theory it should like your explanation above. I
guess the only answer is that there was a bug regarding the one byte fd
control message just like the possibility you listed!
  

>
> Paolo
>
>


-- 
Lei

[Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration.c |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/migration.c b/migration.c
index 4ac466b..68b5b02 100644
--- a/migration.c
+++ b/migration.c
@@ -582,7 +582,7 @@ static void *migration_thread(void *opaque)
             if (pending_size && pending_size >= max_size) {
                 qemu_savevm_state_iterate(s->file);
             } else {
-                int ret;
+                int ret = 0;
 
                 DPRINTF("done iterating\n");
                 qemu_mutex_lock_iothread();
@@ -590,7 +590,10 @@ static void *migration_thread(void *opaque)
                 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
                 old_vm_running = runstate_is_running();
 
-                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                if (!runstate_needs_reset()) {
+                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                }
+
                 if (ret >= 0) {
                     qemu_file_set_rate_limit(s->file, INT_MAX);
                     qemu_savevm_state_complete(s->file);
-- 
1.7.7.6

[Qemu-devel] [PATCH 16/17] migration: adjust migration_thread() process for page flipping

[Lei Li]

Signed-off-by: Lei Li <lilei@linux.vnet.ibm.com>
---
 migration.c |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/migration.c b/migration.c
index e012cd4..7e0ec33 100644
--- a/migration.c
+++ b/migration.c
@@ -582,7 +582,7 @@ static void *migration_thread(void *opaque)
             if (pending_size && pending_size >= max_size) {
                 qemu_savevm_state_iterate(s->file);
             } else {
-                int ret;
+                int ret = 0;
 
                 DPRINTF("done iterating\n");
                 qemu_mutex_lock_iothread();
@@ -590,7 +590,10 @@ static void *migration_thread(void *opaque)
                 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
                 old_vm_running = runstate_is_running();
 
-                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                if (!runstate_needs_reset()) {
+                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                }
+
                 if (ret >= 0) {
                     qemu_file_set_rate_limit(s->file, INT_MAX);
                     qemu_savevm_state_complete(s->file);
-- 
1.7.7.6