[Qemu-devel] Re: [PATCH] Re: Live migration - exec: support to be reintroduced? (r2)

[Charles Duffy <Charles_Duffy@messageone.com>]

[-- Attachment #1: Type: text/plain, Size: 109 bytes --]

The attached version refactors to reduce the amount of duplicate code 
between migrate-tcp and migrate-exec.

[-- Attachment #2: qemu-live_migration_exec_support-r2.patch --]
[-- Type: text/x-patch, Size: 19870 bytes --]

diff --git a/Makefile.target b/Makefile.target
index 031ab45..d844d9c 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -596,7 +596,7 @@ endif
 ifdef CONFIG_WIN32
 OBJS+=block-raw-win32.o
 else
-OBJS+=block-raw-posix.o
+OBJS+=block-raw-posix.o migration-exec.o
 endif
 
 LIBS+=-lz
diff --git a/hw/hw.h b/hw/hw.h
index 99d4b8d..eab7bb4 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -35,6 +35,8 @@ QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc *put_buffer,
                          QEMUFileRateLimit *rate_limit);
 QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd);
+QEMUFile *qemu_popen(FILE *popen_file, const char *mode);
+QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
 void qemu_fflush(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
diff --git a/migration-exec.c b/migration-exec.c
new file mode 100644
index 0000000..5d0b215
--- /dev/null
+++ b/migration-exec.c
@@ -0,0 +1,141 @@
+/*
+ * QEMU live migration
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Dell MessageOne 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Charles Duffy     <charles_duffy@messageone.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "qemu_socket.h"
+#include "migration.h"
+#include "qemu-char.h"
+#include "sysemu.h"
+#include "console.h"
+#include "buffered_file.h"
+#include "block.h"
+
+//#define DEBUG_MIGRATION_EXEC
+
+#ifdef DEBUG_MIGRATION_EXEC
+#define dprintf(fmt, ...) \
+    do { printf("migration-exec: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define dprintf(fmt, ...) \
+    do { } while (0)
+#endif
+
+static int file_errno(FdMigrationState *s)
+{
+    return errno;
+}
+
+static int file_write(FdMigrationState *s, const void * buf, size_t size)
+{
+    return write(s->fd, buf, size);
+}
+
+static int exec_close(FdMigrationState *s)
+{
+    dprintf("exec_close\n");
+    if (s->opaque) {
+        qemu_fclose(s->opaque);
+        s->opaque = NULL;
+        s->fd = -1;
+    }
+    return 0;
+}
+
+MigrationState *exec_start_outgoing_migration(const char *command,
+                                             int64_t bandwidth_limit,
+                                             int async)
+{
+    FdMigrationState *s;
+    FILE *f;
+
+    s = qemu_mallocz(sizeof(*s));
+    if (s == NULL) {
+        dprintf("Unable to allocate FdMigrationState\n");
+        goto err;
+    }
+
+    f = popen(command, "w");
+    if (f == NULL) {
+        dprintf("Unable to popen exec target\n");
+        goto err_after_alloc;
+    }
+
+    s->fd = fileno(f);
+    if (s->fd == -1) {
+        dprintf("Unable to retrieve file descriptor for popen'd handle\n");
+        goto err_after_open;
+    }
+
+    if (fcntl(s->fd, F_SETFD, O_NONBLOCK) == -1) {
+        dprintf("Unable to set nonblocking mode on file descriptor\n");
+        goto err_after_open;
+    }
+
+    s->opaque = qemu_popen(f, "w");
+
+    s->get_error = file_errno;
+    s->write = file_write;
+    s->mig_state.cancel = migrate_fd_cancel;
+    s->mig_state.get_status = migrate_fd_get_status;
+    s->mig_state.release = migrate_fd_release;
+
+    s->state = MIG_STATE_ACTIVE;
+    s->detach = !async;
+    s->bandwidth_limit = bandwidth_limit;
+
+    if (s->detach == 1) {
+        dprintf("detaching from monitor\n");
+        monitor_suspend();
+        s->detach = 2;
+    }
+
+    migrate_fd_connect(s);
+    return &s->mig_state;
+
+err_after_open:
+    pclose(f);
+err_after_alloc:
+    qemu_free(s);
+err:
+    return NULL;
+}
+
+int exec_start_incoming_migration(const char *command)
+{
+    int ret;
+    QEMUFile *f;
+
+    dprintf("Attempting to start an incoming migration\n");
+    f = qemu_popen_cmd(command, "r");
+    if(f == NULL) {
+        dprintf("Unable to apply qemu wrapper to popen file\n");
+        return -errno;
+    }
+    vm_stop(0); /* just in case */
+    ret = qemu_loadvm_state(f);
+    if (ret < 0) {
+        fprintf(stderr, "load of migration failed\n");
+        goto err;
+    }
+    qemu_announce_self();
+    dprintf("successfully loaded vm state\n");
+    vm_start();
+    qemu_fclose(f);
+    return 0;
+
+err:
+    qemu_fclose(f);
+    return -errno;
+}
diff --git a/migration-tcp.c b/migration-tcp.c
index f13dc0d..100644b 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -22,16 +22,6 @@
 
 //#define DEBUG_MIGRATION_TCP
 
-typedef struct FdMigrationState
-{
-    MigrationState mig_state;
-    QEMUFile *file;
-    int64_t bandwidth_limit;
-    int fd;
-    int detach;
-    int state;
-} FdMigrationState;
-
 #ifdef DEBUG_MIGRATION_TCP
 #define dprintf(fmt, ...) \
     do { printf("migration-tcp: " fmt, ## __VA_ARGS__); } while (0)
@@ -40,64 +30,19 @@ typedef struct FdMigrationState
     do { } while (0)
 #endif
 
-static void tcp_cleanup(FdMigrationState *s)
+static int socket_errno(FdMigrationState *s)
 {
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
-    if (s->file) {
-        dprintf("closing file\n");
-        qemu_fclose(s->file);
-    }
-
-    if (s->fd != -1)
-        close(s->fd);
-
-    /* Don't resume monitor until we've flushed all of the buffers */
-    if (s->detach == 2) {
-        monitor_resume();
-        s->detach = 0;
-    }
-
-    s->fd = -1;
-}
-
-static void tcp_error(FdMigrationState *s)
-{
-    dprintf("setting error state\n");
-    s->state = MIG_STATE_ERROR;
-    tcp_cleanup(s);
-}
-
-static void fd_put_notify(void *opaque)
-{
-    FdMigrationState *s = opaque;
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-    qemu_file_put_notify(s->file);
+    return (s->get_error(s));
 }
 
-static ssize_t fd_put_buffer(void *opaque, const void *data, size_t size)
+static int socket_write(FdMigrationState *s, const void * buf, size_t size)
 {
-    FdMigrationState *s = opaque;
-    ssize_t ret;
-
-    do {
-        ret = send(s->fd, data, size, 0);
-    } while (ret == -1 && (socket_error() == EINTR || socket_error() == EWOULDBLOCK));
-
-    if (ret == -1)
-        ret = -socket_error();
-
-    if (ret == -EAGAIN)
-        qemu_set_fd_handler2(s->fd, NULL, NULL, fd_put_notify, s);
-
-    return ret;
+    return send(s->fd, buf, size, 0);
 }
 
-static int fd_close(void *opaque)
+static int tcp_close(FdMigrationState *s)
 {
-    FdMigrationState *s = opaque;
-    dprintf("fd_close\n");
+    dprintf("tcp_close\n");
     if (s->fd != -1) {
         close(s->fd);
         s->fd = -1;
@@ -105,67 +50,6 @@ static int fd_close(void *opaque)
     return 0;
 }
 
-static void fd_wait_for_unfreeze(void *opaque)
-{
-    FdMigrationState *s = opaque;
-    int ret;
-
-    dprintf("wait for unfreeze\n");
-    if (s->state != MIG_STATE_ACTIVE)
-        return;
-
-    do {
-        fd_set wfds;
-
-        FD_ZERO(&wfds);
-        FD_SET(s->fd, &wfds);
-
-        ret = select(s->fd + 1, NULL, &wfds, NULL, NULL);
-    } while (ret == -1 && socket_error() == EINTR);
-}
-
-static void fd_put_ready(void *opaque)
-{
-    FdMigrationState *s = opaque;
-
-    if (s->state != MIG_STATE_ACTIVE) {
-        dprintf("put_ready returning because of non-active state\n");
-        return;
-    }
-
-    dprintf("iterate\n");
-    if (qemu_savevm_state_iterate(s->file) == 1) {
-        dprintf("done iterating\n");
-        vm_stop(0);
-
-        bdrv_flush_all();
-        qemu_savevm_state_complete(s->file);
-        s->state = MIG_STATE_COMPLETED;
-        tcp_cleanup(s);
-    }
-}
-
-static void tcp_connect_migrate(FdMigrationState *s)
-{
-    int ret;
-
-    s->file = qemu_fopen_ops_buffered(s,
-                                      s->bandwidth_limit,
-                                      fd_put_buffer,
-                                      fd_put_ready,
-                                      fd_wait_for_unfreeze,
-                                      fd_close);
-
-    dprintf("beginning savevm\n");
-    ret = qemu_savevm_state_begin(s->file);
-    if (ret < 0) {
-        dprintf("failed, %d\n", ret);
-        tcp_error(s);
-        return;
-    }
-
-    fd_put_ready(s);
-}
 
 static void tcp_wait_for_connect(void *opaque)
 {
@@ -176,60 +60,21 @@ static void tcp_wait_for_connect(void *opaque)
     dprintf("connect completed\n");
     do {
         ret = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &val, &valsize);
-    } while (ret == -1 && socket_error() == EINTR);
+    } while (ret == -1 && (s->get_error(s)) == EINTR);
 
     if (ret < 0) {
-        tcp_error(s);
+        migrate_fd_error(s);
         return;
     }
 
     qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
 
     if (val == 0)
-        tcp_connect_migrate(s);
+        migrate_fd_connect(s);
     else {
         dprintf("error connecting %d\n", val);
-        tcp_error(s);
-    }
-}
-
-static FdMigrationState *to_fms(MigrationState *mig_state)
-{
-    return container_of(mig_state, FdMigrationState, mig_state);
-}
-
-static int tcp_get_status(MigrationState *mig_state)
-{
-    FdMigrationState *s = to_fms(mig_state);
-
-    return s->state;
-}
-
-static void tcp_cancel(MigrationState *mig_state)
-{
-    FdMigrationState *s = to_fms(mig_state);
-
-    if (s->state != MIG_STATE_ACTIVE)
-        return;
-
-    dprintf("cancelling migration\n");
-
-    s->state = MIG_STATE_CANCELLED;
-
-    tcp_cleanup(s);
-}
-
-static void tcp_release(MigrationState *mig_state)
-{
-    FdMigrationState *s = to_fms(mig_state);
-
-    dprintf("releasing state\n");
-   
-    if (s->state == MIG_STATE_ACTIVE) {
-        s->state = MIG_STATE_CANCELLED;
-        tcp_cleanup(s);
+        migrate_fd_error(s);
     }
-    free(s);
 }
 
 MigrationState *tcp_start_outgoing_migration(const char *host_port,
@@ -247,9 +92,12 @@ MigrationState *tcp_start_outgoing_migration(const char *host_port,
     if (s == NULL)
         return NULL;
 
-    s->mig_state.cancel = tcp_cancel;
-    s->mig_state.get_status = tcp_get_status;
-    s->mig_state.release = tcp_release;
+    s->get_error = socket_errno;
+    s->write = socket_write;
+    s->close = tcp_close;
+    s->mig_state.cancel = migrate_fd_cancel;
+    s->mig_state.get_status = migrate_fd_get_status;
+    s->mig_state.release = migrate_fd_release;
 
     s->state = MIG_STATE_ACTIVE;
     s->detach = !async;
@@ -271,7 +119,7 @@ MigrationState *tcp_start_outgoing_migration(const char *host_port,
     do {
         ret = connect(s->fd, (struct sockaddr *)&addr, sizeof(addr));
         if (ret == -1)
-            ret = -socket_error();
+            ret = -(s->get_error(s));
 
         if (ret == -EINPROGRESS || ret == -EWOULDBLOCK)
             qemu_set_fd_handler2(s->fd, NULL, NULL, tcp_wait_for_connect, s);
@@ -283,7 +131,7 @@ MigrationState *tcp_start_outgoing_migration(const char *host_port,
         qemu_free(s);
         return NULL;
     } else if (ret >= 0)
-        tcp_connect_migrate(s);
+        migrate_fd_connect(s);
 
     return &s->mig_state;
 }
diff --git a/migration.c b/migration.c
index 9e6c437..0a3ca1c 100644
--- a/migration.c
+++ b/migration.c
@@ -14,6 +14,19 @@
 #include "qemu-common.h"
 #include "migration.h"
 #include "console.h"
+#include "buffered_file.h"
+#include "sysemu.h"
+#include "block.h"
+
+//#define DEBUG_MIGRATION
+
+#ifdef DEBUG_MIGRATION
+#define dprintf(fmt, ...) \
+    do { printf("migration: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define dprintf(fmt, ...) \
+    do { } while (0)
+#endif
 
 /* Migration speed throttling */
 static uint32_t max_throttle = (32 << 20);
@@ -26,6 +39,10 @@ void qemu_start_incoming_migration(const char *uri)
 
     if (strstart(uri, "tcp:", &p))
         tcp_start_incoming_migration(p);
+#if !defined(WIN32)
+    else if (strstart(uri, "exec:", &p))
+        exec_start_incoming_migration(p);
+#endif
     else
         fprintf(stderr, "unknown migration protocol: %s\n", uri);
 }
@@ -37,6 +54,10 @@ void do_migrate(int detach, const char *uri)
 
     if (strstart(uri, "tcp:", &p))
         s = tcp_start_outgoing_migration(p, max_throttle, detach);
+#if !defined(WIN32)
+    else if (strstart(uri, "exec:", &p))
+        s = exec_start_outgoing_migration(p, max_throttle, detach);
+#endif
     else
         term_printf("unknown migration protocol: %s\n", uri);
 
@@ -101,3 +122,159 @@ void do_info_migrate(void)
     }
 }
 
+/* shared migration helpers */
+
+void migrate_fd_error(FdMigrationState *s)
+{
+    dprintf("setting error state\n");
+    s->state = MIG_STATE_ERROR;
+    migrate_fd_cleanup(s);
+}
+
+void migrate_fd_cleanup(FdMigrationState *s)
+{
+    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
+
+    if (s->file) {
+        dprintf("closing file\n");
+        qemu_fclose(s->file);
+    }
+
+    if (s->fd != -1)
+        close(s->fd);
+
+    /* Don't resume monitor until we've flushed all of the buffers */
+    if (s->detach == 2) {
+        monitor_resume();
+        s->detach = 0;
+    }
+
+    s->fd = -1;
+}
+
+void migrate_fd_put_notify(void *opaque)
+{
+    FdMigrationState *s = opaque;
+
+    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
+    qemu_file_put_notify(s->file);
+}
+
+ssize_t migrate_fd_put_buffer(void *opaque, const void *data, size_t size)
+{
+    FdMigrationState *s = opaque;
+    ssize_t ret;
+
+    do {
+        ret = s->write(s, data, size);
+    } while (ret == -1 && ((s->get_error(s)) == EINTR || (s->get_error(s)) == EWOULDBLOCK));
+
+    if (ret == -1)
+        ret = -(s->get_error(s));
+
+    if (ret == -EAGAIN)
+        qemu_set_fd_handler2(s->fd, NULL, NULL, migrate_fd_put_notify, s);
+
+    return ret;
+}
+
+void migrate_fd_connect(FdMigrationState *s)
+{
+    int ret;
+
+    s->file = qemu_fopen_ops_buffered(s,
+                                      s->bandwidth_limit,
+                                      migrate_fd_put_buffer,
+                                      migrate_fd_put_ready,
+                                      migrate_fd_wait_for_unfreeze,
+                                      migrate_fd_close);
+
+    dprintf("beginning savevm\n");
+    ret = qemu_savevm_state_begin(s->file);
+    if (ret < 0) {
+        dprintf("failed, %d\n", ret);
+        migrate_fd_error(s);
+        return;
+    }
+
+    migrate_fd_put_ready(s);
+}
+
+void migrate_fd_put_ready(void *opaque)
+{
+    FdMigrationState *s = opaque;
+
+    if (s->state != MIG_STATE_ACTIVE) {
+        dprintf("put_ready returning because of non-active state\n");
+        return;
+    }
+
+    dprintf("iterate\n");
+    if (qemu_savevm_state_iterate(s->file) == 1) {
+        dprintf("done iterating\n");
+        vm_stop(0);
+
+        bdrv_flush_all();
+        qemu_savevm_state_complete(s->file);
+        s->state = MIG_STATE_COMPLETED;
+        migrate_fd_cleanup(s);
+    }
+}
+
+int migrate_fd_get_status(MigrationState *mig_state)
+{
+    FdMigrationState *s = migrate_to_fms(mig_state);
+    return s->state;
+}
+
+void migrate_fd_cancel(MigrationState *mig_state)
+{
+    FdMigrationState *s = migrate_to_fms(mig_state);
+
+    if (s->state != MIG_STATE_ACTIVE)
+        return;
+
+    dprintf("cancelling migration\n");
+
+    s->state = MIG_STATE_CANCELLED;
+
+    migrate_fd_cleanup(s);
+}
+
+void migrate_fd_release(MigrationState *mig_state)
+{
+    FdMigrationState *s = migrate_to_fms(mig_state);
+
+    dprintf("releasing state\n");
+   
+    if (s->state == MIG_STATE_ACTIVE) {
+        s->state = MIG_STATE_CANCELLED;
+        migrate_fd_cleanup(s);
+    }
+    free(s);
+}
+
+void migrate_fd_wait_for_unfreeze(void *opaque)
+{
+    FdMigrationState *s = opaque;
+    int ret;
+
+    dprintf("wait for unfreeze\n");
+    if (s->state != MIG_STATE_ACTIVE)
+        return;
+
+    do {
+        fd_set wfds;
+
+        FD_ZERO(&wfds);
+        FD_SET(s->fd, &wfds);
+
+        ret = select(s->fd + 1, NULL, &wfds, NULL, NULL);
+    } while (ret == -1 && (s->get_error(s)) == EINTR);
+}
+
+int migrate_fd_close(void *opaque)
+{
+    FdMigrationState *s = opaque;
+    return s->close(s);
+}
diff --git a/migration.h b/migration.h
index 9947f6a..953ec70 100644
--- a/migration.h
+++ b/migration.h
@@ -29,6 +29,22 @@ struct MigrationState
     void (*release)(MigrationState *s);
 };
 
+typedef struct FdMigrationState FdMigrationState;
+
+struct FdMigrationState
+{
+    MigrationState mig_state;
+    int64_t bandwidth_limit;
+    QEMUFile *file;
+    int fd;
+    int detach;
+    int state;
+    int (*get_error)(struct FdMigrationState*);
+    int (*close)(struct FdMigrationState*);
+    int (*write)(struct FdMigrationState*, const void *, size_t);
+    void *opaque;
+};
+
 void qemu_start_incoming_migration(const char *uri);
 
 void do_migrate(int detach, const char *uri);
@@ -39,11 +55,44 @@ void do_migrate_set_speed(const char *value);
 
 void do_info_migrate(void);
 
+int exec_start_incoming_migration(const char *host_port);
+
+MigrationState *exec_start_outgoing_migration(const char *host_port,
+					     int64_t bandwidth_limit,
+					     int detach);
+
 int tcp_start_incoming_migration(const char *host_port);
 
 MigrationState *tcp_start_outgoing_migration(const char *host_port,
 					     int64_t bandwidth_limit,
 					     int detach);
 
+void migrate_fd_error(FdMigrationState *s);
+
+void migrate_fd_cleanup(FdMigrationState *s);
+
+void migrate_fd_put_notify(void *opaque);
+
+ssize_t migrate_fd_put_buffer(void *opaque, const void *data, size_t size);
+
+void migrate_fd_connect(FdMigrationState *s);
+
+void migrate_fd_put_ready(void *opaque);
+
+int migrate_fd_get_status(MigrationState *mig_state);
+
+void migrate_fd_cancel(MigrationState *mig_state);
+
+void migrate_fd_release(MigrationState *mig_state);
+
+void migrate_fd_wait_for_unfreeze(void *opaque);
+
+int migrate_fd_close(void *opaque);
+
+static inline FdMigrationState *migrate_to_fms(MigrationState *mig_state)
+{
+    return container_of(mig_state, FdMigrationState, mig_state);
+}
+
 #endif
 
diff --git a/vl.c b/vl.c
index 31dd3d7..b49e694 100644
--- a/vl.c
+++ b/vl.c
@@ -2816,6 +2816,12 @@ struct QEMUFile {
     int has_error;
 };
 
+typedef struct QEMUFilePopen
+{
+    FILE *popen_file;
+    QEMUFile *file;
+} QEMUFilePopen;
+
 typedef struct QEMUFileSocket
 {
     int fd;
@@ -2844,6 +2850,64 @@ static int socket_close(void *opaque)
     return 0;
 }
 
+static int popen_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
+{
+    QEMUFilePopen *s = opaque;
+    return fwrite(buf, 1, size, s->popen_file);
+}
+
+static int popen_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+{
+    QEMUFilePopen *s = opaque;
+    return fread(buf, 1, size, s->popen_file);
+}
+
+static int popen_close(void *opaque)
+{
+    QEMUFilePopen *s = opaque;
+    pclose(s->popen_file);
+    qemu_free(s);
+    return 0;
+}
+
+QEMUFile *qemu_popen(FILE *popen_file, const char *mode)
+{
+    QEMUFilePopen *s;
+
+    if (popen_file == NULL || mode == NULL || (mode[0] != 'r' && mode[0] != 'w') || mode[1] != 0) {
+        fprintf(stderr, "qemu_popen: Argument validity check failed\n");
+        return NULL;
+    }
+
+    s = qemu_mallocz(sizeof(QEMUFilePopen));
+    if (!s) {
+        fprintf(stderr, "qemu_popen: malloc failed\n");
+        return NULL;
+    }
+
+    s->popen_file = popen_file;
+
+    if(mode[0] == 'r') {
+        s->file = qemu_fopen_ops(s, NULL, popen_get_buffer, popen_close, NULL);
+    } else {
+        s->file = qemu_fopen_ops(s, popen_put_buffer, NULL, popen_close, NULL);
+    }
+    fprintf(stderr, "qemu_popen: returning result of qemu_fopen_ops\n");
+    return s->file;
+}
+
+QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
+{
+    FILE *popen_file;
+
+    popen_file = popen(command, mode);
+    if(popen_file == NULL) {
+        return NULL;
+    }
+
+    return qemu_popen(popen_file, mode);
+}
+
 QEMUFile *qemu_fopen_socket(int fd)
 {
     QEMUFileSocket *s = qemu_mallocz(sizeof(QEMUFileSocket));