[Qemu-devel] [RFC PATCH 0/6] Run incoming migration in a coroutine

[Qemu-devel] [RFC PATCH 0/6] Run incoming migration in a coroutine

[Paolo Bonzini]

Howdy,

this series moves incoming migration to a coroutine, so that the monitor
remains usable.  I did this is as a step towards implementing an NBD
server inside QEMU (which needs to configured and/or to serve requests
during migration), but I think it is useful in general.

Coroutines work very simply by making the file descriptor non-blocking.
On EAGAIN you call qemu_coroutine_yield(); when the file descriptor
becomes readable, you call qemu_coroutine_enter().

This is mostly an RFC on the approach.  Still, it mostly consists of
cleanups and the first 3 patches could be applied right away.  What
needs some refinement is the hideous passing of file descriptors in
patch 4 and especially the last patch.

Paolo Bonzini (6):
  migration: clean up server sockets and handlers before invoking process_incoming_migration
  migration: close socket QEMUFile from socket_close
  migration: move qemu_fclose to process_incoming_migration
  migration: remove iohandlers before closing the file
  migration: handle EAGAIN while reading QEMUFile
  migration: move process_incoming_migration to a coroutine

 migration-exec.c |  3 +--
 migration-fd.c   |  3 +--
 migration-tcp.c  | 12 ++++++------
 migration-unix.c | 12 ++++++------
 migration.c      | 23 +++++++++++++++++++++--
 migration.h      |  2 +-
 savevm.c         | 34 ++++++++++++++++++++++++++++------
 7 file modificati, 64 inserzioni(+), 25 rimozioni(-)

-- 
1.7.11.2

[Qemu-devel] [PATCH 1/6] migration: clean up server sockets and handlers before invoking process_incoming_migration

[Paolo Bonzini]

We will not accept any other migration from the server socket, so
we can close it.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 migration-exec.c | 2 +-
 migration-fd.c   | 2 +-
 migration-tcp.c  | 7 +++----
 migration-unix.c | 7 +++----
 4 file modificati, 8 inserzioni(+), 10 rimozioni(-)

diff --git a/migration-exec.c b/migration-exec.c
index 6c97db9..061131a 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -97,8 +97,8 @@ static void exec_accept_incoming_migration(void *opaque)
 {
     QEMUFile *f = opaque;
 
-    process_incoming_migration(f);
     qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
+    process_incoming_migration(f);
     qemu_fclose(f);
 }
 
diff --git a/migration-fd.c b/migration-fd.c
index 50138ed..87705c0 100644
--- a/migration-fd.c
+++ b/migration-fd.c
@@ -103,8 +103,8 @@ static void fd_accept_incoming_migration(void *opaque)
 {
     QEMUFile *f = opaque;
 
-    process_incoming_migration(f);
     qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
+    process_incoming_migration(f);
     qemu_fclose(f);
 }
 
diff --git a/migration-tcp.c b/migration-tcp.c
index 440804d..587413e 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -119,12 +119,14 @@ static void tcp_accept_incoming_migration(void *opaque)
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
     } while (c == -1 && socket_error() == EINTR);
+    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
+    close(s);
 
     DPRINTF("accepted migration\n");
 
     if (c == -1) {
         fprintf(stderr, "could not accept migration connection\n");
-        goto out2;
+        goto out;
     }
 
     f = qemu_fopen_socket(c);
@@ -137,9 +139,6 @@ static void tcp_accept_incoming_migration(void *opaque)
     qemu_fclose(f);
 out:
     close(c);
-out2:
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
 }
 
 int tcp_start_incoming_migration(const char *host_port, Error **errp)
diff --git a/migration-unix.c b/migration-unix.c
index 169de88..a407af2 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -129,12 +129,14 @@ static void unix_accept_incoming_migration(void *opaque)
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
     } while (c == -1 && errno == EINTR);
+    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
+    close(s);
 
     DPRINTF("accepted migration\n");
 
     if (c == -1) {
         fprintf(stderr, "could not accept migration connection\n");
-        goto out2;
+        goto out;
     }
 
     f = qemu_fopen_socket(c);
@@ -147,9 +149,6 @@ static void unix_accept_incoming_migration(void *opaque)
     qemu_fclose(f);
 out:
     close(c);
-out2:
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
 }
 
 int unix_start_incoming_migration(const char *path)
-- 
1.7.11.2

[Qemu-devel] [PATCH 2/6] migration: close socket QEMUFile from socket_close

[Paolo Bonzini]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 migration-tcp.c  | 2 ++
 migration-unix.c | 2 ++
 savevm.c         | 1 +
 3 file modificati, 5 inserzioni(+)

diff --git a/migration-tcp.c b/migration-tcp.c
index 587413e..62a6891 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -137,6 +137,8 @@ static void tcp_accept_incoming_migration(void *opaque)
 
     process_incoming_migration(f);
     qemu_fclose(f);
+    return;
+
 out:
     close(c);
 }
diff --git a/migration-unix.c b/migration-unix.c
index a407af2..57aca11 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -147,6 +147,8 @@ static void unix_accept_incoming_migration(void *opaque)
 
     process_incoming_migration(f);
     qemu_fclose(f);
+    return;
+
 out:
     close(c);
 }
diff --git a/savevm.c b/savevm.c
index 6e82b2d..57cae52 100644
--- a/savevm.c
+++ b/savevm.c
@@ -210,6 +210,7 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
 static int socket_close(void *opaque)
 {
     QEMUFileSocket *s = opaque;
+    close(s->fd);
     g_free(s);
     return 0;
 }
-- 
1.7.11.2

[Qemu-devel] [PATCH 3/6] migration: move qemu_fclose to process_incoming_migration

[Paolo Bonzini]

This patch starts moving all tasks coming after process_incoming_migration
to process_incoming_migration itself.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 migration-exec.c | 1 -
 migration-fd.c   | 1 -
 migration-tcp.c  | 1 -
 migration-unix.c | 1 -
 migration.c      | 6 +++++-
 5 file modificati, 5 inserzioni(+), 5 rimozioni(-)

diff --git a/migration-exec.c b/migration-exec.c
index 061131a..84796be 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -99,7 +99,6 @@ static void exec_accept_incoming_migration(void *opaque)
 
     qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
     process_incoming_migration(f);
-    qemu_fclose(f);
 }
 
 int exec_start_incoming_migration(const char *command)
diff --git a/migration-fd.c b/migration-fd.c
index 87705c0..21ad196 100644
--- a/migration-fd.c
+++ b/migration-fd.c
@@ -105,7 +105,6 @@ static void fd_accept_incoming_migration(void *opaque)
 
     qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
     process_incoming_migration(f);
-    qemu_fclose(f);
 }
 
 int fd_start_incoming_migration(const char *infd)
diff --git a/migration-tcp.c b/migration-tcp.c
index 62a6891..62c9309 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -136,7 +136,6 @@ static void tcp_accept_incoming_migration(void *opaque)
     }
 
     process_incoming_migration(f);
-    qemu_fclose(f);
     return;
 
 out:
diff --git a/migration-unix.c b/migration-unix.c
index 57aca11..d8d5dfa 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -146,7 +146,6 @@ static void unix_accept_incoming_migration(void *opaque)
     }
 
     process_incoming_migration(f);
-    qemu_fclose(f);
     return;
 
 out:
diff --git a/migration.c b/migration.c
index 8db1b43..a27ceb3 100644
--- a/migration.c
+++ b/migration.c
@@ -84,7 +84,11 @@ int qemu_start_incoming_migration(const char *uri, Error **errp)
 
 void process_incoming_migration(QEMUFile *f)
 {
-    if (qemu_loadvm_state(f) < 0) {
+    int ret;
+
+    ret = qemu_loadvm_state(f);
+    qemu_fclose(f);
+    if (ret < 0) {
         fprintf(stderr, "load of migration failed\n");
         exit(0);
     }
-- 
1.7.11.2

[Qemu-devel] [PATCH 4/6] migration: remove iohandlers before closing the file

[Paolo Bonzini]

This will be needed as soon as process_incoming_migration will set
handlers on the file.  The patch may be removed if 

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 savevm.c | 3 +++
 1 file modificato, 3 inserzioni(+)

diff --git a/savevm.c b/savevm.c
index 57cae52..8f075e5 100644
--- a/savevm.c
+++ b/savevm.c
@@ -210,6 +210,7 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
 static int socket_close(void *opaque)
 {
     QEMUFileSocket *s = opaque;
+    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
     close(s->fd);
     g_free(s);
     return 0;
@@ -238,6 +239,7 @@ static int stdio_pclose(void *opaque)
 {
     QEMUFileStdio *s = opaque;
     int ret;
+    qemu_set_fd_handler(fileno(s->stdio_file), NULL, NULL, NULL);
     ret = pclose(s->stdio_file);
     if (ret == -1) {
         ret = -errno;
@@ -250,6 +252,7 @@ static int stdio_fclose(void *opaque)
 {
     QEMUFileStdio *s = opaque;
     int ret = 0;
+    qemu_set_fd_handler(fileno(s->stdio_file), NULL, NULL, NULL);
     if (fclose(s->stdio_file) == EOF) {
         ret = -errno;
     }
-- 
1.7.11.2

Re: [Qemu-devel] [PATCH 4/6] migration: remove iohandlers before closing the file

[Anthony Liguori]

Paolo Bonzini <pbonzini@redhat.com> writes:

> This will be needed as soon as process_incoming_migration will set
> handlers on the file.  The patch may be removed if

...?

Regards,

Anthony Liguori
 
>
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  savevm.c | 3 +++
>  1 file modificato, 3 inserzioni(+)
>
> diff --git a/savevm.c b/savevm.c
> index 57cae52..8f075e5 100644
> --- a/savevm.c
> +++ b/savevm.c
> @@ -210,6 +210,7 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
>  static int socket_close(void *opaque)
>  {
>      QEMUFileSocket *s = opaque;
> +    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
>      close(s->fd);
>      g_free(s);
>      return 0;
> @@ -238,6 +239,7 @@ static int stdio_pclose(void *opaque)
>  {
>      QEMUFileStdio *s = opaque;
>      int ret;
> +    qemu_set_fd_handler(fileno(s->stdio_file), NULL, NULL, NULL);
>      ret = pclose(s->stdio_file);
>      if (ret == -1) {
>          ret = -errno;
> @@ -250,6 +252,7 @@ static int stdio_fclose(void *opaque)
>  {
>      QEMUFileStdio *s = opaque;
>      int ret = 0;
> +    qemu_set_fd_handler(fileno(s->stdio_file), NULL, NULL, NULL);
>      if (fclose(s->stdio_file) == EOF) {
>          ret = -errno;
>      }
> -- 
> 1.7.11.2

Re: [Qemu-devel] [PATCH 4/6] migration: remove iohandlers before closing the file

[Paolo Bonzini]

Il 07/08/2012 20:46, Anthony Liguori ha scritto:
> 
>> > This will be needed as soon as process_incoming_migration will set
>> > handlers on the file.  The patch may be removed if
> ...?

... we get access to the file descriptor of a QEMUFile in a better way
(i.e. improve patch 6).

Paolo

[Qemu-devel] [PATCH 5/6] migration: handle EAGAIN while reading QEMUFile

[Paolo Bonzini]

This will never happen right now (the assertion would fail).  The
next patch will set the socket or pipe in non-blocking mode, thus
enabling this part of the code.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 savevm.c | 30 ++++++++++++++++++++++++------
 1 file modificato, 24 inserzioni(+), 6 rimozioni(-)

diff --git a/savevm.c b/savevm.c
index 8f075e5..336b53b 100644
--- a/savevm.c
+++ b/savevm.c
@@ -197,13 +197,22 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
     QEMUFileSocket *s = opaque;
     ssize_t len;
 
-    do {
+    for (;;) {
         len = qemu_recv(s->fd, buf, size, 0);
-    } while (len == -1 && socket_error() == EINTR);
+        if (len != -1) {
+            break;
+        }
+        if (errno == EAGAIN) {
+            assert(qemu_in_coroutine());
+            qemu_coroutine_yield();
+        } else if (errno != EINTR) {
+            break;
+        }
+    }
 
-    if (len == -1)
+    if (len == -1) {
         len = -socket_error();
-
+    }
     return len;
 }
 
@@ -228,10 +237,19 @@ static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
     FILE *fp = s->stdio_file;
     int bytes;
 
-    do {
+    for (;;) {
         clearerr(fp);
         bytes = fread(buf, 1, size, fp);
-    } while ((bytes == 0) && ferror(fp) && (errno == EINTR));
+        if (bytes != 0 || !ferror(fp)) {
+            break;
+        }
+        if (errno == EAGAIN) {
+            assert(qemu_in_coroutine());
+            qemu_coroutine_yield();
+        } else if (errno != EINTR) {
+            break;
+        }
+    }
     return bytes;
 }
 
-- 
1.7.11.2

[Qemu-devel] [PATCH 6/6] migration: move process_incoming_migration to a coroutine

[Paolo Bonzini]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 migration-exec.c |  2 +-
 migration-fd.c   |  2 +-
 migration-tcp.c  |  2 +-
 migration-unix.c |  2 +-
 migration.c      | 17 ++++++++++++++++-
 migration.h      |  2 +-
 6 file modificati, 21 inserzioni(+), 6 rimozioni(-)

diff --git a/migration-exec.c b/migration-exec.c
index 84796be..e8ed144 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -98,7 +98,7 @@ static void exec_accept_incoming_migration(void *opaque)
     QEMUFile *f = opaque;
 
     qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
-    process_incoming_migration(f);
+    process_incoming_migration(f, qemu_stdio_fd(f));
 }
 
 int exec_start_incoming_migration(const char *command)
diff --git a/migration-fd.c b/migration-fd.c
index 21ad196..e9a06d9 100644
--- a/migration-fd.c
+++ b/migration-fd.c
@@ -104,7 +104,7 @@ static void fd_accept_incoming_migration(void *opaque)
     QEMUFile *f = opaque;
 
     qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
-    process_incoming_migration(f);
+    process_incoming_migration(f, qemu_stdio_fd(f));
 }
 
 int fd_start_incoming_migration(const char *infd)
diff --git a/migration-tcp.c b/migration-tcp.c
index 62c9309..765970c 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -135,7 +135,7 @@ static void tcp_accept_incoming_migration(void *opaque)
         goto out;
     }
 
-    process_incoming_migration(f);
+    process_incoming_migration(f, c);
     return;
 
 out:
diff --git a/migration-unix.c b/migration-unix.c
index d8d5dfa..071d4fb 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -145,7 +145,7 @@ static void unix_accept_incoming_migration(void *opaque)
         goto out;
     }
 
-    process_incoming_migration(f);
+    process_incoming_migration(f, c);
     return;
 
 out:
diff --git a/migration.c b/migration.c
index a27ceb3..d821971 100644
--- a/migration.c
+++ b/migration.c
@@ -82,8 +82,9 @@ int qemu_start_incoming_migration(const char *uri, Error **errp)
     return ret;
 }
 
-void process_incoming_migration(QEMUFile *f)
+static void process_incoming_migration_co(void *opaque)
 {
+    QEMUFile *f = opaque;
     int ret;
 
     ret = qemu_loadvm_state(f);
@@ -106,6 +107,20 @@ void process_incoming_migration(QEMUFile *f)
     }
 }
 
+static void enter_migration_coroutine(void *opaque)
+{
+    Coroutine *co = opaque;
+    qemu_coroutine_enter(co, NULL);
+}
+
+void process_incoming_migration(QEMUFile *f, int fd)
+{
+    Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
+    socket_set_nonblock(fd);
+    qemu_set_fd_handler(fd, enter_migration_coroutine, NULL, co);
+    qemu_coroutine_enter(co, f);
+}
+
 /* amount of nanoseconds we are willing to wait for migration to be down.
  * the choice of nanoseconds is because it is the maximum resolution that
  * get_clock() can achieve. It is an internal measure. All user-visible
diff --git a/migration.h b/migration.h
index 57572a6..4a35ea7 100644
--- a/migration.h
+++ b/migration.h
@@ -41,7 +41,7 @@ struct MigrationState
     int64_t total_time;
 };
 
-void process_incoming_migration(QEMUFile *f);
+void process_incoming_migration(QEMUFile *f, int fd);
 
 int qemu_start_incoming_migration(const char *uri, Error **errp);
 
-- 
1.7.11.2