[Qemu-devel] [PATCH] add file: migration support (r2)

[Qemu-devel] [PATCH] add file: migration support (r2)

[Charles Duffy]

[-- Attachment #1: Type: text/plain, Size: 403 bytes --]

This patch adds support for migration to and from file: targets, moves 
common helpers between exec: and file: use cases from migration-exec.c 
to migration.c, and adds a qemu_fdopen() helper parallel to qemu_fopen().

Compared to the previous version submitted, it removes some cruft which 
was accidentally included, and documents the limitations of O_NONBLOCK 
on Linux with local filesystem access.

[-- Attachment #2: qemu-live_migration_file_support-r2.patch --]
[-- Type: text/x-patch, Size: 8770 bytes --]

diff --git a/Makefile b/Makefile
index 3ed8b2a..a5db522 100644
--- a/Makefile
+++ b/Makefile
@@ -78,8 +78,8 @@ OBJS+=usb.o usb-hub.o usb-linux.o usb-hid.o usb-msd.o usb-wacom.o
 OBJS+=usb-serial.o usb-net.o
 OBJS+=sd.o ssi-sd.o
 OBJS+=bt.o bt-host.o bt-vhci.o bt-l2cap.o bt-sdp.o bt-hci.o bt-hid.o usb-bt.o
-OBJS+=buffered_file.o migration.o migration-tcp.o net.o qemu-sockets.o
-OBJS+=qemu-char.o aio.o net-checksum.o savevm.o
+OBJS+=buffered_file.o migration.o migration-file.o migration-tcp.o net.o
+OBJS+=qemu-sockets.o qemu-char.o aio.o net-checksum.o savevm.o
 
 ifdef CONFIG_BRLAPI
 OBJS+= baum.o
diff --git a/hw/hw.h b/hw/hw.h
index eab7bb4..4d5ae42 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -35,6 +35,7 @@ QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc *put_buffer,
                          QEMUFileRateLimit *rate_limit);
 QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd);
+QEMUFile *qemu_fdopen(int fd, const char *mode);
 QEMUFile *qemu_popen(FILE *popen_file, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
 void qemu_fflush(QEMUFile *f);
diff --git a/migration-exec.c b/migration-exec.c
index caeed4b..f625689 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -32,16 +32,6 @@
     do { } while (0)
 #endif
 
-static int file_errno(FdMigrationState *s)
-{
-    return errno;
-}
-
-static int file_write(FdMigrationState *s, const void * buf, size_t size)
-{
-    return write(s->fd, buf, size);
-}
-
 static int exec_close(FdMigrationState *s)
 {
     dprintf("exec_close\n");
@@ -86,8 +76,8 @@ MigrationState *exec_start_outgoing_migration(const char *command,
     s->opaque = qemu_popen(f, "w");
 
     s->close = exec_close;
-    s->get_error = file_errno;
-    s->write = file_write;
+    s->get_error = migrate_file_errno;
+    s->write = migrate_file_write;
     s->mig_state.cancel = migrate_fd_cancel;
     s->mig_state.get_status = migrate_fd_get_status;
     s->mig_state.release = migrate_fd_release;
diff --git a/migration-file.c b/migration-file.c
new file mode 100644
index 0000000..1b90c78
--- /dev/null
+++ b/migration-file.c
@@ -0,0 +1,118 @@
+/*
+ * QEMU live migration
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Dell MessageOne 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Charles Duffy     <charles_duffy@messageone.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "qemu_socket.h"
+#include "migration.h"
+#include "qemu-char.h"
+#include "sysemu.h"
+#include "console.h"
+#include "buffered_file.h"
+#include "block.h"
+
+//#define DEBUG_MIGRATION_FILE
+
+#ifdef DEBUG_MIGRATION_FILE
+#define dprintf(fmt, ...) \
+    do { printf("migration-file: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define dprintf(fmt, ...) \
+    do { } while (0)
+#endif
+
+MigrationState *file_start_outgoing_migration(const char *filename,
+                                             int64_t bandwidth_limit,
+                                             int async)
+{
+    FdMigrationState *s;
+
+    s = qemu_mallocz(sizeof(*s));
+    if (s == NULL) {
+        dprintf("Unable to allocate FdMigrationState\n");
+        goto err;
+    }
+
+    /* Note that while O_NONBLOCK is given here, writes to a standard file on
+     * Linux will not return EAGAIN; thus, migration with a sufficiently high
+     * target speed set will effectively pause qemu during migration.
+     * O_NONBLOCK is still used here, such that writes to a named pipe or
+     * similar object may return EAGAIN if appropriate.
+     */
+    s->fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC|O_NONBLOCK, 0666);
+    if (s->fd == -1) {
+        perror("Unable to open migration target");
+        goto err_after_alloc;
+    }
+
+    s->close = migrate_file_close;
+    s->get_error = migrate_file_errno;
+    s->write = migrate_file_write;
+    s->mig_state.cancel = migrate_fd_cancel;
+    s->mig_state.get_status = migrate_fd_get_status;
+    s->mig_state.release = migrate_fd_release;
+
+    s->state = MIG_STATE_ACTIVE;
+    s->detach = !async;
+    s->bandwidth_limit = bandwidth_limit;
+
+    if (s->detach == 1) {
+        dprintf("detaching from monitor\n");
+        monitor_suspend();
+        s->detach = 2;
+    }
+
+    migrate_fd_connect(s);
+    return &s->mig_state;
+
+err_after_alloc:
+    qemu_free(s);
+err:
+    return NULL;
+}
+
+int file_start_incoming_migration(const char *filename)
+{
+    int fd, ret;
+    QEMUFile *f;
+
+    dprintf("Attempting to start an incoming migration\n");
+    fd = open(filename, O_RDONLY);
+    if(fd < 0) {
+        perror("Unable to open migration source");
+        return -errno;
+    }
+    f = qemu_fdopen(fd, "rb");
+    if(f == NULL) {
+        dprintf("unable to wrap file descriptor for incoming migration\n");
+        ret = -errno;
+        close(fd);
+        return ret;
+    }
+    vm_stop(0); /* just in case */
+    ret = qemu_loadvm_state(f);
+    if (ret < 0) {
+        fprintf(stderr, "load of migration failed\n");
+        goto err;
+    }
+    qemu_announce_self();
+    dprintf("successfully loaded vm state\n");
+    vm_start();
+    qemu_fclose(f);
+    return 0;
+
+err:
+    qemu_fclose(f);
+    return -errno;
+}
diff --git a/migration.c b/migration.c
index 0ef777a..709ee49 100644
--- a/migration.c
+++ b/migration.c
@@ -40,6 +40,8 @@ void qemu_start_incoming_migration(const char *uri)
 
     if (strstart(uri, "tcp:", &p))
         tcp_start_incoming_migration(p);
+    else if (strstart(uri, "file:", &p))
+        file_start_incoming_migration(p);
 #if !defined(WIN32)
     else if (strstart(uri, "exec:", &p))
         exec_start_incoming_migration(p);
@@ -55,6 +57,8 @@ void do_migrate(int detach, const char *uri)
 
     if (strstart(uri, "tcp:", &p))
         s = tcp_start_outgoing_migration(p, max_throttle, detach);
+    else if (strstart(uri, "file:", &p))
+        s = file_start_outgoing_migration(p, max_throttle, detach);
 #if !defined(WIN32)
     else if (strstart(uri, "exec:", &p))
         s = exec_start_outgoing_migration(p, max_throttle, detach);
@@ -279,3 +283,18 @@ int migrate_fd_close(void *opaque)
     FdMigrationState *s = opaque;
     return s->close(s);
 }
+
+int migrate_file_errno(FdMigrationState *s)
+{
+    return errno;
+}
+
+int migrate_file_write(FdMigrationState *s, const void * buf, size_t size)
+{
+    return write(s->fd, buf, size);
+}
+
+int migrate_file_close(FdMigrationState *s)
+{
+    return close(s->fd);
+}
diff --git a/migration.h b/migration.h
index 953ec70..dcbda39 100644
--- a/migration.h
+++ b/migration.h
@@ -61,6 +61,12 @@ MigrationState *exec_start_outgoing_migration(const char *host_port,
 					     int64_t bandwidth_limit,
 					     int detach);
 
+int file_start_incoming_migration(const char *filename);
+
+MigrationState *file_start_outgoing_migration(const char *filename,
+                                             int64_t bandwidth_limit,
+                                             int async);
+
 int tcp_start_incoming_migration(const char *host_port);
 
 MigrationState *tcp_start_outgoing_migration(const char *host_port,
@@ -89,6 +95,12 @@ void migrate_fd_wait_for_unfreeze(void *opaque);
 
 int migrate_fd_close(void *opaque);
 
+int migrate_file_errno(FdMigrationState *s);
+
+int migrate_file_write(FdMigrationState *s, const void * buf, size_t size);
+
+int migrate_file_close(FdMigrationState *s);
+
 static inline FdMigrationState *migrate_to_fms(MigrationState *mig_state)
 {
     return container_of(mig_state, FdMigrationState, mig_state);
diff --git a/savevm.c b/savevm.c
index 1ee5a6f..8f5fd16 100644
--- a/savevm.c
+++ b/savevm.c
@@ -226,7 +226,6 @@ QEMUFile *qemu_popen(FILE *popen_file, const char *mode)
     } else {
         s->file = qemu_fopen_ops(s, popen_put_buffer, NULL, popen_close, NULL);
     }
-    fprintf(stderr, "qemu_popen: returning result of qemu_fopen_ops\n");
     return s->file;
 }
 
@@ -283,6 +282,30 @@ static int file_close(void *opaque)
     return 0;
 }
 
+QEMUFile *qemu_fdopen(int fd, const char *mode)
+{
+    QEMUFileStdio *s;
+
+    s = qemu_mallocz(sizeof(QEMUFileStdio));
+    if (!s)
+        return NULL;
+
+    s->outfile = fdopen(fd, mode);
+    if (!s->outfile)
+        goto fail;
+
+    if (!strcmp(mode, "wb"))
+        return qemu_fopen_ops(s, file_put_buffer, NULL, file_close, NULL);
+    else if (!strcmp(mode, "rb"))
+        return qemu_fopen_ops(s, NULL, file_get_buffer, file_close, NULL);
+
+fail:
+    if (s->outfile)
+        fclose(s->outfile);
+    qemu_free(s);
+    return NULL;
+}
+
 QEMUFile *qemu_fopen(const char *filename, const char *mode)
 {
     QEMUFileStdio *s;

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Jamie Lokier]

Charles Duffy wrote:
> documents the limitations of O_NONBLOCK on Linux with local
> filesystem access.

That's not a Linux limitation.

All OSes ignore O_NONBLOCK for filesystem accesses, including remote
filesystems.  This is a POSIX thing.

The reason is that there's no way to select() on a file.  (You'd need
to select() on readiness to read/write a specific byte range anyway,
so the interface doesn't work.)

The nearest equivalent to select() on file I/O is AIO, which is
probably what should be used if migration file I/O is blocking QEMU.

-- Jamie

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Paul Brook]

On Thursday 13 November 2008, Charles Duffy wrote:
> This patch adds support for migration to and from file: targets, moves
> common helpers between exec: and file: use cases from migration-exec.c
> to migration.c, and adds a qemu_fdopen() helper parallel to qemu_fopen().
>
> Compared to the previous version submitted, it removes some cruft which
> was accidentally included, and documents the limitations of O_NONBLOCK
> on Linux with local filesystem access.

I don't see any documentation.

The documentation should make it clear that this is not intended as a 
substitute for savvm/loadvm. A "live" migration doesn't make sense if you're 
writing to an actual file (rather than a FIFO) as you'll end up with 
unecessarily large files.

Paul

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Anthony Liguori]

Paul Brook wrote:
> On Thursday 13 November 2008, Charles Duffy wrote:
>   
>> This patch adds support for migration to and from file: targets, moves
>> common helpers between exec: and file: use cases from migration-exec.c
>> to migration.c, and adds a qemu_fdopen() helper parallel to qemu_fopen().
>>
>> Compared to the previous version submitted, it removes some cruft which
>> was accidentally included, and documents the limitations of O_NONBLOCK
>> on Linux with local filesystem access.
>>     
>
> I don't see any documentation.
>
> The documentation should make it clear that this is not intended as a 
> substitute for savvm/loadvm. A "live" migration doesn't make sense if you're 
> writing to an actual file (rather than a FIFO) as you'll end up with 
> unecessarily large files.
>   

Forget the large file, it's not "live" which makes it not useful 
compared with savevm/loadvm.  Using rate limiting as a crutch to avoid 
this is wrong because it requires the user to know how much bandwidth 
QEMU can use before blocking.

To do live migration to a file, you need to tie into something like 
posix-aio.  You could actually use a bdrv_raw and expand it like we do 
for a qcow file although that's sort of an abuse of the block driver API.

Regards,

Anthony Liguori

> Paul
>
>
>   

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Jamie Lokier]

Paul Brook wrote:
> On Thursday 13 November 2008, Charles Duffy wrote:
> > This patch adds support for migration to and from file: targets, moves
> > common helpers between exec: and file: use cases from migration-exec.c
> > to migration.c, and adds a qemu_fdopen() helper parallel to qemu_fopen().
> >
> > Compared to the previous version submitted, it removes some cruft which
> > was accidentally included, and documents the limitations of O_NONBLOCK
> > on Linux with local filesystem access.
> 
> I don't see any documentation.
> 
> The documentation should make it clear that this is not intended as a 
> substitute for savvm/loadvm.

> A "live" migration doesn't make sense if you're writing to an actual
> file (rather than a FIFO) as you'll end up with unecessarily large
> files.

That's true.  It would be better to use a slightly different format
when writing to a file, which lays out the RAM linearly and overwrites
re-dirtied pages during the migration.

I'm not sure if "live" migration to a file is really useful.  "Dead"
migration is useful for suspending a VM to disk.

-- Jamie

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Paul Brook]

> I'm not sure if "live" migration to a file is really useful.  "Dead"
> migration is useful for suspending a VM to disk.

Right. That's the point I was trying to make.

Paul

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Jamie Lokier]

Paul Brook wrote:
> > I'm not sure if "live" migration to a file is really useful.  "Dead"
> > migration is useful for suspending a VM to disk.
> 
> Right. That's the point I was trying to make.

I thought of a use for live migration to and from a file.  It's when
you have to restart a host while minimising downtime of all its
guests.  The liveness reduces that downtime.  That could be rebooting,
or simply replacing QEMU/KVM by a newer version.  It would require a
lot of guest memory (collectively), to be worth it.

-- Jamie

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Paul Brook]

On Tuesday 18 November 2008, Jamie Lokier wrote:
> Paul Brook wrote:
> > > I'm not sure if "live" migration to a file is really useful.  "Dead"
> > > migration is useful for suspending a VM to disk.
> >
> > Right. That's the point I was trying to make.
>
> I thought of a use for live migration to and from a file.  It's when
> you have to restart a host while minimising downtime of all its
> guests.  The liveness reduces that downtime.

I wouldn't be surprised if this actually took longer, because you have to read 
more data in when you do the restore.

Paul

Re: [Qemu-devel] [PATCH] add file: migration support (r2)

[Anthony Liguori]

Paul Brook wrote:
> On Tuesday 18 November 2008, Jamie Lokier wrote:
>   
>> Paul Brook wrote:
>>     
>>>> I'm not sure if "live" migration to a file is really useful.  "Dead"
>>>> migration is useful for suspending a VM to disk.
>>>>         
>>> Right. That's the point I was trying to make.
>>>       
>> I thought of a use for live migration to and from a file.  It's when
>> you have to restart a host while minimising downtime of all its
>> guests.  The liveness reduces that downtime.
>>     
>
> I wouldn't be surprised if this actually took longer, because you have to read 
> more data in when you do the restore.
>   

file: migration is essentially live checkpointing.  It takes longer to 
save but the key is that you do the checkpointing without incurring down 
time.

The real use case would be live migrating to a file (or BDRV) and then 
immediately resuming the source VM after the migration has completed.  
The effect would be live checkpointing.

But without an implementation of file: or bdrv: that are actually live, 
it really is just an academic conversation :-)

Regards,

Anthony Liguori

> Paul
>
>
>