[Qemu-devel] [PULL 0/5] Migration pull request

[Qemu-devel] [PULL 0/5] Migration pull request

[Juan Quintela]

Hi

This are the migration patches ready for inclusion:
- autoconverge improvements: (felipe franciosi)
- create savem.h

Please, Apply.

The following changes since commit 0748b3526e8cb78b9cd64208426bfc3d54a72b04:

  Merge remote-tracking branch 'kwolf/tags/for-upstream' into staging (2017-05-30 14:15:15 +0100)

are available in the git repository at:

  git://github.com/juanquintela/qemu.git tags/migration/20170531

for you to fetch changes up to b4a3c64b16856a018869bfd4a9ed3b2a74554541:

  migration: use dirty_rate_high_cnt more aggressively (2017-05-31 09:39:20 +0200)

----------------------------------------------------------------
migration/next for 20170531

----------------------------------------------------------------
Felipe Franciosi (4):
      migration: keep bytes_xfer_prev init'd to zero
      migration: set dirty_pages_rate before autoconverge logic
      migration: set bytes_xfer_* outside of autoconverge logic
      migration: use dirty_rate_high_cnt more aggressively

Juan Quintela (1):
      migration: Create savevm.h for functions exported from savevm.c

 include/sysemu/sysemu.h  | 47 -----------------------------------------------
 migration/colo.c         |  1 +
 migration/migration.c    |  1 +
 migration/postcopy-ram.c |  1 +
 migration/ram.c          | 23 +++++++++++------------
 migration/savevm.c       | 29 +++++++++++++++++++++++++----
 migration/savevm.h       | 41 +++++++++++++++++++++++++++++++++++++++++
 7 files changed, 80 insertions(+), 63 deletions(-)
 create mode 100644 migration/savevm.h

[Qemu-devel] [PULL 1/5] migration: Create savevm.h for functions exported from savevm.c

[Juan Quintela]

This removes last trace of migration functions from sysemu/sysemu.h.

Signed-off-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Laurent Vivier <lvivier@redhat.com>
---
 include/sysemu/sysemu.h  | 47 -----------------------------------------------
 migration/colo.c         |  1 +
 migration/migration.c    |  1 +
 migration/postcopy-ram.c |  1 +
 migration/savevm.c       | 29 +++++++++++++++++++++++++----
 migration/savevm.h       | 41 +++++++++++++++++++++++++++++++++++++++++
 6 files changed, 69 insertions(+), 51 deletions(-)
 create mode 100644 migration/savevm.h

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 69046eb..723c8dc 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -97,53 +97,6 @@ int load_vmstate(const char *name, Error **errp);
 
 void qemu_announce_self(void);
 
-/* Subcommands for QEMU_VM_COMMAND */
-enum qemu_vm_cmd {
-    MIG_CMD_INVALID = 0,   /* Must be 0 */
-    MIG_CMD_OPEN_RETURN_PATH,  /* Tell the dest to open the Return path */
-    MIG_CMD_PING,              /* Request a PONG on the RP */
-
-    MIG_CMD_POSTCOPY_ADVISE,       /* Prior to any page transfers, just
-                                      warn we might want to do PC */
-    MIG_CMD_POSTCOPY_LISTEN,       /* Start listening for incoming
-                                      pages as it's running. */
-    MIG_CMD_POSTCOPY_RUN,          /* Start execution */
-
-    MIG_CMD_POSTCOPY_RAM_DISCARD,  /* A list of pages to discard that
-                                      were previously sent during
-                                      precopy but are dirty. */
-    MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
-    MIG_CMD_MAX
-};
-
-#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24)
-
-bool qemu_savevm_state_blocked(Error **errp);
-void qemu_savevm_state_begin(QEMUFile *f);
-void qemu_savevm_state_header(QEMUFile *f);
-int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy);
-void qemu_savevm_state_cleanup(void);
-void qemu_savevm_state_complete_postcopy(QEMUFile *f);
-void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only);
-void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
-                               uint64_t *res_non_postcopiable,
-                               uint64_t *res_postcopiable);
-void qemu_savevm_command_send(QEMUFile *f, enum qemu_vm_cmd command,
-                              uint16_t len, uint8_t *data);
-void qemu_savevm_send_ping(QEMUFile *f, uint32_t value);
-void qemu_savevm_send_open_return_path(QEMUFile *f);
-int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len);
-void qemu_savevm_send_postcopy_advise(QEMUFile *f);
-void qemu_savevm_send_postcopy_listen(QEMUFile *f);
-void qemu_savevm_send_postcopy_run(QEMUFile *f);
-
-void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
-                                           uint16_t len,
-                                           uint64_t *start_list,
-                                           uint64_t *length_list);
-
-int qemu_loadvm_state(QEMUFile *f);
-
 extern int autostart;
 
 typedef enum {
diff --git a/migration/colo.c b/migration/colo.c
index 8c13a3c..3dd1390 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -16,6 +16,7 @@
 #include "qemu-file-channel.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
+#include "savevm.h"
 #include "migration/colo.h"
 #include "migration/block.h"
 #include "io/channel-buffer.h"
diff --git a/migration/migration.c b/migration/migration.c
index ad29e53..7087d1a 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -19,6 +19,7 @@
 #include "qemu/main-loop.h"
 #include "migration/blocker.h"
 #include "migration/migration.h"
+#include "savevm.h"
 #include "qemu-file-channel.h"
 #include "migration/qemu-file.h"
 #include "migration/vmstate.h"
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
index a37620d..3f9ae1b 100644
--- a/migration/postcopy-ram.c
+++ b/migration/postcopy-ram.c
@@ -22,6 +22,7 @@
 #include "exec/target_page.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
+#include "savevm.h"
 #include "postcopy-ram.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/balloon.h"
diff --git a/migration/savevm.c b/migration/savevm.c
index a4532b6..a2d4f9c 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -36,6 +36,7 @@
 #include "qemu/timer.h"
 #include "migration/migration.h"
 #include "qemu-file-channel.h"
+#include "savevm.h"
 #include "postcopy-ram.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
@@ -63,6 +64,26 @@ const unsigned int postcopy_ram_discard_version = 0;
 
 static bool skip_section_footers;
 
+/* Subcommands for QEMU_VM_COMMAND */
+enum qemu_vm_cmd {
+    MIG_CMD_INVALID = 0,   /* Must be 0 */
+    MIG_CMD_OPEN_RETURN_PATH,  /* Tell the dest to open the Return path */
+    MIG_CMD_PING,              /* Request a PONG on the RP */
+
+    MIG_CMD_POSTCOPY_ADVISE,       /* Prior to any page transfers, just
+                                      warn we might want to do PC */
+    MIG_CMD_POSTCOPY_LISTEN,       /* Start listening for incoming
+                                      pages as it's running. */
+    MIG_CMD_POSTCOPY_RUN,          /* Start execution */
+
+    MIG_CMD_POSTCOPY_RAM_DISCARD,  /* A list of pages to discard that
+                                      were previously sent during
+                                      precopy but are dirty. */
+    MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
+    MIG_CMD_MAX
+};
+
+#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24)
 static struct mig_cmd_args {
     ssize_t     len; /* -1 = variable */
     const char *name;
@@ -807,10 +828,10 @@ static void save_section_footer(QEMUFile *f, SaveStateEntry *se)
  * @len: Length of associated data
  * @data: Data associated with command.
  */
-void qemu_savevm_command_send(QEMUFile *f,
-                              enum qemu_vm_cmd command,
-                              uint16_t len,
-                              uint8_t *data)
+static void qemu_savevm_command_send(QEMUFile *f,
+                                     enum qemu_vm_cmd command,
+                                     uint16_t len,
+                                     uint8_t *data)
 {
     trace_savevm_command_send(command, len);
     qemu_put_byte(f, QEMU_VM_COMMAND);
diff --git a/migration/savevm.h b/migration/savevm.h
new file mode 100644
index 0000000..eb44877
--- /dev/null
+++ b/migration/savevm.h
@@ -0,0 +1,41 @@
+/*
+ * QEMU save vm functions
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009-2017 Red Hat Inc
+ *
+ * Authors:
+ *  Juan Quintela <quintela@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef MIGRATION_SAVEVM_H
+#define MIGRATION_SAVEVM_H
+
+bool qemu_savevm_state_blocked(Error **errp);
+void qemu_savevm_state_begin(QEMUFile *f);
+void qemu_savevm_state_header(QEMUFile *f);
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy);
+void qemu_savevm_state_cleanup(void);
+void qemu_savevm_state_complete_postcopy(QEMUFile *f);
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only);
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable);
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value);
+void qemu_savevm_send_open_return_path(QEMUFile *f);
+int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len);
+void qemu_savevm_send_postcopy_advise(QEMUFile *f);
+void qemu_savevm_send_postcopy_listen(QEMUFile *f);
+void qemu_savevm_send_postcopy_run(QEMUFile *f);
+
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list);
+
+int qemu_loadvm_state(QEMUFile *f);
+
+#endif
-- 
2.9.4

[Qemu-devel] [PULL 2/5] migration: keep bytes_xfer_prev init'd to zero

[Juan Quintela]

From: Felipe Franciosi <felipe@nutanix.com>

The first time migration_bitmap_sync() is called, bytes_xfer_prev is set
to ram_state.bytes_transferred which is, at this point, zero. The next
time migration_bitmap_sync() is called, an iteration has happened and
bytes_xfer_prev is set to 'x' bytes. Most likely, more than one second
has passed, so the auto converge logic will be triggered and
bytes_xfer_now will also be set to 'x' bytes.

This condition is currently masked by dirty_rate_high_cnt, which will
wait for a few iterations before throttling. It would otherwise always
assume zero bytes have been copied and therefore throttle the guest
(possibly) prematurely.

Given bytes_xfer_prev is only used by the auto convergence logic, it
makes sense to only set its value after a check has been made against
bytes_xfer_now.

Signed-off-by: Felipe Franciosi <felipe@nutanix.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
 migration/ram.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/migration/ram.c b/migration/ram.c
index c07a9c0..36bf720 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -673,10 +673,6 @@ static void migration_bitmap_sync(RAMState *rs)
 
     rs->bitmap_sync_count++;
 
-    if (!rs->bytes_xfer_prev) {
-        rs->bytes_xfer_prev = ram_bytes_transferred();
-    }
-
     if (!rs->time_last_bitmap_sync) {
         rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     }
-- 
2.9.4

[Qemu-devel] [PULL 3/5] migration: set dirty_pages_rate before autoconverge logic

[Juan Quintela]

From: Felipe Franciosi <felipe@nutanix.com>

Currently, a "period" in the RAM migration logic is at least a second
long and accounts for what happened since the last period (or the
beginning of the migration). The dirty_pages_rate counter is calculated
at the end this logic.

If the auto convergence capability is enabled from the start of the
migration, it won't be able to use this counter the first time around.
This calculates dirty_pages_rate as soon as a period is deemed over,
which allows for it to be used immediately.

Signed-off-by: Felipe Franciosi <felipe@nutanix.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
 migration/ram.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/migration/ram.c b/migration/ram.c
index 36bf720..495ecbe 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -694,6 +694,10 @@ static void migration_bitmap_sync(RAMState *rs)
 
     /* more than 1 second = 1000 millisecons */
     if (end_time > rs->time_last_bitmap_sync + 1000) {
+        /* calculate period counters */
+        rs->dirty_pages_rate = rs->num_dirty_pages_period * 1000
+            / (end_time - rs->time_last_bitmap_sync);
+
         if (migrate_auto_converge()) {
             /* The following detection logic can be refined later. For now:
                Check to see if the dirtied bytes is 50% more than the approx.
@@ -702,15 +706,14 @@ static void migration_bitmap_sync(RAMState *rs)
                throttling */
             bytes_xfer_now = ram_bytes_transferred();
 
-            if (rs->dirty_pages_rate &&
-               (rs->num_dirty_pages_period * TARGET_PAGE_SIZE >
+            if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE >
                    (bytes_xfer_now - rs->bytes_xfer_prev) / 2) &&
-               (rs->dirty_rate_high_cnt++ >= 2)) {
+                (rs->dirty_rate_high_cnt++ >= 2)) {
                     trace_migration_throttle();
                     rs->dirty_rate_high_cnt = 0;
                     mig_throttle_guest_down();
-             }
-             rs->bytes_xfer_prev = bytes_xfer_now;
+            }
+            rs->bytes_xfer_prev = bytes_xfer_now;
         }
 
         if (migrate_use_xbzrle()) {
@@ -723,8 +726,8 @@ static void migration_bitmap_sync(RAMState *rs)
             rs->iterations_prev = rs->iterations;
             rs->xbzrle_cache_miss_prev = rs->xbzrle_cache_miss;
         }
-        rs->dirty_pages_rate = rs->num_dirty_pages_period * 1000
-            / (end_time - rs->time_last_bitmap_sync);
+
+        /* reset period counters */
         rs->time_last_bitmap_sync = end_time;
         rs->num_dirty_pages_period = 0;
     }
-- 
2.9.4

[Qemu-devel] [PULL 4/5] migration: set bytes_xfer_* outside of autoconverge logic

[Juan Quintela]

From: Felipe Franciosi <felipe@nutanix.com>

The bytes_xfer_now/prev counters are only used by the auto convergence
logic. However, they are used alongside the dirty_pages_rate counter,
which is calculated (and required) outside of this logic. The problem
with this approach is that if the auto convergence capability is changed
while a migration is ongoing, the relationship of the counters will be
broken.

This moves the management of bytes_xfer_now/prev counters outside of the
auto convergence logic to address this issue.

Signed-off-by: Felipe Franciosi <felipe@nutanix.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
 migration/ram.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/migration/ram.c b/migration/ram.c
index 495ecbe..1a3d9e6 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -697,6 +697,7 @@ static void migration_bitmap_sync(RAMState *rs)
         /* calculate period counters */
         rs->dirty_pages_rate = rs->num_dirty_pages_period * 1000
             / (end_time - rs->time_last_bitmap_sync);
+        bytes_xfer_now = ram_bytes_transferred();
 
         if (migrate_auto_converge()) {
             /* The following detection logic can be refined later. For now:
@@ -704,7 +705,6 @@ static void migration_bitmap_sync(RAMState *rs)
                amount of bytes that just got transferred since the last time we
                were in this routine. If that happens twice, start or increase
                throttling */
-            bytes_xfer_now = ram_bytes_transferred();
 
             if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE >
                    (bytes_xfer_now - rs->bytes_xfer_prev) / 2) &&
@@ -713,7 +713,6 @@ static void migration_bitmap_sync(RAMState *rs)
                     rs->dirty_rate_high_cnt = 0;
                     mig_throttle_guest_down();
             }
-            rs->bytes_xfer_prev = bytes_xfer_now;
         }
 
         if (migrate_use_xbzrle()) {
@@ -730,6 +729,7 @@ static void migration_bitmap_sync(RAMState *rs)
         /* reset period counters */
         rs->time_last_bitmap_sync = end_time;
         rs->num_dirty_pages_period = 0;
+        rs->bytes_xfer_prev = bytes_xfer_now;
     }
     if (migrate_use_events()) {
         qapi_event_send_migration_pass(rs->bitmap_sync_count, NULL);
-- 
2.9.4

[Qemu-devel] [PULL 5/5] migration: use dirty_rate_high_cnt more aggressively

[Juan Quintela]

From: Felipe Franciosi <felipe@nutanix.com>

The commit message from 070afca25 suggests that dirty_rate_high_cnt
should be used more aggressively to start throttling after two
iterations instead of four. The code, however, only changes the auto
convergence behaviour to throttle after three iterations. This makes the
behaviour more aggressive by kicking off throttling after two iterations
as originally intended.

Signed-off-by: Felipe Franciosi <felipe@nutanix.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
 migration/ram.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migration/ram.c b/migration/ram.c
index 1a3d9e6..26e03a5 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -708,7 +708,7 @@ static void migration_bitmap_sync(RAMState *rs)
 
             if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE >
                    (bytes_xfer_now - rs->bytes_xfer_prev) / 2) &&
-                (rs->dirty_rate_high_cnt++ >= 2)) {
+                (++rs->dirty_rate_high_cnt >= 2)) {
                     trace_migration_throttle();
                     rs->dirty_rate_high_cnt = 0;
                     mig_throttle_guest_down();
-- 
2.9.4

Re: [Qemu-devel] [PULL 0/5] Migration pull request

[Peter Maydell]

On 31 May 2017 at 10:15, Juan Quintela <quintela@redhat.com> wrote:
> Hi
>
> This are the migration patches ready for inclusion:
> - autoconverge improvements: (felipe franciosi)
> - create savem.h
>
> Please, Apply.
>
> The following changes since commit 0748b3526e8cb78b9cd64208426bfc3d54a72b04:
>
>   Merge remote-tracking branch 'kwolf/tags/for-upstream' into staging (2017-05-30 14:15:15 +0100)
>
> are available in the git repository at:
>
>   git://github.com/juanquintela/qemu.git tags/migration/20170531
>
> for you to fetch changes up to b4a3c64b16856a018869bfd4a9ed3b2a74554541:
>
>   migration: use dirty_rate_high_cnt more aggressively (2017-05-31 09:39:20 +0200)
>
> ----------------------------------------------------------------
> migration/next for 20170531
>
> ----------------------------------------------------------------
> Felipe Franciosi (4):
>       migration: keep bytes_xfer_prev init'd to zero
>       migration: set dirty_pages_rate before autoconverge logic
>       migration: set bytes_xfer_* outside of autoconverge logic
>       migration: use dirty_rate_high_cnt more aggressively
>
> Juan Quintela (1):
>       migration: Create savevm.h for functions exported from savevm.c

Applied, thanks.

-- PMM