[PATCH RFC 08/20] libxl/migration: add precopy tuning parameters

[Joshua Otto <jtotto@uwaterloo.ca>]

In the context of the live migration algorithm, the precopy iteration
count refers to the number of page-copying iterations performed prior to
the suspension of the guest and transmission of the final set of dirty
pages.  Similarly, the precopy dirty threshold refers to the dirty page
count below which we judge it more profitable to proceed to
stop-and-copy rather than continue with the precopy.  These would be
helpful tuning parameters to work with when migrating particularly busy
guests, as they enable an administrator to reap the available benefits
of the precopy algorithm (the transmission of guest pages _not_ in the
writable working set can be completed without guest downtime) while
reducing the total amount of time required for the migration (as
iterations of the precopy loop that will certainly be redundant can be
skipped in favour of an earlier suspension).

To expose these tuning parameters to users:
- introduce a new libxl API function, libxl_domain_live_migrate(),
  taking the same parameters as libxl_domain_suspend() _and_
  precopy_iterations and precopy_dirty_threshold parameters, and
  consider these parameters in the precopy policy

  (though a pair of new parameters on their own might not warrant an
  entirely new API function, it is added in anticipation of a number of
  additional migration-only parameters that would be cumbersome on the
  whole to tack on to the existing suspend API)

- switch xl migrate to the new libxl_domain_live_migrate() and add new
  --postcopy-iterations and --postcopy-threshold parameters to pass
  through

Signed-off-by: Joshua Otto <jtotto@uwaterloo.ca>
---
 tools/libxl/libxl.h          | 10 ++++++++++
 tools/libxl/libxl_dom_save.c | 20 +++++++++++---------
 tools/libxl/libxl_domain.c   | 27 +++++++++++++++++++++++++--
 tools/libxl/libxl_internal.h |  2 ++
 tools/xl/xl_cmdtable.c       | 22 +++++++++++++---------
 tools/xl/xl_migrate.c        | 31 +++++++++++++++++++++++++++----
 6 files changed, 88 insertions(+), 24 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 833f866..84ac96a 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1375,6 +1375,16 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
 #define LIBXL_SUSPEND_DEBUG 1
 #define LIBXL_SUSPEND_LIVE 2
 
+int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int fd,
+                              int flags, /* LIBXL_SUSPEND_* */
+                              unsigned int precopy_iterations,
+                              unsigned int precopy_dirty_threshold,
+                              const libxl_asyncop_how *ao_how)
+                              LIBXL_EXTERNAL_CALLERS_ONLY;
+
+#define LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT 5
+#define LIBXL_LM_DIRTY_THRESHOLD_DEFAULT 50
+
 /* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
  *   If this parameter is true, use co-operative resume. The guest
  *   must support this.
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 6d28cce..10d5012 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -332,19 +332,21 @@ int libxl__save_emulator_xenstore_data(libxl__domain_save_state *dss,
  * This is the live migration precopy policy - it's called periodically during
  * the precopy phase of live migrations, and is responsible for deciding when
  * the precopy phase should terminate and what should be done next.
- *
- * The policy implemented here behaves identically to the policy previously
- * hard-coded into xc_domain_save() - it proceeds to the stop-and-copy phase of
- * the live migration when there are either fewer than 50 dirty pages, or more
- * than 5 precopy rounds have completed.
  */
 static int libxl__save_live_migration_simple_precopy_policy(
     struct precopy_stats stats, void *user)
 {
-    return ((stats.dirty_count >= 0 && stats.dirty_count < 50) ||
-            stats.iteration >= 5)
-        ? XGS_POLICY_STOP_AND_COPY
-        : XGS_POLICY_CONTINUE_PRECOPY;
+    libxl__save_helper_state *shs = user;
+    libxl__domain_save_state *dss = shs->caller_state;
+
+    if (stats.dirty_count >= 0 &&
+        stats.dirty_count <= dss->precopy_dirty_threshold)
+        return XGS_POLICY_STOP_AND_COPY;
+
+    if (stats.iteration >= dss->precopy_iterations)
+        return XGS_POLICY_STOP_AND_COPY;
+
+    return XGS_POLICY_CONTINUE_PRECOPY;
 }
 
 /*----- main code for saving, in order of execution -----*/
diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c
index 08eccd0..b1cf643 100644
--- a/tools/libxl/libxl_domain.c
+++ b/tools/libxl/libxl_domain.c
@@ -486,8 +486,10 @@ static void domain_suspend_cb(libxl__egc *egc,
 
 }
 
-int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
-                         const libxl_asyncop_how *ao_how)
+static int do_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
+                             unsigned int precopy_iterations,
+                             unsigned int precopy_dirty_threshold,
+                             const libxl_asyncop_how *ao_how)
 {
     AO_CREATE(ctx, domid, ao_how);
     int rc;
@@ -510,6 +512,8 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
     dss->live = flags & LIBXL_SUSPEND_LIVE;
     dss->debug = flags & LIBXL_SUSPEND_DEBUG;
     dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
+    dss->precopy_iterations = precopy_iterations;
+    dss->precopy_dirty_threshold = precopy_dirty_threshold;
 
     rc = libxl__fd_flags_modify_save(gc, dss->fd,
                                      ~(O_NONBLOCK|O_NDELAY), 0,
@@ -523,6 +527,25 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
     return AO_CREATE_FAIL(rc);
 }
 
+int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
+                         const libxl_asyncop_how *ao_how)
+{
+    return do_domain_suspend(ctx, domid, fd, flags,
+                             LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT,
+                             LIBXL_LM_DIRTY_THRESHOLD_DEFAULT, ao_how);
+}
+
+int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
+                              unsigned int precopy_iterations,
+                              unsigned int precopy_dirty_threshold,
+                              const libxl_asyncop_how *ao_how)
+{
+    flags |= LIBXL_SUSPEND_LIVE;
+
+    return do_domain_suspend(ctx, domid, fd, flags, precopy_iterations,
+                             precopy_dirty_threshold, ao_how);
+}
+
 int libxl_domain_pause(libxl_ctx *ctx, uint32_t domid)
 {
     int ret;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index f1d8f9a..45d607a 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3292,6 +3292,8 @@ struct libxl__domain_save_state {
     int live;
     int debug;
     int checkpointed_stream;
+    unsigned int precopy_iterations;
+    unsigned int precopy_dirty_threshold;
     const libxl_domain_remus_info *remus;
     /* private */
     int rc;
diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c
index 7d97811..6df66fb 100644
--- a/tools/xl/xl_cmdtable.c
+++ b/tools/xl/xl_cmdtable.c
@@ -157,15 +157,19 @@ struct cmd_spec cmd_table[] = {
       &main_migrate, 0, 1,
       "Migrate a domain to another host",
       "[options] <Domain> <host>",
-      "-h              Print this help.\n"
-      "-C <config>     Send <config> instead of config file from creation.\n"
-      "-s <sshcommand> Use <sshcommand> instead of ssh.  String will be passed\n"
-      "                to sh. If empty, run <host> instead of ssh <host> xl\n"
-      "                migrate-receive [-d -e]\n"
-      "-e              Do not wait in the background (on <host>) for the death\n"
-      "                of the domain.\n"
-      "--debug         Print huge (!) amount of debug during the migration process.\n"
-      "-p              Do not unpause domain after migrating it."
+      "-h                   Print this help.\n"
+      "-C <config>          Send <config> instead of config file from creation.\n"
+      "-s <sshcommand>      Use <sshcommand> instead of ssh.  String will be passed\n"
+      "                     to sh. If empty, run <host> instead of ssh <host> xl\n"
+      "                     migrate-receive [-d -e]\n"
+      "-e                   Do not wait in the background (on <host>) for the death\n"
+      "                     of the domain.\n"
+      "--debug              Print huge (!) amount of debug during the migration process.\n"
+      "-p                   Do not unpause domain after migrating it.\n"
+      "--precopy-iterations Perform at most this many iterations of the precopy\n"
+      "                     memory migration loop before suspending the domain.\n"
+      "--precopy-threshold  If fewer than this many pages are dirty at the end of a\n"
+      "                     copy round, exit the precopy loop and suspend the domain."
     },
     { "restore",
       &main_restore, 0, 1,
diff --git a/tools/xl/xl_migrate.c b/tools/xl/xl_migrate.c
index 1f0e87d..1bb3fb4 100644
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -177,7 +177,9 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
 }
 
 static void migrate_domain(uint32_t domid, const char *rune, int debug,
-                           const char *override_config_file)
+                           const char *override_config_file,
+                           unsigned int precopy_iterations,
+                           unsigned int precopy_dirty_threshold)
 {
     pid_t child = -1;
     int rc;
@@ -205,7 +207,9 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
 
     if (debug)
         flags |= LIBXL_SUSPEND_DEBUG;
-    rc = libxl_domain_suspend(ctx, domid, send_fd, flags, NULL);
+    rc = libxl_domain_live_migrate(ctx, domid, send_fd, flags,
+                                   precopy_iterations, precopy_dirty_threshold,
+                                   NULL);
     if (rc) {
         fprintf(stderr, "migration sender: libxl_domain_suspend failed"
                 " (rc=%d)\n", rc);
@@ -537,13 +541,17 @@ int main_migrate(int argc, char **argv)
     char *rune = NULL;
     char *host;
     int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
+    int precopy_iterations = LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT,
+        precopy_dirty_threshold = LIBXL_LM_DIRTY_THRESHOLD_DEFAULT;
     static struct option opts[] = {
         {"debug", 0, 0, 0x100},
         {"live", 0, 0, 0x200},
+        {"precopy-iterations", 1, 0, 'i'},
+        {"precopy-threshold", 1, 0, 'd'},
         COMMON_LONG_OPTS
     };
 
-    SWITCH_FOREACH_OPT(opt, "FC:s:ep", opts, "migrate", 2) {
+    SWITCH_FOREACH_OPT(opt, "FC:s:epi:d:", opts, "migrate", 2) {
     case 'C':
         config_filename = optarg;
         break;
@@ -560,6 +568,20 @@ int main_migrate(int argc, char **argv)
     case 'p':
         pause_after_migration = 1;
         break;
+    case 'i':
+        precopy_iterations = atoi(optarg);
+        if (precopy_iterations < 0) {
+            fprintf(stderr, "negative precopy iterations not supported\n");
+            return EXIT_FAILURE;
+        }
+        break;
+    case 'd':
+        precopy_dirty_threshold = atoi(optarg);
+        if (precopy_dirty_threshold < 0) {
+            fprintf(stderr, "negative dirty threshold not supported\n");
+            return EXIT_FAILURE;
+        }
+        break;
     case 0x100: /* --debug */
         debug = 1;
         break;
@@ -596,7 +618,8 @@ int main_migrate(int argc, char **argv)
                   pause_after_migration ? " -p" : "");
     }
 
-    migrate_domain(domid, rune, debug, config_filename);
+    migrate_domain(domid, rune, debug, config_filename, precopy_iterations,
+                   precopy_dirty_threshold);
     return EXIT_SUCCESS;
 }
 
-- 
2.7.4


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel