[PATCH 4 of 6] tools: set migration constraints from cmdline

[Olaf Hering <olaf@aepfle.de>]

# HG changeset patch
# User Olaf Hering <olaf@aepfle.de>
# Date 1363623457 -3600
# Node ID a897276ed24d434daf5a5d3d8543343a23df2f8a
# Parent  b3ed5c4b5078e5bf90d23f7b09eda727af0d7e74
tools: set migration constraints from cmdline

Add new options to xm/xl migrate to control the process of migration.
The intention is to optionally abort the migration if it takes too long
to migrate a busy guest due to the high number of new dirty pages.
Currently the guest is suspended to transfer the remaining dirty pages.
This transfer can take a long time, which can confuse the guest if its
suspended for too long.
The new options allow to override the built-in default values, which are
not changed by this patch.

--max_iters <number>  Number of iterations before final suspend (default: 30)

--max_factor <factor> Max amount of memory to transfer before final suspend (default: 3*RAM)

--abort_if_busy       Abort migration instead of doing final suspend.



The changes to libxl change the API, handle LIBXL_API_VERSION == 0x040200.

TODO:
 eventually add also --min_remaining (default value 50) in a seperate patch

v7:
  - remove short options
  - update description of --abort_if_busy in xl.1
  - extend description of --abort_if_busy in xl help
  - add comment to libxl_domain_suspend declaration, props is optional

v6:
 - update the LIBXL_API_VERSION handling for libxl_domain_suspend
   change it to an inline function if LIBXL_API_VERSION is defined to 4.2.0
 - rename libxl_save_properties to libxl_domain_suspend_properties
 - rename ->xlflags to ->flags within that struct

v5:
 - adjust libxl_domain_suspend prototype, move flags, max_iters,
   max_factor into a new, optional struct libxl_save_properties
 - rename XCFLAGS_DOMSAVE_NOSUSPEND to XCFLAGS_DOMSAVE_ABORT_IF_BUSY
 - rename LIBXL_SUSPEND_NO_FINAL_SUSPEND to LIBXL_SUSPEND_ABORT_IF_BUSY
 - rename variables no_suspend to abort_if_busy
 - rename option -N/--no_suspend to -A/--abort_if_busy
 - update xl.1, extend description of -A option

v4:
 - update default for no_suspend from None to 0 in XendCheckpoint.py:save
 - update logoutput in setMigrateConstraints
 - change xm migrate defaults from None to 0
 - add new options to xl.1
 - fix syntax error in XendDomain.py:domain_migrate_constraints_set
 - fix xm migrate -N option name to match xl migrate

v3:
 - move logic errors in libxl__domain_suspend and fixed help text in
   cmd_table to separate patches
 - fix syntax error in XendCheckpoint.py
 - really pass max_iters and max_factor in libxl__xc_domain_save
 - make libxl_domain_suspend_0x040200 declaration globally visible
 - bump libxenlight.so SONAME from 2.0 to 2.1 due to changed
   libxl_domain_suspend

v2:
 - use LIBXL_API_VERSION and define libxl_domain_suspend_0x040200
 - fix logic error in min_reached check in xc_domain_save
 - add longopts
 - update --help text
 - correct description of migrate --help text

Signed-off-by: Olaf Hering <olaf@aepfle.de>

diff -r b3ed5c4b5078 -r a897276ed24d docs/man/xl.pod.1
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -391,6 +391,21 @@ Send <config> instead of config file fro
 
 Print huge (!) amount of debug during the migration process.
 
+=item B<--max_iters> I<number>
+
+Number of iterations before final suspend (default: 30)
+
+=item B<--max_factor> I<factor>
+
+Max amount of memory to transfer before final suspend (default: 3*RAM)
+
+=item B<--abort_if_busy>
+
+Abort migration instead of doing final suspend/transfer/resume if the
+guest has still dirty pages after the number of iterations and/or the
+amount of RAM transferred. This avoids long periods of time where the
+guest is suspended.
+
 =back
 
 =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxc/xc_domain_save.c
--- a/tools/libxc/xc_domain_save.c
+++ b/tools/libxc/xc_domain_save.c
@@ -804,6 +804,7 @@ int xc_domain_save(xc_interface *xch, in
     int rc = 1, frc, i, j, last_iter = 0, iter = 0;
     int live  = (flags & XCFLAGS_LIVE);
     int debug = (flags & XCFLAGS_DEBUG);
+    int abort_if_busy = (flags & XCFLAGS_DOMSAVE_ABORT_IF_BUSY);
     int superpages = !!hvm;
     int race = 0, sent_last_iter, skip_this_iter = 0;
     unsigned int sent_this_iter = 0;
@@ -1527,10 +1528,20 @@ int xc_domain_save(xc_interface *xch, in
 
         if ( live )
         {
+            int min_reached = sent_this_iter + skip_this_iter < 50;
             if ( (iter >= max_iters) ||
-                 (sent_this_iter+skip_this_iter < 50) ||
+                 min_reached ||
                  (total_sent > dinfo->p2m_size*max_factor) )
             {
+                if ( !min_reached && abort_if_busy )
+                {
+                    ERROR("Live migration aborted, as requested. (guest too busy?)"
+                     " total_sent %lu iter %d, max_iters %u max_factor %u",
+                      total_sent, iter, max_iters, max_factor);
+                    rc = 1;
+                    goto out;
+                }
+
                 DPRINTF("Start last iteration\n");
                 last_iter = 1;
 
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxc/xenguest.h
--- a/tools/libxc/xenguest.h
+++ b/tools/libxc/xenguest.h
@@ -28,6 +28,7 @@
 #define XCFLAGS_HVM       (1 << 2)
 #define XCFLAGS_STDVGA    (1 << 3)
 #define XCFLAGS_CHECKPOINT_COMPRESS    (1 << 4)
+#define XCFLAGS_DOMSAVE_ABORT_IF_BUSY  (1 << 5)
 
 #define X86_64_B_SIZE   64 
 #define X86_32_B_SIZE   32
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/Makefile
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -5,7 +5,7 @@
 XEN_ROOT = $(CURDIR)/../..
 include $(XEN_ROOT)/tools/Rules.mk
 
-MAJOR = 2.0
+MAJOR = 2.1
 MINOR = 0
 
 XLUMAJOR = 1.0
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/libxl.c
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -756,7 +756,8 @@ static void domain_suspend_cb(libxl__egc
 
 }
 
-int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
+int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
+                         const libxl_domain_suspend_properties *props,
                          const libxl_asyncop_how *ao_how)
 {
     AO_CREATE(ctx, domid, ao_how);
@@ -777,8 +778,13 @@ int libxl_domain_suspend(libxl_ctx *ctx,
     dss->domid = domid;
     dss->fd = fd;
     dss->type = type;
-    dss->live = flags & LIBXL_SUSPEND_LIVE;
-    dss->debug = flags & LIBXL_SUSPEND_DEBUG;
+    if (props) {
+        dss->live = props->flags & LIBXL_SUSPEND_LIVE;
+        dss->debug = props->flags & LIBXL_SUSPEND_DEBUG;
+        dss->max_iters = props->max_iters;
+        dss->max_factor = props->max_factor;
+        dss->xlflags = props->flags;
+    }
 
     libxl__domain_suspend(egc, dss);
     return AO_INPROGRESS;
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/libxl.h
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -507,12 +507,31 @@ int libxl_domain_create_restore(libxl_ct
 void libxl_domain_config_init(libxl_domain_config *d_config);
 void libxl_domain_config_dispose(libxl_domain_config *d_config);
 
+typedef struct {
+    int flags; /* LIBXL_SUSPEND_* */
+    int max_iters;
+    int max_factor;
+} libxl_domain_suspend_properties;
+
 int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
-                         int flags, /* LIBXL_SUSPEND_* */
+                         const libxl_domain_suspend_properties *props, /* optional */
                          const libxl_asyncop_how *ao_how)
                          LIBXL_EXTERNAL_CALLERS_ONLY;
+#ifdef LIBXL_API_VERSION
+#if LIBXL_API_VERSION == 0x040200
+static inline int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd,
+                          int flags, const libxl_asyncop_how *ao_how)
+{
+    libxl_domain_suspend_properties props = { .flags = flags };
+    return libxl_domain_suspend(ctx, domid, fd, &props, ao_how);
+}
+#define libxl_domain_suspend libxl_domain_suspend_0x040200
+#endif
+#endif
+
 #define LIBXL_SUSPEND_DEBUG 1
 #define LIBXL_SUSPEND_LIVE 2
+#define LIBXL_SUSPEND_ABORT_IF_BUSY 4
 
 /* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
  *   If this parameter is true, use co-operative resume. The guest
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/libxl_dom.c
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1297,6 +1297,7 @@ void libxl__domain_suspend(libxl__egc *e
 
     dss->xcflags = (live ? XCFLAGS_LIVE : 0)
           | (debug ? XCFLAGS_DEBUG : 0)
+          | (dss->xlflags & LIBXL_SUSPEND_ABORT_IF_BUSY ? XCFLAGS_DOMSAVE_ABORT_IF_BUSY : 0)
           | (dss->hvm ? XCFLAGS_HVM : 0);
 
     dss->suspend_eventchn = -1;
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/libxl_internal.h
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2249,6 +2249,9 @@ struct libxl__domain_suspend_state {
     xc_evtchn *xce; /* event channel handle */
     int suspend_eventchn;
     int hvm;
+    int max_iters;
+    int max_factor;
+    int xlflags;
     int xcflags;
     int guest_responded;
     const char *dm_savefile;
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/libxl_save_callout.c
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -108,8 +108,8 @@ void libxl__xc_domain_save(libxl__egc *e
     }
 
     const unsigned long argnums[] = {
-        dss->domid, 0, 0, dss->xcflags, dss->hvm, vm_generationid_addr,
-        toolstack_data_fd, toolstack_data_len,
+        dss->domid, dss->max_iters, dss->max_factor, dss->xcflags, dss->hvm,
+        vm_generationid_addr, toolstack_data_fd, toolstack_data_len,
         cbflags,
     };
 
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/xl_cmdimpl.c
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -3201,7 +3201,7 @@ static int save_domain(uint32_t domid, c
 
     save_domain_core_writeconfig(fd, filename, config_data, config_len);
 
-    int rc = libxl_domain_suspend(ctx, domid, fd, 0, NULL);
+    int rc = libxl_domain_suspend(ctx, domid, fd, NULL, NULL);
     close(fd);
 
     if (rc < 0)
@@ -3361,6 +3361,7 @@ static void migrate_do_preamble(int send
 }
 
 static void migrate_domain(uint32_t domid, const char *rune, int debug,
+                           int max_iters, int max_factor, int abort_if_busy,
                            const char *override_config_file)
 {
     pid_t child = -1;
@@ -3369,7 +3370,12 @@ static void migrate_domain(uint32_t domi
     char *away_domname;
     char rc_buf;
     uint8_t *config_data;
-    int config_len, flags = LIBXL_SUSPEND_LIVE;
+    int config_len;
+    libxl_domain_suspend_properties props = {
+        .flags = LIBXL_SUSPEND_LIVE,
+        .max_iters = max_iters,
+        .max_factor = max_factor,
+    };
 
     save_domain_core_begin(domid, override_config_file,
                            &config_data, &config_len);
@@ -3388,8 +3394,11 @@ static void migrate_domain(uint32_t domi
     xtl_stdiostream_adjust_flags(logger, XTL_STDIOSTREAM_HIDE_PROGRESS, 0);
 
     if (debug)
-        flags |= LIBXL_SUSPEND_DEBUG;
-    rc = libxl_domain_suspend(ctx, domid, send_fd, flags, NULL);
+        props.flags |= LIBXL_SUSPEND_DEBUG;
+    if (abort_if_busy)
+        props.flags |= LIBXL_SUSPEND_ABORT_IF_BUSY;
+
+    rc = libxl_domain_suspend(ctx, domid, send_fd, &props, NULL);
     if (rc) {
         fprintf(stderr, "migration sender: libxl_domain_suspend failed"
                 " (rc=%d)\n", rc);
@@ -3782,13 +3791,17 @@ int main_migrate(int argc, char **argv)
     char *rune = NULL;
     char *host;
     int opt, daemonize = 1, monitor = 1, debug = 0;
+    int max_iters = 0, max_factor = 0, abort_if_busy = 0;
     static struct option opts[] = {
         {"debug", 0, 0, 0x100},
+        {"max_iters", 1, 0, 0x101},
+        {"max_factor", 1, 0, 0x102},
+        {"abort_if_busy", 0, 0, 0x103},
         COMMON_LONG_OPTS,
         {0, 0, 0, 0}
     };
 
-    SWITCH_FOREACH_OPT(opt, "FC:s:e", opts, "migrate", 2) {
+    SWITCH_FOREACH_OPT(opt, "FC:s:eM:m:A", opts, "migrate", 2) {
     case 'C':
         config_filename = optarg;
         break;
@@ -3805,6 +3818,15 @@ int main_migrate(int argc, char **argv)
     case 0x100:
         debug = 1;
         break;
+    case 0x101:
+        max_iters = atoi(optarg);
+        break;
+    case 0x102:
+        max_factor = atoi(optarg);
+        break;
+    case 0x103:
+        abort_if_busy = 1;
+        break;
     }
 
     domid = find_domain(argv[optind]);
@@ -3820,7 +3842,7 @@ int main_migrate(int argc, char **argv)
             return 1;
     }
 
-    migrate_domain(domid, rune, debug, config_filename);
+    migrate_domain(domid, rune, debug, max_iters, max_factor, abort_if_busy, config_filename);
     return 0;
 }
 
diff -r b3ed5c4b5078 -r a897276ed24d tools/libxl/xl_cmdtable.c
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -147,14 +147,18 @@ struct cmd_spec cmd_table[] = {
       &main_migrate, 0, 1,
       "Migrate a domain to another host",
       "[options] <Domain> <host>",
-      "-h              Print this help.\n"
-      "-C <config>     Send <config> instead of config file from creation.\n"
-      "-s <sshcommand> Use <sshcommand> instead of ssh.  String will be passed\n"
-      "                to sh. If empty, run <host> instead of ssh <host> xl\n"
-      "                migrate-receive [-d -e]\n"
-      "-e              Do not wait in the background (on <host>) for the death\n"
-      "                of the domain.\n"
-      "--debug         Print huge (!) amount of debug during the migration process."
+      "-h                    Print this help.\n"
+      "-C <config>           Send <config> instead of config file from creation.\n"
+      "-s <sshcommand>       Use <sshcommand> instead of ssh.  String will be passed\n"
+      "                      to sh. If empty, run <host> instead of ssh <host> xl\n"
+      "                      migrate-receive [-d -e]\n"
+      "-e                    Do not wait in the background (on <host>) for the death\n"
+      "                      of the domain.\n"
+      "--debug               Print huge (!) amount of debug during the migration process.\n"
+      "--max_iters <number>  Number of iterations before final suspend (default: 30)\n"
+      "--max_factor <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n"
+      "--abort_if_busy       Abort migration instead of doing final suspend, if number\n"
+      "                      of iterations or amount of transfered memory is exceeded."
     },
     { "dump-core",
       &main_dump_core, 0, 1,
diff -r b3ed5c4b5078 -r a897276ed24d tools/python/xen/xend/XendCheckpoint.py
--- a/tools/python/xen/xend/XendCheckpoint.py
+++ b/tools/python/xen/xend/XendCheckpoint.py
@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst
         # enabled. Passing "0" simply uses the defaults compiled into
         # libxenguest; see the comments and/or code in xc_linux_save() for
         # more information.
+        max_iters = dominfo.info.get('max_iters', "0")
+        max_factor = dominfo.info.get('max_factor', "0")
+        abort_if_busy = dominfo.info.get('abort_if_busy', "0")
+        if max_iters == "None":
+            max_iters = "0"
+        if max_factor == "None":
+            max_factor = "0"
+        if abort_if_busy == "None":
+            abort_if_busy = "0"
         cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd),
-               str(dominfo.getDomid()), "0", "0", 
-               str(int(live) | (int(hvm) << 2)) ]
+               str(dominfo.getDomid()),
+               max_iters, max_factor,
+               str( int(live) | (int(hvm) << 2) | (int(abort_if_busy) << 5) ) ]
         log.debug("[xc_save]: %s", string.join(cmd))
 
         def saveInputHandler(line, tochild):
diff -r b3ed5c4b5078 -r a897276ed24d tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py
+++ b/tools/python/xen/xend/XendDomain.py
@@ -1832,6 +1832,18 @@ class XendDomain:
             log.exception(ex)
             raise XendError(str(ex))
 
+    def domain_migrate_constraints_set(self, domid, max_iters, max_factor, abort_if_busy):
+        """Set the Migrate Constraints of this domain.
+        @param domid: Domain ID or Name
+        @param max_iters: Number of iterations before final suspend
+        @param max_factor: Max amount of memory to transfer before final suspend
+        @param abort_if_busy: Abort migration instead of doing final suspend
+        """
+        dominfo = self.domain_lookup_nr(domid)
+        if not dominfo:
+            raise XendInvalidDomain(str(domid))
+        dominfo.setMigrateConstraints(max_iters, max_factor, abort_if_busy)
+
     def domain_maxmem_set(self, domid, mem):
         """Set the memory limit for a domain.
 
diff -r b3ed5c4b5078 -r a897276ed24d tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py
+++ b/tools/python/xen/xend/XendDomainInfo.py
@@ -1459,6 +1459,18 @@ class XendDomainInfo:
         pci_conf = self.info['devices'][dev_uuid][1]
         return map(pci_dict_to_bdf_str, pci_conf['devs'])
 
+    def setMigrateConstraints(self, max_iters, max_factor, abort_if_busy):
+        """Set the Migrate Constraints of this domain.
+        @param max_iters: Number of iterations before final suspend
+        @param max_factor: Max amount of memory to transfer before final suspend
+        @param abort_if_busy: Abort migration instead of doing final suspend
+        """
+        log.debug("Setting migration constraints of domain %s (%s) to '%s' '%s' '%s'.",
+                  self.info['name_label'], str(self.domid), max_iters, max_factor, abort_if_busy)
+        self.info['max_iters'] = str(max_iters)
+        self.info['max_factor'] = str(max_factor)
+        self.info['abort_if_busy'] = str(abort_if_busy)
+
     def setMemoryTarget(self, target):
         """Set the memory target of this domain.
         @param target: In MiB.
diff -r b3ed5c4b5078 -r a897276ed24d tools/python/xen/xm/migrate.py
--- a/tools/python/xen/xm/migrate.py
+++ b/tools/python/xen/xm/migrate.py
@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c
           fn=set_true, default=0,
           use="Change home server for managed domains.")
 
+gopts.opt('max_iters',val='max_iters',
+          fn=set_int, default=0,
+          use="Number of iterations before final suspend (default: 30).")
+
+gopts.opt('max_factor', val='max_factor',
+          fn=set_int, default=0,
+          use="Max amount of memory to transfer before final suspend (default: 3*RAM).")
+
+gopts.opt('abort_if_busy',
+          fn=set_true, default=0,
+          use="Abort migration instead of doing final suspend.")
+
 def help():
     return str(gopts)
     
@@ -80,6 +92,10 @@ def main(argv):
         server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live),
                                  other_config)
     else:
+        server.xend.domain.migrate_constraints_set(dom,
+                                                   opts.vals.max_iters,
+                                                   opts.vals.max_factor,
+                                                   opts.vals.abort_if_busy)
         server.xend.domain.migrate(dom, dst, opts.vals.live,
                                    opts.vals.port,
                                    opts.vals.node,