[PATCH 3/6] xl: Implement XENMEM_claim_pages support via 'claim_mode' global config

[Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>]

The XENMEM_claim_pages operates per domain and its usage is suppose to
be done system wide. As such this patch introduces a global
configuration option 'claim_mode' that by default is disabled.

The problem is that when a guest is launched the process of allocating
memory from the hypervisor is not atomic and for large guests can take a while.
During which time other guests (especially self-ballooning) can take
balloon up / down making the free memory information available to the
toolstack stale.  With the "claim_mode" we can "stake a claim" for
exact number of pages we need to allocate the guest and we are guaranteed that
they will be there when we start allocating the guest.

There are two options:
 "on" - Use the default memory and also include the ephereal tmem pool in this
      calculation. This can affect temporarily negatively 'tmem' enabled guests
      but provides more freeable memory.
 "off" - (default) No claim attempt is made.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 docs/man/xl.conf.pod.5      | 25 +++++++++++++++++++++++++
 tools/examples/xl.conf      |  5 +++++
 tools/libxl/libxl.c         |  5 +++++
 tools/libxl/libxl.h         |  2 ++
 tools/libxl/libxl_dom.c     |  3 ++-
 tools/libxl/libxl_types.idl |  7 ++++++-
 tools/libxl/xl.c            |  5 +++++
 tools/libxl/xl.h            |  1 +
 tools/libxl/xl_cmdimpl.c    |  5 +++++
 9 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5
index 82c6b20..fded0ab 100644
--- a/docs/man/xl.conf.pod.5
+++ b/docs/man/xl.conf.pod.5
@@ -98,6 +98,31 @@ Configures the name of the first block device to be used for temporary
 block device allocations by the toolstack.
 The default choice is "xvda".
 
+=item B<claim_mode="off|on">
+
+If enabled when a guest is created it will provide an guarantee whether the
+guest can be launched or not due to memory exhaustion. This is needed when
+using tmem type guests that can balloon up and down (self-balloon) extremely
+quickly and the list of free memory information is stale the moment it is
+displayed. When this mode is used there will be initially a claim set for the
+amount of pages needed which will be fulfilled as the guest is created.
+If the claim fails the guest creation process will also fail.
+
+Default: C<off>
+
+=over 4
+
+=item B<"off">
+
+No action is taken and there is no stake put for the memory.
+
+=item B<"on">
+
+Normal memory and freeable pool of ephereal pages (tmem) is used when
+calculating whether there is enough memory free to launch a guest.
+
+=back
+
 =back
 
 =head1 SEE ALSO
diff --git a/tools/examples/xl.conf b/tools/examples/xl.conf
index 28ab796..2b64f7e 100644
--- a/tools/examples/xl.conf
+++ b/tools/examples/xl.conf
@@ -20,3 +20,8 @@
 # if disabled the old behaviour will be used, and hotplug scripts will be
 # launched by udev.
 #run_hotplug_scripts=1
+#
+# stake a claim of memory when launching a guest. This guarantees immediate
+# feedback whether the guest can be launched due to memory exhaustion
+# (which can take a long time to find out if launching huge guests).
+#claim_mode="on"
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 73e0dc3..8c6e1a1 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -4788,6 +4788,11 @@ int libxl_tmem_freeable(libxl_ctx *ctx)
     return rc;
 }
 
+int libxl_parse_claim_mode(const char *s, unsigned int *flag)
+{
+    return libxl_claim_mode_from_string(s, (libxl_claim_mode *)flag);
+}
+
 int libxl_get_freecpus(libxl_ctx *ctx, libxl_bitmap *cpumap)
 {
     int ncpus;
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 030aa86..538bf93 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -579,6 +579,8 @@ int libxl_wait_for_free_memory(libxl_ctx *ctx, uint32_t domid, uint32_t memory_k
 /* wait for the memory target of a domain to be reached */
 int libxl_wait_for_memory_target(libxl_ctx *ctx, uint32_t domid, int wait_secs);
 
+/* Parse the claim_mode options */
+int libxl_parse_claim_mode(const char *s, unsigned int *flag);
 
 int libxl_vncviewer_exec(libxl_ctx *ctx, uint32_t domid, int autopass);
 int libxl_console_exec(libxl_ctx *ctx, uint32_t domid, int cons_num, libxl_console_type type);
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index de555ee..4715378 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -367,6 +367,7 @@ int libxl__build_pv(libxl__gc *gc, uint32_t domid,
     dom->console_domid = state->console_domid;
     dom->xenstore_evtchn = state->store_port;
     dom->xenstore_domid = state->store_domid;
+    dom->claim_mode = info->claim_mode;
 
     if ( (ret = xc_dom_boot_xen_init(dom, ctx->xch, domid)) != 0 ) {
         LOGE(ERROR, "xc_dom_boot_xen_init failed");
@@ -601,7 +602,7 @@ int libxl__build_hvm(libxl__gc *gc, uint32_t domid,
      */
     args.mem_size = (uint64_t)(info->max_memkb - info->video_memkb) << 10;
     args.mem_target = (uint64_t)(info->target_memkb - info->video_memkb) << 10;
-
+    args.claim_mode = info->claim_mode;
     if (libxl__domain_firmware(gc, info, &args)) {
         LOG(ERROR, "initializing domain firmware failed");
         goto out;
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 5b080ed..e417851 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -94,6 +94,11 @@ libxl_tsc_mode = Enumeration("tsc_mode", [
     (3, "native_paravirt"),
     ])
 
+libxl_claim_mode = Enumeration("claim_mode", [
+    (0, "off"),
+    (1, "on"),
+    ]);
+
 # Consistent with the values defined for HVM_PARAM_TIMER_MODE.
 libxl_timer_mode = Enumeration("timer_mode", [
     (0, "delay_for_missed_ticks"),
@@ -293,7 +298,7 @@ libxl_domain_build_info = Struct("domain_build_info",[
     ("ioports",          Array(libxl_ioport_range, "num_ioports")),
     ("irqs",             Array(uint32, "num_irqs")),
     ("iomem",            Array(libxl_iomem_range, "num_iomem")),
-
+    ("claim_mode",	 libxl_claim_mode),
     ("u", KeyedUnion(None, libxl_domain_type, "type",
                 [("hvm", Struct(None, [("firmware",         string),
                                        ("bios",             libxl_bios_type),
diff --git a/tools/libxl/xl.c b/tools/libxl/xl.c
index ecbcd3b..37d56c9 100644
--- a/tools/libxl/xl.c
+++ b/tools/libxl/xl.c
@@ -44,6 +44,7 @@ char *lockfile;
 char *default_vifscript = NULL;
 char *default_bridge = NULL;
 enum output_format default_output_format = OUTPUT_FORMAT_JSON;
+unsigned int global_claim_mode = 0;
 
 static xentoollog_level minmsglevel = XTL_PROGRESS;
 
@@ -102,6 +103,10 @@ static void parse_global_config(const char *configfile,
     }
     if (!xlu_cfg_get_string (config, "blkdev_start", &buf, 0))
         blkdev_start = strdup(buf);
+
+    if (!xlu_cfg_get_string(config, "claim_mode", &buf, 0))
+        libxl_parse_claim_mode(buf, &global_claim_mode);
+
     xlu_cfg_destroy(config);
 }
 
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index be6f38b..b1d434e 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -145,6 +145,7 @@ int xl_child_pid(xlchildnum); /* returns 0 if child struct is not in use */
 extern int autoballoon;
 extern int run_hotplug_scripts;
 extern int dryrun_only;
+extern unsigned int global_claim_mode;
 extern char *lockfile;
 extern char *default_vifscript;
 extern char *default_bridge;
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index a98705e..27298ea 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -757,6 +757,11 @@ static void parse_config_data(const char *config_source,
     if (!xlu_cfg_get_long (config, "maxmem", &l, 0))
         b_info->max_memkb = l * 1024;
 
+    if (global_claim_mode)
+        b_info->claim_mode = (libxl_claim_mode)global_claim_mode;
+    else
+        b_info->claim_mode = 0;
+
     if (xlu_cfg_get_string (config, "on_poweroff", &buf, 0))
         buf = "destroy";
     if (!parse_action_on_shutdown(buf, &d_config->on_poweroff)) {
-- 
1.8.0.2