[PATCH 3/6] xend: Implement XENMEM_claim_pages support via 'claim-mode' global config

[Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>]

The XENMEM_claim_pages hypercall operates per domain and it should be
used system wide. As such this patch introduces a global configuration
option 'claim-mode' that by default is disabled.

If this option is enabled then when a guest is created there will be an
guarantee that there is memory available for the guest. This is an particularly
acute problem on hosts with memory over-provisioned guests that use
tmem and have self-balloon enabled (which is the default option for them).
The self-balloon mechanism can deflate/inflate the balloon quickly and the
amount of free memory (which 'xm info' can show) is stale the moment
it is printed. When claim is enabled a reservation for the amount of
memory ('memory' in guest config) is set, which is then reduced as the
domain's memory is populated and eventually reaches zero.

If the reservation cannot be meet the guest creation fails immediately instead
of taking seconds/minutes (depending on the size of the guest) while the guest
is populated.

Note that to enable tmem type guest, one need to provide 'tmem' on the
Xen hypervisor argument and as well on the Linux kernel command line.

There are two boolean options:

(false) No claim is made. Memory population during guest creation will be
attempted as normal and may fail due to memory exhaustion.

(true) Normal memory and freeable pool of ephemeral pages (tmem) is used when
calculating whether there is enough memory free to launch a guest.
This guarantees immediate feedback whether the guest can be launched due
to memory exhaustion (which can take a long time to find out if launching
massively huge guests) and in parallel.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 tools/examples/xend-config.sxp       |  5 +++++
 tools/python/xen/lowlevel/xc/xc.c    | 29 +++++++++++++++++++----------
 tools/python/xen/xend/XendOptions.py |  8 ++++++++
 tools/python/xen/xend/balloon.py     |  4 ++++
 tools/python/xen/xend/image.py       | 13 ++++++++++---
 5 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/tools/examples/xend-config.sxp b/tools/examples/xend-config.sxp
index 0896a27..4d5816c 100644
--- a/tools/examples/xend-config.sxp
+++ b/tools/examples/xend-config.sxp
@@ -302,3 +302,8 @@
 # command lsscsi, e.g. ('16:0:0:0' '15:0') 
 # (pscsi-device-mask ('*'))
 
+# Reserve a claim of memory when launching a guest. This guarantees immediate
+# feedback whether the guest can be launched due to memory exhaustion
+# (which can take a long time to find out if launching huge guests).
+# see xl.conf(5) for details.
+# (claim-mode no)
diff --git a/tools/python/xen/lowlevel/xc/xc.c b/tools/python/xen/lowlevel/xc/xc.c
index e220f68..3540313 100644
--- a/tools/python/xen/lowlevel/xc/xc.c
+++ b/tools/python/xen/lowlevel/xc/xc.c
@@ -455,6 +455,7 @@ static PyObject *pyxc_linux_build(XcObject *self,
     int store_evtchn, console_evtchn;
     int vhpt = 0;
     int superpages = 0;
+    int claim = 0;
     unsigned int mem_mb;
     unsigned long store_mfn = 0;
     unsigned long console_mfn = 0;
@@ -467,14 +468,15 @@ static PyObject *pyxc_linux_build(XcObject *self,
                                 "console_evtchn", "image",
                                 /* optional */
                                 "ramdisk", "cmdline", "flags",
-                                "features", "vhpt", "superpages", NULL };
+                                "features", "vhpt", "superpages",
+                                "claim_mode", NULL };
 
-    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iiiis|ssisii", kwd_list,
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iiiis|ssisiii", kwd_list,
                                       &domid, &store_evtchn, &mem_mb,
                                       &console_evtchn, &image,
                                       /* optional */
                                       &ramdisk, &cmdline, &flags,
-                                      &features, &vhpt, &superpages) )
+                                      &features, &vhpt, &superpages, &claim) )
         return NULL;
 
     xc_dom_loginit(self->xc_handle);
@@ -486,6 +488,8 @@ static PyObject *pyxc_linux_build(XcObject *self,
 
     dom->superpages = superpages;
 
+    dom->claim_enabled = claim;
+
     if ( xc_dom_linux_build(self->xc_handle, dom, domid, mem_mb, image,
                             ramdisk, flags, store_evtchn, &store_mfn,
                             console_evtchn, &console_mfn) != 0 ) {
@@ -944,16 +948,16 @@ static PyObject *pyxc_hvm_build(XcObject *self,
 #endif
     int i;
     char *image;
-    int memsize, target=-1, vcpus = 1, acpi = 0, apic = 1;
+    int memsize, target=-1, vcpus = 1, acpi = 0, apic = 1, claim = 0;
     PyObject *vcpu_avail_handle = NULL;
     uint8_t vcpu_avail[(HVM_MAX_VCPUS + 7)/8];
-
+    struct xc_hvm_build_args pargs = {};
     static char *kwd_list[] = { "domid",
                                 "memsize", "image", "target", "vcpus", 
-                                "vcpu_avail", "acpi", "apic", NULL };
-    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iis|iiOii", kwd_list,
+                                "vcpu_avail", "acpi", "apic", "claim_mode", NULL };
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iis|iiOiii", kwd_list,
                                       &dom, &memsize, &image, &target, &vcpus,
-                                      &vcpu_avail_handle, &acpi, &apic) )
+                                      &vcpu_avail_handle, &acpi, &apic, &claim) )
         return NULL;
 
     memset(vcpu_avail, 0, sizeof(vcpu_avail));
@@ -984,8 +988,13 @@ static PyObject *pyxc_hvm_build(XcObject *self,
     if ( target == -1 )
         target = memsize;
 
-    if ( xc_hvm_build_target_mem(self->xc_handle, dom, memsize,
-                                 target, image) != 0 )
+    memset(&pargs, 0, sizeof(struct xc_hvm_build_args));
+    pargs.mem_size = (uint64_t)memsize << 20;
+    pargs.mem_target = (uint64_t)target << 20;
+    pargs.image_file_name = image;
+    pargs.claim_enabled = claim;
+
+    if ( xc_hvm_build(self->xc_handle, dom, &pargs) != 0 )
         return pyxc_error_to_exception(self->xc_handle);
 
 #if !defined(__ia64__)
diff --git a/tools/python/xen/xend/XendOptions.py b/tools/python/xen/xend/XendOptions.py
index cc6f38e..275efdc 100644
--- a/tools/python/xen/xend/XendOptions.py
+++ b/tools/python/xen/xend/XendOptions.py
@@ -154,6 +154,12 @@ class XendOptions:
     use loose check automatically if necessary."""
     pci_dev_assign_strict_check_default = True
 
+    """Reserve a claim of memory when launching a guest. This guarantees
+    immediate feedback whether the guest can be launched due to memory
+    exhaustion (which can take a long time to find out if launching huge
+    guests)."""
+    claim_mode_default = False
+
     def __init__(self):
         self.configure()
 
@@ -436,6 +442,8 @@ class XendOptions:
     def get_pscsi_device_mask(self):
         return self.get_config_value("pscsi-device-mask",
                                       self.xend_pscsi_device_mask)
+    def get_claim_mode(self):
+        return self.get_config_bool("claim-mode", self.claim_mode_default)
 
 class XendOptionsFile(XendOptions):
 
diff --git a/tools/python/xen/xend/balloon.py b/tools/python/xen/xend/balloon.py
index 89965d7..dcd4caa 100644
--- a/tools/python/xen/xend/balloon.py
+++ b/tools/python/xen/xend/balloon.py
@@ -116,6 +116,10 @@ def free(need_mem, dominfo):
             dom0_ballooning = xoptions.get_enable_dom0_ballooning()
         dom0_alloc = get_dom0_current_alloc()
 
+        # let it handle it all via the claim option
+        if (xoptions.get_claim_mode() and not dom0_ballooning):
+            return
+
         retries = 0
         sleep_time = SLEEP_TIME_GROWTH
         new_alloc = 0
diff --git a/tools/python/xen/xend/image.py b/tools/python/xen/xend/image.py
index 832c168..7d3716d 100644
--- a/tools/python/xen/xend/image.py
+++ b/tools/python/xen/xend/image.py
@@ -84,6 +84,7 @@ class ImageHandler:
 
     ostype = None
     superpages = 0
+    claim_mode = 0
     memory_sharing = 0
 
     def __init__(self, vm, vmConfig):
@@ -95,7 +96,9 @@ class ImageHandler:
         self.kernel = None
         self.ramdisk = None
         self.cmdline = None
-
+        xoptions = XendOptions.instance()
+        if (xoptions.get_claim_mode() == True):
+            self.claim_mode = 1
         self.configure(vmConfig)
 
     def configure(self, vmConfig):
@@ -729,6 +732,7 @@ class LinuxImageHandler(ImageHandler):
         log.debug("features       = %s", self.vm.getFeatures())
         log.debug("flags          = %d", self.flags)
         log.debug("superpages     = %d", self.superpages)
+        log.debug("claim_mode     = %d", self.claim_mode)
         if arch.type == "ia64":
             log.debug("vhpt          = %d", self.vhpt)
 
@@ -742,7 +746,8 @@ class LinuxImageHandler(ImageHandler):
                               features       = self.vm.getFeatures(),
                               flags          = self.flags,
                               vhpt           = self.vhpt,
-                              superpages     = self.superpages)
+                              superpages     = self.superpages,
+                              claim_mode     = self.claim_mode)
 
     def getBitSize(self):
         return xc.getBitSize(image    = self.kernel,
@@ -956,6 +961,7 @@ class HVMImageHandler(ImageHandler):
         log.debug("vcpu_avail     = %li", self.vm.getVCpuAvail())
         log.debug("acpi           = %d", self.acpi)
         log.debug("apic           = %d", self.apic)
+        log.debug("claim_mode     = %d", self.claim_mode)
 
         rc = xc.hvm_build(domid          = self.vm.getDomid(),
                           image          = self.loader,
@@ -964,7 +970,8 @@ class HVMImageHandler(ImageHandler):
                           vcpus          = self.vm.getVCpuCount(),
                           vcpu_avail     = self.vm.getVCpuAvail(),
                           acpi           = self.acpi,
-                          apic           = self.apic)
+                          apic           = self.apic,
+                          claim_mode     = self.claim_mode)
         rc['notes'] = { 'SUSPEND_CANCEL': 1 }
 
         rc['store_mfn'] = xc.hvm_get_param(self.vm.getDomid(),
-- 
1.8.0.2