[PATCH v6 07/10] libxc: allocate domain memory for vnuma enabled

[Elena Ufimtseva <ufimtseva@gmail.com>]

vNUMA-aware domain memory allocation based on provided
vnode to pnode map. If this map is not defined, use
default allocation. Default allocation will not specify
any physical node when allocating memory.
Domain creation will fail if at least one node was not defined.

Signed-off-by: Elena Ufimtseva <ufimtseva@gmail.com>
---
 tools/libxc/xc_dom.h     |   10 ++++++
 tools/libxc/xc_dom_x86.c |   76 ++++++++++++++++++++++++++++++++++------------
 tools/libxc/xg_private.h |    1 +
 3 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/tools/libxc/xc_dom.h b/tools/libxc/xc_dom.h
index 71a3701..4cc08bc 100644
--- a/tools/libxc/xc_dom.h
+++ b/tools/libxc/xc_dom.h
@@ -164,6 +164,16 @@ struct xc_dom_image {
 
     /* kernel loader */
     struct xc_dom_arch *arch_hooks;
+
+   /*
+    * vNUMA topology and memory allocation structure.
+    * Defines the way to allocate memory on per NUMA
+    * physical defined by vnode_to_pnode.
+    */
+    uint32_t nr_nodes;
+    uint64_t *numa_memszs;
+    unsigned int *vnode_to_pnode;
+
     /* allocate up to virt_alloc_end */
     int (*allocate) (struct xc_dom_image * dom, xen_vaddr_t up_to);
 };
diff --git a/tools/libxc/xc_dom_x86.c b/tools/libxc/xc_dom_x86.c
index 40d3408..23267ed 100644
--- a/tools/libxc/xc_dom_x86.c
+++ b/tools/libxc/xc_dom_x86.c
@@ -756,26 +756,6 @@ static int x86_shadow(xc_interface *xch, domid_t domid)
     return rc;
 }
 
-int arch_boot_alloc(struct xc_dom_image *dom)
-{
-        int rc = 0;
-        xen_pfn_t allocsz, i;
-
-        /* allocate guest memory */
-        for ( i = rc = allocsz = 0;
-              (i < dom->total_pages) && !rc;
-              i += allocsz )
-        {
-            allocsz = dom->total_pages - i;
-            if ( allocsz > 1024*1024 )
-                allocsz = 1024*1024;
-            rc = xc_domain_populate_physmap_exact(
-                dom->xch, dom->guest_domid, allocsz,
-                0, 0, &dom->p2m_host[i]);
-        }
-        return rc;
-}
-
 int arch_setup_meminit(struct xc_dom_image *dom)
 {
     int rc;
@@ -832,6 +812,13 @@ int arch_setup_meminit(struct xc_dom_image *dom)
         for ( pfn = 0; pfn < dom->total_pages; pfn++ )
             dom->p2m_host[pfn] = pfn;
 
+        /* allocate guest memory */
+        if ( dom->nr_nodes == 0 ) {
+            xc_dom_panic(dom->xch, XC_INTERNAL_ERROR,
+                         "%s: Cannot allocate domain memory for 0 vnodes\n",
+                         __FUNCTION__);
+            return -EINVAL;
+        }
         rc = arch_boot_alloc(dom);
         if ( rc )
             return rc;
@@ -841,6 +828,55 @@ int arch_setup_meminit(struct xc_dom_image *dom)
         (void)xc_domain_claim_pages(dom->xch, dom->guest_domid,
                                     0 /* cancels the claim */);
     }
+    return rc;
+}
+
+/*
+ * Allocates domain memory taking into account
+ * defined vnuma topology and vnode_to_pnode map.
+ * Any pv guest will have at least one vnuma node
+ * with vnuma_memszs[0] = domain memory and the rest
+ * topology initialized with default values.
+ */
+int arch_boot_alloc(struct xc_dom_image *dom)
+{
+    int rc;
+    unsigned int n, memflags;
+    unsigned long long vnode_pages;
+    unsigned long long allocsz = 0, node_pfn_base, i;
+
+    rc = allocsz = node_pfn_base = 0;
+
+    allocsz = 0;
+    for ( n = 0; n < dom->nr_nodes; n++ )
+    {
+        memflags = 0;
+        if ( dom->vnode_to_pnode[n] != VNUMA_NO_NODE )
+        {
+            memflags |= XENMEMF_exact_node(dom->vnode_to_pnode[n]);
+            memflags |= XENMEMF_exact_node_request;
+        }
+        /* memeszs are in megabytes, calc pages from it for this node. */
+        vnode_pages = (dom->numa_memszs[n] << 20) >> PAGE_SHIFT_X86;
+        for ( i = 0; i < vnode_pages; i += allocsz )
+        {
+            allocsz = vnode_pages - i;
+            if ( allocsz > 1024*1024 )
+                allocsz = 1024*1024;
+
+            rc = xc_domain_populate_physmap_exact(dom->xch, dom->guest_domid,
+                                            allocsz, 0, memflags,
+                                            &dom->p2m_host[node_pfn_base + i]);
+            if ( rc )
+            {
+                xc_dom_panic(dom->xch, XC_INTERNAL_ERROR,
+                        "%s: Failed allocation of %Lu pages for vnode %d on pnode %d out of %lu\n",
+                        __FUNCTION__, vnode_pages, n, dom->vnode_to_pnode[n], dom->total_pages);
+                return rc;
+            }
+        }
+        node_pfn_base += i;
+    }
 
     return rc;
 }
diff --git a/tools/libxc/xg_private.h b/tools/libxc/xg_private.h
index e593364..21e4a20 100644
--- a/tools/libxc/xg_private.h
+++ b/tools/libxc/xg_private.h
@@ -123,6 +123,7 @@ typedef uint64_t l4_pgentry_64_t;
 #define ROUNDUP(_x,_w) (((unsigned long)(_x)+(1UL<<(_w))-1) & ~((1UL<<(_w))-1))
 #define NRPAGES(x) (ROUNDUP(x, PAGE_SHIFT) >> PAGE_SHIFT)
 
+#define VNUMA_NO_NODE ~((unsigned int)0)
 
 /* XXX SMH: following skanky macros rely on variable p2m_size being set */
 /* XXX TJD: also, "guest_width" should be the guest's sizeof(unsigned long) */
-- 
1.7.10.4