From: Elena Ufimtseva <ufimtseva@gmail.com>
To: xen-devel@lists.xen.org
Cc: Ian.Campbell@citrix.com, stefano.stabellini@eu.citrix.com,
george.dunlap@eu.citrix.com, msw@linux.com,
dario.faggioli@citrix.com, lccycc123@gmail.com,
ian.jackson@eu.citrix.com, Elena Ufimtseva <ufimtseva@gmail.com>
Subject: [PATCH v2 5/7] libxl: vNUMA configuration parser
Date: Wed, 13 Nov 2013 22:27:04 -0500 [thread overview]
Message-ID: <1384399624-24125-1-git-send-email-ufimtseva@gmail.com> (raw)
Parses guest vNUMA related options, verifies and
sets default values for errorneous parameters.
To enable vNUMA, vnodes is a must in the config,
all other options can be omitted.
all possible config options are:
vnodes: number of vnuma nodes:
vnodes = 2
vnumamem: vnodes memory in MBytes:
vnumamem = [2048, 2048]
vdistance: distance table or part of it.
Can use minimal form with same-node distance first and
second - all other nodes distances:
vdistance = [10, 20]
will expand to square symmetrical matrix of any n x n:
[10, 20, 20]
[20, 10, 20]
[20, 20, 10]
If not defined or format is incorrect, default [10, 20] will be used.
Latest kernel (3.13-rc1) oopses on non-symmetrical distance table and
next version will introduce extended distance table in config.
vnuma_vcpumap: vcpu to vnode mapping. The index is a vcpu
number starting from 0, the value is the node number, starting
from zero.
vnuma_vcpumap =[1, 0, 1, 0]
node 1 - vcpus 0 and 2
node 0 - vcpus 1 and 3
If verification fails, default vcpu interleaved map over vnodes
will be used.
vnuma_vnodemap - vnode tp physical node mapping.
Suggested map will be verified and if the requested mapping
cannot be completed (no NUMA hardware, not correct mapping,
not enough memory), will try to set this with help of automatic
numa placement. If this fails, will be used not node specific
allocation.
vnuma_vnodemap = [1, 0]
vnode0 -> pnode1
vnode1 -> pnode0
Signed-off-by: Elena Ufimtseva <ufimtseva@gmail.com>
---
tools/libxl/xl_cmdimpl.c | 268 +++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 266 insertions(+), 2 deletions(-)
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 5bd9b15..3963b75 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -40,6 +40,7 @@
#include "libxl_json.h"
#include "libxlutil.h"
#include "xl.h"
+#include "libxl_vnuma.h"
#define CHK_ERRNO( call ) ({ \
int chk_errno = (call); \
@@ -572,6 +573,74 @@ vcpp_out:
return rc;
}
+static void vnuma_info_release(libxl_domain_build_info *info)
+{
+ info->nr_vnodes = 0;
+ if ( info->vnuma_memszs ) free(info->vnuma_memszs);
+ if ( info->vdistance ) free(info->vdistance);
+ if ( info->vcpu_to_vnode ) free(info->vcpu_to_vnode);
+ if ( info->vnode_numamap ) free(info->vnode_numamap);
+}
+
+static int get_list_item_uint(XLU_ConfigList *list, unsigned int i)
+{
+ const char *buf;
+ char *ep;
+ unsigned long ul;
+ int rc = -EINVAL;
+ buf = xlu_cfg_get_listitem(list, i);
+ if (!buf)
+ return rc;
+ ul = strtoul(buf, &ep, 10);
+ if (ep == buf)
+ return rc;
+ if (ul >= UINT16_MAX)
+ return rc;
+ return (int)ul;
+}
+
+static void vdistance_default(unsigned int *vdistance,
+ unsigned int nr_vnodes,
+ unsigned int samenode,
+ unsigned int othernode)
+{
+ int i, j;
+ for (i = 0; i < nr_vnodes; i++)
+ for (j = 0; j < nr_vnodes; j++)
+ *(vdistance + j * nr_vnodes + i) = i == j ? samenode : othernode;
+}
+
+static void vcputovnode_default(unsigned int *vcpu_to_vnode,
+ unsigned int nr_vnodes,
+ unsigned int max_vcpus)
+{
+ int i;
+ if (vcpu_to_vnode == NULL)
+ return;
+ for(i = 0; i < max_vcpus; i++)
+ vcpu_to_vnode[i] = i % nr_vnodes;
+}
+
+/* Split domain memory between vNUMA nodes equally */
+static int split_vnumamem(libxl_domain_build_info *b_info)
+{
+ unsigned long long vnodemem = 0;
+ unsigned long n;
+ unsigned int i;
+ /* In MBytes */
+ vnodemem = (b_info->max_memkb >> 10) / b_info->nr_vnodes;
+ if (vnodemem < MIN_VNODE_SIZE)
+ return -1;
+ /* reminder in MBytes */
+ n = (b_info->max_memkb >> 10) % b_info->nr_vnodes;
+ /* get final sizes in MBytes */
+ for(i = 0; i < (b_info->nr_vnodes - 1); i++)
+ b_info->vnuma_memszs[i] = vnodemem;
+ /* add the reminder to the last node */
+ b_info->vnuma_memszs[i] = vnodemem + n;
+ return 0;
+}
+
static void parse_config_data(const char *config_source,
const char *config_data,
int config_len,
@@ -906,7 +975,11 @@ static void parse_config_data(const char *config_source,
{
char *cmdline = NULL;
const char *root = NULL, *extra = "";
-
+ XLU_ConfigList *vnumamemcfg, *vdistancecfg, *vnodemap, *vcpumap;
+ int nr_vnuma_regions, nr_vdist, nr_vnodemap;
+ unsigned long long vnuma_memparsed = 0;
+ unsigned long ul;
+
xlu_cfg_replace_string (config, "kernel", &b_info->u.pv.kernel, 0);
xlu_cfg_get_string (config, "root", &root, 0);
@@ -923,7 +996,198 @@ static void parse_config_data(const char *config_source,
fprintf(stderr, "Failed to allocate memory for cmdline\n");
exit(1);
}
-
+
+ if (!xlu_cfg_get_long (config, "vnodes", &l, 0)) {
+ if (l > MAX_VNUMA_NODES) {
+ fprintf(stderr, "Too many vnuma nodes, max %d is allowed.\n", MAX_VNUMA_NODES);
+ exit(1);
+ }
+ b_info->nr_vnodes = l;
+ if (b_info->nr_vnodes != 0 || b_info->max_vcpus >= b_info->nr_vnodes) {
+ if (!xlu_cfg_get_list(config, "vnumamem",
+ &vnumamemcfg, &nr_vnuma_regions, 0)) {
+ /*
+ * If number of regions parsed != number of nodes, check
+ * the memory configuration anyways and if its ok we adjust total
+ * number of nodes. The memory parsed is in MBytes.
+ */
+ if (nr_vnuma_regions != b_info->nr_vnodes)
+ fprintf(stderr, "Number of numa regions is incorrect. Will attepmt to adjust.\n");
+ b_info->vnuma_memszs = calloc(b_info->nr_vnodes,
+ sizeof(*b_info->vnuma_memszs));
+ if (b_info->vnuma_memszs == NULL) {
+ fprintf(stderr, "unable to allocate memory for vnuma ranges.\n");
+ exit(1);
+ }
+ char *ep;
+ /*
+ * Will parse only nr_vnodes times, even if we have more/less regions.
+ * Take care of it later if less or discard if too many regions.
+ */
+ for (i = 0; i < b_info->nr_vnodes; i++) {
+ buf = xlu_cfg_get_listitem(vnumamemcfg, i);
+ if (!buf) {
+ fprintf(stderr,
+ "xl: Unable to get element %d in vnuma memroy list.\n", i);
+ break;
+ }
+ ul = strtoul(buf, &ep, 10);
+ if (ep == buf) {
+ fprintf(stderr,
+ "xl: Invalid argument parsing vnumamem: %s.\n", buf);
+ break;
+ }
+ /* 32Mb is a min size for a node, taken from Linux */
+ if (ul >= UINT32_MAX || ul < MIN_VNODE_SIZE) {
+ fprintf(stderr, "xl: vnuma memory %lu is not withing %u - %u range.\n",
+ ul, MIN_VNODE_SIZE, UINT32_MAX);
+ break;
+ }
+ /* memory in MBytes */
+ b_info->vnuma_memszs[i] = ul;
+ }
+ /* Total memory for vNUMA parsed to verify */
+ for(i = 0; i < nr_vnuma_regions; i++)
+ vnuma_memparsed = vnuma_memparsed + (b_info->vnuma_memszs[i]);
+ /* Now we have all inputs. Check for misconfigurations and adjust if needed */
+ /* Amount of memory for vnodes same as total? */
+ if((vnuma_memparsed << 10) == (b_info->max_memkb)) {
+ if(b_info->nr_vnodes != nr_vnuma_regions) {
+ fprintf(stderr, "xl: vnuma memory regions looks incorrect, will use first %d.\n",
+ b_info->nr_vnodes);
+ }
+ } else {
+ fprintf(stderr, "WARNING: vNUMA memory %llu Kbytes is %s than total memory.\n",
+ vnuma_memparsed << 10,
+ (vnuma_memparsed << 10)> b_info->max_memkb ? "more" : "less");
+ fprintf(stderr, "Each vnode will get equal memory size of %lu Kbytes.\n",
+ b_info->max_memkb / b_info->nr_vnodes);
+ if (split_vnumamem(b_info) < 0) {
+ vnuma_info_release(b_info);
+ exit(1);
+ }
+ }
+ } else {
+ b_info->vnuma_memszs = calloc(b_info->nr_vnodes,
+ sizeof(*b_info->vnuma_memszs));
+ if (b_info->vnuma_memszs == NULL) {
+ fprintf(stderr, "unable to allocate memory for vnuma ranges.\n");
+ exit(1);
+ }
+ fprintf(stderr, "WARNING: vNUMA memory ranges were not specified.\n");
+ fprintf(stderr, "Using default equal vnode memory size %lu Kbytes to cover %lu Kbytes.\n",
+ b_info->max_memkb / b_info->nr_vnodes,
+ b_info->max_memkb);
+ if (split_vnumamem(b_info) < 0) {
+ vnuma_info_release(b_info);
+ exit(1);
+ }
+ }
+ b_info->vdistance = calloc(b_info->nr_vnodes * b_info->nr_vnodes,
+ sizeof(*b_info->vdistance));
+ if (b_info->vdistance == NULL) {
+ vnuma_info_release(b_info);
+ exit(1);
+ }
+ if(!xlu_cfg_get_list(config, "vdistance", &vdistancecfg, &nr_vdist, 0) &&
+ nr_vdist == 2) {
+ /*
+ * If only two elements are in the vdistance list, consider
+ * first as value for same node distance, the second as the
+ * rest of distances.
+ * The following is required right now to avoid non-symmetrical
+ * distance table as it may break latest kernel.
+ * TODO: Better way to analyze extended distance table, possibly
+ * OS specific.
+ */
+ int d1, d2;
+ d1 = get_list_item_uint(vdistancecfg, 0);
+ d2 = get_list_item_uint(vdistancecfg, 1);
+ if (d1 >= 0 && d2 >= 0)
+ vdistance_default(b_info->vdistance, b_info->nr_vnodes, d1, d2);
+ } else
+ vdistance_default(b_info->vdistance, b_info->nr_vnodes, 10, 20);
+
+ b_info->vcpu_to_vnode = (unsigned int *)calloc(b_info->max_vcpus,
+ sizeof(*b_info->vcpu_to_vnode));
+ if (b_info->vcpu_to_vnode == NULL)
+ exit(1);
+
+ b_info->vcpu_to_vnode = (unsigned int *)calloc(b_info->max_vcpus,
+ sizeof(*b_info->vcpu_to_vnode));
+
+ if (b_info->vcpu_to_vnode == NULL) {
+ vnuma_info_release(b_info);
+ exit(1);
+ }
+ if (!xlu_cfg_get_list(config, "vnuma_vcpumap",
+ &vcpumap, &nr_vnodemap, 0)) {
+ if (nr_vnodemap == b_info->max_vcpus) {
+ unsigned int vcpumask = 0, vcpu, smask;
+ smask = ~(~0 << nr_vnodemap);
+ for (i = 0; i < nr_vnodemap; i++) {
+ vcpu = get_list_item_uint(vcpumap, i);
+ if (vcpu >= 0) {
+ vcpumask |= (1 << i);
+ b_info->vcpu_to_vnode[i] = vcpu;
+ }
+ }
+ /* Did it covered all vnodes in the vcpu mask? */
+ if ( !((vcpumask & smask) + 1) == (1 << nr_vnodemap) ) {
+ /* no, using default */
+ vcputovnode_default(b_info->vcpu_to_vnode,
+ b_info->nr_vnodes,
+ b_info->max_vcpus);
+ }
+ } else
+ vcputovnode_default(b_info->vcpu_to_vnode,
+ b_info->nr_vnodes,
+ b_info->max_vcpus);
+ }
+ else
+ vcputovnode_default(b_info->vcpu_to_vnode,
+ b_info->nr_vnodes,
+ b_info->max_vcpus);
+ /* There is mapping to NUMA physical nodes? */
+ if (!xlu_cfg_get_list(config, "vnuma_vnodemap",&vnodemap,
+ &nr_vnodemap, 0)) {
+ /*
+ * If not specified or incorred, will be defined
+ * later based on the machine architecture, configuration
+ * and memory availble when creating domain.
+ */
+ if (nr_vnodemap == b_info->nr_vnodes) {
+ b_info->vnode_numamap = (unsigned int *)calloc(b_info->nr_vnodes,
+ sizeof(*b_info->vnode_numamap));
+ if (b_info->vnode_numamap == NULL) {
+ vnuma_info_release(b_info);
+ exit(1);
+ }
+
+ unsigned int vnodemask = 0, pnode, smask;
+ smask = ~(~0 << nr_vnodemap);
+ for (i = 0; i < nr_vnodemap; i++) {
+ pnode = get_list_item_uint(vnodemap, i);
+ if (pnode >= 0) {
+ vnodemask |= (1 << i);
+ b_info->vnode_numamap[i] = pnode;
+ }
+ }
+ /* Did it covered all vnodes in the mask? */
+ if ( !((vnodemask & smask) + 1) == (1 << nr_vnodemap) ) {
+ fprintf(stderr, "WARNING: Not all vnodes were covered vnuma_vnodemap.\n");
+ free(b_info->vnode_numamap);
+ b_info->vnode_numamap = NULL;
+ }
+ }
+ }
+ }
+ else
+ b_info->nr_vnodes=0;
+ }
+ else
+ b_info->nr_vnodes = 0;
+
xlu_cfg_replace_string (config, "bootloader", &b_info->u.pv.bootloader, 0);
switch (xlu_cfg_get_list_as_string_list(config, "bootloader_args",
&b_info->u.pv.bootloader_args, 1))
--
1.7.10.4
reply other threads:[~2013-11-14 3:27 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1384399624-24125-1-git-send-email-ufimtseva@gmail.com \
--to=ufimtseva@gmail.com \
--cc=Ian.Campbell@citrix.com \
--cc=dario.faggioli@citrix.com \
--cc=george.dunlap@eu.citrix.com \
--cc=ian.jackson@eu.citrix.com \
--cc=lccycc123@gmail.com \
--cc=msw@linux.com \
--cc=stefano.stabellini@eu.citrix.com \
--cc=xen-devel@lists.xen.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).