[XEN][vNUMA][PATCH 2/9] Config options

[XEN][vNUMA][PATCH 2/9] Config options

[Dulloor]

[-- Attachment #1: Type: text/plain, Size: 211 bytes --]

Implement the following config options :

strategy = “str”, where str is confine/stripe/split/auto
vnodes = <num-nodes>
stripesz = <size-in-pages>

-dulloor

Signed-off-by : Dulloor <dulloor@gmail.com>

[-- Attachment #2: xen-02-config-options.patch --]
[-- Type: text/x-patch, Size: 4479 bytes --]

diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -227,6 +227,25 @@ typedef union
     start_info_t s;
 } start_info_any_t;
 
+/**
+ * struct xc_dom_numa_info : Carries information required for NUMA memory 
+ * allocation for the guests.
+ */
+#define XC_DOM_NUMA_AUTO     0  /* Let the allocator choose */
+#define XC_DOM_NUMA_CONFINE  1
+#define XC_DOM_NUMA_SPLIT    2
+#define XC_DOM_NUMA_STRIPE   3
+#define XC_DOM_NUMA_NONE     4
+
+#define XC_DOM_NUMA_DEF_STRIPE 32    /* in 4K pages */  
+
+typedef struct xc_domain_numa_config
+{
+    uint32_t strategy;      /* By default, DONTCARE (for now) */
+    uint32_t nr_nodes;      /* For SPLIT/STRIPE */
+    uint32_t stripe_size;   /* For STRIPE only */
+} xc_domain_numa_config_t;
+
 
 int xc_domain_create(xc_interface *xch,
                      uint32_t ssidref,
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -102,6 +102,7 @@ typedef struct {
     uint32_t shadow_memkb;
     const char *kernel;
     int hvm;
+    struct xc_domain_numa_config numa_config;
     union {
         struct {
             bool pae;
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -208,6 +208,7 @@ static void init_build_info(libxl_domain
     } else {
         b_info->u.pv.slack_memkb = 8 * 1024;
     }
+    b_info->numa_config.strategy = XC_DOM_NUMA_NONE;
 }
 
 static void init_dm_info(libxl_device_model_info *dm_info,
@@ -316,6 +317,37 @@ static void init_console_info(libxl_devi
         console->build_state = state;
 }
 
+static char *numa_val_to_str(uint32_t val)
+{
+    switch (val)
+    {
+        case XC_DOM_NUMA_AUTO:
+                return "AUTO";
+        case XC_DOM_NUMA_CONFINE:
+                return "CONFINE";
+        case XC_DOM_NUMA_SPLIT:
+                return "SPLIT";
+        case XC_DOM_NUMA_STRIPE:
+                return "STRIPE";
+        default:
+                return "NONE";
+    }
+}
+
+static uint32_t numa_str_to_val(const char *str)
+{
+    if (!strcasecmp(str, "AUTO"))
+        return XC_DOM_NUMA_AUTO;
+    if (!strcasecmp(str, "CONFINE"))
+        return XC_DOM_NUMA_CONFINE;
+    if (!strcasecmp(str, "SPLIT"))
+        return XC_DOM_NUMA_SPLIT;
+    if (!strcasecmp(str, "STRIPE"))
+        return XC_DOM_NUMA_STRIPE;
+
+    return XC_DOM_NUMA_NONE;
+}
+
 static void printf_info(int domid,
                         libxl_domain_create_info *c_info,
                         libxl_domain_build_info *b_info,
@@ -363,6 +395,10 @@ static void printf_info(int domid,
     printf("\t(tsc_mode %d)\n", b_info->tsc_mode);
     printf("\t(max_memkb %d)\n", b_info->max_memkb);
     printf("\t(target_memkb %d)\n", b_info->target_memkb);
+    printf("\t(numa_strategy %s)\n", 
+                            numa_val_to_str(b_info->numa_config.strategy));
+    printf("\t(numa_nodes %d)\n", b_info->numa_config.nr_nodes);
+    printf("\t(stripe_size %d)\n", b_info->numa_config.stripe_size);
 
     printf("\t(image\n");
     if (c_info->hvm) {
@@ -557,6 +593,14 @@ static void parse_config_data(const char
     if (!xlu_cfg_get_long (config, "videoram", &l))
         b_info->video_memkb = l * 1024;
 
+    if (!xlu_cfg_get_string (config, "strategy", &buf)) {
+        b_info->numa_config.strategy = numa_str_to_val(buf);
+        if (!xlu_cfg_get_long (config, "vnodes", &l))
+            b_info->numa_config.nr_nodes = l;
+        if (!xlu_cfg_get_long (config, "stripesz", &l))
+            b_info->numa_config.stripe_size = l;
+    }
+
     if (!xlu_cfg_get_string (config, "kernel", &buf))
         b_info->kernel = strdup(buf);
 
@@ -1066,6 +1110,9 @@ static int create_domain(struct domain_c
 
     parse_config_data(config_file, config_data, config_len, &info1, &info2, &disks, &num_disks, &vifs, &num_vifs, &vif2s, &num_vif2s, &pcidevs, &num_pcidevs, &vfbs, &num_vfbs, &vkbs, &num_vkbs, &dm_info);
 
+    if (debug)
+        printf_info(-1, &info1, &info2, disks, num_disks, vifs, num_vifs, pcidevs, num_pcidevs, vfbs, num_vfbs, vkbs, num_vkbs, &dm_info);
+
     if (dom_info->dryrun)
         return 0;
 
@@ -1083,9 +1130,6 @@ static int create_domain(struct domain_c
         }
     }
 
-    if (debug)
-        printf_info(-1, &info1, &info2, disks, num_disks, vifs, num_vifs, pcidevs, num_pcidevs, vfbs, num_vfbs, vkbs, num_vkbs, &dm_info);
-
 start:
     domid = 0;
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

[vNUMA v2][PATCH 1/8] Config options

[Dulloor]

[-- Attachment #1: Type: text/plain, Size: 211 bytes --]

Implement the following config options :

strategy = “str”, where str is confine/stripe/split/auto
vnodes = <num-nodes>
stripesz = <size-in-pages>

-dulloor

Signed-off-by : Dulloor <dulloor@gmail.com>

[-- Attachment #2: xen-01-config-options.patch --]
[-- Type: text/x-patch, Size: 3834 bytes --]

vNUMA : Guest NUMA config options

diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -227,6 +227,25 @@ typedef union
     start_info_t s;
 } start_info_any_t;
 
+/**
+ * struct xc_dom_numa_info : Carries information required for NUMA memory 
+ * allocation for the guests.
+ */
+#define XC_DOM_NUMA_AUTO     0  /* Let the allocator choose */
+#define XC_DOM_NUMA_CONFINE  1
+#define XC_DOM_NUMA_SPLIT    2
+#define XC_DOM_NUMA_STRIPE   3
+#define XC_DOM_NUMA_NONE     4
+
+#define XC_DOM_NUMA_DEF_STRIPE 32    /* in 4K pages */  
+
+typedef struct xc_domain_numa_config
+{
+    uint32_t strategy;      /* By default, DONTCARE (for now) */
+    uint32_t nr_nodes;      /* For SPLIT/STRIPE */
+    uint32_t stripe_size;   /* For STRIPE only */
+} xc_domain_numa_config_t;
+
 
 int xc_domain_create(xc_interface *xch,
                      uint32_t ssidref,
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -129,6 +129,7 @@ typedef struct {
     bool disable_migrate;
     libxl_file_reference kernel;
     int hvm;
+    struct xc_domain_numa_config numa_config;
     union {
         struct {
             bool pae;
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -253,6 +253,7 @@ static void init_build_info(libxl_domain
     } else {
         b_info->u.pv.slack_memkb = 8 * 1024;
     }
+    b_info->numa_config.strategy = XC_DOM_NUMA_NONE;
 }
 
 static void random_uuid(uint8_t *uuid)
@@ -365,6 +366,37 @@ static void init_console_info(libxl_devi
         console->build_state = state;
 }
 
+static char *numa_val_to_str(uint32_t val)
+{
+    switch (val)
+    {
+        case XC_DOM_NUMA_AUTO:
+                return "AUTO";
+        case XC_DOM_NUMA_CONFINE:
+                return "CONFINE";
+        case XC_DOM_NUMA_SPLIT:
+                return "SPLIT";
+        case XC_DOM_NUMA_STRIPE:
+                return "STRIPE";
+        default:
+                return "NONE";
+    }
+}
+
+static uint32_t numa_str_to_val(const char *str)
+{
+    if (!strcasecmp(str, "AUTO"))
+        return XC_DOM_NUMA_AUTO;
+    if (!strcasecmp(str, "CONFINE"))
+        return XC_DOM_NUMA_CONFINE;
+    if (!strcasecmp(str, "SPLIT"))
+        return XC_DOM_NUMA_SPLIT;
+    if (!strcasecmp(str, "STRIPE"))
+        return XC_DOM_NUMA_STRIPE;
+
+    return XC_DOM_NUMA_NONE;
+}
+
 static void printf_info(int domid,
                         struct domain_config *d_config,
                         libxl_device_model_info *dm_info)
@@ -403,6 +435,10 @@ static void printf_info(int domid,
     printf("\t(max_memkb %d)\n", b_info->max_memkb);
     printf("\t(target_memkb %d)\n", b_info->target_memkb);
     printf("\t(nomigrate %d)\n", b_info->disable_migrate);
+    printf("\t(numa_strategy %s)\n", 
+                            numa_val_to_str(b_info->numa_config.strategy));
+    printf("\t(numa_nodes %d)\n", b_info->numa_config.nr_nodes);
+    printf("\t(stripe_size %d)\n", b_info->numa_config.stripe_size);
 
     if (!c_info->hvm && b_info->u.pv.bootloader) {
         printf("\t(bootloader %s)\n", b_info->u.pv.bootloader);
@@ -650,6 +686,14 @@ static void parse_config_data(const char
     if (!xlu_cfg_get_long (config, "videoram", &l))
         b_info->video_memkb = l * 1024;
 
+    if (!xlu_cfg_get_string (config, "strategy", &buf)) {
+        b_info->numa_config.strategy = numa_str_to_val(buf);
+        if (!xlu_cfg_get_long (config, "vnodes", &l))
+            b_info->numa_config.nr_nodes = l;
+        if (!xlu_cfg_get_long (config, "stripesz", &l))
+            b_info->numa_config.stripe_size = l;
+    }
+
     if (!xlu_cfg_get_string (config, "kernel", &buf))
         b_info->kernel.path = strdup(buf);
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [vNUMA v2][PATCH 1/8] Config options

[Ian Campbell]

On Sun, 2010-08-01 at 23:01 +0100, Dulloor wrote:
> Implement the following config options :
> 
> strategy = “str”, where str is confine/stripe/split/auto
> vnodes = <num-nodes>
> stripesz = <size-in-pages>

I think these would all be better with a "numa_" prefix or something
similar. "strategy" in particular is a bit generic.

Ian.

Re: [vNUMA v2][PATCH 1/8] Config options

[Andre Przywara]

Dulloor wrote:
> Implement the following config options :
> 
> strategy = "str", where str is confine/stripe/split/auto
> vnodes = <num-nodes>
> stripesz = <size-in-pages>
As Ian already said, I'd also prefer NUMA related names instead of
the generic "strategy". Also "vnodes" may be a bit misleading, what about
"guestnodes"?

> +typedef struct xc_domain_numa_config
> +{
> +    uint32_t strategy;      /* By default, DONTCARE (for now) */
> +    uint32_t nr_nodes;      /* For SPLIT/STRIPE */
> +    uint32_t stripe_size;   /* For STRIPE only */
Are 32 bit here sufficient? Although for the stripe size probably 4GB
are more than needed, I'd prefer to use 64bit (or long) for each
memory-related variable.
> +} xc_domain_numa_config_t;
> +

> +++ b/tools/libxl/xl_cmdimpl.c
> +static uint32_t numa_str_to_val(const char *str)
> +{
> +    if (!strcasecmp(str, "AUTO"))
> +        return XC_DOM_NUMA_AUTO;
> +    if (!strcasecmp(str, "CONFINE"))
> +        return XC_DOM_NUMA_CONFINE;
> +    if (!strcasecmp(str, "SPLIT"))
> +        return XC_DOM_NUMA_SPLIT;
> +    if (!strcasecmp(str, "STRIPE"))
> +        return XC_DOM_NUMA_STRIPE;
> +
> +    return XC_DOM_NUMA_NONE;
Shouldn't the function return something like "unknown" here?
This would allow to detect typos in the config file.

> @@ -650,6 +686,14 @@ static void parse_config_data(const char
>      if (!xlu_cfg_get_long (config, "videoram", &l))
>          b_info->video_memkb = l * 1024;
>  
> +    if (!xlu_cfg_get_string (config, "strategy", &buf)) {
> +        b_info->numa_config.strategy = numa_str_to_val(buf);
Here one chould check the returned value for "unknown" to detect
illegal strategy types.
> +        if (!xlu_cfg_get_long (config, "vnodes", &l))
> +            b_info->numa_config.nr_nodes = l;
> +        if (!xlu_cfg_get_long (config, "stripesz", &l))
> +            b_info->numa_config.stripe_size = l;
> +    }
> +
>      if (!xlu_cfg_get_string (config, "kernel", &buf))
>          b_info->kernel.path = strdup(buf);
>  

Regards,
Andre.

-- 
Andre Przywara
AMD-Operating System Research Center (OSRC), Dresden, Germany
Tel: +49 351 448-3567-12

Re: [vNUMA v2][PATCH 1/8] Config options

[Dulloor]

On Tue, Aug 3, 2010 at 5:40 AM, Andre Przywara <andre.przywara@amd.com> wrote:
> Dulloor wrote:
>>
>> Implement the following config options :
>>
>> strategy = "str", where str is confine/stripe/split/auto
>> vnodes = <num-nodes>
>> stripesz = <size-in-pages>
>
> As Ian already said, I'd also prefer NUMA related names instead of
> the generic "strategy". Also "vnodes" may be a bit misleading, what about
> "guestnodes"?
That's right. How about "numa_strategy", "numa_vnodes", and "numa_stripesz" ?

>
>> +typedef struct xc_domain_numa_config
>> +{
>> +    uint32_t strategy;      /* By default, DONTCARE (for now) */
>> +    uint32_t nr_nodes;      /* For SPLIT/STRIPE */
>> +    uint32_t stripe_size;   /* For STRIPE only */
>
> Are 32 bit here sufficient? Although for the stripe size probably 4GB
> are more than needed, I'd prefer to use 64bit (or long) for each
> memory-related variable.
Here, stripe_size is in 4K pages, so effectively we have 44 bits,
which should suffice.
But, for consistency, I will change that to 64-bit.

>>
>> +} xc_domain_numa_config_t;
>> +
>
>> +++ b/tools/libxl/xl_cmdimpl.c
>> +static uint32_t numa_str_to_val(const char *str)
>> +{
>> +    if (!strcasecmp(str, "AUTO"))
>> +        return XC_DOM_NUMA_AUTO;
>> +    if (!strcasecmp(str, "CONFINE"))
>> +        return XC_DOM_NUMA_CONFINE;
>> +    if (!strcasecmp(str, "SPLIT"))
>> +        return XC_DOM_NUMA_SPLIT;
>> +    if (!strcasecmp(str, "STRIPE"))
>> +        return XC_DOM_NUMA_STRIPE;
>> +
>> +    return XC_DOM_NUMA_NONE;
>
> Shouldn't the function return something like "unknown" here?
> This would allow to detect typos in the config file.
I  was thinking that if someone misconfigures (or for typos), we fall back
to the current case. But, it makes sense not to do that. Will change this.

>
>> @@ -650,6 +686,14 @@ static void parse_config_data(const char
>>     if (!xlu_cfg_get_long (config, "videoram", &l))
>>         b_info->video_memkb = l * 1024;
>>  +    if (!xlu_cfg_get_string (config, "strategy", &buf)) {
>> +        b_info->numa_config.strategy = numa_str_to_val(buf);
>
> Here one chould check the returned value for "unknown" to detect
> illegal strategy types.
OK.

>>
>> +        if (!xlu_cfg_get_long (config, "vnodes", &l))
>> +            b_info->numa_config.nr_nodes = l;
>> +        if (!xlu_cfg_get_long (config, "stripesz", &l))
>> +            b_info->numa_config.stripe_size = l;
>> +    }
>> +
>>     if (!xlu_cfg_get_string (config, "kernel", &buf))
>>         b_info->kernel.path = strdup(buf);
>>
>
> Regards,
> Andre.
>
> --
> Andre Przywara
> AMD-Operating System Research Center (OSRC), Dresden, Germany
> Tel: +49 351 448-3567-12
>
>

Re: [vNUMA v2][PATCH 1/8] Config options

[Andre Przywara]

Dulloor wrote:
> On Tue, Aug 3, 2010 at 5:40 AM, Andre Przywara <andre.przywara@amd.com> wrote:
>> Dulloor wrote:
>>> Implement the following config options :
>>>
>>> strategy = "str", where str is confine/stripe/split/auto
>>> vnodes = <num-nodes>
>>> stripesz = <size-in-pages>
>> As Ian already said, I'd also prefer NUMA related names instead of
>> the generic "strategy". Also "vnodes" may be a bit misleading, what about
>> "guestnodes"?
> That's right. How about "numa_strategy", "numa_vnodes", and "numa_stripesz" ?
The numa_vnodes looks quite cumbersome and isn't very intuitive in 
understanding. I'd prefer numa_guestnodes (although this is a bit long) 
cause it is clear in what it means. But I can also live with numa_vnodes 
if this is put in the example config file together with an appropriate 
comment.
> 
>>> +typedef struct xc_domain_numa_config
>>> +{
>>> +    uint32_t strategy;      /* By default, DONTCARE (for now) */
>>> +    uint32_t nr_nodes;      /* For SPLIT/STRIPE */
>>> +    uint32_t stripe_size;   /* For STRIPE only */
>> Are 32 bit here sufficient? Although for the stripe size probably 4GB
>> are more than needed, I'd prefer to use 64bit (or long) for each
>> memory-related variable.
> Here, stripe_size is in 4K pages, so effectively we have 44 bits,
> which should suffice.
> But, for consistency, I will change that to 64-bit.
Thanks. I think we all should have learned from the "640 KB is enough" 
story ;-)
> 
>>> +} xc_domain_numa_config_t;
>>> +
>>> +++ b/tools/libxl/xl_cmdimpl.c
>>> +static uint32_t numa_str_to_val(const char *str)
>>> +{
>>> +    if (!strcasecmp(str, "AUTO"))
>>> +        return XC_DOM_NUMA_AUTO;
>>> +    if (!strcasecmp(str, "CONFINE"))
>>> +        return XC_DOM_NUMA_CONFINE;
>>> +    if (!strcasecmp(str, "SPLIT"))
>>> +        return XC_DOM_NUMA_SPLIT;
>>> +    if (!strcasecmp(str, "STRIPE"))
>>> +        return XC_DOM_NUMA_STRIPE;
>>> +
>>> +    return XC_DOM_NUMA_NONE;
>> Shouldn't the function return something like "unknown" here?
>> This would allow to detect typos in the config file.
> I  was thinking that if someone misconfigures (or for typos), we fall back
> to the current case. But, it makes sense not to do that. Will change this.
I was already seeing the mails on xen-devel asking why 
numa_strategy=strip doesn't make a difference ;-) If we have only a few 
possible strategies, we should fail on illegal ones. That would also 
make later extension easier to use, as they fail on older setups and the 
user gets a hint on this.

Regards,
Andre.


-- 
Andre Przywara
AMD-Operating System Research Center (OSRC), Dresden, Germany
Tel: +49 351 448-3567-12

Re: [vNUMA v2][PATCH 1/8] Config options

[Andre Przywara]

Dulloor wrote:
> Implement the following config options :
> 
> strategy = “str”, where str is confine/stripe/split/auto
While playing around with it, I am always confused with the similarity 
between split und stripe. Can we replace stripe with interleave? This 
term seems to be used in hardware and other tools like numactl, so it 
would be nice to not add another name for it.

Regards,
Andre.

-- 
Andre Przywara
AMD-Operating System Research Center (OSRC), Dresden, Germany
Tel: +49 351 448-3567-12

Re: [vNUMA v2][PATCH 1/8] Config options

[Dulloor]

On Mon, Aug 16, 2010 at 9:19 AM, Andre Przywara <andre.przywara@amd.com> wrote:
> Dulloor wrote:
>>
>> Implement the following config options :
>>
>> strategy = “str”, where str is confine/stripe/split/auto
>
> While playing around with it, I am always confused with the similarity
> between split und stripe. Can we replace stripe with interleave? This term
> seems to be used in hardware and other tools like numactl, so it would be
> nice to not add another name for it.
Makes sense. Will do that.

>
> Regards,
> Andre.
>
> --
> Andre Przywara
> AMD-Operating System Research Center (OSRC), Dresden, Germany
> Tel: +49 351 448-3567-12
>
>