[PATCH]Add free memory size of every NUMA node in phsical info

[PATCH]Add free memory size of every NUMA node in phsical info

[Duan, Ronghui]

[-- Attachment #1.1: Type: text/plain, Size: 257 bytes --]

Returns free memory size per node in "xm info". This info can help users
who want to bind their guest domain in one node of their NUMA machines
thought set CPU affinity. I also write IA64 part support which I would
send to XEN-IA64 mail-list. Thanks.


[-- Attachment #1.2: Type: text/html, Size: 1515 bytes --]

[-- Attachment #2: get_node_memory_x86_dom0.patch --]
[-- Type: application/octet-stream, Size: 1029 bytes --]

Add free memory size of every node in physinfo to help user to get usage of
memory of their NUMA machine.

Signed-off-by: Duan Ronghui <ronghui.duan@intel.com>

diff -r 6ce1a5e00cad include/xen/interface/sysctl.h
--- a/include/xen/interface/sysctl.h	Tue Feb 26 09:11:35 2008 +0800
+++ b/include/xen/interface/sysctl.h	Tue Feb 26 09:13:06 2008 +0800
@@ -103,6 +103,7 @@ struct xen_sysctl_physinfo {
      * If OUT is greater than IN then the cpu_to_node array is truncated!
      */
     uint32_t max_cpu_id;
+    uint32_t max_node_id;
     /*
      * If not NULL, this array is filled with node identifier for each cpu.
      * If a cpu has no node information (e.g., cpu not present) then the
@@ -112,6 +113,7 @@ struct xen_sysctl_physinfo {
      * elements of the array will not be written by the sysctl.
      */
     XEN_GUEST_HANDLE_64(uint32) cpu_to_node;
+    XEN_GUEST_HANDLE_64(uint64) free_memory_on_node;
 };
 typedef struct xen_sysctl_physinfo xen_sysctl_physinfo_t;
 DEFINE_XEN_GUEST_HANDLE(xen_sysctl_physinfo_t);

[-- Attachment #3: get_node_memory_x86.patch --]
[-- Type: application/octet-stream, Size: 6436 bytes --]

Add free memory size of every node in physinfo to help user to get usage of
memory of their NUMA machine.

Signed-off-by: Duan Ronghui <ronghui.duan@intel.com>
diff -r 09b53f27a18b tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c	Thu Feb 21 18:02:42 2008 +0000
+++ b/tools/python/xen/lowlevel/xc/xc.c	Tue Feb 26 09:18:55 2008 +0800
@@ -761,14 +761,18 @@ static PyObject *pyxc_physinfo(XcObject 
 static PyObject *pyxc_physinfo(XcObject *self)
 {
 #define MAX_CPU_ID 255
+#define MAX_NODE_ID 255
     xc_physinfo_t info;
     char cpu_cap[128], *p=cpu_cap, *q=cpu_cap;
-    int i, j, max_cpu_id;
-    PyObject *ret_obj, *node_to_cpu_obj;
+    int i, j, max_cpu_id, max_node_id;
+    PyObject *ret_obj, *node_to_cpu_obj, *node_to_memory_obj;
     xc_cpu_to_node_t map[MAX_CPU_ID + 1];
+    uint64_t avail_mem[MAX_NODE_ID + 1];
 
     set_xen_guest_handle(info.cpu_to_node, map);
+    set_xen_guest_handle(info.free_memory_on_node, avail_mem);
     info.max_cpu_id = MAX_CPU_ID;
+    info.max_node_id = MAX_NODE_ID;
 
     if ( xc_physinfo(self->xc_handle, &info) != 0 )
         return pyxc_error_to_exception();
@@ -788,7 +792,7 @@ static PyObject *pyxc_physinfo(XcObject 
                             "max_cpu_id",       info.max_cpu_id,
                             "threads_per_core", info.threads_per_core,
                             "cores_per_socket", info.cores_per_socket,
-                            "nr_cpus",          info.nr_cpus, 
+                            "nr_cpus",          info.nr_cpus,
                             "total_memory",     pages_to_kib(info.total_pages),
                             "free_memory",      pages_to_kib(info.free_pages),
                             "scrub_memory",     pages_to_kib(info.scrub_pages),
@@ -812,8 +816,18 @@ static PyObject *pyxc_physinfo(XcObject 
         PyList_Append(node_to_cpu_obj, cpus); 
     }
 
+    max_node_id = info.max_node_id;
+    if ( max_node_id > MAX_NODE_ID )
+        max_node_id = MAX_NODE_ID;
+
+    node_to_memory_obj = PyList_New(0);
+    for ( i = 0; i <= max_node_id; i++ )
+        PyList_Append(node_to_memory_obj,
+            PyInt_FromLong(pages_to_kib(avail_mem[i])));
+
     PyDict_SetItemString(ret_obj, "node_to_cpu", node_to_cpu_obj);
- 
+    PyDict_SetItemString(ret_obj, "node_to_memory", node_to_memory_obj);
+
     return ret_obj;
 #undef MAX_CPU_ID
 }
diff -r 09b53f27a18b tools/python/xen/xend/XendNode.py
--- a/tools/python/xen/xend/XendNode.py	Thu Feb 21 18:02:42 2008 +0000
+++ b/tools/python/xen/xend/XendNode.py	Tue Feb 26 09:18:55 2008 +0800
@@ -574,6 +574,20 @@ class XendNode:
             str='none\n'
         return str[:-1];
 
+    def format_node_to_memory(self, pinfo):
+        str=''
+        whitespace=''
+        try:
+            node_to_memory=pinfo['node_to_memory']
+            for i in range(0, pinfo['nr_nodes']):
+                str+='%snode%d:%d\n' % (whitespace,
+                                        i,
+                                        node_to_memory[i] / 1024)
+                whitespace='%25s' % ''
+        except:
+            str='none\n'
+        return str[:-1];
+
     def physinfo(self):
         info = self.xc.physinfo()
 
@@ -583,6 +597,7 @@ class XendNode:
         info['total_memory'] = info['total_memory'] / 1024
         info['free_memory']  = info['free_memory'] / 1024
         info['node_to_cpu']  = self.format_node_to_cpu(info)
+        info['node_to_memory'] = self.format_node_to_memory(info)
 
         ITEM_ORDER = ['nr_cpus',
                       'nr_nodes',
@@ -592,7 +607,8 @@ class XendNode:
                       'hw_caps',
                       'total_memory',
                       'free_memory',
-                      'node_to_cpu'
+                      'node_to_cpu',
+                      'node_to_memory'
                       ]
 
         return [[k, info[k]] for k in ITEM_ORDER]
diff -r 09b53f27a18b xen/arch/x86/sysctl.c
--- a/xen/arch/x86/sysctl.c	Thu Feb 21 18:02:42 2008 +0000
+++ b/xen/arch/x86/sysctl.c	Tue Feb 26 09:21:57 2008 +0800
@@ -19,6 +19,7 @@
 #include <xen/trace.h>
 #include <xen/console.h>
 #include <xen/iocap.h>
+#include <xen/nodemask.h>
 #include <asm/irq.h>
 #include <asm/hvm/hvm.h>
 #include <asm/hvm/support.h>
@@ -75,6 +76,24 @@ long arch_do_sysctl(
             }
         }
 
+	max_array_ent = pi->max_node_id;
+	pi->max_node_id = last_node(node_online_map);
+	max_array_ent = min_t(uint32_t, max_array_ent, pi->max_node_id);
+
+	ret = -EFAULT;
+	if(!guest_handle_is_null(pi->free_memory_on_node))
+	{
+	    for (i = 0; i <= max_array_ent; i++)
+	    {
+		uint64_t pages = avail_domheap_pages_region(i, 0, 0);
+		if (copy_to_guest_offset(pi->free_memory_on_node, i, &pages, 1))
+		{
+		    ret = -EFAULT;
+		    break;
+		}
+	    }
+	}
+
         ret = copy_to_guest(u_sysctl, sysctl, 1) ? -EFAULT : 0;
     }
     break;
diff -r 09b53f27a18b xen/include/public/sysctl.h
--- a/xen/include/public/sysctl.h	Thu Feb 21 18:02:42 2008 +0000
+++ b/xen/include/public/sysctl.h	Tue Feb 26 09:18:55 2008 +0800
@@ -103,6 +103,7 @@ struct xen_sysctl_physinfo {
      * If OUT is greater than IN then the cpu_to_node array is truncated!
      */
     uint32_t max_cpu_id;
+    uint32_t max_node_id;
     /*
      * If not NULL, this array is filled with node identifier for each cpu.
      * If a cpu has no node information (e.g., cpu not present) then the
@@ -112,6 +113,7 @@ struct xen_sysctl_physinfo {
      * elements of the array will not be written by the sysctl.
      */
     XEN_GUEST_HANDLE_64(uint32) cpu_to_node;
+    XEN_GUEST_HANDLE_64(uint64) free_memory_on_node;
 };
 typedef struct xen_sysctl_physinfo xen_sysctl_physinfo_t;
 DEFINE_XEN_GUEST_HANDLE(xen_sysctl_physinfo_t);
diff -r 09b53f27a18b xen/include/xen/nodemask.h
--- a/xen/include/xen/nodemask.h	Thu Feb 21 18:02:42 2008 +0000
+++ b/xen/include/xen/nodemask.h	Tue Feb 26 09:18:55 2008 +0800
@@ -224,6 +224,15 @@ static inline int __next_node(int n, con
 static inline int __next_node(int n, const nodemask_t *srcp)
 {
 	return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
+}
+
+#define last_node(src) __last_node(&(src))
+static inline int __last_node(const nodemask_t *srcp)
+{
+    int node, pnode = MAX_NUMNODES;
+    for (node = first_node(*srcp); node < MAX_NUMNODES; node = next_node(node, *srcp))
+        pnode = node;
+    return pnode;
 }
 
 #define nodemask_of_node(node)						\

[-- Attachment #4: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH]Add free memory size of every NUMA node in phsical info

[Daniel P. Berrange]

On Tue, Feb 26, 2008 at 10:02:36AM +0800, Duan, Ronghui wrote:
> Returns free memory size per node in "xm info". This info can help users
> who want to bind their guest domain in one node of their NUMA machines
> thought set CPU affinity. I also write IA64 part support which I would
> send to XEN-IA64 mail-list. Thanks.

AFAICT, changing the ABI of the 'physinfo' sysctl is completely unneccessary.

The per-node NUMA free memory information is already available to Dom0
via the existing 'availheap' sysctl:

  #define XEN_SYSCTL_availheap         9
  struct xen_sysctl_availheap {
      /* IN variables. */
      uint32_t min_bitwidth;  /* Smallest address width (zero if don't care). */
      uint32_t max_bitwidth;  /* Largest address width (zero if don't care). */
      int32_t  node;          /* NUMA node of interest (-1 for all nodes). */
      /* OUT variables. */
      uint64_aligned_t avail_bytes;/* Bytes available in the specified region. */
  };


If you want to display this info in 'xm info', then simply invoke this
availheap sysctl call to fetch the data.

Dan.
-- 
|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392 2496 -=|
|=-           Perl modules: http://search.cpan.org/~danberr/              -=|
|=-               Projects: http://freshmeat.net/~danielpb/               -=|
|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505  -=| 

RE: [PATCH]Add free memory size of every NUMA node in phsical info

[Duan, Ronghui]

I see that, the reason I don't use that function is there need one more
time hypercall, I only reuse the function which have been realized in
hypervisor. Thanks for your advice.

>-----Original Message-----
>From: Daniel P. Berrange [mailto:berrange@redhat.com]
>Sent: Tuesday, February 26, 2008 10:40 AM
>To: Duan, Ronghui
>Cc: xen-devel@lists.xensource.com
>Subject: Re: [Xen-devel] [PATCH]Add free memory size of every NUMA node
in
>phsical info
>
>On Tue, Feb 26, 2008 at 10:02:36AM +0800, Duan, Ronghui wrote:
>> Returns free memory size per node in "xm info". This info can help
users
>> who want to bind their guest domain in one node of their NUMA
machines
>> thought set CPU affinity. I also write IA64 part support which I
would
>> send to XEN-IA64 mail-list. Thanks.
>
>AFAICT, changing the ABI of the 'physinfo' sysctl is completely
>unneccessary.
>
>The per-node NUMA free memory information is already available to Dom0
>via the existing 'availheap' sysctl:
>
>  #define XEN_SYSCTL_availheap         9
>  struct xen_sysctl_availheap {
>      /* IN variables. */
>      uint32_t min_bitwidth;  /* Smallest address width (zero if don't
>care). */
>      uint32_t max_bitwidth;  /* Largest address width (zero if don't
care).
>*/
>      int32_t  node;          /* NUMA node of interest (-1 for all
nodes).
>*/
>      /* OUT variables. */
>      uint64_aligned_t avail_bytes;/* Bytes available in the specified
>region. */
>  };
>
>
>If you want to display this info in 'xm info', then simply invoke this
>availheap sysctl call to fetch the data.
>
>Dan.
>--
>|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392
2496 -
>=|
>|=-           Perl modules: http://search.cpan.org/~danberr/
-
>=|
>|=-               Projects: http://freshmeat.net/~danielpb/
-
>=|
>|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B
9505  -
>=|

Re: [PATCH]Add free memory size of every NUMA node in phsical info

[Daniel P. Berrange]

On Tue, Feb 26, 2008 at 10:46:17AM +0800, Duan, Ronghui wrote:
> I see that, the reason I don't use that function is there need one more
> time hypercall, I only reuse the function which have been realized in
> hypervisor. Thanks for your advice.

The performance impact of doing 1 extra hypercall for 'availheap' is
completely irrelevant in this context. The time for 1 extra hypercall
is dwarfed (by several orders of magnitude) by the overhead due to
'xm' and 'xend' being in python & using XML-RPC. We should just use
the existing hypercall & not worry about time overhead we'll never
notice.

Dan.
-- 
|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392 2496 -=|
|=-           Perl modules: http://search.cpan.org/~danberr/              -=|
|=-               Projects: http://freshmeat.net/~danielpb/               -=|
|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505  -=| 

RE: [PATCH]Add free memory size of every NUMA node in phsical info

[Duan, Ronghui]

Thanks for your advice.

As far as I know, xc_availheap will return specific node's free memory,
in this context we need all nodes' info, so under the most conditions if
use xc_availheap, there may be 4~8 times hypercall, one time per node.
So in my methods, I allocate more memory to let one hpercall to return a
list instead of a value. It is something related with time complexity
and space complexity. This info has the same attribute with
"cpu_to_node" in physical info. So I think it is better to get them
using the same methods.

Thanks again.

>-----Original Message-----
>From: Daniel P. Berrange [mailto:berrange@redhat.com]
>Sent: Tuesday, February 26, 2008 11:02 AM
>To: Duan, Ronghui
>Cc: xen-devel@lists.xensource.com
>Subject: Re: [Xen-devel] [PATCH]Add free memory size of every NUMA node
in
>phsical info
>
>On Tue, Feb 26, 2008 at 10:46:17AM +0800, Duan, Ronghui wrote:
>> I see that, the reason I don't use that function is there need one
more
>> time hypercall, I only reuse the function which have been realized in
>> hypervisor. Thanks for your advice.
>
>The performance impact of doing 1 extra hypercall for 'availheap' is
>completely irrelevant in this context. The time for 1 extra hypercall
>is dwarfed (by several orders of magnitude) by the overhead due to
>'xm' and 'xend' being in python & using XML-RPC. We should just use
>the existing hypercall & not worry about time overhead we'll never
>notice.
>
>Dan.
>--
>|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392
2496 -
>=|
>|=-           Perl modules: http://search.cpan.org/~danberr/
-
>=|
>|=-               Projects: http://freshmeat.net/~danielpb/
-
>=|
>|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B
9505  -
>=|

RE: [PATCH]Add free memory size of every NUMA node inphsical info

[Duan, Ronghui]

[-- Attachment #1.1: Type: text/plain, Size: 652 bytes --]

Thank for Dan's advice, rewrite it using current interface.

 

________________________________

From: xen-devel-bounces@lists.xensource.com
[mailto:xen-devel-bounces@lists.xensource.com] On Behalf Of Duan,
Ronghui
Sent: Tuesday, February 26, 2008 10:03 AM
To: xen-devel@lists.xensource.com
Subject: [Xen-devel] [PATCH]Add free memory size of every NUMA node
inphsical info

 

Returns free memory size per node in "xm info". This info can help users
who want to bind their guest domain in one node of their NUMA machines
thought set CPU affinity. I also write IA64 part support which I would
send to XEN-IA64 mail-list. Thanks.


[-- Attachment #1.2: Type: text/html, Size: 3439 bytes --]

[-- Attachment #2: get_node_memory.patch --]
[-- Type: application/octet-stream, Size: 2818 bytes --]

Add free memory size of every node in physinfo to help user to get usage of
memory of their NUMA machine.

Signed-off-by: Duan Ronghui <ronghui.duan@intel.com>

diff -r 09b53f27a18b tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c	Thu Feb 21 18:02:42 2008 +0000
+++ b/tools/python/xen/lowlevel/xc/xc.c	Tue Feb 26 14:34:36 2008 +0800
@@ -764,7 +764,8 @@ static PyObject *pyxc_physinfo(XcObject 
     xc_physinfo_t info;
     char cpu_cap[128], *p=cpu_cap, *q=cpu_cap;
     int i, j, max_cpu_id;
-    PyObject *ret_obj, *node_to_cpu_obj;
+    uint64_t free_heap;
+    PyObject *ret_obj, *node_to_cpu_obj, *node_to_memory_obj;
     xc_cpu_to_node_t map[MAX_CPU_ID + 1];
 
     set_xen_guest_handle(info.cpu_to_node, map);
@@ -812,7 +813,17 @@ static PyObject *pyxc_physinfo(XcObject 
         PyList_Append(node_to_cpu_obj, cpus); 
     }
 
+    node_to_memory_obj = PyList_New(0);
+
+    for ( i = 0; i < info.nr_nodes; i++ )
+    {
+	xc_availheap(self->xc_handle, 0, 0, i, &free_heap);
+	PyList_Append(node_to_memory_obj,
+	    PyInt_FromLong(free_heap / 1024));
+    }
+	
     PyDict_SetItemString(ret_obj, "node_to_cpu", node_to_cpu_obj);
+    PyDict_SetItemString(ret_obj, "node_to_memory", node_to_memory_obj);
  
     return ret_obj;
 #undef MAX_CPU_ID
diff -r 09b53f27a18b tools/python/xen/xend/XendNode.py
--- a/tools/python/xen/xend/XendNode.py	Thu Feb 21 18:02:42 2008 +0000
+++ b/tools/python/xen/xend/XendNode.py	Tue Feb 26 15:41:51 2008 +0800
@@ -573,6 +573,20 @@ class XendNode:
         except:
             str='none\n'
         return str[:-1];
+    def format_node_to_memory(self, pinfo):
+        str=''
+        whitespace=''
+        try:
+            node_to_memory=pinfo['node_to_memory']
+            for i in range(0, pinfo['nr_nodes']):
+                str+='%snode%d:%d\n' % (whitespace,
+                                        i,
+                                        node_to_memory[i] / 1024)
+                whitespace='%25s' % ''
+        except:
+            str='none\n'
+        return str[:-1];
+
 
     def physinfo(self):
         info = self.xc.physinfo()
@@ -583,6 +597,7 @@ class XendNode:
         info['total_memory'] = info['total_memory'] / 1024
         info['free_memory']  = info['free_memory'] / 1024
         info['node_to_cpu']  = self.format_node_to_cpu(info)
+        info['node_to_memory'] = self.format_node_to_memory(info)
 
         ITEM_ORDER = ['nr_cpus',
                       'nr_nodes',
@@ -592,7 +607,8 @@ class XendNode:
                       'hw_caps',
                       'total_memory',
                       'free_memory',
-                      'node_to_cpu'
+                      'node_to_cpu',
+                      'node_to_memory'
                       ]
 
         return [[k, info[k]] for k in ITEM_ORDER]

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH]Add free memory size of every NUMA node inphsical info

[Keir Fraser]

[-- Attachment #1.1: Type: text/plain, Size: 933 bytes --]

Looks better, thanks. I¹ll check this one in.

 -- Keir


On 26/2/08 07:58, "Duan, Ronghui" <ronghui.duan@intel.com> wrote:

> Thank for Dan¹s advice, rewrite it using current interface.
>  
> 
> 
> From: xen-devel-bounces@lists.xensource.com
> [mailto:xen-devel-bounces@lists.xensource.com] On Behalf Of Duan, Ronghui
> Sent: Tuesday, February 26, 2008 10:03 AM
> To: xen-devel@lists.xensource.com
> Subject: [Xen-devel] [PATCH]Add free memory size of every NUMA node inphsical
> info
>  
> Returns free memory size per node in ³xm info². This info can help users who
> want to bind their guest domain in one node of their NUMA machines thought set
> CPU affinity. I also write IA64 part support which I would send to XEN-IA64
> mail-list. Thanks.
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel



[-- Attachment #1.2: Type: text/html, Size: 2370 bytes --]

[-- Attachment #2: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH]Add free memory size of every NUMA node inphsical info

[Daniel P. Berrange]

On Tue, Feb 26, 2008 at 03:58:20PM +0800, Duan, Ronghui wrote:
> Thank for Dan's advice, rewrite it using current interface.

Thanks for changing that - it looks much nicer now & avoids ABI break :-)

Dan.
-- 
|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392 2496 -=|
|=-           Perl modules: http://search.cpan.org/~danberr/              -=|
|=-               Projects: http://freshmeat.net/~danielpb/               -=|
|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505  -=|