Re: [Xen-staging] [xen-unstable] Extended the physinfo sysctl to export NUMA cpu_to_node topology info.

Re: [Xen-staging] [xen-unstable] Extended the physinfo sysctl to export NUMA cpu_to_node topology info.

[Alex Williamson]

On Fri, 2007-07-06 at 16:14 +0100, Xen staging patchbot-unstable wrote:
> # HG changeset patch
> # User kfraser@localhost.localdomain
> # Date 1183734727 -3600
> # Node ID 89d2192942befb0daf56e730be61d3a3c06926dd
> # Parent  538c3d8aa4b14833174423ec506284279d5a33ab
> Extended the physinfo sysctl to export NUMA cpu_to_node topology info.
> Print this in 'xm info'.
...
>  xen/include/public/sysctl.h                         |   20 +++++

   How is this working for anyone without sync'ing the linux sysctl.h?
I can't even get xend to start.  Thanks,

	Alex


Sync Xen sysctl.h with Linux

Signed-off-by: Alex Williamson <alex.williamson@hp.com>
---

diff -r 353e0152bb50 include/xen/interface/sysctl.h
--- a/include/xen/interface/sysctl.h	Mon Jul 09 11:00:44 2007 -0600
+++ b/include/xen/interface/sysctl.h	Mon Jul 09 21:30:17 2007 -0600
@@ -34,7 +34,7 @@
 #include "xen.h"
 #include "domctl.h"
 
-#define XEN_SYSCTL_INTERFACE_VERSION 0x00000003
+#define XEN_SYSCTL_INTERFACE_VERSION 0x00000004
 
 /*
  * Read console content from Xen buffer ring.
@@ -76,6 +76,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_sysctl_tbuf_
  */
 #define XEN_SYSCTL_physinfo          3
 struct xen_sysctl_physinfo {
+    /* IN variables. */
     uint32_t threads_per_core;
     uint32_t cores_per_socket;
     uint32_t sockets_per_node;
@@ -85,6 +86,23 @@ struct xen_sysctl_physinfo {
     uint64_aligned_t free_pages;
     uint64_aligned_t scrub_pages;
     uint32_t hw_cap[8];
+
+    /* IN/OUT variables. */
+    /*
+     * IN: maximum addressable entry in the caller-provided cpu_to_node array.
+     * OUT: largest cpu identifier in the system.
+     * If OUT is greater than IN then the cpu_to_node array is truncated!
+     */
+    uint32_t max_cpu_id;
+    /*
+     * If not NULL, this array is filled with node identifier for each cpu.
+     * If a cpu has no node information (e.g., cpu not present) then the
+     * sentinel value ~0u is written.
+     * The size of this array is specified by the caller in @max_cpu_id.
+     * If the actual @max_cpu_id is smaller than the array then the trailing
+     * elements of the array will not be written by the sysctl.
+     */
+    XEN_GUEST_HANDLE_64(uint32_t) cpu_to_node;
 };
 typedef struct xen_sysctl_physinfo xen_sysctl_physinfo_t;
 DEFINE_XEN_GUEST_HANDLE(xen_sysctl_physinfo_t);
@@ -140,9 +158,7 @@ typedef struct xen_sysctl_getdomaininfol
 typedef struct xen_sysctl_getdomaininfolist xen_sysctl_getdomaininfolist_t;
 DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomaininfolist_t);
 
-/*
- * Inject debug keys into Xen.
- */
+/* Inject debug keys into Xen. */
 #define XEN_SYSCTL_debug_keys        7
 struct xen_sysctl_debug_keys {
     /* IN variables. */
@@ -151,6 +167,23 @@ struct xen_sysctl_debug_keys {
 };
 typedef struct xen_sysctl_debug_keys xen_sysctl_debug_keys_t;
 DEFINE_XEN_GUEST_HANDLE(xen_sysctl_debug_keys_t);
+
+/* Get physical CPU information. */
+#define XEN_SYSCTL_getcpuinfo        8
+struct xen_sysctl_cpuinfo {
+    uint64_t idletime;
+};
+typedef struct xen_sysctl_cpuinfo xen_sysctl_cpuinfo_t;
+DEFINE_XEN_GUEST_HANDLE(xen_sysctl_cpuinfo_t); 
+struct xen_sysctl_getcpuinfo {
+    /* IN variables. */
+    uint32_t max_cpus;
+    XEN_GUEST_HANDLE_64(xen_sysctl_cpuinfo_t) info;
+    /* OUT variables. */
+    uint32_t nr_cpus;
+}; 
+typedef struct xen_sysctl_getcpuinfo xen_sysctl_getcpuinfo_t;
+DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getcpuinfo_t); 
 
 struct xen_sysctl {
     uint32_t cmd;
@@ -163,6 +196,7 @@ struct xen_sysctl {
         struct xen_sysctl_perfc_op          perfc_op;
         struct xen_sysctl_getdomaininfolist getdomaininfolist;
         struct xen_sysctl_debug_keys        debug_keys;
+        struct xen_sysctl_getcpuinfo        getcpuinfo;
         uint8_t                             pad[128];
     } u;
 };

[PATCH] Re: [Xen-staging] [xen-unstable] Extended the physinfo sysctl to export NUMA cpu_to_node topology info.

[Alex Williamson]

On Fri, 2007-07-06 at 16:14 +0100, Xen staging patchbot-unstable wrote:
> # HG changeset patch
> # User kfraser@localhost.localdomain
> # Date 1183734727 -3600
> # Node ID 89d2192942befb0daf56e730be61d3a3c06926dd
> # Parent  538c3d8aa4b14833174423ec506284279d5a33ab
> Extended the physinfo sysctl to export NUMA cpu_to_node topology info.
> Print this in 'xm info'.
...
> diff -r 538c3d8aa4b1 -r 89d2192942be tools/python/xen/xend/XendNode.py
> --- a/tools/python/xen/xend/XendNode.py	Fri Jul 06 15:01:20 2007 +0100
> +++ b/tools/python/xen/xend/XendNode.py	Fri Jul 06 16:12:07 2007 +0100
...
> +    def format_pairs(self,pairs):
> +            if not pairs:
> +                return "no cpus"

  We add support here for nodes w/o cpus, but the existing code blindly
assumes all nodes have cpus later on:

        info['nr_cpus'] = (info['nr_nodes'] *
                           info['sockets_per_node'] *
                           info['cores_per_socket'] *
                           info['threads_per_core'])

How about we count the cpus in the node_to_cpu list to avoid mistakes?
Here's the output on a system where this gave strange results before:

nr_cpus                : 8
nr_nodes               : 3
sockets_per_node       : 4
cores_per_socket       : 1
threads_per_core       : 1
...
node_to_cpu            : node0:0-3
                         node1:4-7
                         node2:no cpus

BTW, I don't know a case where it fails, but assuming sockets_per_node
is a constant for the system seems risky.  Thanks,

	Alex


Count CPUs rather than relying on a static nodes/sockets/cores/threads
relationship.

Signed-off-by: Alex Williamson <alex.williamson@hp.com>
---

diff -r ef0a5778e3ac tools/python/xen/xend/XendNode.py
--- a/tools/python/xen/xend/XendNode.py	Mon Jul 09 10:59:56 2007 -0600
+++ b/tools/python/xen/xend/XendNode.py	Mon Jul 09 23:12:33 2007 -0600
@@ -580,14 +580,17 @@ class XendNode:
             str='none\n'
         return str[:-1];
 
+    def count_cpus(self, pinfo):
+        count=0
+        node_to_cpu=pinfo['node_to_cpu']
+        for i in range(0, pinfo['nr_nodes']):
+            count+=len(node_to_cpu[i])
+        return count;
 
     def physinfo(self):
         info = self.xc.physinfo()
 
-        info['nr_cpus'] = (info['nr_nodes'] *
-                           info['sockets_per_node'] *
-                           info['cores_per_socket'] *
-                           info['threads_per_core'])
+        info['nr_cpus'] = self.count_cpus(info)
         info['cpu_mhz'] = info['cpu_khz'] / 1000
         
         # physinfo is in KiB, need it in MiB

Re: Re: [Xen-staging] [xen-unstable] Extended the physinfo sysctl to export NUMA cpu_to_node topology info.

[Keir Fraser]

On 10/7/07 04:57, "Alex Williamson" <alex.williamson@hp.com> wrote:

>    How is this working for anyone without sync'ing the linux sysctl.h?
> I can't even get xend to start.  Thanks,
> 
> Alex

Oh, I forgot that you xencomm translate the sysctls. That's a shame. I'll
copy over the header file.

 -- Keir