[PATCH][2nd try][RESEND] xend: add cpus config option

[Ryan Harper <ryanh@us.ibm.com>]

Updated to latest tip, commented out cpus entry in xmexample2 as per
Ian's request.  Previous description of patch and discussion [1]here.

Please apply.

1. http://lists.xensource.com/archives/html/xen-devel/2005-11/msg00763.html

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@us.ibm.com


diffstat output:
 docs/man/xmdomain.cfg.pod.5             |   10 ++++++
 tools/examples/xmexample.vmx            |    6 ++-
 tools/examples/xmexample.vti            |    6 ++-
 tools/examples/xmexample1               |    6 ++-
 tools/examples/xmexample2               |    8 +++-
 tools/examples/xmexample3               |    8 +++-
 tools/python/xen/xend/XendDomainInfo.py |   53 +++++++++++++++++++++++++++++---
 tools/python/xen/xm/create.py           |    8 ++++
 tools/python/xen/xm/main.py             |    4 +-
 9 files changed, 91 insertions(+), 18 deletions(-)

Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -r 690b9e98b6c3 docs/man/xmdomain.cfg.pod.5
--- a/docs/man/xmdomain.cfg.pod.5	Wed Nov 30 16:12:41 2005
+++ b/docs/man/xmdomain.cfg.pod.5	Wed Nov 30 10:21:08 2005
@@ -156,6 +156,16 @@
 the first cpu, 1 the second, and so on.  This defaults to -1, which
 means Xen is free to pick which CPU to start on.
 
+=item B<cpus>
+
+Specifies a list of CPUs on which the domains' VCPUs are allowed to
+execute upon.  The syntax supports ranges (0-3), and negation, ^1.
+For instance:
+
+    cpus = "0-3,5,^1"
+
+Will result in CPUs 0, 2, 3, 5 being available for use by the domain.
+
 =item B<extra>
 
 Extra information to append to the end of the kernel parameter line.
diff -r 690b9e98b6c3 tools/examples/xmexample.vmx
--- a/tools/examples/xmexample.vmx	Wed Nov 30 16:12:41 2005
+++ b/tools/examples/xmexample.vmx	Wed Nov 30 10:21:08 2005
@@ -30,8 +30,10 @@
 # the number of cpus guest platform has, default=1
 vcpus=1
 
-# Which CPU to start domain on? 
-#cpu = -1   # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = ""         # leave to Xen to pick
+#cpus = "0"        # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
 
 # Optionally define mac and/or bridge for the network interfaces.
 # Random MACs are assigned if not given.
diff -r 690b9e98b6c3 tools/examples/xmexample.vti
--- a/tools/examples/xmexample.vti	Wed Nov 30 16:12:41 2005
+++ b/tools/examples/xmexample.vti	Wed Nov 30 10:21:08 2005
@@ -23,8 +23,10 @@
 # A name for your domain. All domains must have different names.
 name = "ExampleVMXDomain"
 
-# Which CPU to start domain on? 
-#cpu = -1   # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = ""         # leave to Xen to pick
+#cpus = "0"        # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
 
 # Disable vif for now
 nics=0
diff -r 690b9e98b6c3 tools/examples/xmexample1
--- a/tools/examples/xmexample1	Wed Nov 30 16:12:41 2005
+++ b/tools/examples/xmexample1	Wed Nov 30 10:21:08 2005
@@ -22,8 +22,10 @@
 # A name for your domain. All domains must have different names.
 name = "ExampleDomain"
 
-# Which CPU to start domain on? 
-#cpu = -1   # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = ""         # leave to Xen to pick
+#cpus = "0"        # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
 
 # Number of Virtual CPUS to use, default is 1
 #vcpus = 1
diff -r 690b9e98b6c3 tools/examples/xmexample2
--- a/tools/examples/xmexample2	Wed Nov 30 16:12:41 2005
+++ b/tools/examples/xmexample2	Wed Nov 30 10:21:08 2005
@@ -51,9 +51,11 @@
 # so we use the vmid to create a name.
 name = "VM%d" % vmid
 
-# Which CPU to start domain on? 
-#cpu = -1   # leave to Xen to pick
-cpu = vmid  # set based on vmid (mod number of CPUs)
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = ""         # leave to Xen to pick
+#cpus = "0"        # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
+#cpus = "%s" % vmid # set based on vmid (mod number of CPUs)
 
 # Number of Virtual CPUS to use, default is 1
 #vcpus = 1
diff -r 690b9e98b6c3 tools/examples/xmexample3
--- a/tools/examples/xmexample3	Wed Nov 30 16:12:41 2005
+++ b/tools/examples/xmexample3	Wed Nov 30 10:21:08 2005
@@ -51,9 +51,11 @@
 # so we use the vmid to create a name.
 name = "VM%d" % vmid
 
-# Which CPU to start domain on? 
-#cpu = -1   # leave to Xen to pick
-cpu = vmid  # set based on vmid (mod number of CPUs)
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = ""         # leave to Xen to pick
+#cpus = "0"        # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
+cpus = "%s" % vmid # set based on vmid (mod number of CPUs)
 
 #----------------------------------------------------------------------------
 # Define network interfaces.
diff -r 690b9e98b6c3 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py	Wed Nov 30 16:12:41 2005
+++ b/tools/python/xen/xend/XendDomainInfo.py	Wed Nov 30 10:21:08 2005
@@ -287,6 +287,7 @@
         result[e[0]] = get_cfg(e[0], e[1])
 
     result['cpu']       = get_cfg('cpu',       int)
+    result['cpus']      = get_cfg('cpus',      str)
     result['image']     = get_cfg('image')
 
     try:
@@ -299,6 +300,43 @@
         raise VmError(
             'Invalid configuration setting: vcpus = %s: %s' %
             (sxp.child_value(result['image'], 'vcpus', 1), str(exn)))
+
+    try:
+        # support legacy config files with 'cpu' parameter
+        # NB: prepending to list to support previous behavior
+        #     where 'cpu' parameter pinned VCPU0.
+        if result['cpu']:
+           if result['cpus']:
+               result['cpus'] = "%s,%s" % (str(result['cpu']), result['cpus'])
+           else:
+               result['cpus'] = str(result['cpu'])
+
+        # convert 'cpus' string to list of ints
+        # 'cpus' supports a list of ranges (0-3), seperated by
+        # commas, and negation, (^1).  
+        # Precedence is settled by  order of the string:
+        #     "0-3,^1"   -> [0,2,3]
+        #     "0-3,^1,1" -> [0,1,2,3]
+        if result['cpus']:
+            cpus = []
+            for c in result['cpus'].split(','):
+                if c.find('-') != -1:             
+                    (x,y) = c.split('-')
+                    for i in range(int(x),int(y)+1):
+                        cpus.append(int(i))
+                else:
+                    # remove this element from the list 
+                    if c[0] == '^':
+                        cpus = [x for x in cpus if x != int(c[1])]
+                    else:
+                        cpus.append(int(c))
+
+            result['cpus'] = cpus
+        
+    except ValueError, exn:
+        raise VmError(
+            'Invalid configuration setting: cpus = %s: %s' %
+            (result['cpus'], exn))
 
     result['backend'] = []
     for c in sxp.children(config, 'backend'):
@@ -480,6 +518,7 @@
             defaultInfo('on_reboot',    lambda: "restart")
             defaultInfo('on_crash',     lambda: "restart")
             defaultInfo('cpu',          lambda: None)
+            defaultInfo('cpus',         lambda: [])
             defaultInfo('cpu_weight',   lambda: 1.0)
 
             # some domains don't have a config file (e.g. dom0 )
@@ -1105,13 +1144,19 @@
 
             xc.domain_setcpuweight(self.domid, self.info['cpu_weight'])
 
+            # repin domain vcpus if a restricted cpus list is provided
+            # this is done prior to memory allocation to aide in memory
+            # distribution for NUMA systems.
+            cpus = self.info['cpus']
+            if cpus is not None and len(cpus) > 0:
+                for v in range(0, self.info['max_vcpu_id']+1):
+                    # pincpu takes a list of ints
+                    cpu = [ int( cpus[v % len(cpus)] ) ]
+                    xc.domain_pincpu(self.domid, v, cpu)
+ 
             m = self.image.getDomainMemory(self.info['memory'] * 1024)
             xc.domain_setmaxmem(self.domid, m)
             xc.domain_memory_increase_reservation(self.domid, m, 0, 0)
-
-            cpu = self.info['cpu']
-            if cpu is not None and cpu != -1:
-                xc.domain_pincpu(self.domid, 0, 1 << cpu)
 
             self.createChannels()
 
diff -r 690b9e98b6c3 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py	Wed Nov 30 16:12:41 2005
+++ b/tools/python/xen/xm/create.py	Wed Nov 30 10:21:08 2005
@@ -154,7 +154,11 @@
 
 gopts.var('cpu', val='CPU',
           fn=set_int, default=None,
-          use="CPU to run the domain on.")
+          use="CPU to run the VCPU0 on.")
+
+gopts.var('cpus', val='CPUS',
+          fn=set_int, default=None,
+          use="CPUS to run the domain on.")
 
 gopts.var('lapic', val='LAPIC',
           fn=set_int, default=0,
@@ -572,6 +576,8 @@
     
     if vals.cpu is not None:
         config.append(['cpu', vals.cpu])
+    if vals.cpus is not None:
+        config.append(['cpus', vals.cpus])
     if vals.cpu_weight is not None:
         config.append(['cpu_weight', vals.cpu_weight])
     if vals.blkif:
diff -r 690b9e98b6c3 tools/python/xen/xm/main.py
--- a/tools/python/xen/xm/main.py	Wed Nov 30 16:12:41 2005
+++ b/tools/python/xen/xm/main.py	Wed Nov 30 10:21:08 2005
@@ -436,7 +436,9 @@
             for x in server.xend_node()[1:]:
                 if len(x) > 1 and x[0] == 'nr_cpus':
                     nr_cpus = int(x[1])
-                    cpumap = filter(lambda x: x < nr_cpus, cpumap)
+                    # normalize cpumap by modulus nr_cpus, and drop duplicates
+                    cpumap = dict.fromkeys(
+                                map(lambda x: x % nr_cpus, cpumap)).keys()
                     if len(cpumap) == nr_cpus:
                         return "any cpu"
                     break