[PATCH] 2nd try: 2/2 VCPU creation and allocation

[Ryan Harper <ryanh@us.ibm.com>]

Allocation map.

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@us.ibm.com


diffstat output:
 tools/examples/xmexample1               |    4 +
 tools/examples/xmexample2               |    3 +
 tools/python/xen/lowlevel/xc/xc.c       |    2 
 tools/python/xen/xend/XendDomainInfo.py |   16 ++++-
 tools/python/xen/xm/create.py           |    6 ++
 xen/common/dom0_ops.c                   |   90 +++++++++++++++++---------------
 xen/common/domain.c                     |   28 +++++++++
 xen/common/schedule.c                   |   11 ++-
 xen/include/public/dom0_ops.h           |    2 
 xen/include/xen/domain.h                |    2 
 xen/include/xen/sched.h                 |    2 
 11 files changed, 114 insertions(+), 52 deletions(-)

Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -r 12d1e93653c2 tools/examples/xmexample1
--- a/tools/examples/xmexample1	Tue Oct 11 00:13:12 2005
+++ b/tools/examples/xmexample1	Mon Oct 10 19:35:55 2005
@@ -27,6 +27,10 @@
 
 # Number of Virtual CPUS to use, default is 1
 #vcpus = 1
+
+# A bitmap of which physical cpus are vcpus allowed to use.
+# ex1: 0x2 <-- bit 1 set means all vcpus will be created on CPU1
+#allocmap = 0xffffffff  # default value, vcpus can run on any cpu.
 
 #----------------------------------------------------------------------------
 # Define network interfaces.
diff -r 12d1e93653c2 tools/examples/xmexample2
--- a/tools/examples/xmexample2	Tue Oct 11 00:13:12 2005
+++ b/tools/examples/xmexample2	Mon Oct 10 19:35:55 2005
@@ -58,6 +58,9 @@
 # Number of Virtual CPUS to use, default is 1
 #vcpus = 1
 vcpus = 4 # make your domain a 4-way
+
+# A bitmap of which physical cpus are vcpus allowed to use.
+allocmap = 0x2  # start all of your VCPUs on CPU1
 
 #----------------------------------------------------------------------------
 # Define network interfaces.
diff -r 12d1e93653c2 tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c	Tue Oct 11 00:13:12 2005
+++ b/tools/python/xen/lowlevel/xc/xc.c	Mon Oct 10 19:35:55 2005
@@ -185,7 +185,7 @@
 
     static char *kwd_list[] = { "dom", "vcpu", "cpumap", NULL };
 
-    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|ii", kwd_list, 
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|iL", kwd_list, 
                                       &dom, &vcpu, &cpumap) )
         return NULL;
 
diff -r 12d1e93653c2 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py	Tue Oct 11 00:13:12 2005
+++ b/tools/python/xen/xend/XendDomainInfo.py	Mon Oct 10 19:35:55 2005
@@ -266,6 +266,7 @@
     result['maxmem']    = get_cfg('maxmem',    int)
     result['maxmem_kb'] = get_cfg('maxmem_kb', int)
     result['cpu']       = get_cfg('cpu',       int)
+    result['allocmap']  = get_cfg('allocmap',  int)
     result['image']     = get_cfg('image')
 
     try:
@@ -438,6 +439,7 @@
             defaultInfo('cpu_weight',   lambda: 1.0)
             defaultInfo('vcpus',        lambda: 1)
             defaultInfo('vcpu_avail',   lambda: (1 << self.info['vcpus']) - 1)
+            defaultInfo('allocmap'  ,   lambda: None)
             defaultInfo('bootloader',   lambda: None)
             defaultInfo('backend',      lambda: [])
             defaultInfo('device',       lambda: [])
@@ -1018,6 +1020,16 @@
             self.image.handleBootloading()
 
         xc.domain_setcpuweight(self.domid, self.info['cpu_weight'])
+        cpu = self.info['cpu']
+        if cpu is not None and cpu != -1:
+            xc.domain_pincpu(self.domid, 0, 1 << cpu)
+
+        # set the domain alloc map for future vcpus,
+        # repin VCPU0 according to the alloc map
+        allocmap = self.info['allocmap']
+        if self.domid and allocmap:
+            xc.domain_pincpu(self.domid, -1, allocmap)  # domain allocmap
+            xc.domain_pincpu(self.domid, 0, allocmap)   # repin VCPU0
 
         # increase the total number of vcpus in domain
         xc.domain_vcpus_increase(self.domid, int(self.info['vcpus']));
@@ -1026,10 +1038,6 @@
         m = self.image.getDomainMemory(self.info['memory_KiB'])
         xc.domain_setmaxmem(self.domid, m)
         xc.domain_memory_increase_reservation(self.domid, m, 0, 0)
-
-        cpu = self.info['cpu']
-        if cpu is not None and cpu != -1:
-            xc.domain_pincpu(self.domid, 0, 1 << cpu)
 
         self.info['start_time'] = time.time()
 
diff -r 12d1e93653c2 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py	Tue Oct 11 00:13:12 2005
+++ b/tools/python/xen/xm/create.py	Mon Oct 10 19:35:55 2005
@@ -151,6 +151,10 @@
 gopts.var('cpu', val='CPU',
           fn=set_int, default=None,
           use="CPU to run the domain on.")
+
+gopts.var('allocmap', val='ALLOCMAP',
+          fn=set_int, default=None,
+          use="Set default cpumap used for allocating vcpus.")
 
 gopts.var('vcpus', val='VCPUS',
           fn=set_int, default=1,
@@ -562,6 +566,8 @@
         config.append(['maxmem', vals.maxmem])
     if vals.cpu is not None:
         config.append(['cpu', vals.cpu])
+    if vals.allocmap is not None:
+        config.append(['allocmap', vals.allocmap])
     if vals.cpu_weight is not None:
         config.append(['cpu_weight', vals.cpu_weight])
     if vals.blkif:
diff -r 12d1e93653c2 xen/common/dom0_ops.c
--- a/xen/common/dom0_ops.c	Tue Oct 11 00:13:12 2005
+++ b/xen/common/dom0_ops.c	Mon Oct 10 19:35:55 2005
@@ -275,7 +275,7 @@
     {
         domid_t dom = op->u.pincpudomain.domain;
         struct domain *d = find_domain_by_id(dom);
-        struct vcpu *v;
+        struct vcpu *v = NULL;
         cpumap_t cpumap;
 
 
@@ -285,27 +285,33 @@
             break;
         }
         
-        if ( (op->u.pincpudomain.vcpu >= MAX_VIRT_CPUS) ||
-             !d->vcpu[op->u.pincpudomain.vcpu] )
-        {
-            ret = -EINVAL;
-            put_domain(d);
-            break;
-        }
-
-        v = d->vcpu[op->u.pincpudomain.vcpu];
-        if ( v == NULL )
-        {
-            ret = -ESRCH;
-            put_domain(d);
-            break;
-        }
-
-        if ( v == current )
-        {
-            ret = -EINVAL;
-            put_domain(d);
-            break;
+        /* don't bail on vcpu = -1 as that sets domain cpumap */
+        if ( (op->u.pincpudomain.vcpu != -1) && 
+              ((op->u.pincpudomain.vcpu >= MAX_VIRT_CPUS) ||
+              !d->vcpu[op->u.pincpudomain.vcpu])
+           )
+        {
+            ret = -EINVAL;
+            put_domain(d);
+            break;
+        }
+
+        /* don't get a struct vcpu pointer for -1 op */
+        if (op->u.pincpudomain.vcpu != -1) {
+           v = d->vcpu[op->u.pincpudomain.vcpu];
+           if ( v == NULL )
+           {
+               ret = -ESRCH;
+               put_domain(d);
+               break;
+           }
+
+           if ( v == current )
+           {
+               ret = -EINVAL;
+               put_domain(d);
+               break;
+           }
         }
 
         if ( copy_from_user(&cpumap, op->u.pincpudomain.cpumap,
@@ -316,24 +322,28 @@
             break;
         }
 
-        /* update cpumap for this vcpu */
-        v->cpumap = cpumap;
-
-        if ( cpumap == CPUMAP_RUNANYWHERE )
-        {
-            clear_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
-        }
-        else
-        {
-            /* pick a new cpu from the usable map */
-            int new_cpu = (int)find_first_set_bit(cpumap) % num_online_cpus();
-
-            vcpu_pause(v);
-            vcpu_migrate_cpu(v, new_cpu);
-            set_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
-            vcpu_unpause(v);
-        }
-
+        /* update domain vcpu alloc map */
+        if ( v == NULL ) {
+            d->allocmap = cpumap;
+        } else {
+            /* update cpumap for this vcpu */
+            v->cpumap = cpumap;
+
+           if ( cpumap == CPUMAP_RUNANYWHERE )
+           {
+               clear_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
+           }
+           else
+           {
+               /* pick a new cpu from the usable map */
+               int new_cpu = get_next_processor(d, v, &cpumap);
+
+               vcpu_pause(v);
+               vcpu_migrate_cpu(v, new_cpu);
+               set_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
+               vcpu_unpause(v);
+           }
+        }
         put_domain(d);
     }
     break;
diff -r 12d1e93653c2 xen/common/domain.c
--- a/xen/common/domain.c	Tue Oct 11 00:13:12 2005
+++ b/xen/common/domain.c	Mon Oct 10 19:35:55 2005
@@ -41,6 +41,7 @@
     atomic_set(&d->refcnt, 1);
     atomic_set(&v->pausecnt, 0);
 
+    d->allocmap  = CPUMAP_RUNANYWHERE;
     d->domain_id = dom_id;
     v->processor = cpu;
 
@@ -380,7 +381,7 @@
     v = d->vcpu[vcpuid];
 
     atomic_set(&v->pausecnt, 0);
-    v->cpumap = CPUMAP_RUNANYWHERE;
+    v->cpumap = d->allocmap;
 
     memcpy(&v->arch, &idle0_vcpu.arch, sizeof(v->arch));
 
@@ -469,6 +470,31 @@
 
     return -ENOSYS;
 }
+
+/* find the least loaded processor , ignorning vcpu v, in cpumap_t *map */
+int get_next_processor(struct domain* d, struct vcpu *v, cpumap_t *map) {
+    struct vcpu  *vc = NULL;
+    int pro, i, cnt[NR_CPUS] = { 0 };
+
+    /* count the processor layout for this dom, except for vcpu v 
+     * whose processor field may not have been set yet. */
+    for_each_vcpu( d, vc ) {
+        if (vc->vcpu_id != v->vcpu_id)
+            cnt[vc->processor]++;
+    }
+
+    /* start from the first allowable cpu, guard against bogus cpus */
+    pro = (int)find_first_set_bit(*map) % num_online_cpus();
+
+    /* pick least loaded processor in the map */
+    for ( i = pro; i < num_online_cpus(); i++ ) {
+        if ( test_bit(i, &*map) && (cnt[i] <= cnt[pro]) )
+            pro = i;
+    }
+
+    return pro;
+}
+
 
 /*
  * Local variables:
diff -r 12d1e93653c2 xen/common/schedule.c
--- a/xen/common/schedule.c	Tue Oct 11 00:13:12 2005
+++ b/xen/common/schedule.c	Mon Oct 10 19:35:55 2005
@@ -125,12 +125,13 @@
         v->next_in_list  = vc->next_in_list;
         vc->next_in_list = v;
 
-        if (test_bit(_VCPUF_cpu_pinned, &vc->vcpu_flags)) {
-            v->processor = (vc->processor + 1) % num_online_cpus();
+
+        /* XXX: if previous vcpu was pinned, mark new vcpu as pinned why? */
+        if (test_bit(_VCPUF_cpu_pinned, &vc->vcpu_flags))
             set_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
-        } else {
-            v->processor = (vc->processor + 1) % num_online_cpus();
-        }
+
+        v->processor = get_next_processor(d, v, &d->allocmap);
+
     }
 
     return v;
diff -r 12d1e93653c2 xen/include/public/dom0_ops.h
--- a/xen/include/public/dom0_ops.h	Tue Oct 11 00:13:12 2005
+++ b/xen/include/public/dom0_ops.h	Mon Oct 10 19:35:55 2005
@@ -181,7 +181,7 @@
 typedef struct {
     /* IN variables. */
     domid_t      domain;
-    u16          vcpu;
+    s16          vcpu;
     cpumap_t     *cpumap;
 } dom0_pincpudomain_t;
 
diff -r 12d1e93653c2 xen/include/xen/domain.h
--- a/xen/include/xen/domain.h	Tue Oct 11 00:13:12 2005
+++ b/xen/include/xen/domain.h	Mon Oct 10 19:35:55 2005
@@ -27,4 +27,6 @@
 
 extern void dump_pageframe_info(struct domain *d);
 
+int get_next_processor(struct domain *d, struct vcpu *v, cpumap_t *map);
+
 #endif /* __XEN_DOMAIN_H__ */
diff -r 12d1e93653c2 xen/include/xen/sched.h
--- a/xen/include/xen/sched.h	Tue Oct 11 00:13:12 2005
+++ b/xen/include/xen/sched.h	Mon Oct 10 19:35:55 2005
@@ -134,6 +134,8 @@
 
     /* Bitmask of CPUs which are holding onto this domain's state. */
     cpumask_t        cpumask;
+
+    cpumap_t         allocmap;        /* vcpu allocation bitmap */
 
     struct arch_domain arch;