[PATCH] vcpu-set increases vcpus beyond the value set at domain creation

[Ryan Grimm <grimm@us.ibm.com>]

Hi,

This patch allows a domain's vcpus to increase beyond the max (up to
CONFIG_NR_CPUS) set at creation time by making 3 changes:

1.  xen/common/dom0_ops.c - removes the check in DOM0_MAX_CPUS that
doesnt allow vcpus to be alloc'd after the first vcpu has been
initialiazed.  

2.  xend/XendDomainInfo.py - when vcpu-set is given a value greater than
the current vcpu count, the dom0_op above is called with the new value
and the store is updated.

3.  drivers/xen/core/smpboot.c - the cpu_possible_map and
cpu_present_map made to reflect the values outlined in
include/linux/cpumask.h when CONFIG_HOTPLUG_CPU is set:

    cpu_possible_map - all NR_CPUS bits set
    cpu_present_map  - has bit 'cpu' set iff cpu is populated
    cpu_online_map   - has bit 'cpu' set iff cpu available to
      scheduler

This has been tested on x86_32 and x86_64 with and without
CONFIG_HOTPLUG_CPU.  

Does anyone know what potential races are being avoided by the check in
dom0_ops.c that this patch removes?

Any comments on the patch?

Thanks,
Ryan

Signed-off-by: Ryan Grimm <grimm@us.ibm.com>
  
diff -r 1346a69694be -r 334b38a81c7d linux-2.6-xen-sparse/drivers/xen/core/smpboot.c
--- a/linux-2.6-xen-sparse/drivers/xen/core/smpboot.c	Wed Feb 15 14:20:32 2006
+++ b/linux-2.6-xen-sparse/drivers/xen/core/smpboot.c	Wed Feb 15 11:54:51 2006
@@ -57,6 +57,7 @@
 
 #ifdef CONFIG_HOTPLUG_CPU
 DEFINE_PER_CPU(int, cpu_state) = { 0 };
+static int vcpu_allocated(unsigned int vcpu);
 #endif
 
 static DEFINE_PER_CPU(int, resched_irq);
@@ -81,15 +82,17 @@
 
 void __init prefill_possible_map(void)
 {
-	int i, rc;
+	int i;
 
 	if (!cpus_empty(cpu_possible_map))
 		return;
 
 	for (i = 0; i < NR_CPUS; i++) {
-		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
+#ifndef CONFIG_HOTPLUG_CPU
+		int rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
 		if (rc == -ENOENT)
 			break;
+#endif
 		cpu_set(i, cpu_possible_map);
 	}
 }
@@ -257,11 +260,15 @@
 #ifdef CONFIG_HOTPLUG_CPU
 		if (xen_start_info->flags & SIF_INITDOMAIN)
 			cpu_set(cpu, cpu_present_map);
+		if (vcpu_allocated(cpu)) {
+			vcpu_prepare(cpu);
+			cpu_set(cpu, cpu_present_map);
+		}
 #else
 		cpu_set(cpu, cpu_present_map);
-#endif
-
 		vcpu_prepare(cpu);
+#endif
+
 	}
 
 	/* Currently, Xen gives no dynamic NUMA/HT info. */
@@ -289,17 +296,15 @@
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-/*
- * Initialize cpu_present_map late to skip SMP boot code in init/main.c.
- * But do it early enough to catch critical for_each_present_cpu() loops
- * in i386-specific code.
- */
-static int __init initialize_cpu_present_map(void)
-{
-	cpu_present_map = cpu_possible_map;
-	return 0;
-}
-core_initcall(initialize_cpu_present_map);
+static int vcpu_allocated(unsigned int vcpu)
+{
+	int rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, vcpu, NULL);
+	if (rc >= 0)
+		rc = 1;
+	else	
+		rc = 0;
+	return rc;
+}
 
 static void vcpu_hotplug(unsigned int cpu)
 {
@@ -312,11 +317,17 @@
 	sprintf(dir, "cpu/%d", cpu);
 	err = xenbus_scanf(XBT_NULL, dir, "availability", "%s", state);
 	if (err != 1) {
-		printk(KERN_ERR "XENBUS: Unable to read cpu state\n");
+		printk(KERN_ERR "XENBUS: Unable to read cpu %d state\n", cpu);
 		return;
 	}
 
 	if (strcmp(state, "online") == 0) {
+		if (!vcpu_allocated(cpu))
+			return;
+		if (!cpu_present(cpu)) {
+			vcpu_prepare(cpu);
+			cpu_set(cpu, cpu_present_map);
+		}
 		(void)cpu_up(cpu);
 	} else if (strcmp(state, "offline") == 0) {
 		(void)cpu_down(cpu);
diff -r 1346a69694be -r 334b38a81c7d tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py	Wed Feb 15 14:20:32 2006
+++ b/tools/python/xen/xend/XendDomainInfo.py	Wed Feb 15 11:54:51 2006
@@ -750,6 +750,9 @@
 
 
     def setVCpuCount(self, vcpus):
+        if vcpus > self.getVCpuCount():
+            xc.domain_max_vcpus(self.domid, vcpus)
+            self.info['vcpus'] = vcpus
         self.info['vcpu_avail'] = (1 << vcpus) - 1
         self.storeVm('vcpu_avail', self.info['vcpu_avail'])
         self.writeDom(self.vcpuDomDetails())
diff -r 1346a69694be -r 334b38a81c7d xen/common/dom0_ops.c
--- a/xen/common/dom0_ops.c	Wed Feb 15 14:20:32 2006
+++ b/xen/common/dom0_ops.c	Wed Feb 15 11:54:51 2006
@@ -236,15 +236,6 @@
         if ( (d = find_domain_by_id(op->u.max_vcpus.domain)) == NULL )
             break;
 
-        /*
-         * Can only create new VCPUs while the domain is not fully constructed
-         * (and hence not runnable). Xen needs auditing for races before
-         * removing this check.
-         */
-        ret = -EINVAL;
-        if ( test_bit(_VCPUF_initialised, &d->vcpu[0]->vcpu_flags) )
-            goto maxvcpu_out;
-
         /* We cannot reduce maximum VCPUs. */
         ret = -EINVAL;
         if ( (max != MAX_VIRT_CPUS) && (d->vcpu[max] != NULL) )