[PATCH] vcpu-set increases vcpus beyond the value set at domain creation

[PATCH] vcpu-set increases vcpus beyond the value set at domain creation

[Ryan Grimm]

Hi,

This patch allows a domain's vcpus to increase beyond the max (up to
CONFIG_NR_CPUS) set at creation time by making 3 changes:

1.  xen/common/dom0_ops.c - removes the check in DOM0_MAX_CPUS that
doesnt allow vcpus to be alloc'd after the first vcpu has been
initialiazed.  

2.  xend/XendDomainInfo.py - when vcpu-set is given a value greater than
the current vcpu count, the dom0_op above is called with the new value
and the store is updated.

3.  drivers/xen/core/smpboot.c - the cpu_possible_map and
cpu_present_map made to reflect the values outlined in
include/linux/cpumask.h when CONFIG_HOTPLUG_CPU is set:

    cpu_possible_map - all NR_CPUS bits set
    cpu_present_map  - has bit 'cpu' set iff cpu is populated
    cpu_online_map   - has bit 'cpu' set iff cpu available to
      scheduler

This has been tested on x86_32 and x86_64 with and without
CONFIG_HOTPLUG_CPU.  

Does anyone know what potential races are being avoided by the check in
dom0_ops.c that this patch removes?

Any comments on the patch?

Thanks,
Ryan

Signed-off-by: Ryan Grimm <grimm@us.ibm.com>
  
diff -r 1346a69694be -r 334b38a81c7d linux-2.6-xen-sparse/drivers/xen/core/smpboot.c
--- a/linux-2.6-xen-sparse/drivers/xen/core/smpboot.c	Wed Feb 15 14:20:32 2006
+++ b/linux-2.6-xen-sparse/drivers/xen/core/smpboot.c	Wed Feb 15 11:54:51 2006
@@ -57,6 +57,7 @@
 
 #ifdef CONFIG_HOTPLUG_CPU
 DEFINE_PER_CPU(int, cpu_state) = { 0 };
+static int vcpu_allocated(unsigned int vcpu);
 #endif
 
 static DEFINE_PER_CPU(int, resched_irq);
@@ -81,15 +82,17 @@
 
 void __init prefill_possible_map(void)
 {
-	int i, rc;
+	int i;
 
 	if (!cpus_empty(cpu_possible_map))
 		return;
 
 	for (i = 0; i < NR_CPUS; i++) {
-		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
+#ifndef CONFIG_HOTPLUG_CPU
+		int rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
 		if (rc == -ENOENT)
 			break;
+#endif
 		cpu_set(i, cpu_possible_map);
 	}
 }
@@ -257,11 +260,15 @@
 #ifdef CONFIG_HOTPLUG_CPU
 		if (xen_start_info->flags & SIF_INITDOMAIN)
 			cpu_set(cpu, cpu_present_map);
+		if (vcpu_allocated(cpu)) {
+			vcpu_prepare(cpu);
+			cpu_set(cpu, cpu_present_map);
+		}
 #else
 		cpu_set(cpu, cpu_present_map);
-#endif
-
 		vcpu_prepare(cpu);
+#endif
+
 	}
 
 	/* Currently, Xen gives no dynamic NUMA/HT info. */
@@ -289,17 +296,15 @@
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-/*
- * Initialize cpu_present_map late to skip SMP boot code in init/main.c.
- * But do it early enough to catch critical for_each_present_cpu() loops
- * in i386-specific code.
- */
-static int __init initialize_cpu_present_map(void)
-{
-	cpu_present_map = cpu_possible_map;
-	return 0;
-}
-core_initcall(initialize_cpu_present_map);
+static int vcpu_allocated(unsigned int vcpu)
+{
+	int rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, vcpu, NULL);
+	if (rc >= 0)
+		rc = 1;
+	else	
+		rc = 0;
+	return rc;
+}
 
 static void vcpu_hotplug(unsigned int cpu)
 {
@@ -312,11 +317,17 @@
 	sprintf(dir, "cpu/%d", cpu);
 	err = xenbus_scanf(XBT_NULL, dir, "availability", "%s", state);
 	if (err != 1) {
-		printk(KERN_ERR "XENBUS: Unable to read cpu state\n");
+		printk(KERN_ERR "XENBUS: Unable to read cpu %d state\n", cpu);
 		return;
 	}
 
 	if (strcmp(state, "online") == 0) {
+		if (!vcpu_allocated(cpu))
+			return;
+		if (!cpu_present(cpu)) {
+			vcpu_prepare(cpu);
+			cpu_set(cpu, cpu_present_map);
+		}
 		(void)cpu_up(cpu);
 	} else if (strcmp(state, "offline") == 0) {
 		(void)cpu_down(cpu);
diff -r 1346a69694be -r 334b38a81c7d tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py	Wed Feb 15 14:20:32 2006
+++ b/tools/python/xen/xend/XendDomainInfo.py	Wed Feb 15 11:54:51 2006
@@ -750,6 +750,9 @@
 
 
     def setVCpuCount(self, vcpus):
+        if vcpus > self.getVCpuCount():
+            xc.domain_max_vcpus(self.domid, vcpus)
+            self.info['vcpus'] = vcpus
         self.info['vcpu_avail'] = (1 << vcpus) - 1
         self.storeVm('vcpu_avail', self.info['vcpu_avail'])
         self.writeDom(self.vcpuDomDetails())
diff -r 1346a69694be -r 334b38a81c7d xen/common/dom0_ops.c
--- a/xen/common/dom0_ops.c	Wed Feb 15 14:20:32 2006
+++ b/xen/common/dom0_ops.c	Wed Feb 15 11:54:51 2006
@@ -236,15 +236,6 @@
         if ( (d = find_domain_by_id(op->u.max_vcpus.domain)) == NULL )
             break;
 
-        /*
-         * Can only create new VCPUs while the domain is not fully constructed
-         * (and hence not runnable). Xen needs auditing for races before
-         * removing this check.
-         */
-        ret = -EINVAL;
-        if ( test_bit(_VCPUF_initialised, &d->vcpu[0]->vcpu_flags) )
-            goto maxvcpu_out;
-
         /* We cannot reduce maximum VCPUs. */
         ret = -EINVAL;
         if ( (max != MAX_VIRT_CPUS) && (d->vcpu[max] != NULL) )

Re: [PATCH] vcpu-set increases vcpus beyond the value set at domain creation

[Keir Fraser]

On 15 Feb 2006, at 23:04, Ryan Grimm wrote:

> This patch allows a domain's vcpus to increase beyond the max (up to
> CONFIG_NR_CPUS) set at creation time by making 3 changes:

I'd prefer to keep the current Xen mechanisms, but extend xend and/or 
config file formats so that we can distinguish max_vcpus from 
initial_vcpus. Currently the two values are conflated. Then you can set 
max_vcpus as high as you like, but xenstore will tell the guest how 
many CPUs to bring up during boot.

If we want a 'hard limit' check in Xen (kind of like we have a 
per-domain memory limit) to ensure that guests do not sneakily bring up 
CPUs that we didn't ask them to, then we can add that but it's an 
orthogonal change (i.e., different patch) to what you are trying to do 
here.

  -- Keir