Re: cpusets not cpu hotplug aware

[Nathan Lynch <ntl@pobox.com>]

Nathan Lynch wrote:
> Paul Jackson wrote:
> > 
> > If there is some decent way I can get the cpus_allowed of the top
> > cpuset to track the cpu_online_map, then we avoid this discontinuity
> > in system behaviour.
> 
> 
> How about this?  I've verified it fixes the issue but I'm nervous
> about the locking.
> 
> 
> --- cpuhp-sched_setaffinity.orig/kernel/cpuset.c
> +++ cpuhp-sched_setaffinity/kernel/cpuset.c
> @@ -2033,6 +2033,31 @@ out:
>  	return err;
>  }
>  
> +static int cpuset_handle_cpuhp(struct notifier_block *nb,
> +				unsigned long phase, void *_cpu)
> +{
> +	unsigned long cpu = (unsigned long)_cpu;
> +
> +	mutex_lock(&manage_mutex);
> +	lock_cpu_hotplug();
> +	mutex_lock(&callback_mutex);
> +
> +	switch (phase) {
> +	case CPU_ONLINE:
> +		cpu_set(cpu, top_cpuset.cpus_allowed);
> +		break;
> +	case CPU_DEAD:
> +		cpu_clear(cpu, top_cpuset.cpus_allowed);
> +		break;
> +	}
> +
> +	mutex_unlock(&callback_mutex);
> +	unlock_cpu_hotplug();
> +	mutex_unlock(&manage_mutex);
> +
> +	return 0;
> +}

Actually the lock/unlock_cpu_hotplug aren't necessary,
cpu_add_remove_lock is already held in this context.


Update cpus_allowed in top_cpuset when cpus are hotplugged;
otherwise binding a task to a newly hotplugged cpu fails since the
toplevel cpuset has a static copy of whatever cpu_online_map was at
boot time.

Signed-off-by: Nathan Lynch <ntl@pobox.com>

--- cpuhp-sched_setaffinity.orig/kernel/cpuset.c
+++ cpuhp-sched_setaffinity/kernel/cpuset.c
@@ -2033,6 +2033,29 @@ out:
 	return err;
 }
 
+static int cpuset_handle_cpuhp(struct notifier_block *nb,
+				unsigned long phase, void *_cpu)
+{
+	unsigned long cpu = (unsigned long)_cpu;
+
+	mutex_lock(&manage_mutex);
+	mutex_lock(&callback_mutex);
+
+	switch (phase) {
+	case CPU_ONLINE:
+		cpu_set(cpu, top_cpuset.cpus_allowed);
+		break;
+	case CPU_DEAD:
+		cpu_clear(cpu, top_cpuset.cpus_allowed);
+		break;
+	}
+
+	mutex_unlock(&callback_mutex);
+	mutex_unlock(&manage_mutex);
+
+	return 0;
+}
+
 /**
  * cpuset_init_smp - initialize cpus_allowed
  *
@@ -2043,6 +2066,8 @@ void __init cpuset_init_smp(void)
 {
 	top_cpuset.cpus_allowed = cpu_online_map;
 	top_cpuset.mems_allowed = node_online_map;
+
+	hotcpu_notifier(cpuset_handle_cpuhp, 0);
 }
 
 /**