Re: [PATCH v8 1/3] smpboot: allow excluding cpus from the smpboot threads

[Frederic Weisbecker <fweisbec@gmail.com>]

On Tue, Apr 14, 2015 at 03:37:31PM -0400, Chris Metcalf wrote:
> diff --git a/kernel/smpboot.c b/kernel/smpboot.c
> index c697f73d82d6..c5d53a335387 100644
> --- a/kernel/smpboot.c
> +++ b/kernel/smpboot.c
> @@ -92,6 +92,9 @@ enum {
>  	HP_THREAD_PARKED,
>  };
>  
> +/* Statically allocated and used under smpboot_threads_lock. */
> +static struct cpumask tmp_mask;
> +

Better allocate the cpumask on need rather than have it resident on memory.
struct cpumask can be large. Plus we need to worry about locking it.

>  /**
>   * smpboot_thread_fn - percpu hotplug thread loop function
>   * @data:	thread data pointer
> @@ -232,7 +235,8 @@ void smpboot_unpark_threads(unsigned int cpu)
>  
>  	mutex_lock(&smpboot_threads_lock);
>  	list_for_each_entry(cur, &hotplug_threads, list)
> -		smpboot_unpark_thread(cur, cpu);
> +		if (cur->cpumask == NULL || cpumask_test_cpu(cpu, cur->cpumask))
> +			smpboot_unpark_thread(cur, cpu);
>  	mutex_unlock(&smpboot_threads_lock);
>  }
>  
> @@ -258,6 +262,16 @@ static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
>  {
>  	unsigned int cpu;
>  
> +	/* Unpark any threads that were voluntarily parked. */
> +	if (ht->cpumask) {
> +		cpumask_andnot(&tmp_mask, cpu_online_mask, ht->cpumask);
> +		for_each_cpu(cpu, &tmp_mask) {
> +			struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
> +			if (tsk)
> +				kthread_unpark(tsk);
> +		}
> +	}

Why do you need to do that? smpboot_destroy_threads() doesn't work on parked threads?
But kthread_stop() does an explicit unparking.

> +
>  	/* We need to destroy also the parked threads of offline cpus */
>  	for_each_possible_cpu(cpu) {
>  		struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
> @@ -289,7 +303,9 @@ int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
>  			smpboot_destroy_threads(plug_thread);
>  			goto out;
>  		}
> -		smpboot_unpark_thread(plug_thread, cpu);
> +		if (plug_thread->cpumask == NULL ||
> +		    cpumask_test_cpu(cpu, plug_thread->cpumask))
> +			smpboot_unpark_thread(plug_thread, cpu);
>  	}
>  	list_add(&plug_thread->list, &hotplug_threads);
>  out:
> @@ -316,6 +332,43 @@ void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)
>  }
>  EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread);
>  
> +/**
> + * smpboot_update_cpumask_percpu_thread - Adjust which per_cpu hotplug threads stay parked
> + * @plug_thread:	Hotplug thread descriptor
> + * @new:		Revised mask to use
> + *
> + * The cpumask field in the smp_hotplug_thread must not be updated directly
> + * by the client, but only by calling this function.  A non-NULL cpumask must
> + * have been provided at registration time to be able to use this function.
> + */
> +void smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread,
> +					  const struct cpumask *new)
> +{
> +	unsigned int cpu;
> +	struct cpumask *old = plug_thread->cpumask;
> +
> +	BUG_ON(old == NULL);

Ouch. So the caller must have passed an explicit mask to be able to modify it?
We can't do that.

> +
> +	get_online_cpus();
> +	mutex_lock(&smpboot_threads_lock);
> +
> +	/* Park threads that were exclusively enabled on the old mask. */
> +	cpumask_andnot(&tmp_mask, old, new);
> +	for_each_cpu_and(cpu, &tmp_mask, cpu_online_mask)
> +		smpboot_park_thread(plug_thread, cpu);
> +
> +	/* Unpark threads that are exclusively enabled on the new mask. */
> +	cpumask_andnot(&tmp_mask, new, old);
> +	for_each_cpu_and(cpu, &tmp_mask, cpu_online_mask)
> +		smpboot_unpark_thread(plug_thread, cpu);
> +
> +	cpumask_copy(old, new);

So unfortunately I had to see the result to realize my mistake on one detail.
With this scheme, it's not clear who allocates and who releases the cpumasks.
If the caller of smpboot_register_percpu_thread() allocates the cpumask, then he
should release it itself after calling smpboot_unregister_percpu_thread(). But
if the cpumask is NULL and we call smpboot_update_cpumask_percpu_thread(), it's
not clear to the caller if we make a copy, if he can release it after calling
the function, etc...

So the client should not touch the cpumask field of struct smp_hotplug_thread at all
and it should pass the cpumask to smpboot_register_percpu_thread() and smpboot_update_cpumask_percpu_thread().

smpboot subsystem then does its own copy to the struct smp_hotplug_thread which it releases from
smpboot_unregister_percpu_thread().

This way we prevent from any nasty side effet or headscratch about who is responsible
of allocations and releases.

> +
> +	mutex_unlock(&smpboot_threads_lock);
> +	put_online_cpus();
> +}
> +EXPORT_SYMBOL_GPL(smpboot_update_cpumask_percpu_thread);
> +
>  static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD);
>  
>  /*
> -- 
> 2.1.2
>