Re: [PATCH v10 1/3] smpboot: allow excluding cpus from the smpboot threads

[Frederic Weisbecker <fweisbec@gmail.com>]

On Thu, Apr 30, 2015 at 03:39:24PM -0400, Chris Metcalf wrote:
> This change allows some cores to be excluded from running the
> smp_hotplug_thread tasks.  The following commit to update
> kernel/watchdog.c to use this functionality is the motivating
> example, and more information on the motivation is provided there.
> 
> A new smp_hotplug_thread field is introduced, "cpumask", which
> is cpumask field managed by the smpboot subsystem that indicates whether
> or not the given smp_hotplug_thread should run on that core; the
> cpumask is checked when deciding whether to unpark the thread.
> 
> To limit the cpumask to less than cpu_possible, you must call
> smpboot_update_cpumask_percpu_thread() after registering.
> 
> Signed-off-by: Chris Metcalf <cmetcalf@ezchip.com>
> ---
>  include/linux/smpboot.h |  5 +++++
>  kernel/smpboot.c        | 55 ++++++++++++++++++++++++++++++++++++++++++++++++-
>  2 files changed, 59 insertions(+), 1 deletion(-)
> 
> diff --git a/include/linux/smpboot.h b/include/linux/smpboot.h
> index d600afb21926..7c42153edfac 100644
> --- a/include/linux/smpboot.h
> +++ b/include/linux/smpboot.h
> @@ -27,6 +27,8 @@ struct smpboot_thread_data;
>   * @pre_unpark:		Optional unpark function, called before the thread is
>   *			unparked (cpu online). This is not guaranteed to be
>   *			called on the target cpu of the thread. Careful!
> + * @cpumask:		Internal state.  To update which threads are unparked,
> + *			call smpboot_update_cpumask_percpu_thread().
>   * @selfparking:	Thread is not parked by the park function.
>   * @thread_comm:	The base name of the thread
>   */
> @@ -41,11 +43,14 @@ struct smp_hotplug_thread {
>  	void				(*park)(unsigned int cpu);
>  	void				(*unpark)(unsigned int cpu);
>  	void				(*pre_unpark)(unsigned int cpu);
> +	struct cpumask			cpumask;

I believe it should be allocated dynamically, otherwise it gets the size of NR_CPUS
instead of nr_cpus_bits. It's not _that_ much space spared but think there should be
several struct smp_hotplug_thread registered.

>  	bool				selfparking;
>  	const char			*thread_comm;
>  };
>  
>  int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread);
>  void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread);
> +int smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread,
> +					 const struct cpumask *);
>  
>  #endif
> diff --git a/kernel/smpboot.c b/kernel/smpboot.c
> index c697f73d82d6..209750ab7031 100644
> --- a/kernel/smpboot.c
> +++ b/kernel/smpboot.c
> @@ -232,7 +232,8 @@ void smpboot_unpark_threads(unsigned int cpu)
>  
>  	mutex_lock(&smpboot_threads_lock);
>  	list_for_each_entry(cur, &hotplug_threads, list)
> -		smpboot_unpark_thread(cur, cpu);
> +		if (cpumask_test_cpu(cpu, &cur->cpumask))
> +			smpboot_unpark_thread(cur, cpu);
>  	mutex_unlock(&smpboot_threads_lock);
>  }
>  
> @@ -258,6 +259,15 @@ static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
>  {
>  	unsigned int cpu;
>  
> +	/* Unpark any threads that were voluntarily parked. */
> +	for_each_cpu_not(cpu, &ht->cpumask) {
> +		if (cpu_online(cpu)) {
> +			struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
> +			if (tsk)
> +				kthread_unpark(tsk);

I'm still not clear why we are doing that. kthread_stop() should be able
to handle parked kthreads, otherwise it needs to be fixed.

> +		}
> +	}
> +
>  	/* We need to destroy also the parked threads of offline cpus */
>  	for_each_possible_cpu(cpu) {
>  		struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
> @@ -281,6 +291,7 @@ int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
>  	unsigned int cpu;
>  	int ret = 0;
>  
> +	cpumask_copy(&plug_thread->cpumask, cpu_possible_mask);
>  	get_online_cpus();
>  	mutex_lock(&smpboot_threads_lock);
>  	for_each_online_cpu(cpu) {
> @@ -316,6 +327,48 @@ void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)
>  }
>  EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread);
>  
> +/**
> + * smpboot_update_cpumask_percpu_thread - Adjust which per_cpu hotplug threads stay parked
> + * @plug_thread:	Hotplug thread descriptor
> + * @new:		Revised mask to use
> + *
> + * The cpumask field in the smp_hotplug_thread must not be updated directly
> + * by the client, but only by calling this function.
> + */
> +int smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread,
> +					 const struct cpumask *new)
> +{
> +	struct cpumask *old = &plug_thread->cpumask;
> +	cpumask_var_t tmp;
> +	unsigned int cpu;
> +
> +	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
> +		return -ENOMEM;
> +
> +	get_online_cpus();
> +	mutex_lock(&smpboot_threads_lock);
> +
> +	/* Park threads that were exclusively enabled on the old mask. */
> +	cpumask_andnot(tmp, old, new);
> +	for_each_cpu_and(cpu, tmp, cpu_online_mask)
> +		smpboot_park_thread(plug_thread, cpu);
> +
> +	/* Unpark threads that are exclusively enabled on the new mask. */
> +	cpumask_andnot(tmp, new, old);
> +	for_each_cpu_and(cpu, tmp, cpu_online_mask)
> +		smpboot_unpark_thread(plug_thread, cpu);
> +
> +	cpumask_copy(old, new);
> +
> +	mutex_unlock(&smpboot_threads_lock);
> +	put_online_cpus();
> +
> +	free_cpumask_var(tmp);
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(smpboot_update_cpumask_percpu_thread);
> +
>  static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD);
>  
>  /*
> -- 
> 2.1.2
>