Re: [PATCH 02/10] KVM: PPC: Book3S HV: Allow KVM guests to stop secondary threads coming online

[Benjamin Herrenschmidt <benh@kernel.crashing.org>]

On Fri, 2012-09-21 at 15:35 +1000, Paul Mackerras wrote:
> When a Book3S HV KVM guest is running, we need the host to be in
> single-thread mode, that is, all of the cores (or at least all of
> the cores where the KVM guest could run) to be running only one
> active hardware thread.  This is because of the hardware restriction
> in POWER processors that all of the hardware threads in the core
> must be in the same logical partition.  Complying with this restriction
> is much easier if, from the host kernel's point of view, only one
> hardware thread is active.
> 
> This adds two hooks in the SMP hotplug code to allow the KVM code to
> make sure that secondary threads (i.e. hardware threads other than
> thread 0) cannot come online while any KVM guest exists.  The KVM
> code still has to check that any core where it runs a guest has the
> secondary threads offline, but having done that check it can now be
> sure that they will not come online while the guest is running.
> 
> Signed-off-by: Paul Mackerras <paulus@samba.org>

Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

> ---
>  arch/powerpc/include/asm/smp.h |    8 +++++++
>  arch/powerpc/kernel/smp.c      |   46 ++++++++++++++++++++++++++++++++++++++++
>  arch/powerpc/kvm/book3s_hv.c   |   12 +++++++++--
>  3 files changed, 64 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h
> index ebc24dc..b625a1a 100644
> --- a/arch/powerpc/include/asm/smp.h
> +++ b/arch/powerpc/include/asm/smp.h
> @@ -66,6 +66,14 @@ void generic_cpu_die(unsigned int cpu);
>  void generic_mach_cpu_die(void);
>  void generic_set_cpu_dead(unsigned int cpu);
>  int generic_check_cpu_restart(unsigned int cpu);
> +
> +extern void inhibit_secondary_onlining(void);
> +extern void uninhibit_secondary_onlining(void);
> +
> +#else /* HOTPLUG_CPU */
> +static inline void inhibit_secondary_onlining(void) {}
> +static inline void uninhibit_secondary_onlining(void) {}
> +
>  #endif
>  
>  #ifdef CONFIG_PPC64
> diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
> index 0321007..c45f51d 100644
> --- a/arch/powerpc/kernel/smp.c
> +++ b/arch/powerpc/kernel/smp.c
> @@ -410,6 +410,45 @@ int generic_check_cpu_restart(unsigned int cpu)
>  {
>  	return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;
>  }
> +
> +static atomic_t secondary_inhibit_count;
> +
> +/*
> + * Don't allow secondary CPU threads to come online
> + */
> +void inhibit_secondary_onlining(void)
> +{
> +	/*
> +	 * This makes secondary_inhibit_count stable during cpu
> +	 * online/offline operations.
> +	 */
> +	get_online_cpus();
> +
> +	atomic_inc(&secondary_inhibit_count);
> +	put_online_cpus();
> +}
> +EXPORT_SYMBOL_GPL(inhibit_secondary_onlining);
> +
> +/*
> + * Allow secondary CPU threads to come online again
> + */
> +void uninhibit_secondary_onlining(void)
> +{
> +	get_online_cpus();
> +	atomic_dec(&secondary_inhibit_count);
> +	put_online_cpus();
> +}
> +EXPORT_SYMBOL_GPL(uninhibit_secondary_onlining);
> +
> +static int secondaries_inhibited(void)
> +{
> +	return atomic_read(&secondary_inhibit_count);
> +}
> +
> +#else /* HOTPLUG_CPU */
> +
> +#define secondaries_inhibited()		0
> +
>  #endif
>  
>  static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
> @@ -428,6 +467,13 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle)
>  {
>  	int rc, c;
>  
> +	/*
> +	 * Don't allow secondary threads to come online if inhibited
> +	 */
> +	if (threads_per_core > 1 && secondaries_inhibited() &&
> +	    cpu % threads_per_core != 0)
> +		return -EBUSY;
> +
>  	if (smp_ops == NULL ||
>  	    (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
>  		return -EINVAL;
> diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
> index bebf9cb..6fe1410 100644
> --- a/arch/powerpc/kvm/book3s_hv.c
> +++ b/arch/powerpc/kvm/book3s_hv.c
> @@ -47,6 +47,7 @@
>  #include <asm/page.h>
>  #include <asm/hvcall.h>
>  #include <asm/switch_to.h>
> +#include <asm/smp.h>
>  #include <linux/gfp.h>
>  #include <linux/vmalloc.h>
>  #include <linux/highmem.h>
> @@ -918,8 +919,6 @@ static int kvmppc_run_core(struct kvmppc_vcore *vc)
>  	/*
>  	 * Make sure we are running on thread 0, and that
>  	 * secondary threads are offline.
> -	 * XXX we should also block attempts to bring any
> -	 * secondary threads online.
>  	 */
>  	if (threads_per_core > 1 && !on_primary_thread()) {
>  		list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
> @@ -1632,11 +1631,20 @@ int kvmppc_core_init_vm(struct kvm *kvm)
>  
>  	kvm->arch.using_mmu_notifiers = !!cpu_has_feature(CPU_FTR_ARCH_206);
>  	spin_lock_init(&kvm->arch.slot_phys_lock);
> +
> +	/*
> +	 * Don't allow secondary CPU threads to come online
> +	 * while any KVM VMs exist.
> +	 */
> +	inhibit_secondary_onlining();
> +
>  	return 0;
>  }
>  
>  void kvmppc_core_destroy_vm(struct kvm *kvm)
>  {
> +	uninhibit_secondary_onlining();
> +
>  	if (kvm->arch.rma) {
>  		kvm_release_rma(kvm->arch.rma);
>  		kvm->arch.rma = NULL;