Re: [PATCH v9 10/18] KVM: s390: pv: add mmu_notifier

[Janosch Frank <frankja@linux.ibm.com>]

On 3/30/22 14:25, Claudio Imbrenda wrote:
> Add an mmu_notifier for protected VMs. The callback function is
> triggered when the mm is torn down, and will attempt to convert all
> protected vCPUs to non-protected. This allows the mm teardown to use
> the destroy page UVC instead of export.
> 
> Also make KVM select CONFIG_MMU_NOTIFIER, needed to use mmu_notifiers.
> 
> Signed-off-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Acked-by: Janosch Frank <frankja@linux.ibm.com>

> ---
>   arch/s390/include/asm/kvm_host.h |  2 ++
>   arch/s390/kvm/Kconfig            |  1 +
>   arch/s390/kvm/kvm-s390.c         |  5 ++++-
>   arch/s390/kvm/pv.c               | 26 ++++++++++++++++++++++++++
>   4 files changed, 33 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index a22c9266ea05..1bccb8561ba9 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -19,6 +19,7 @@
>   #include <linux/kvm.h>
>   #include <linux/seqlock.h>
>   #include <linux/module.h>
> +#include <linux/mmu_notifier.h>
>   #include <asm/debug.h>
>   #include <asm/cpu.h>
>   #include <asm/fpu/api.h>
> @@ -921,6 +922,7 @@ struct kvm_s390_pv {
>   	u64 guest_len;
>   	unsigned long stor_base;
>   	void *stor_var;
> +	struct mmu_notifier mmu_notifier;
>   };
>   
>   struct kvm_arch{
> diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
> index 2e84d3922f7c..33f4ff909476 100644
> --- a/arch/s390/kvm/Kconfig
> +++ b/arch/s390/kvm/Kconfig
> @@ -34,6 +34,7 @@ config KVM
>   	select SRCU
>   	select KVM_VFIO
>   	select INTERVAL_TREE
> +	select MMU_NOTIFIER
>   	help
>   	  Support hosting paravirtualized guest machines using the SIE
>   	  virtualization capability on the mainframe. This should work
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 21fcca09e9bf..446f89db93a1 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -32,6 +32,7 @@
>   #include <linux/sched/signal.h>
>   #include <linux/string.h>
>   #include <linux/pgtable.h>
> +#include <linux/mmu_notifier.h>
>   
>   #include <asm/asm-offsets.h>
>   #include <asm/lowcore.h>
> @@ -2833,8 +2834,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
>   	 * can mess with the pv state. To avoid lockdep_assert_held from
>   	 * complaining we do not use kvm_s390_pv_is_protected.
>   	 */
> -	if (kvm_s390_pv_get_handle(kvm))
> +	if (kvm_s390_pv_get_handle(kvm)) {
>   		kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
> +		mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
> +	}
>   	debug_unregister(kvm->arch.dbf);
>   	free_page((unsigned long)kvm->arch.sie_page2);
>   	if (!kvm_is_ucontrol(kvm))
> diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c
> index 76ef33a277d3..788b96b36931 100644
> --- a/arch/s390/kvm/pv.c
> +++ b/arch/s390/kvm/pv.c
> @@ -14,6 +14,7 @@
>   #include <asm/mman.h>
>   #include <linux/pagewalk.h>
>   #include <linux/sched/mm.h>
> +#include <linux/mmu_notifier.h>
>   #include "kvm-s390.h"
>   
>   static void kvm_s390_clear_pv_state(struct kvm *kvm)
> @@ -192,6 +193,26 @@ int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
>   	return -EIO;
>   }
>   
> +static void kvm_s390_pv_mmu_notifier_release(struct mmu_notifier *subscription,
> +					     struct mm_struct *mm)
> +{
> +	struct kvm *kvm = container_of(subscription, struct kvm, arch.pv.mmu_notifier);
> +	u16 dummy;
> +
> +	/*
> +	 * No locking is needed since this is the last thread of the last user of this
> +	 * struct mm.
> +	 * When the struct kvm gets deinitialized, this notifier is also
> +	 * unregistered. This means that if this notifier runs, then the
> +	 * struct kvm is still valid.
> +	 */
> +	kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
> +}
> +
> +static const struct mmu_notifier_ops kvm_s390_pv_mmu_notifier_ops = {
> +	.release = kvm_s390_pv_mmu_notifier_release,
> +};
> +
>   int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
>   {
>   	struct uv_cb_cgc uvcb = {
> @@ -233,6 +254,11 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
>   		return -EIO;
>   	}
>   	kvm->arch.gmap->guest_handle = uvcb.guest_handle;
> +	/* Add the notifier only once. No races because we hold kvm->lock */
> +	if (kvm->arch.pv.mmu_notifier.ops != &kvm_s390_pv_mmu_notifier_ops) {
> +		kvm->arch.pv.mmu_notifier.ops = &kvm_s390_pv_mmu_notifier_ops;
> +		mmu_notifier_register(&kvm->arch.pv.mmu_notifier, kvm->mm);
> +	}
>   	return 0;
>   }
>