Re: [PATCH v4 1/2] KVM: s390: add stat counter for shadow gmap events

[Claudio Imbrenda <imbrenda@linux.ibm.com>]

On Mon,  9 Oct 2023 11:32:52 +0200
Nico Boehr <nrb@linux.ibm.com> wrote:

> The shadow gmap tracks memory of nested guests (guest-3). In certain
> scenarios, the shadow gmap needs to be rebuilt, which is a costly operation
> since it involves a SIE exit into guest-1 for every entry in the respective
> shadow level.
> 
> Add kvm stat counters when new shadow structures are created at various
> levels. Also add a counter gmap_shadow_create when a completely fresh
> shadow gmap is created as well as a counter gmap_shadow_reuse when an
> existing gmap is being reused.
> 
> Note that when several levels are shadowed at once, counters on all
> affected levels will be increased.
> 
> Also note that not all page table levels need to be present and a ASCE
> can directly point to e.g. a segment table. In this case, a new segment
> table will always be equivalent to a new shadow gmap and hence will be
> counted as gmap_shadow_create and not as gmap_shadow_segment.
> 
> Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

> ---
>  arch/s390/include/asm/kvm_host.h | 7 +++++++
>  arch/s390/kvm/gaccess.c          | 7 +++++++
>  arch/s390/kvm/kvm-s390.c         | 9 ++++++++-
>  arch/s390/kvm/vsie.c             | 5 ++++-
>  4 files changed, 26 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 427f9528a7b6..67a298b6cf6e 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -777,6 +777,13 @@ struct kvm_vm_stat {
>  	u64 inject_service_signal;
>  	u64 inject_virtio;
>  	u64 aen_forward;
> +	u64 gmap_shadow_create;
> +	u64 gmap_shadow_reuse;
> +	u64 gmap_shadow_r1_entry;
> +	u64 gmap_shadow_r2_entry;
> +	u64 gmap_shadow_r3_entry;
> +	u64 gmap_shadow_sg_entry;
> +	u64 gmap_shadow_pg_entry;
>  };
>  
>  struct kvm_arch_memory_slot {
> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> index 6d6bc19b37dc..ff8349d17b33 100644
> --- a/arch/s390/kvm/gaccess.c
> +++ b/arch/s390/kvm/gaccess.c
> @@ -1382,6 +1382,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>  				  unsigned long *pgt, int *dat_protection,
>  				  int *fake)
>  {
> +	struct kvm *kvm;
>  	struct gmap *parent;
>  	union asce asce;
>  	union vaddress vaddr;
> @@ -1390,6 +1391,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>  
>  	*fake = 0;
>  	*dat_protection = 0;
> +	kvm = sg->private;
>  	parent = sg->parent;
>  	vaddr.addr = saddr;
>  	asce.val = sg->orig_asce;
> @@ -1450,6 +1452,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>  		rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
>  		if (rc)
>  			return rc;
> +		kvm->stat.gmap_shadow_r1_entry++;
>  	}
>  		fallthrough;
>  	case ASCE_TYPE_REGION2: {
> @@ -1478,6 +1481,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>  		rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
>  		if (rc)
>  			return rc;
> +		kvm->stat.gmap_shadow_r2_entry++;
>  	}
>  		fallthrough;
>  	case ASCE_TYPE_REGION3: {
> @@ -1515,6 +1519,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>  		rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
>  		if (rc)
>  			return rc;
> +		kvm->stat.gmap_shadow_r3_entry++;
>  	}
>  		fallthrough;
>  	case ASCE_TYPE_SEGMENT: {
> @@ -1548,6 +1553,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>  		rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
>  		if (rc)
>  			return rc;
> +		kvm->stat.gmap_shadow_sg_entry++;
>  	}
>  	}
>  	/* Return the parent address of the page table */
> @@ -1618,6 +1624,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
>  	pte.p |= dat_protection;
>  	if (!rc)
>  		rc = gmap_shadow_page(sg, saddr, __pte(pte.val));
> +	vcpu->kvm->stat.gmap_shadow_pg_entry++;
>  	ipte_unlock(vcpu->kvm);
>  	mmap_read_unlock(sg->mm);
>  	return rc;
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index b3f17e014cab..b42493110d76 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -66,7 +66,14 @@ const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
>  	STATS_DESC_COUNTER(VM, inject_pfault_done),
>  	STATS_DESC_COUNTER(VM, inject_service_signal),
>  	STATS_DESC_COUNTER(VM, inject_virtio),
> -	STATS_DESC_COUNTER(VM, aen_forward)
> +	STATS_DESC_COUNTER(VM, aen_forward),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_create),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
>  };
>  
>  const struct kvm_stats_header kvm_vm_stats_header = {
> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> index 61499293c2ac..02dcbe82a8e5 100644
> --- a/arch/s390/kvm/vsie.c
> +++ b/arch/s390/kvm/vsie.c
> @@ -1214,8 +1214,10 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
>  	 * we're holding has been unshadowed. If the gmap is still valid,
>  	 * we can safely reuse it.
>  	 */
> -	if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat))
> +	if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat)) {
> +		vcpu->kvm->stat.gmap_shadow_reuse++;
>  		return 0;
> +	}
>  
>  	/* release the old shadow - if any, and mark the prefix as unmapped */
>  	release_gmap_shadow(vsie_page);
> @@ -1223,6 +1225,7 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
>  	if (IS_ERR(gmap))
>  		return PTR_ERR(gmap);
>  	gmap->private = vcpu->kvm;
> +	vcpu->kvm->stat.gmap_shadow_create++;
>  	WRITE_ONCE(vsie_page->gmap, gmap);
>  	return 0;
>  }