Re: [PATCH v1 2/3] KVM: s390: add stat counter for shadow gmap events

Janosch Frank
2023-05-09T11:59:57Z

On 5/9/23 13:12, Nico Boehr wrote:
> The shadow gmap tracks memory of nested guests (guest-3). In certain
> scenarios, the shadow gmap needs to be rebuilt, which is a costly operation
> since it involves a SIE exit into guest-1 for every entry in the respective
> shadow level.
>
> Add kvm stat counters when new shadow structures are created at various
> levels. Also add a counter gmap_shadow_acquire when a completely fresh
> shadow gmap is created.
>
> Note that there is no counter for the region first level. This is because
> the region first level is the highest level and hence is never referenced
> by another table. Creating a new region first table is therefore always
> equivalent to a new shadow gmap and hence is counted as
> gmap_shadow_acquire.
>
> Also note that not all page table levels need to be present and a ASCE
> can directly point to e.g. a segment table. In this case, a new segment
> table will always be equivalent to a new shadow gmap and hence will be
> counted as gmap_shadow_acquire and not as gmap_shadow_segment.
>
> Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
> ---
> arch/s390/include/asm/kvm_host.h | 5 +++++
> arch/s390/kvm/gaccess.c | 6 ++++++
> arch/s390/kvm/kvm-s390.c | 7 ++++++-
> arch/s390/kvm/vsie.c | 1 +
> 4 files changed, 18 insertions(+), 1 deletion(-)
>
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 3c3fe45085ec..7f70e3bbb44c 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -777,6 +777,11 @@ struct kvm_vm_stat {
> u64 inject_service_signal;
> u64 inject_virtio;
> u64 aen_forward;
> + u64 gmap_shadow_acquire;
> + u64 gmap_shadow_r2;
> + u64 gmap_shadow_r3;
> + u64 gmap_shadow_segment;
> + u64 gmap_shadow_page;

This needs to be gmap_shadow_pgt and then we need a separate shadow page
counter that's beeing incremented in kvm_s390_shadow_fault().

I'm wondering if we should name them after the entries to reduce
confusion especially when we get huge pages in the future.

gmap_shadow_acquire
gmap_shadow_r1_te (ptr to r2 table)
gmap_shadow_r2_te (ptr to r3 table)
gmap_shadow_r3_te (ptr to segment table)
gmap_shadow_sg_te (ptr to page table)
gmap_shadow_pg_te (single page table entry)

> };
>
> struct kvm_arch_memory_slot {
> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> index 3eb85f254881..8348a0095f3a 100644
> --- a/arch/s390/kvm/gaccess.c
> +++ b/arch/s390/kvm/gaccess.c
> @@ -1382,6 +1382,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> unsigned long *pgt, int *dat_protection,
> int *fake)
> {
> + struct kvm *kvm;
> struct gmap *parent;
> union asce asce;
> union vaddress vaddr;
> @@ -1390,6 +1391,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>
> *fake = 0;
> *dat_protection = 0;
> + kvm = sg->private;
> parent = sg->parent;
> vaddr.addr = saddr;
> asce.val = sg->orig_asce;
> @@ -1450,6 +1452,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_r2++;
> }
> fallthrough;
> case ASCE_TYPE_REGION2: {
> @@ -1478,6 +1481,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_r3++;
> }
> fallthrough;
> case ASCE_TYPE_REGION3: {
> @@ -1515,6 +1519,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_segment++;
> }
> fallthrough;
> case ASCE_TYPE_SEGMENT: {
> @@ -1548,6 +1553,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_page++;
> }
> }
> /* Return the parent address of the page table */
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 17b81659cdb2..b012645a5a7c 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -66,7 +66,12 @@ const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
> STATS_DESC_COUNTER(VM, inject_pfault_done),
> STATS_DESC_COUNTER(VM, inject_service_signal),
> STATS_DESC_COUNTER(VM, inject_virtio),
> - STATS_DESC_COUNTER(VM, aen_forward)
> + STATS_DESC_COUNTER(VM, aen_forward),
> + STATS_DESC_COUNTER(VM, gmap_shadow_acquire),
> + STATS_DESC_COUNTER(VM, gmap_shadow_r2),
> + STATS_DESC_COUNTER(VM, gmap_shadow_r3),
> + STATS_DESC_COUNTER(VM, gmap_shadow_segment),
> + STATS_DESC_COUNTER(VM, gmap_shadow_page),
> };
>
> const struct kvm_stats_header kvm_vm_stats_header = {
> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> index 8d6b765abf29..beb3be037722 100644
> --- a/arch/s390/kvm/vsie.c
> +++ b/arch/s390/kvm/vsie.c
> @@ -1221,6 +1221,7 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
> if (IS_ERR(gmap))
> return PTR_ERR(gmap);
> gmap->private = vcpu->kvm;
> + vcpu->kvm->stat.gmap_shadow_acquire++;
> WRITE_ONCE(vsie_page->gmap, gmap);
> return 0;
> }