Linux KVM/arm64 development list
 help / color / mirror / Atom feed
From: Jing Zhang <jingzhangos@google.com>
To: KVM <kvm@vger.kernel.org>, KVMARM <kvmarm@lists.cs.columbia.edu>,
	 Paolo Bonzini <pbonzini@redhat.com>,
	Marc Zyngier <maz@kernel.org>, Will Deacon <will@kernel.org>,
	 David Matlack <dmatlack@google.com>,
	Peter Shier <pshier@google.com>,
	 Oliver Upton <oupton@google.com>,
	Sean Christopherson <seanjc@google.com>
Subject: [PATCH v1 3/3] KVM: arm64: Add histogram stats for handling time of arch specific exit reasons
Date: Wed, 22 Sep 2021 01:08:51 +0000	[thread overview]
Message-ID: <20210922010851.2312845-3-jingzhangos@google.com> (raw)
In-Reply-To: <20210922010851.2312845-1-jingzhangos@google.com>

These logarithmic histogram stats are useful for monitoring performance
of handling for different kinds of VCPU exit reasons.

Signed-off-by: Jing Zhang <jingzhangos@google.com>
---
 arch/arm64/include/asm/kvm_host.h | 36 ++++++++++++
 arch/arm64/kvm/arm.c              |  4 ++
 arch/arm64/kvm/guest.c            | 43 ++++++++++++++
 arch/arm64/kvm/handle_exit.c      | 95 +++++++++++++++++++++++++++++++
 4 files changed, 178 insertions(+)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 4d65de22add3..f1a29ca3d4f3 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -417,6 +417,9 @@ struct kvm_vcpu_arch {
 
 	/* Arch specific exit reason */
 	enum arm_exit_reason exit_reason;
+
+	/* The timestamp for the last VCPU exit */
+	u64 last_exit_time;
 };
 
 /* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
@@ -605,6 +608,8 @@ struct kvm_vm_stat {
 	struct kvm_vm_stat_generic generic;
 };
 
+#define ARM_EXIT_HIST_CNT	64
+
 struct kvm_vcpu_stat {
 	struct kvm_vcpu_stat_generic generic;
 	u64 mmio_exit_user;
@@ -641,6 +646,36 @@ struct kvm_vcpu_stat {
 		u64 exit_fp_asimd;
 		u64 exit_pac;
 	};
+	/* Histogram stats for handling time of arch specific exit reasons */
+	struct {
+		u64 exit_unknown_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_irq_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_el1_serror_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_hyp_gone_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_il_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_wfi_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_wfe_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_cp15_32_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_cp15_64_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_cp14_32_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_cp14_ls_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_cp14_64_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_hvc32_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_smc32_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_hvc64_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_smc64_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_sys64_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_sve_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_iabt_low_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_dabt_low_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_softstp_low_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_watchpt_low_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_breakpt_low_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_bkpt32_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_brk64_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_fp_asimd_hist[ARM_EXIT_HIST_CNT];
+		u64 exit_pac_hist[ARM_EXIT_HIST_CNT];
+	};
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
@@ -715,6 +750,7 @@ void force_vm_exit(const cpumask_t *mask);
 
 int handle_exit(struct kvm_vcpu *vcpu, int exception_index);
 void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index);
+void update_hist_exit_stats(struct kvm_vcpu *vcpu);
 
 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu);
 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index fe102cd2e518..156f80b699d3 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -795,6 +795,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 	ret = 1;
 	run->exit_reason = KVM_EXIT_UNKNOWN;
 	while (ret > 0) {
+		/* Update histogram stats for exit reasons */
+		update_hist_exit_stats(vcpu);
+
 		/*
 		 * Check conditions before entering the guest
 		 */
@@ -903,6 +906,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 		 */
 		guest_exit();
 		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
+		vcpu->arch.last_exit_time = ktime_to_ns(ktime_get());
 
 		/* Exit types that need handling before we can be preempted */
 		handle_exit_early(vcpu, ret);
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index abd9327d7110..bbf51578fdec 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -75,6 +75,49 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
 	STATS_DESC_COUNTER(VCPU, exit_brk64),
 	STATS_DESC_COUNTER(VCPU, exit_fp_asimd),
 	STATS_DESC_COUNTER(VCPU, exit_pac),
+	/* Histogram stats for handling time of arch specific exit reasons */
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_unknown_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_irq_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_el1_serror_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_hyp_gone_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_il_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_wfi_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_wfe_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_cp15_32_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_cp15_64_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_cp14_32_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_cp14_ls_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_cp14_64_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_hvc32_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_smc32_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_hvc64_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_smc64_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_sys64_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_sve_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_iabt_low_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_dabt_low_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_softstp_low_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_watchpt_low_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_breakpt_low_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_bkpt32_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_brk64_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(
+			VCPU, exit_fp_asimd_hist, ARM_EXIT_HIST_CNT),
+	STATS_DESC_LOGHIST_TIME_NSEC(VCPU, exit_pac_hist, ARM_EXIT_HIST_CNT),
 };
 
 const struct kvm_stats_header kvm_vcpu_stats_header = {
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index e83cd52078b2..5e642a6275c1 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -395,3 +395,98 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr,
 	panic("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%016lx\n",
 	      spsr, elr_virt, esr, far, hpfar, par, vcpu);
 }
+
+void update_hist_exit_stats(struct kvm_vcpu *vcpu)
+{
+	u64 val = ktime_to_ns(ktime_get()) - vcpu->arch.last_exit_time;
+
+	if (unlikely(!vcpu->arch.last_exit_time))
+		return;
+
+	switch (vcpu->arch.exit_reason) {
+	case ARM_EXIT_UNKNOWN:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_unknown_hist, val);
+		break;
+	case ARM_EXIT_IRQ:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_irq_hist, val);
+		break;
+	case ARM_EXIT_EL1_SERROR:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_el1_serror_hist, val);
+		break;
+	case ARM_EXIT_HYP_GONE:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_hyp_gone_hist, val);
+		break;
+	case ARM_EXIT_IL:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_il_hist, val);
+		break;
+	case ARM_EXIT_WFI:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_wfi_hist, val);
+		break;
+	case ARM_EXIT_WFE:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_wfe_hist, val);
+		break;
+	case ARM_EXIT_CP15_32:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_cp15_32_hist, val);
+		break;
+	case ARM_EXIT_CP15_64:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_cp15_64_hist, val);
+		break;
+	case ARM_EXIT_CP14_32:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_cp14_32_hist, val);
+		break;
+	case ARM_EXIT_CP14_LS:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_cp14_ls_hist, val);
+		break;
+	case ARM_EXIT_CP14_64:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_cp14_64_hist, val);
+		break;
+	case ARM_EXIT_HVC32:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_hvc32_hist, val);
+		break;
+	case ARM_EXIT_SMC32:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_smc32_hist, val);
+		break;
+	case ARM_EXIT_HVC64:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_hvc64_hist, val);
+		break;
+	case ARM_EXIT_SMC64:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_smc64_hist, val);
+		break;
+	case ARM_EXIT_SYS64:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_sys64_hist, val);
+		break;
+	case ARM_EXIT_SVE:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_sve_hist, val);
+		break;
+	case ARM_EXIT_IABT_LOW:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_iabt_low_hist, val);
+		break;
+	case ARM_EXIT_DABT_LOW:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_dabt_low_hist, val);
+		break;
+	case ARM_EXIT_SOFTSTP_LOW:
+		KVM_STATS_LOG_HIST_UPDATE(
+				vcpu->stat.exit_softstp_low_hist, val);
+		break;
+	case ARM_EXIT_WATCHPT_LOW:
+		KVM_STATS_LOG_HIST_UPDATE(
+				vcpu->stat.exit_watchpt_low_hist, val);
+		break;
+	case ARM_EXIT_BREAKPT_LOW:
+		KVM_STATS_LOG_HIST_UPDATE(
+				vcpu->stat.exit_breakpt_low_hist, val);
+		break;
+	case ARM_EXIT_BKPT32:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_bkpt32_hist, val);
+		break;
+	case ARM_EXIT_BRK64:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_brk64_hist, val);
+		break;
+	case ARM_EXIT_FP_ASIMD:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_fp_asimd_hist, val);
+		break;
+	case ARM_EXIT_PAC:
+		KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.exit_pac_hist, val);
+		break;
+	}
+}
-- 
2.33.0.464.g1972c5931b-goog

_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm

  parent reply	other threads:[~2021-09-22  1:09 UTC|newest]

Thread overview: 14+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-09-22  1:08 [PATCH v1 1/3] KVM: arm64: Add arch specific exit reasons Jing Zhang
2021-09-22  1:08 ` [PATCH v1 2/3] KVM: arm64: Add counter stats for " Jing Zhang
2021-09-22  1:08 ` Jing Zhang [this message]
2021-09-22 11:22   ` [PATCH v1 3/3] KVM: arm64: Add histogram stats for handling time of " Marc Zyngier
2021-09-22 15:37     ` Paolo Bonzini
2021-09-22 16:09       ` Jing Zhang
2021-09-22 18:13       ` Sean Christopherson
2021-09-22 18:53         ` Marc Zyngier
2021-09-22 23:22           ` David Matlack
2021-09-30 14:04             ` Marc Zyngier
2021-09-30 18:05               ` Sean Christopherson
2021-09-23  6:36           ` Paolo Bonzini
2021-09-23  7:45             ` Marc Zyngier
2021-09-23  9:44               ` Paolo Bonzini

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210922010851.2312845-3-jingzhangos@google.com \
    --to=jingzhangos@google.com \
    --cc=dmatlack@google.com \
    --cc=kvm@vger.kernel.org \
    --cc=kvmarm@lists.cs.columbia.edu \
    --cc=maz@kernel.org \
    --cc=oupton@google.com \
    --cc=pbonzini@redhat.com \
    --cc=pshier@google.com \
    --cc=seanjc@google.com \
    --cc=will@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox