[PATCH v3 14/22] KVM: arm64: Write fast path PMU register handlers

Colton Lewis
2025-06-26T20:06:07Z

We may want a partitioned PMU but not have FEAT_FGT to untrap the
specific registers that would normally be. Add a handler for those
registers in the fast path so we can still get a performance boost
from partitioning.

Signed-off-by: Colton Lewis <coltonlewis@google.com>
---
arch/arm64/include/asm/arm_pmuv3.h | 37 ++++-
arch/arm64/include/asm/kvm_pmu.h | 10 ++
arch/arm64/kvm/hyp/include/hyp/switch.h | 174 ++++++++++++++++++++++++
arch/arm64/kvm/pmu.c | 16 +++
arch/arm64/kvm/sys_regs.c | 16 ---
5 files changed, 236 insertions(+), 17 deletions(-)

diff --git a/arch/arm64/include/asm/arm_pmuv3.h b/arch/arm64/include/asm/arm_pmuv3.h
index 3bddde5f4ebb..12004fd04018 100644
--- a/arch/arm64/include/asm/arm_pmuv3.h
+++ b/arch/arm64/include/asm/arm_pmuv3.h
@@ -41,6 +41,16 @@ static inline unsigned long read_pmevtypern(int n)
return 0;
}

+static inline void write_pmxevcntr(u64 val)
+{
+ write_sysreg(val, pmxevcntr_el0);
+}
+
+static inline u64 read_pmxevcntr(void)
+{
+ return read_sysreg(pmxevcntr_el0);
+}
+
static inline unsigned long read_pmmir(void)
{
return read_cpuid(PMMIR_EL1);
@@ -107,21 +117,41 @@ static inline void write_pmcntenset(u64 val)
write_sysreg(val, pmcntenset_el0);
}

+static inline u64 read_pmcntenset(void)
+{
+ return read_sysreg(pmcntenset_el0);
+}
+
static inline void write_pmcntenclr(u64 val)
{
write_sysreg(val, pmcntenclr_el0);
}

+static inline u64 read_pmcntenclr(void)
+{
+ return read_sysreg(pmcntenclr_el0);
+}
+
static inline void write_pmintenset(u64 val)
{
write_sysreg(val, pmintenset_el1);
}

+static inline u64 read_pmintenset(void)
+{
+ return read_sysreg(pmintenset_el1);
+}
+
static inline void write_pmintenclr(u64 val)
{
write_sysreg(val, pmintenclr_el1);
}

+static inline u64 read_pmintenclr(void)
+{
+ return read_sysreg(pmintenclr_el1);
+}
+
static inline void write_pmccfiltr(u64 val)
{
write_sysreg(val, pmccfiltr_el0);
@@ -162,11 +192,16 @@ static inline u64 read_pmovsclr(void)
return read_sysreg(pmovsclr_el0);
}

-static inline void write_pmuserenr(u32 val)
+static inline void write_pmuserenr(u64 val)
{
write_sysreg(val, pmuserenr_el0);
}

+static inline u64 read_pmuserenr(void)
+{
+ return read_sysreg(pmuserenr_el0);
+}
+
static inline void write_pmuacr(u64 val)
{
write_sysreg_s(val, SYS_PMUACR_EL1);
diff --git a/arch/arm64/include/asm/kvm_pmu.h b/arch/arm64/include/asm/kvm_pmu.h
index 73b7161e3f4e..62c8032a548f 100644
--- a/arch/arm64/include/asm/kvm_pmu.h
+++ b/arch/arm64/include/asm/kvm_pmu.h
@@ -81,6 +81,8 @@ struct kvm_pmu_events *kvm_get_pmu_events(void);
void kvm_set_pmu_events(u64 set, struct perf_event_attr *attr);
void kvm_clr_pmu_events(u64 clr);
bool kvm_set_pmuserenr(u64 val);
+bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags);
+bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu);
void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu);
void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu);
void kvm_vcpu_pmu_resync_el0(void);
@@ -214,6 +216,14 @@ static inline bool kvm_set_pmuserenr(u64 val)
{
return false;
}
+static inline bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
+{
+ return false;
+}
+static inline bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
+{
+ return false;
+}
static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {}
static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {}
static inline void kvm_vcpu_reload_pmu(struct kvm_vcpu *vcpu) {}
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 47d2db8446df..4920b7da9ce8 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -25,12 +25,14 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
+#include <asm/kvm_pmu.h>
#include <asm/kvm_nested.h>
#include <asm/fpsimd.h>
#include <asm/debug-monitors.h>
#include <asm/processor.h>
#include <asm/traps.h>

+#include <../../sys_regs.h>
#include "arm_psci.h"

struct kvm_exception_table_entry {
@@ -782,6 +784,175 @@ static bool handle_ampere1_tcr(struct kvm_vcpu *vcpu)
return true;
}

+/**
+ * handle_pmu_reg() - Handle fast access to most PMU regs
+ * @vcpu: Ponter to kvm_vcpu struct
+ * @p: System register parameters (read/write, Op0, Op1, CRm, CRn, Op2)
+ * @reg: VCPU register identifier
+ * @rt: Target general register
+ * @val: Value to write
+ * @readfn: Sysreg read function
+ * @writefn: Sysreg write function
+ *
+ * Handle fast access to most PMU regs. Writethrough to the physical
+ * register. This function is a wrapper for the simplest case, but
+ * sadly there aren't many of those.
+ *
+ * Always return true. The boolean makes usage more consistent with
+ * similar functions.
+ *
+ * Return: True
+ */
+static bool handle_pmu_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
+ enum vcpu_sysreg reg, u8 rt, u64 val,
+ u64 (*readfn)(void), void (*writefn)(u64))
+{
+ if (p->is_write) {
+ __vcpu_assign_sys_reg(vcpu, reg, val);
+ writefn(val);
+ } else {
+ vcpu_set_reg(vcpu, rt, readfn());
+ }
+
+ return true;
+}
+
+/**
+ * kvm_hyp_handle_pmu_regs() - Fast handler for PMU registers
+ * @vcpu: Pointer to vcpu struct
+ *
+ * This handler immediately writes through certain PMU registers when
+ * we have a partitioned PMU (that is, MDCR_EL2.HPMN is set to reserve
+ * a range of counters for the guest) but the machine does not have
+ * FEAT_FGT to selectively untrap the registers we want.
+ *
+ * Return: True if the exception was successfully handled, false otherwise
+ */
+static bool kvm_hyp_handle_pmu_regs(struct kvm_vcpu *vcpu)
+{
+ struct sys_reg_params p;
+ u64 esr;
+ u32 sysreg;
+ u8 rt;
+ u64 val;
+ u8 idx;
+ bool ret;
+
+ if (!kvm_vcpu_pmu_is_partitioned(vcpu)
+ || pmu_access_el0_disabled(vcpu))
+ return false;
+
+ esr = kvm_vcpu_get_esr(vcpu);
+ p = esr_sys64_to_params(esr);
+ sysreg = esr_sys64_to_sysreg(esr);
+ rt = kvm_vcpu_sys_get_rt(vcpu);
+ val = vcpu_get_reg(vcpu, rt);
+
+ switch (sysreg) {
+ case SYS_PMCR_EL0:
+ val &= ARMV8_PMU_PMCR_MASK;
+
+ if (p.is_write) {
+ write_pmcr(val);
+ __vcpu_assign_sys_reg(vcpu, PMCR_EL0, read_pmcr());
+ } else {
+ val = u64_replace_bits(
+ read_pmcr(),
+ vcpu->kvm->arch.nr_pmu_counters,
+ ARMV8_PMU_PMCR_N);
+ vcpu_set_reg(vcpu, rt, val);
+ }
+
+ ret = true;
+ break;
+ case SYS_PMUSERENR_EL0:
+ val &= ARMV8_PMU_USERENR_MASK;
+ ret = handle_pmu_reg(vcpu, &p, PMUSERENR_EL0, rt, val,
+ &read_pmuserenr, &write_pmuserenr);
+ break;
+ case SYS_PMSELR_EL0:
+ val &= PMSELR_EL0_SEL_MASK;
+ ret = handle_pmu_reg(vcpu, &p, PMSELR_EL0, rt, val,
+ &read_pmselr, &write_pmselr);
+ break;
+ case SYS_PMINTENCLR_EL1:
+ val &= kvm_pmu_accessible_counter_mask(vcpu);
+ if (p.is_write) {
+ __vcpu_rmw_sys_reg(vcpu, PMINTENSET_EL1, &=, ~val);
+ write_pmintenclr(val);
+ } else {
+ vcpu_set_reg(vcpu, rt, read_pmintenclr());
+ }
+ ret = true;
+ break;
+ case SYS_PMINTENSET_EL1:
+ val &= kvm_pmu_accessible_counter_mask(vcpu);
+ if (p.is_write) {
+ __vcpu_rmw_sys_reg(vcpu, PMINTENSET_EL1, |=, val);
+ write_pmintenset(val);
+ } else {
+ vcpu_set_reg(vcpu, rt, read_pmintenset());
+ }
+ ret = true;
+ break;
+ case SYS_PMCNTENCLR_EL0:
+ val &= kvm_pmu_accessible_counter_mask(vcpu);
+ if (p.is_write) {
+ __vcpu_rmw_sys_reg(vcpu, PMCNTENSET_EL0, &=, ~val);
+ write_pmcntenclr(val);
+ } else {
+ vcpu_set_reg(vcpu, rt, read_pmcntenclr());
+ }
+ ret = true;
+ break;
+ case SYS_PMCNTENSET_EL0:
+ val &= kvm_pmu_accessible_counter_mask(vcpu);
+ if (p.is_write) {
+ __vcpu_rmw_sys_reg(vcpu, PMCNTENSET_EL0, |=, val);
+ write_pmcntenset(val);
+ } else {
+ vcpu_set_reg(vcpu, rt, read_pmcntenset());
+ }
+ ret = true;
+ break;
+ case SYS_PMCCNTR_EL0:
+ ret = handle_pmu_reg(vcpu, &p, PMCCNTR_EL0, rt, val,
+ &read_pmccntr, &write_pmccntr);
+ break;
+ case SYS_PMXEVCNTR_EL0:
+ idx = FIELD_GET(PMSELR_EL0_SEL, read_pmselr());
+
+ if (idx >= vcpu->kvm->arch.nr_pmu_counters)
+ return false;
+
+ ret = handle_pmu_reg(vcpu, &p, PMEVCNTR0_EL0 + idx, rt, val,
+ &read_pmxevcntr, &write_pmxevcntr);
+ break;
+ case SYS_PMEVCNTRn_EL0(0) ... SYS_PMEVCNTRn_EL0(30):
+ idx = ((p.CRm & 3) << 3) | (p.Op2 & 7);
+
+ if (idx >= vcpu->kvm->arch.nr_pmu_counters)
+ return false;
+
+ if (p.is_write) {
+ write_pmevcntrn(idx, val);
+ __vcpu_assign_sys_reg(vcpu, PMEVCNTR0_EL0 + idx, val);
+ } else {
+ vcpu_set_reg(vcpu, rt, read_pmevcntrn(idx));
+ }
+
+ ret = true;
+ break;
+ default:
+ ret = false;
+ }
+
+ if (ret)
+ __kvm_skip_instr(vcpu);
+
+ return ret;
+}
+
static inline bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
{
if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&
@@ -799,6 +970,9 @@ static inline bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
if (kvm_handle_cntxct(vcpu))
return true;

+ if (kvm_hyp_handle_pmu_regs(vcpu))
+ return true;
+
return false;
}

diff --git a/arch/arm64/kvm/pmu.c b/arch/arm64/kvm/pmu.c
index 67216451b8ce..338e7eebf0d1 100644
--- a/arch/arm64/kvm/pmu.c
+++ b/arch/arm64/kvm/pmu.c
@@ -891,3 +891,19 @@ u64 kvm_vcpu_read_pmcr(struct kvm_vcpu *vcpu)

return u64_replace_bits(pmcr, n, ARMV8_PMU_PMCR_N);
}
+
+bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
+{
+ u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
+ bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
+
+ if (!enabled)
+ kvm_inject_undefined(vcpu);
+
+ return !enabled;
+}
+
+bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
+{
+ return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
+}
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index cfbce4537b4c..b80cf6194fa3 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -842,22 +842,6 @@ static u64 reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
return __vcpu_sys_reg(vcpu, r->reg);
}

-static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
-{
- u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
- bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
-
- if (!enabled)
- kvm_inject_undefined(vcpu);
-
- return !enabled;
-}
-
-static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
-{
- return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
-}
-
static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
{
return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
--
2.50.0.727.gbf7dc18ff4-goog