[PATCH 00/18] KVM: ARM64: Add guest PMU support

[PATCH 00/18] KVM: ARM64: Add guest PMU support

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

This patchset adds guest PMU support for KVM on ARM64. It takes
trap-and-emulate approach. When guest wants to monitor one event, it
will be trapped by KVM and KVM will call perf_event API to create a perf
event and call relevant perf_event APIs to get the count value of event.

Use perf to test this patchset in guest. When using "perf list", it
shows the list of the hardware events and hardware cache events perf
supports. Then use "perf stat -e EVENT" to monitor some event. For
example, use "perf stat -e cycles" to count cpu cycles and
"perf stat -e cache-misses" to count cache misses.

This can be fetched from [1] and the relevant QEMU version for test can
be fetched from [2].

Thanks,
Shannon

[1] https://git.linaro.org/people/shannon.zhao/linux-mainline.git  KVM_ARM64_PMU_v1
[2] https://git.linaro.org/people/shannon.zhao/qemu.git  PMU

Shannon Zhao (18):
  ARM64: Move PMU register related defines to asm/pmu.h
  KVM: ARM64: Add initial support for PMU
  KVM: ARM64: Add offset defines for PMU registers
  KVM: ARM64: Add reset and access handlers for PMCR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMSELR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMCEID0_EL0 and
    PMCEID1_EL0 register
  KVM: ARM64: PMU: Add perf event map and introduce perf event creating
    function
  KVM: ARM64: Add reset and access handlers for PMXEVTYPER_EL0 register
  KVM: ARM64: Add reset and access handlers for PMXEVCNTR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMCCNTR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMCNTENSET_EL0 and
    PMCNTENCLR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMINTENSET_EL1 and
    PMINTENCLR_EL1 register
  KVM: ARM64: Add reset and access handlers for PMOVSSET_EL0 and
    PMOVSCLR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMUSERENR_EL0 register
  KVM: ARM64: Add reset and access handlers for PMSWINC_EL0 register
  KVM: ARM64: Add access handlers for PMEVCNTRn_EL0 and PMEVTYPERn_EL0
    register
  KVM: ARM64: Add PMU overflow interrupt routing
  KVM: ARM64: Add KVM_CAP_ARM_PMU and KVM_ARM_PMU_SET_IRQ

 arch/arm/kvm/arm.c                |  14 ++
 arch/arm64/include/asm/kvm_asm.h  |  59 ++++-
 arch/arm64/include/asm/kvm_host.h |   2 +
 arch/arm64/include/asm/pmu.h      |  47 ++++
 arch/arm64/kernel/perf_event.c    |  35 ---
 arch/arm64/kvm/Kconfig            |   7 +
 arch/arm64/kvm/Makefile           |   1 +
 arch/arm64/kvm/reset.c            |   3 +
 arch/arm64/kvm/sys_regs.c         | 448 ++++++++++++++++++++++++++++++++++++--
 include/kvm/arm_pmu.h             |  87 ++++++++
 include/uapi/linux/kvm.h          |   4 +
 virt/kvm/arm/pmu.c                | 414 +++++++++++++++++++++++++++++++++++
 12 files changed, 1064 insertions(+), 57 deletions(-)
 create mode 100644 include/kvm/arm_pmu.h
 create mode 100644 virt/kvm/arm/pmu.c

-- 
2.1.0

[PATCH 01/18] ARM64: Move PMU register related defines to asm/pmu.h

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

To use the ARMv8 PMU related register defines from the KVM code,
we move the relevant definitions to asm/pmu.h header file.

Signed-off-by: Anup Patel <anup.patel@linaro.org>
Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/include/asm/pmu.h   | 45 ++++++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/perf_event.c | 35 --------------------------------
 2 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
index b7710a5..b9f394a 100644
--- a/arch/arm64/include/asm/pmu.h
+++ b/arch/arm64/include/asm/pmu.h
@@ -19,6 +19,51 @@
 #ifndef __ASM_PMU_H
 #define __ASM_PMU_H
 
+#define ARMV8_MAX_COUNTERS      32
+#define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
+
+/*
+ * Per-CPU PMCR: config reg
+ */
+#define ARMV8_PMCR_E		(1 << 0) /* Enable all counters */
+#define ARMV8_PMCR_P		(1 << 1) /* Reset all counters */
+#define ARMV8_PMCR_C		(1 << 2) /* Cycle counter reset */
+#define ARMV8_PMCR_D		(1 << 3) /* CCNT counts every 64th cpu cycle */
+#define ARMV8_PMCR_X		(1 << 4) /* Export to ETM */
+#define ARMV8_PMCR_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
+#define	ARMV8_PMCR_N_SHIFT	11	 /* Number of counters supported */
+#define	ARMV8_PMCR_N_MASK	0x1f
+#define	ARMV8_PMCR_MASK		0x3f	 /* Mask for writable bits */
+
+/*
+ * PMCNTEN: counters enable reg
+ */
+#define	ARMV8_CNTEN_MASK	0xffffffff	/* Mask for writable bits */
+
+/*
+ * PMINTEN: counters interrupt enable reg
+ */
+#define	ARMV8_INTEN_MASK	0xffffffff	/* Mask for writable bits */
+
+/*
+ * PMOVSR: counters overflow flag status reg
+ */
+#define	ARMV8_OVSR_MASK		0xffffffff	/* Mask for writable bits */
+#define	ARMV8_OVERFLOWED_MASK	ARMV8_OVSR_MASK
+
+/*
+ * PMXEVTYPER: Event selection reg
+ */
+#define	ARMV8_EVTYPE_MASK	0xc80003ff	/* Mask for writable bits */
+#define	ARMV8_EVTYPE_EVENT	0x3ff		/* Mask for EVENT bits */
+
+/*
+ * Event filters for PMUv3
+ */
+#define	ARMV8_EXCLUDE_EL1	(1 << 31)
+#define	ARMV8_EXCLUDE_EL0	(1 << 30)
+#define	ARMV8_INCLUDE_EL2	(1 << 27)
+
 #ifdef CONFIG_HW_PERF_EVENTS
 
 /* The events for a given PMU register set. */
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index fd26e57..0790cac 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -824,9 +824,6 @@ static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 #define	ARMV8_IDX_COUNTER0	1
 #define	ARMV8_IDX_COUNTER_LAST	(ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 
-#define	ARMV8_MAX_COUNTERS	32
-#define	ARMV8_COUNTER_MASK	(ARMV8_MAX_COUNTERS - 1)
-
 /*
  * ARMv8 low level PMU access
  */
@@ -837,38 +834,6 @@ static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 #define	ARMV8_IDX_TO_COUNTER(x)	\
 	(((x) - ARMV8_IDX_COUNTER0) & ARMV8_COUNTER_MASK)
 
-/*
- * Per-CPU PMCR: config reg
- */
-#define ARMV8_PMCR_E		(1 << 0) /* Enable all counters */
-#define ARMV8_PMCR_P		(1 << 1) /* Reset all counters */
-#define ARMV8_PMCR_C		(1 << 2) /* Cycle counter reset */
-#define ARMV8_PMCR_D		(1 << 3) /* CCNT counts every 64th cpu cycle */
-#define ARMV8_PMCR_X		(1 << 4) /* Export to ETM */
-#define ARMV8_PMCR_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
-#define	ARMV8_PMCR_N_SHIFT	11	 /* Number of counters supported */
-#define	ARMV8_PMCR_N_MASK	0x1f
-#define	ARMV8_PMCR_MASK		0x3f	 /* Mask for writable bits */
-
-/*
- * PMOVSR: counters overflow flag status reg
- */
-#define	ARMV8_OVSR_MASK		0xffffffff	/* Mask for writable bits */
-#define	ARMV8_OVERFLOWED_MASK	ARMV8_OVSR_MASK
-
-/*
- * PMXEVTYPER: Event selection reg
- */
-#define	ARMV8_EVTYPE_MASK	0xc80003ff	/* Mask for writable bits */
-#define	ARMV8_EVTYPE_EVENT	0x3ff		/* Mask for EVENT bits */
-
-/*
- * Event filters for PMUv3
- */
-#define	ARMV8_EXCLUDE_EL1	(1 << 31)
-#define	ARMV8_EXCLUDE_EL0	(1 << 30)
-#define	ARMV8_INCLUDE_EL2	(1 << 27)
-
 static inline u32 armv8pmu_pmcr_read(void)
 {
 	u32 val;
-- 
2.1.0

[PATCH 01/18] ARM64: Move PMU register related defines to asm/pmu.h

[Will Deacon]

On Mon, Jul 06, 2015 at 03:17:31AM +0100, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> To use the ARMv8 PMU related register defines from the KVM code,
> we move the relevant definitions to asm/pmu.h header file.
> 
> Signed-off-by: Anup Patel <anup.patel@linaro.org>
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/include/asm/pmu.h   | 45 ++++++++++++++++++++++++++++++++++++++++++
>  arch/arm64/kernel/perf_event.c | 35 --------------------------------
>  2 files changed, 45 insertions(+), 35 deletions(-)

Whilst I'm ok with the idea, we're currently in the process of moving the
ARM and arm64 PMU drivers out into drivers/perf/, so this is likely to
conflict horribly with that work.

Christoffer/Marc: if you end up wanting to queue this for 4.3, give me a
shout and I'll try to create a stable branch you can use as a base (but
it's likely to contain a whole heap of stuff you don't want...).

Will

[PATCH 02/18] KVM: ARM64: Add initial support for PMU

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Here we plan to support virtual PMU for guest by full software
emulation, so define some basic structs and functions preparing for
futher steps. Define struct kvm_pmc for performance monitor counter and
struct kvm_pmu for performance monitor unit for each vcpu. According to
ARMv8 spec, the PMU contains at most 32(ARMV8_MAX_COUNTERS) counters.
Initialize PMU for each vcpu when kvm initialize vcpus, while reset PMU
when kvm reset vcpus.
Since this only supports ARM64 (or PMUv3), add a separate config symbol
for it.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm/kvm/arm.c                |  4 +++
 arch/arm64/include/asm/kvm_host.h |  2 ++
 arch/arm64/include/asm/pmu.h      |  2 ++
 arch/arm64/kvm/Kconfig            |  7 +++++
 arch/arm64/kvm/Makefile           |  1 +
 arch/arm64/kvm/reset.c            |  3 +++
 include/kvm/arm_pmu.h             | 54 ++++++++++++++++++++++++++++++++++++++
 virt/kvm/arm/pmu.c                | 55 +++++++++++++++++++++++++++++++++++++++
 8 files changed, 128 insertions(+)
 create mode 100644 include/kvm/arm_pmu.h
 create mode 100644 virt/kvm/arm/pmu.c

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index bc738d2..4e82625 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -28,6 +28,7 @@
 #include <linux/sched.h>
 #include <linux/kvm.h>
 #include <trace/events/kvm.h>
+#include <kvm/arm_pmu.h>
 
 #define CREATE_TRACE_POINTS
 #include "trace.h"
@@ -278,6 +279,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 	/* Set up the timer */
 	kvm_timer_vcpu_init(vcpu);
 
+	/* Set up the PMU */
+	kvm_pmu_init(vcpu);
+
 	return 0;
 }
 
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 2709db2..3c88873 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -42,6 +42,7 @@
 
 #include <kvm/arm_vgic.h>
 #include <kvm/arm_arch_timer.h>
+#include <kvm/arm_pmu.h>
 
 #define KVM_VCPU_MAX_FEATURES 3
 
@@ -116,6 +117,7 @@ struct kvm_vcpu_arch {
 	/* VGIC state */
 	struct vgic_cpu vgic_cpu;
 	struct arch_timer_cpu timer_cpu;
+	struct kvm_pmu pmu;
 
 	/*
 	 * Anything that is not used directly from assembly code goes
diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
index b9f394a..95681e6 100644
--- a/arch/arm64/include/asm/pmu.h
+++ b/arch/arm64/include/asm/pmu.h
@@ -19,6 +19,8 @@
 #ifndef __ASM_PMU_H
 #define __ASM_PMU_H
 
+#include <linux/perf_event.h>
+
 #define ARMV8_MAX_COUNTERS      32
 #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
 
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index bfffe8f..eae3a1b 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -31,6 +31,7 @@ config KVM
 	select KVM_VFIO
 	select HAVE_KVM_EVENTFD
 	select HAVE_KVM_IRQFD
+	select KVM_ARM_PMU
 	---help---
 	  Support hosting virtualized guest machines.
 
@@ -52,4 +53,10 @@ config KVM_ARM_MAX_VCPUS
 	  large, so only choose a reasonable number that you expect to
 	  actually use.
 
+config KVM_ARM_PMU
+	bool
+	depends on KVM_ARM_HOST
+	---help---
+	  Adds support for the Performance Monitoring in virtual machines.
+
 endif # VIRTUALIZATION
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index f90f4aa..78db4ee 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -27,3 +27,4 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
 kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
+kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 0b43265..ee2c2e9 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -107,5 +107,8 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 	/* Reset timer */
 	kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
 
+	/* Reset PMU */
+	kvm_pmu_vcpu_reset(vcpu);
+
 	return 0;
 }
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
new file mode 100644
index 0000000..27d14ca
--- /dev/null
+++ b/include/kvm/arm_pmu.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2015 Linaro Ltd.
+ * Author: Shannon Zhao <shannon.zhao@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ASM_ARM_KVM_PMU_H
+#define __ASM_ARM_KVM_PMU_H
+
+#include <asm/pmu.h>
+#include <linux/irq_work.h>
+
+struct kvm_pmc {
+	u8 idx;
+	u64 counter;
+	u64 eventsel;
+	bool interrupt;
+	bool enable;
+	struct perf_event *perf_event;
+	struct kvm_vcpu *vcpu;
+};
+
+struct kvm_pmu {
+#ifdef CONFIG_KVM_ARM_PMU
+	/* PMU IRQ Number per VCPU */
+	int irq_num;
+	int overflow_status;
+	/* IRQ pending flag */
+	bool irq_pending;
+	struct irq_work irq_work;
+	struct kvm_pmc pmc[ARMV8_MAX_COUNTERS];
+#endif
+};
+
+#ifdef CONFIG_KVM_ARM_PMU
+void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
+void kvm_pmu_init(struct kvm_vcpu *vcpu);
+#else
+static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
+static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
+#endif
+
+#endif
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
new file mode 100644
index 0000000..dc252d0
--- /dev/null
+++ b/virt/kvm/arm/pmu.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2015 Linaro Ltd.
+ * Author: Shannon Zhao <shannon.zhao@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/cpu.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <kvm/arm_pmu.h>
+
+/**
+ * kvm_pmu_vcpu_reset - reset pmu state for cpu
+ * @vcpu: The vcpu pointer
+ *
+ */
+void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+
+	pmu->irq_pending = false;
+}
+
+/**
+ * kvm_pmu_init - Initialize global PMU state for per vcpu
+ * @vcpu: The vcpu pointer
+ *
+ * Initialize PMC and Set all the PMU IRQ numbers to invalid
+ * value so that user space has to explicitly provide PMU
+ * IRQ numbers using set device address ioctl.
+ */
+void kvm_pmu_init(struct kvm_vcpu *vcpu)
+{
+	int i;
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+
+	memset(pmu, 0, sizeof(*pmu));
+	for (i = 0; i < ARMV8_MAX_COUNTERS; i++) {
+		pmu->pmc[i].idx = i;
+		pmu->pmc[i].vcpu = vcpu;
+	}
+
+	pmu->irq_num = -1;
+}
-- 
2.1.0

[PATCH 02/18] KVM: ARM64: Add initial support for PMU

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:32AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Here we plan to support virtual PMU for guest by full software
> emulation, so define some basic structs and functions preparing for
> futher steps. Define struct kvm_pmc for performance monitor counter and
> struct kvm_pmu for performance monitor unit for each vcpu. According to
> ARMv8 spec, the PMU contains at most 32(ARMV8_MAX_COUNTERS) counters.
> Initialize PMU for each vcpu when kvm initialize vcpus, while reset PMU
> when kvm reset vcpus.
> Since this only supports ARM64 (or PMUv3), add a separate config symbol
> for it.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm/kvm/arm.c                |  4 +++
>  arch/arm64/include/asm/kvm_host.h |  2 ++
>  arch/arm64/include/asm/pmu.h      |  2 ++
>  arch/arm64/kvm/Kconfig            |  7 +++++
>  arch/arm64/kvm/Makefile           |  1 +
>  arch/arm64/kvm/reset.c            |  3 +++
>  include/kvm/arm_pmu.h             | 54 ++++++++++++++++++++++++++++++++++++++
>  virt/kvm/arm/pmu.c                | 55 +++++++++++++++++++++++++++++++++++++++
>  8 files changed, 128 insertions(+)
>  create mode 100644 include/kvm/arm_pmu.h
>  create mode 100644 virt/kvm/arm/pmu.c
> 
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index bc738d2..4e82625 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -28,6 +28,7 @@
>  #include <linux/sched.h>
>  #include <linux/kvm.h>
>  #include <trace/events/kvm.h>
> +#include <kvm/arm_pmu.h>
>  
>  #define CREATE_TRACE_POINTS
>  #include "trace.h"
> @@ -278,6 +279,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
>  	/* Set up the timer */
>  	kvm_timer_vcpu_init(vcpu);
>  
> +	/* Set up the PMU */
> +	kvm_pmu_init(vcpu);
> +
>  	return 0;
>  }
>  
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 2709db2..3c88873 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -42,6 +42,7 @@
>  
>  #include <kvm/arm_vgic.h>
>  #include <kvm/arm_arch_timer.h>
> +#include <kvm/arm_pmu.h>
>  
>  #define KVM_VCPU_MAX_FEATURES 3
>  
> @@ -116,6 +117,7 @@ struct kvm_vcpu_arch {
>  	/* VGIC state */
>  	struct vgic_cpu vgic_cpu;
>  	struct arch_timer_cpu timer_cpu;
> +	struct kvm_pmu pmu;
>  
>  	/*
>  	 * Anything that is not used directly from assembly code goes
> diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
> index b9f394a..95681e6 100644
> --- a/arch/arm64/include/asm/pmu.h
> +++ b/arch/arm64/include/asm/pmu.h
> @@ -19,6 +19,8 @@
>  #ifndef __ASM_PMU_H
>  #define __ASM_PMU_H
>  
> +#include <linux/perf_event.h>
> +
>  #define ARMV8_MAX_COUNTERS      32
>  #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
>  
> diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
> index bfffe8f..eae3a1b 100644
> --- a/arch/arm64/kvm/Kconfig
> +++ b/arch/arm64/kvm/Kconfig
> @@ -31,6 +31,7 @@ config KVM
>  	select KVM_VFIO
>  	select HAVE_KVM_EVENTFD
>  	select HAVE_KVM_IRQFD
> +	select KVM_ARM_PMU
>  	---help---
>  	  Support hosting virtualized guest machines.
>  
> @@ -52,4 +53,10 @@ config KVM_ARM_MAX_VCPUS
>  	  large, so only choose a reasonable number that you expect to
>  	  actually use.
>  
> +config KVM_ARM_PMU
> +	bool
> +	depends on KVM_ARM_HOST
> +	---help---
> +	  Adds support for the Performance Monitoring in virtual machines.

Adds support for a virtual Performance Monitoring Unit (PMU) in virtual
machines.

> +
>  endif # VIRTUALIZATION
> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
> index f90f4aa..78db4ee 100644
> --- a/arch/arm64/kvm/Makefile
> +++ b/arch/arm64/kvm/Makefile
> @@ -27,3 +27,4 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
>  kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
> +kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> index 0b43265..ee2c2e9 100644
> --- a/arch/arm64/kvm/reset.c
> +++ b/arch/arm64/kvm/reset.c
> @@ -107,5 +107,8 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>  	/* Reset timer */
>  	kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
>  
> +	/* Reset PMU */
> +	kvm_pmu_vcpu_reset(vcpu);
> +
>  	return 0;
>  }
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> new file mode 100644
> index 0000000..27d14ca
> --- /dev/null
> +++ b/include/kvm/arm_pmu.h
> @@ -0,0 +1,54 @@
> +/*
> + * Copyright (C) 2015 Linaro Ltd.
> + * Author: Shannon Zhao <shannon.zhao@linaro.org>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#ifndef __ASM_ARM_KVM_PMU_H
> +#define __ASM_ARM_KVM_PMU_H
> +
> +#include <asm/pmu.h>
> +#include <linux/irq_work.h>
> +
> +struct kvm_pmc {
> +	u8 idx;

which index is this?  Does it map to something architecturally or is it
just the array index number?

> +	u64 counter;
> +	u64 eventsel;

does this map to PMEVTYPER<n>_EL0.evtCount?

> +	bool interrupt;

is this that an interrupt which is pending or which is enabled?

> +	bool enable;
> +	struct perf_event *perf_event;
> +	struct kvm_vcpu *vcpu;
> +};
> +
> +struct kvm_pmu {
> +#ifdef CONFIG_KVM_ARM_PMU
> +	/* PMU IRQ Number per VCPU */
> +	int irq_num;
> +	int overflow_status;
> +	/* IRQ pending flag */
> +	bool irq_pending;
> +	struct irq_work irq_work;
> +	struct kvm_pmc pmc[ARMV8_MAX_COUNTERS];
> +#endif
> +};
> +
> +#ifdef CONFIG_KVM_ARM_PMU
> +void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
> +void kvm_pmu_init(struct kvm_vcpu *vcpu);
> +#else
> +static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
> +static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
> +#endif
> +
> +#endif
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> new file mode 100644
> index 0000000..dc252d0
> --- /dev/null
> +++ b/virt/kvm/arm/pmu.c
> @@ -0,0 +1,55 @@
> +/*
> + * Copyright (C) 2015 Linaro Ltd.
> + * Author: Shannon Zhao <shannon.zhao@linaro.org>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <linux/cpu.h>
> +#include <linux/kvm.h>
> +#include <linux/kvm_host.h>
> +#include <kvm/arm_pmu.h>
> +
> +/**
> + * kvm_pmu_vcpu_reset - reset pmu state for cpu
> + * @vcpu: The vcpu pointer
> + *
> + */
> +void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +
> +	pmu->irq_pending = false;
> +}
> +
> +/**
> + * kvm_pmu_init - Initialize global PMU state for per vcpu

how is this both global and per VCPU?

> + * @vcpu: The vcpu pointer
> + *
> + * Initialize PMC and Set all the PMU IRQ numbers to invalid

s/Set/set/

> + * value so that user space has to explicitly provide PMU
> + * IRQ numbers using set device address ioctl.
> + */
> +void kvm_pmu_init(struct kvm_vcpu *vcpu)
> +{
> +	int i;
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +
> +	memset(pmu, 0, sizeof(*pmu));
> +	for (i = 0; i < ARMV8_MAX_COUNTERS; i++) {
> +		pmu->pmc[i].idx = i;
> +		pmu->pmc[i].vcpu = vcpu;
> +	}
> +
> +	pmu->irq_num = -1;
> +}
> -- 
> 2.1.0
> 

[PATCH 02/18] KVM: ARM64: Add initial support for PMU

[Shannon Zhao]

On 2015/7/17 2:25, Christoffer Dall wrote:
> On Mon, Jul 06, 2015 at 10:17:32AM +0800, shannon.zhao at linaro.org wrote:
>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>
>> Here we plan to support virtual PMU for guest by full software
>> emulation, so define some basic structs and functions preparing for
>> futher steps. Define struct kvm_pmc for performance monitor counter and
>> struct kvm_pmu for performance monitor unit for each vcpu. According to
>> ARMv8 spec, the PMU contains at most 32(ARMV8_MAX_COUNTERS) counters.
>> Initialize PMU for each vcpu when kvm initialize vcpus, while reset PMU
>> when kvm reset vcpus.
>> Since this only supports ARM64 (or PMUv3), add a separate config symbol
>> for it.
>>
>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>> ---
>>  arch/arm/kvm/arm.c                |  4 +++
>>  arch/arm64/include/asm/kvm_host.h |  2 ++
>>  arch/arm64/include/asm/pmu.h      |  2 ++
>>  arch/arm64/kvm/Kconfig            |  7 +++++
>>  arch/arm64/kvm/Makefile           |  1 +
>>  arch/arm64/kvm/reset.c            |  3 +++
>>  include/kvm/arm_pmu.h             | 54 ++++++++++++++++++++++++++++++++++++++
>>  virt/kvm/arm/pmu.c                | 55 +++++++++++++++++++++++++++++++++++++++
>>  8 files changed, 128 insertions(+)
>>  create mode 100644 include/kvm/arm_pmu.h
>>  create mode 100644 virt/kvm/arm/pmu.c
>>
>> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> index bc738d2..4e82625 100644
>> --- a/arch/arm/kvm/arm.c
>> +++ b/arch/arm/kvm/arm.c
>> @@ -28,6 +28,7 @@
>>  #include <linux/sched.h>
>>  #include <linux/kvm.h>
>>  #include <trace/events/kvm.h>
>> +#include <kvm/arm_pmu.h>
>>  
>>  #define CREATE_TRACE_POINTS
>>  #include "trace.h"
>> @@ -278,6 +279,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
>>  	/* Set up the timer */
>>  	kvm_timer_vcpu_init(vcpu);
>>  
>> +	/* Set up the PMU */
>> +	kvm_pmu_init(vcpu);
>> +
>>  	return 0;
>>  }
>>  
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index 2709db2..3c88873 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -42,6 +42,7 @@
>>  
>>  #include <kvm/arm_vgic.h>
>>  #include <kvm/arm_arch_timer.h>
>> +#include <kvm/arm_pmu.h>
>>  
>>  #define KVM_VCPU_MAX_FEATURES 3
>>  
>> @@ -116,6 +117,7 @@ struct kvm_vcpu_arch {
>>  	/* VGIC state */
>>  	struct vgic_cpu vgic_cpu;
>>  	struct arch_timer_cpu timer_cpu;
>> +	struct kvm_pmu pmu;
>>  
>>  	/*
>>  	 * Anything that is not used directly from assembly code goes
>> diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
>> index b9f394a..95681e6 100644
>> --- a/arch/arm64/include/asm/pmu.h
>> +++ b/arch/arm64/include/asm/pmu.h
>> @@ -19,6 +19,8 @@
>>  #ifndef __ASM_PMU_H
>>  #define __ASM_PMU_H
>>  
>> +#include <linux/perf_event.h>
>> +
>>  #define ARMV8_MAX_COUNTERS      32
>>  #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
>>  
>> diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
>> index bfffe8f..eae3a1b 100644
>> --- a/arch/arm64/kvm/Kconfig
>> +++ b/arch/arm64/kvm/Kconfig
>> @@ -31,6 +31,7 @@ config KVM
>>  	select KVM_VFIO
>>  	select HAVE_KVM_EVENTFD
>>  	select HAVE_KVM_IRQFD
>> +	select KVM_ARM_PMU
>>  	---help---
>>  	  Support hosting virtualized guest machines.
>>  
>> @@ -52,4 +53,10 @@ config KVM_ARM_MAX_VCPUS
>>  	  large, so only choose a reasonable number that you expect to
>>  	  actually use.
>>  
>> +config KVM_ARM_PMU
>> +	bool
>> +	depends on KVM_ARM_HOST
>> +	---help---
>> +	  Adds support for the Performance Monitoring in virtual machines.
> 
> Adds support for a virtual Performance Monitoring Unit (PMU) in virtual
> machines.
> 
>> +
>>  endif # VIRTUALIZATION
>> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
>> index f90f4aa..78db4ee 100644
>> --- a/arch/arm64/kvm/Makefile
>> +++ b/arch/arm64/kvm/Makefile
>> @@ -27,3 +27,4 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
>>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
>>  kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
>>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
>> +kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
>> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
>> index 0b43265..ee2c2e9 100644
>> --- a/arch/arm64/kvm/reset.c
>> +++ b/arch/arm64/kvm/reset.c
>> @@ -107,5 +107,8 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>>  	/* Reset timer */
>>  	kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
>>  
>> +	/* Reset PMU */
>> +	kvm_pmu_vcpu_reset(vcpu);
>> +
>>  	return 0;
>>  }
>> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
>> new file mode 100644
>> index 0000000..27d14ca
>> --- /dev/null
>> +++ b/include/kvm/arm_pmu.h
>> @@ -0,0 +1,54 @@
>> +/*
>> + * Copyright (C) 2015 Linaro Ltd.
>> + * Author: Shannon Zhao <shannon.zhao@linaro.org>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License version 2 as
>> + * published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU General Public License
>> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
>> + */
>> +
>> +#ifndef __ASM_ARM_KVM_PMU_H
>> +#define __ASM_ARM_KVM_PMU_H
>> +
>> +#include <asm/pmu.h>
>> +#include <linux/irq_work.h>
>> +
>> +struct kvm_pmc {
>> +	u8 idx;
> 
> which index is this?  Does it map to something architecturally or is it
> just the array index number?
> 

It's the index of the PMU counter.

>> +	u64 counter;
>> +	u64 eventsel;
> 
> does this map to PMEVTYPER<n>_EL0.evtCount?
> 
No, it just stores the event type.

>> +	bool interrupt;
> 
> is this that an interrupt which is pending or which is enabled?
> 
>> +	bool enable;
>> +	struct perf_event *perf_event;
>> +	struct kvm_vcpu *vcpu;
>> +};
>> +
>> +struct kvm_pmu {
>> +#ifdef CONFIG_KVM_ARM_PMU
>> +	/* PMU IRQ Number per VCPU */
>> +	int irq_num;
>> +	int overflow_status;
>> +	/* IRQ pending flag */
>> +	bool irq_pending;
>> +	struct irq_work irq_work;
>> +	struct kvm_pmc pmc[ARMV8_MAX_COUNTERS];
>> +#endif
>> +};
>> +
>> +#ifdef CONFIG_KVM_ARM_PMU
>> +void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
>> +void kvm_pmu_init(struct kvm_vcpu *vcpu);
>> +#else
>> +static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
>> +static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
>> +#endif
>> +
>> +#endif
>> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
>> new file mode 100644
>> index 0000000..dc252d0
>> --- /dev/null
>> +++ b/virt/kvm/arm/pmu.c
>> @@ -0,0 +1,55 @@
>> +/*
>> + * Copyright (C) 2015 Linaro Ltd.
>> + * Author: Shannon Zhao <shannon.zhao@linaro.org>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License version 2 as
>> + * published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU General Public License
>> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
>> + */
>> +
>> +#include <linux/cpu.h>
>> +#include <linux/kvm.h>
>> +#include <linux/kvm_host.h>
>> +#include <kvm/arm_pmu.h>
>> +
>> +/**
>> + * kvm_pmu_vcpu_reset - reset pmu state for cpu
>> + * @vcpu: The vcpu pointer
>> + *
>> + */
>> +void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
>> +{
>> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
>> +
>> +	pmu->irq_pending = false;
>> +}
>> +
>> +/**
>> + * kvm_pmu_init - Initialize global PMU state for per vcpu
> 
> how is this both global and per VCPU?
> 
>> + * @vcpu: The vcpu pointer
>> + *
>> + * Initialize PMC and Set all the PMU IRQ numbers to invalid
> 
> s/Set/set/
> 
>> + * value so that user space has to explicitly provide PMU
>> + * IRQ numbers using set device address ioctl.
>> + */
>> +void kvm_pmu_init(struct kvm_vcpu *vcpu)
>> +{
>> +	int i;
>> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
>> +
>> +	memset(pmu, 0, sizeof(*pmu));
>> +	for (i = 0; i < ARMV8_MAX_COUNTERS; i++) {
>> +		pmu->pmc[i].idx = i;
>> +		pmu->pmc[i].vcpu = vcpu;
>> +	}
>> +
>> +	pmu->irq_num = -1;
>> +}
>> -- 
>> 2.1.0
>>

-- 
Shannon

[PATCH 02/18] KVM: ARM64: Add initial support for PMU

[Christoffer Dall]

On Fri, Jul 17, 2015 at 04:13:35PM +0800, Shannon Zhao wrote:
> 
> 
> On 2015/7/17 2:25, Christoffer Dall wrote:
> > On Mon, Jul 06, 2015 at 10:17:32AM +0800, shannon.zhao at linaro.org wrote:
> >> From: Shannon Zhao <shannon.zhao@linaro.org>
> >>
> >> Here we plan to support virtual PMU for guest by full software
> >> emulation, so define some basic structs and functions preparing for
> >> futher steps. Define struct kvm_pmc for performance monitor counter and
> >> struct kvm_pmu for performance monitor unit for each vcpu. According to
> >> ARMv8 spec, the PMU contains at most 32(ARMV8_MAX_COUNTERS) counters.
> >> Initialize PMU for each vcpu when kvm initialize vcpus, while reset PMU
> >> when kvm reset vcpus.
> >> Since this only supports ARM64 (or PMUv3), add a separate config symbol
> >> for it.
> >>
> >> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> >> ---
> >>  arch/arm/kvm/arm.c                |  4 +++
> >>  arch/arm64/include/asm/kvm_host.h |  2 ++
> >>  arch/arm64/include/asm/pmu.h      |  2 ++
> >>  arch/arm64/kvm/Kconfig            |  7 +++++
> >>  arch/arm64/kvm/Makefile           |  1 +
> >>  arch/arm64/kvm/reset.c            |  3 +++
> >>  include/kvm/arm_pmu.h             | 54 ++++++++++++++++++++++++++++++++++++++
> >>  virt/kvm/arm/pmu.c                | 55 +++++++++++++++++++++++++++++++++++++++
> >>  8 files changed, 128 insertions(+)
> >>  create mode 100644 include/kvm/arm_pmu.h
> >>  create mode 100644 virt/kvm/arm/pmu.c
> >>
> >> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> >> index bc738d2..4e82625 100644
> >> --- a/arch/arm/kvm/arm.c
> >> +++ b/arch/arm/kvm/arm.c
> >> @@ -28,6 +28,7 @@
> >>  #include <linux/sched.h>
> >>  #include <linux/kvm.h>
> >>  #include <trace/events/kvm.h>
> >> +#include <kvm/arm_pmu.h>
> >>  
> >>  #define CREATE_TRACE_POINTS
> >>  #include "trace.h"
> >> @@ -278,6 +279,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
> >>  	/* Set up the timer */
> >>  	kvm_timer_vcpu_init(vcpu);
> >>  
> >> +	/* Set up the PMU */
> >> +	kvm_pmu_init(vcpu);
> >> +
> >>  	return 0;
> >>  }
> >>  
> >> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> >> index 2709db2..3c88873 100644
> >> --- a/arch/arm64/include/asm/kvm_host.h
> >> +++ b/arch/arm64/include/asm/kvm_host.h
> >> @@ -42,6 +42,7 @@
> >>  
> >>  #include <kvm/arm_vgic.h>
> >>  #include <kvm/arm_arch_timer.h>
> >> +#include <kvm/arm_pmu.h>
> >>  
> >>  #define KVM_VCPU_MAX_FEATURES 3
> >>  
> >> @@ -116,6 +117,7 @@ struct kvm_vcpu_arch {
> >>  	/* VGIC state */
> >>  	struct vgic_cpu vgic_cpu;
> >>  	struct arch_timer_cpu timer_cpu;
> >> +	struct kvm_pmu pmu;
> >>  
> >>  	/*
> >>  	 * Anything that is not used directly from assembly code goes
> >> diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
> >> index b9f394a..95681e6 100644
> >> --- a/arch/arm64/include/asm/pmu.h
> >> +++ b/arch/arm64/include/asm/pmu.h
> >> @@ -19,6 +19,8 @@
> >>  #ifndef __ASM_PMU_H
> >>  #define __ASM_PMU_H
> >>  
> >> +#include <linux/perf_event.h>
> >> +
> >>  #define ARMV8_MAX_COUNTERS      32
> >>  #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
> >>  
> >> diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
> >> index bfffe8f..eae3a1b 100644
> >> --- a/arch/arm64/kvm/Kconfig
> >> +++ b/arch/arm64/kvm/Kconfig
> >> @@ -31,6 +31,7 @@ config KVM
> >>  	select KVM_VFIO
> >>  	select HAVE_KVM_EVENTFD
> >>  	select HAVE_KVM_IRQFD
> >> +	select KVM_ARM_PMU
> >>  	---help---
> >>  	  Support hosting virtualized guest machines.
> >>  
> >> @@ -52,4 +53,10 @@ config KVM_ARM_MAX_VCPUS
> >>  	  large, so only choose a reasonable number that you expect to
> >>  	  actually use.
> >>  
> >> +config KVM_ARM_PMU
> >> +	bool
> >> +	depends on KVM_ARM_HOST
> >> +	---help---
> >> +	  Adds support for the Performance Monitoring in virtual machines.
> > 
> > Adds support for a virtual Performance Monitoring Unit (PMU) in virtual
> > machines.
> > 
> >> +
> >>  endif # VIRTUALIZATION
> >> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
> >> index f90f4aa..78db4ee 100644
> >> --- a/arch/arm64/kvm/Makefile
> >> +++ b/arch/arm64/kvm/Makefile
> >> @@ -27,3 +27,4 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
> >>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
> >>  kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
> >>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
> >> +kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
> >> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> >> index 0b43265..ee2c2e9 100644
> >> --- a/arch/arm64/kvm/reset.c
> >> +++ b/arch/arm64/kvm/reset.c
> >> @@ -107,5 +107,8 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
> >>  	/* Reset timer */
> >>  	kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
> >>  
> >> +	/* Reset PMU */
> >> +	kvm_pmu_vcpu_reset(vcpu);
> >> +
> >>  	return 0;
> >>  }
> >> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> >> new file mode 100644
> >> index 0000000..27d14ca
> >> --- /dev/null
> >> +++ b/include/kvm/arm_pmu.h
> >> @@ -0,0 +1,54 @@
> >> +/*
> >> + * Copyright (C) 2015 Linaro Ltd.
> >> + * Author: Shannon Zhao <shannon.zhao@linaro.org>
> >> + *
> >> + * This program is free software; you can redistribute it and/or modify
> >> + * it under the terms of the GNU General Public License version 2 as
> >> + * published by the Free Software Foundation.
> >> + *
> >> + * This program is distributed in the hope that it will be useful,
> >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> >> + * GNU General Public License for more details.
> >> + *
> >> + * You should have received a copy of the GNU General Public License
> >> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> >> + */
> >> +
> >> +#ifndef __ASM_ARM_KVM_PMU_H
> >> +#define __ASM_ARM_KVM_PMU_H
> >> +
> >> +#include <asm/pmu.h>
> >> +#include <linux/irq_work.h>
> >> +
> >> +struct kvm_pmc {
> >> +	u8 idx;
> > 
> > which index is this?  Does it map to something architecturally or is it
> > just the array index number?
> > 
> 
> It's the index of the PMU counter.
> 

sorry, I don't feel this answers my question? ;)

> >> +	u64 counter;
> >> +	u64 eventsel;
> > 
> > does this map to PMEVTYPER<n>_EL0.evtCount?
> > 
> No, it just stores the event type.
> 

can you specify more clearly what this maps to/how I derive its meaning?

Is this a perf (Linux) event type number, or an architecturally defined
thing?


Thanks,
-Christoffer

[PATCH 02/18] KVM: ARM64: Add initial support for PMU

[Shannon Zhao]

On 2015/7/17 17:58, Christoffer Dall wrote:
> On Fri, Jul 17, 2015 at 04:13:35PM +0800, Shannon Zhao wrote:
>>
>>
>> On 2015/7/17 2:25, Christoffer Dall wrote:
>>> On Mon, Jul 06, 2015 at 10:17:32AM +0800, shannon.zhao at linaro.org wrote:
>>>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>>>
>>>> Here we plan to support virtual PMU for guest by full software
>>>> emulation, so define some basic structs and functions preparing for
>>>> futher steps. Define struct kvm_pmc for performance monitor counter and
>>>> struct kvm_pmu for performance monitor unit for each vcpu. According to
>>>> ARMv8 spec, the PMU contains at most 32(ARMV8_MAX_COUNTERS) counters.
>>>> Initialize PMU for each vcpu when kvm initialize vcpus, while reset PMU
>>>> when kvm reset vcpus.
>>>> Since this only supports ARM64 (or PMUv3), add a separate config symbol
>>>> for it.
>>>>
>>>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>>>> ---
>>>>   arch/arm/kvm/arm.c                |  4 +++
>>>>   arch/arm64/include/asm/kvm_host.h |  2 ++
>>>>   arch/arm64/include/asm/pmu.h      |  2 ++
>>>>   arch/arm64/kvm/Kconfig            |  7 +++++
>>>>   arch/arm64/kvm/Makefile           |  1 +
>>>>   arch/arm64/kvm/reset.c            |  3 +++
>>>>   include/kvm/arm_pmu.h             | 54 ++++++++++++++++++++++++++++++++++++++
>>>>   virt/kvm/arm/pmu.c                | 55 +++++++++++++++++++++++++++++++++++++++
>>>>   8 files changed, 128 insertions(+)
>>>>   create mode 100644 include/kvm/arm_pmu.h
>>>>   create mode 100644 virt/kvm/arm/pmu.c
>>>>
>>>> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>>>> index bc738d2..4e82625 100644
>>>> --- a/arch/arm/kvm/arm.c
>>>> +++ b/arch/arm/kvm/arm.c
>>>> @@ -28,6 +28,7 @@
>>>>   #include <linux/sched.h>
>>>>   #include <linux/kvm.h>
>>>>   #include <trace/events/kvm.h>
>>>> +#include <kvm/arm_pmu.h>
>>>>
>>>>   #define CREATE_TRACE_POINTS
>>>>   #include "trace.h"
>>>> @@ -278,6 +279,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
>>>>   	/* Set up the timer */
>>>>   	kvm_timer_vcpu_init(vcpu);
>>>>
>>>> +	/* Set up the PMU */
>>>> +	kvm_pmu_init(vcpu);
>>>> +
>>>>   	return 0;
>>>>   }
>>>>
>>>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>>>> index 2709db2..3c88873 100644
>>>> --- a/arch/arm64/include/asm/kvm_host.h
>>>> +++ b/arch/arm64/include/asm/kvm_host.h
>>>> @@ -42,6 +42,7 @@
>>>>
>>>>   #include <kvm/arm_vgic.h>
>>>>   #include <kvm/arm_arch_timer.h>
>>>> +#include <kvm/arm_pmu.h>
>>>>
>>>>   #define KVM_VCPU_MAX_FEATURES 3
>>>>
>>>> @@ -116,6 +117,7 @@ struct kvm_vcpu_arch {
>>>>   	/* VGIC state */
>>>>   	struct vgic_cpu vgic_cpu;
>>>>   	struct arch_timer_cpu timer_cpu;
>>>> +	struct kvm_pmu pmu;
>>>>
>>>>   	/*
>>>>   	 * Anything that is not used directly from assembly code goes
>>>> diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
>>>> index b9f394a..95681e6 100644
>>>> --- a/arch/arm64/include/asm/pmu.h
>>>> +++ b/arch/arm64/include/asm/pmu.h
>>>> @@ -19,6 +19,8 @@
>>>>   #ifndef __ASM_PMU_H
>>>>   #define __ASM_PMU_H
>>>>
>>>> +#include <linux/perf_event.h>
>>>> +
>>>>   #define ARMV8_MAX_COUNTERS      32
>>>>   #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
>>>>
>>>> diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
>>>> index bfffe8f..eae3a1b 100644
>>>> --- a/arch/arm64/kvm/Kconfig
>>>> +++ b/arch/arm64/kvm/Kconfig
>>>> @@ -31,6 +31,7 @@ config KVM
>>>>   	select KVM_VFIO
>>>>   	select HAVE_KVM_EVENTFD
>>>>   	select HAVE_KVM_IRQFD
>>>> +	select KVM_ARM_PMU
>>>>   	---help---
>>>>   	  Support hosting virtualized guest machines.
>>>>
>>>> @@ -52,4 +53,10 @@ config KVM_ARM_MAX_VCPUS
>>>>   	  large, so only choose a reasonable number that you expect to
>>>>   	  actually use.
>>>>
>>>> +config KVM_ARM_PMU
>>>> +	bool
>>>> +	depends on KVM_ARM_HOST
>>>> +	---help---
>>>> +	  Adds support for the Performance Monitoring in virtual machines.
>>>
>>> Adds support for a virtual Performance Monitoring Unit (PMU) in virtual
>>> machines.
>>>
>>>> +
>>>>   endif # VIRTUALIZATION
>>>> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
>>>> index f90f4aa..78db4ee 100644
>>>> --- a/arch/arm64/kvm/Makefile
>>>> +++ b/arch/arm64/kvm/Makefile
>>>> @@ -27,3 +27,4 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
>>>>   kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
>>>>   kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
>>>>   kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
>>>> +kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
>>>> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
>>>> index 0b43265..ee2c2e9 100644
>>>> --- a/arch/arm64/kvm/reset.c
>>>> +++ b/arch/arm64/kvm/reset.c
>>>> @@ -107,5 +107,8 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>>>>   	/* Reset timer */
>>>>   	kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
>>>>
>>>> +	/* Reset PMU */
>>>> +	kvm_pmu_vcpu_reset(vcpu);
>>>> +
>>>>   	return 0;
>>>>   }
>>>> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
>>>> new file mode 100644
>>>> index 0000000..27d14ca
>>>> --- /dev/null
>>>> +++ b/include/kvm/arm_pmu.h
>>>> @@ -0,0 +1,54 @@
>>>> +/*
>>>> + * Copyright (C) 2015 Linaro Ltd.
>>>> + * Author: Shannon Zhao <shannon.zhao@linaro.org>
>>>> + *
>>>> + * This program is free software; you can redistribute it and/or modify
>>>> + * it under the terms of the GNU General Public License version 2 as
>>>> + * published by the Free Software Foundation.
>>>> + *
>>>> + * This program is distributed in the hope that it will be useful,
>>>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>>>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>>>> + * GNU General Public License for more details.
>>>> + *
>>>> + * You should have received a copy of the GNU General Public License
>>>> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
>>>> + */
>>>> +
>>>> +#ifndef __ASM_ARM_KVM_PMU_H
>>>> +#define __ASM_ARM_KVM_PMU_H
>>>> +
>>>> +#include <asm/pmu.h>
>>>> +#include <linux/irq_work.h>
>>>> +
>>>> +struct kvm_pmc {
>>>> +	u8 idx;
>>>
>>> which index is this?  Does it map to something architecturally or is it
>>> just the array index number?
>>>
>>
>> It's the index of the PMU counter.
>>
>
> sorry, I don't feel this answers my question? ;)
>

This index is only used for kvm_pmu_perf_overflow to locate the index of 
this PMU counter and kvm_pmu_perf_overflow will set the corresponding 
bit of overflow_status to show whether this counter overflows.

See patch 17/18:

+static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
+				  struct perf_sample_data *data,
+				  struct pt_regs *regs)
+{
+	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
+	struct kvm_vcpu *vcpu = pmc->vcpu;
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+
+	if (pmc->interrupt == true) {
+		__set_bit(pmc->idx, (unsigned long *)&pmu->overflow_status);

>>>> +	u64 counter;
>>>> +	u64 eventsel;
>>>
>>> does this map to PMEVTYPER<n>_EL0.evtCount?
>>>
>> No, it just stores the event type.
>>
>
> can you specify more clearly what this maps to/how I derive its meaning?
>
> Is this a perf (Linux) event type number, or an architecturally defined
> thing?
>

Yes, it stores a perf event type number. This is used for 
kvm_pmu_set_counter_event_type to check whether event type of this time 
is same with the previous one. If they are same, it will return and 
doesn't create a new perf event for it and just reuses the previous one.

See PATCH 07/18:

+void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned 
long data,
+				    unsigned long select_idx)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+	struct perf_event *event;
+	struct perf_event_attr attr;
+	unsigned config, type = PERF_TYPE_RAW;
+
+	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
+		return;

>
> Thanks,
> -Christoffer
>

-- 
Shannon

[PATCH 02/18] KVM: ARM64: Add initial support for PMU

[Christoffer Dall]

On Fri, Jul 17, 2015 at 07:34:58PM +0800, Shannon Zhao wrote:
> 
> 
> On 2015/7/17 17:58, Christoffer Dall wrote:
> >On Fri, Jul 17, 2015 at 04:13:35PM +0800, Shannon Zhao wrote:
> >>
> >>
> >>On 2015/7/17 2:25, Christoffer Dall wrote:
> >>>On Mon, Jul 06, 2015 at 10:17:32AM +0800, shannon.zhao at linaro.org wrote:
> >>>>From: Shannon Zhao <shannon.zhao@linaro.org>
> >>>>
> >>>>Here we plan to support virtual PMU for guest by full software
> >>>>emulation, so define some basic structs and functions preparing for
> >>>>futher steps. Define struct kvm_pmc for performance monitor counter and
> >>>>struct kvm_pmu for performance monitor unit for each vcpu. According to
> >>>>ARMv8 spec, the PMU contains at most 32(ARMV8_MAX_COUNTERS) counters.
> >>>>Initialize PMU for each vcpu when kvm initialize vcpus, while reset PMU
> >>>>when kvm reset vcpus.
> >>>>Since this only supports ARM64 (or PMUv3), add a separate config symbol
> >>>>for it.
> >>>>
> >>>>Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> >>>>---
> >>>>  arch/arm/kvm/arm.c                |  4 +++
> >>>>  arch/arm64/include/asm/kvm_host.h |  2 ++
> >>>>  arch/arm64/include/asm/pmu.h      |  2 ++
> >>>>  arch/arm64/kvm/Kconfig            |  7 +++++
> >>>>  arch/arm64/kvm/Makefile           |  1 +
> >>>>  arch/arm64/kvm/reset.c            |  3 +++
> >>>>  include/kvm/arm_pmu.h             | 54 ++++++++++++++++++++++++++++++++++++++
> >>>>  virt/kvm/arm/pmu.c                | 55 +++++++++++++++++++++++++++++++++++++++
> >>>>  8 files changed, 128 insertions(+)
> >>>>  create mode 100644 include/kvm/arm_pmu.h
> >>>>  create mode 100644 virt/kvm/arm/pmu.c
> >>>>
> >>>>diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> >>>>index bc738d2..4e82625 100644
> >>>>--- a/arch/arm/kvm/arm.c
> >>>>+++ b/arch/arm/kvm/arm.c
> >>>>@@ -28,6 +28,7 @@
> >>>>  #include <linux/sched.h>
> >>>>  #include <linux/kvm.h>
> >>>>  #include <trace/events/kvm.h>
> >>>>+#include <kvm/arm_pmu.h>
> >>>>
> >>>>  #define CREATE_TRACE_POINTS
> >>>>  #include "trace.h"
> >>>>@@ -278,6 +279,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
> >>>>  	/* Set up the timer */
> >>>>  	kvm_timer_vcpu_init(vcpu);
> >>>>
> >>>>+	/* Set up the PMU */
> >>>>+	kvm_pmu_init(vcpu);
> >>>>+
> >>>>  	return 0;
> >>>>  }
> >>>>
> >>>>diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> >>>>index 2709db2..3c88873 100644
> >>>>--- a/arch/arm64/include/asm/kvm_host.h
> >>>>+++ b/arch/arm64/include/asm/kvm_host.h
> >>>>@@ -42,6 +42,7 @@
> >>>>
> >>>>  #include <kvm/arm_vgic.h>
> >>>>  #include <kvm/arm_arch_timer.h>
> >>>>+#include <kvm/arm_pmu.h>
> >>>>
> >>>>  #define KVM_VCPU_MAX_FEATURES 3
> >>>>
> >>>>@@ -116,6 +117,7 @@ struct kvm_vcpu_arch {
> >>>>  	/* VGIC state */
> >>>>  	struct vgic_cpu vgic_cpu;
> >>>>  	struct arch_timer_cpu timer_cpu;
> >>>>+	struct kvm_pmu pmu;
> >>>>
> >>>>  	/*
> >>>>  	 * Anything that is not used directly from assembly code goes
> >>>>diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
> >>>>index b9f394a..95681e6 100644
> >>>>--- a/arch/arm64/include/asm/pmu.h
> >>>>+++ b/arch/arm64/include/asm/pmu.h
> >>>>@@ -19,6 +19,8 @@
> >>>>  #ifndef __ASM_PMU_H
> >>>>  #define __ASM_PMU_H
> >>>>
> >>>>+#include <linux/perf_event.h>
> >>>>+
> >>>>  #define ARMV8_MAX_COUNTERS      32
> >>>>  #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
> >>>>
> >>>>diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
> >>>>index bfffe8f..eae3a1b 100644
> >>>>--- a/arch/arm64/kvm/Kconfig
> >>>>+++ b/arch/arm64/kvm/Kconfig
> >>>>@@ -31,6 +31,7 @@ config KVM
> >>>>  	select KVM_VFIO
> >>>>  	select HAVE_KVM_EVENTFD
> >>>>  	select HAVE_KVM_IRQFD
> >>>>+	select KVM_ARM_PMU
> >>>>  	---help---
> >>>>  	  Support hosting virtualized guest machines.
> >>>>
> >>>>@@ -52,4 +53,10 @@ config KVM_ARM_MAX_VCPUS
> >>>>  	  large, so only choose a reasonable number that you expect to
> >>>>  	  actually use.
> >>>>
> >>>>+config KVM_ARM_PMU
> >>>>+	bool
> >>>>+	depends on KVM_ARM_HOST
> >>>>+	---help---
> >>>>+	  Adds support for the Performance Monitoring in virtual machines.
> >>>
> >>>Adds support for a virtual Performance Monitoring Unit (PMU) in virtual
> >>>machines.
> >>>
> >>>>+
> >>>>  endif # VIRTUALIZATION
> >>>>diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
> >>>>index f90f4aa..78db4ee 100644
> >>>>--- a/arch/arm64/kvm/Makefile
> >>>>+++ b/arch/arm64/kvm/Makefile
> >>>>@@ -27,3 +27,4 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
> >>>>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
> >>>>  kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
> >>>>  kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
> >>>>+kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
> >>>>diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> >>>>index 0b43265..ee2c2e9 100644
> >>>>--- a/arch/arm64/kvm/reset.c
> >>>>+++ b/arch/arm64/kvm/reset.c
> >>>>@@ -107,5 +107,8 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
> >>>>  	/* Reset timer */
> >>>>  	kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
> >>>>
> >>>>+	/* Reset PMU */
> >>>>+	kvm_pmu_vcpu_reset(vcpu);
> >>>>+
> >>>>  	return 0;
> >>>>  }
> >>>>diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> >>>>new file mode 100644
> >>>>index 0000000..27d14ca
> >>>>--- /dev/null
> >>>>+++ b/include/kvm/arm_pmu.h
> >>>>@@ -0,0 +1,54 @@
> >>>>+/*
> >>>>+ * Copyright (C) 2015 Linaro Ltd.
> >>>>+ * Author: Shannon Zhao <shannon.zhao@linaro.org>
> >>>>+ *
> >>>>+ * This program is free software; you can redistribute it and/or modify
> >>>>+ * it under the terms of the GNU General Public License version 2 as
> >>>>+ * published by the Free Software Foundation.
> >>>>+ *
> >>>>+ * This program is distributed in the hope that it will be useful,
> >>>>+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
> >>>>+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> >>>>+ * GNU General Public License for more details.
> >>>>+ *
> >>>>+ * You should have received a copy of the GNU General Public License
> >>>>+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> >>>>+ */
> >>>>+
> >>>>+#ifndef __ASM_ARM_KVM_PMU_H
> >>>>+#define __ASM_ARM_KVM_PMU_H
> >>>>+
> >>>>+#include <asm/pmu.h>
> >>>>+#include <linux/irq_work.h>
> >>>>+
> >>>>+struct kvm_pmc {
> >>>>+	u8 idx;
> >>>
> >>>which index is this?  Does it map to something architecturally or is it
> >>>just the array index number?
> >>>
> >>
> >>It's the index of the PMU counter.
> >>
> >
> >sorry, I don't feel this answers my question? ;)
> >
> 
> This index is only used for kvm_pmu_perf_overflow to locate the
> index of this PMU counter and kvm_pmu_perf_overflow will set the
> corresponding bit of overflow_status to show whether this counter
> overflows.
> 
> See patch 17/18:
> 
> +static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
> +				  struct perf_sample_data *data,
> +				  struct pt_regs *regs)
> +{
> +	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
> +	struct kvm_vcpu *vcpu = pmc->vcpu;
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +
> +	if (pmc->interrupt == true) {
> +		__set_bit(pmc->idx, (unsigned long *)&pmu->overflow_status);
> 
> >>>>+	u64 counter;
> >>>>+	u64 eventsel;
> >>>
> >>>does this map to PMEVTYPER<n>_EL0.evtCount?
> >>>
> >>No, it just stores the event type.
> >>
> >
> >can you specify more clearly what this maps to/how I derive its meaning?
> >
> >Is this a perf (Linux) event type number, or an architecturally defined
> >thing?
> >
> 
> Yes, it stores a perf event type number. This is used for
> kvm_pmu_set_counter_event_type to check whether event type of this
> time is same with the previous one. If they are same, it will return
> and doesn't create a new perf event for it and just reuses the
> previous one.
> 
> See PATCH 07/18:
> 
> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned
> long data,
> +				    unsigned long select_idx)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +	struct perf_event *event;
> +	struct perf_event_attr attr;
> +	unsigned config, type = PERF_TYPE_RAW;
> +
> +	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
> +		return;
> 

ok, thanks for the explanation of both:

I think it would be helpful, especially for the review, to have a
one-line comment on the struct definition, along the lines of:


struct kvm_pmc {
	u8 idx;		/* index into the pmu->pmc array */
	u64 eventsel;	/* perf event number, see ARMV8_EVTYPE_EVENT */
	...
};

Thanks,
-Christoffer

[PATCH 03/18] KVM: ARM64: Add offset defines for PMU registers

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

We are about to trap and emulate acccesses to each PMU register
individually. This adds the context offsets for the AArch64 PMU
registers and their AArch32 counterparts.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/include/asm/kvm_asm.h | 59 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 3c5fe68..21b5d3b 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -56,14 +56,36 @@
 #define DBGWVR15_EL1	86
 #define MDCCINT_EL1	87	/* Monitor Debug Comms Channel Interrupt Enable Reg */
 
+/* Performance Monitors Registers */
+#define PMCR_EL0	88	/* Control Register */
+#define PMOVSSET_EL0	89	/* Overflow Flag Status Set Register */
+#define PMOVSCLR_EL0	90	/* Overflow Flag Status Clear Register */
+#define PMCCNTR_EL0	91	/* Cycle Counter Register */
+#define PMSELR_EL0	92	/* Event Counter Selection Register */
+#define PMCEID0_EL0	93	/* Common Event Identification Register 0 */
+#define PMCEID1_EL0	94	/* Common Event Identification Register 1 */
+#define PMEVCNTR0_EL0	95	/* Event Counter Register (0-30) */
+#define PMEVTYPER0_EL0	96	/* Event Type Register (0-30) */
+#define PMEVCNTR30_EL0	155
+#define PMEVTYPER30_EL0	156
+#define PMXEVCNTR_EL0	157	/* Selected Event Count Register */
+#define PMXEVTYPER_EL0	158	/* Selected Event Type Register */
+#define PMCNTENSET_EL0	159	/* Count Enable Set Register */
+#define PMCNTENCLR_EL0	160	/* Count Enable Clear Register */
+#define PMINTENSET_EL1	161	/* Interrupt Enable Set Register */
+#define PMINTENCLR_EL1	162	/* Interrupt Enable Clear Register */
+#define PMUSERENR_EL0	163	/* User Enable Register */
+#define PMCCFILTR_EL0	164	/* Cycle Count Filter Register */
+#define PMSWINC_EL0	165	/* Software Increment Register */
+
 /* 32bit specific registers. Keep them at the end of the range */
-#define	DACR32_EL2	88	/* Domain Access Control Register */
-#define	IFSR32_EL2	89	/* Instruction Fault Status Register */
-#define	FPEXC32_EL2	90	/* Floating-Point Exception Control Register */
-#define	DBGVCR32_EL2	91	/* Debug Vector Catch Register */
-#define	TEECR32_EL1	92	/* ThumbEE Configuration Register */
-#define	TEEHBR32_EL1	93	/* ThumbEE Handler Base Register */
-#define	NR_SYS_REGS	94
+#define	DACR32_EL2	166	/* Domain Access Control Register */
+#define	IFSR32_EL2	167	/* Instruction Fault Status Register */
+#define	FPEXC32_EL2	168	/* Floating-Point Exception Control Register */
+#define	DBGVCR32_EL2	169	/* Debug Vector Catch Register */
+#define	TEECR32_EL1	170	/* ThumbEE Configuration Register */
+#define	TEEHBR32_EL1	171	/* ThumbEE Handler Base Register */
+#define	NR_SYS_REGS	172
 
 /* 32bit mapping */
 #define c0_MPIDR	(MPIDR_EL1 * 2)	/* MultiProcessor ID Register */
@@ -85,6 +107,24 @@
 #define c6_IFAR		(c6_DFAR + 1)	/* Instruction Fault Address Register */
 #define c7_PAR		(PAR_EL1 * 2)	/* Physical Address Register */
 #define c7_PAR_high	(c7_PAR + 1)	/* PAR top 32 bits */
+
+/* Performance Monitors*/
+#define c9_PMCR		(PMCR_EL0 * 2)
+#define c9_PMOVSSET	(PMOVSSET_EL0 * 2)
+#define c9_PMOVSCLR	(PMOVSCLR_EL0 * 2)
+#define c9_PMCCNTR	(PMCCNTR_EL0 * 2)
+#define c9_PMSELR	(PMSELR_EL0 * 2)
+#define c9_PMCEID0	(PMCEID0_EL0 * 2)
+#define c9_PMCEID1	(PMCEID1_EL0 * 2)
+#define c9_PMXEVCNTR	(PMXEVCNTR_EL0 * 2)
+#define c9_PMXEVTYPER	(PMXEVTYPER_EL0 * 2)
+#define c9_PMCNTENSET	(PMCNTENSET_EL0 * 2)
+#define c9_PMCNTENCLR	(PMCNTENCLR_EL0 * 2)
+#define c9_PMINTENSET	(PMINTENSET_EL1 * 2)
+#define c9_PMINTENCLR	(PMINTENCLR_EL1 * 2)
+#define c9_PMUSERENR	(PMUSERENR_EL0 * 2)
+#define c9_PMSWINC	(PMSWINC_EL0 * 2)
+
 #define c10_PRRR	(MAIR_EL1 * 2)	/* Primary Region Remap Register */
 #define c10_NMRR	(c10_PRRR + 1)	/* Normal Memory Remap Register */
 #define c12_VBAR	(VBAR_EL1 * 2)	/* Vector Base Address Register */
@@ -96,6 +136,11 @@
 #define c10_AMAIR1	(c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
 #define c14_CNTKCTL	(CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
 
+/* Performance Monitors*/
+#define c14_PMEVCNTR0  (PMEVCNTR0_EL0 * 2)
+#define c14_PMEVTYPR0  (PMEVTYPER0_EL0 * 2)
+#define c14_PMCCFILTR  (PMCCFILTR_EL0 * 2)
+
 #define cp14_DBGDSCRext	(MDSCR_EL1 * 2)
 #define cp14_DBGBCR0	(DBGBCR0_EL1 * 2)
 #define cp14_DBGBVR0	(DBGBVR0_EL1 * 2)
-- 
2.1.0

[PATCH 03/18] KVM: ARM64: Add offset defines for PMU registers

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:33AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> We are about to trap and emulate acccesses to each PMU register
> individually. This adds the context offsets for the AArch64 PMU
> registers and their AArch32 counterparts.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/include/asm/kvm_asm.h | 59 +++++++++++++++++++++++++++++++++++-----
>  1 file changed, 52 insertions(+), 7 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
> index 3c5fe68..21b5d3b 100644
> --- a/arch/arm64/include/asm/kvm_asm.h
> +++ b/arch/arm64/include/asm/kvm_asm.h
> @@ -56,14 +56,36 @@
>  #define DBGWVR15_EL1	86
>  #define MDCCINT_EL1	87	/* Monitor Debug Comms Channel Interrupt Enable Reg */
>  
> +/* Performance Monitors Registers */
> +#define PMCR_EL0	88	/* Control Register */
> +#define PMOVSSET_EL0	89	/* Overflow Flag Status Set Register */
> +#define PMOVSCLR_EL0	90	/* Overflow Flag Status Clear Register */
> +#define PMCCNTR_EL0	91	/* Cycle Counter Register */
> +#define PMSELR_EL0	92	/* Event Counter Selection Register */
> +#define PMCEID0_EL0	93	/* Common Event Identification Register 0 */
> +#define PMCEID1_EL0	94	/* Common Event Identification Register 1 */
> +#define PMEVCNTR0_EL0	95	/* Event Counter Register (0-30) */

why do we need these when we trap-and-emulate and we have the kvm_pmc
structs?  Is that because the kvm_pmc structs are only used when we
actually have an active counter running and registered with perf?

> +#define PMEVTYPER0_EL0	96	/* Event Type Register (0-30) */
> +#define PMEVCNTR30_EL0	155
> +#define PMEVTYPER30_EL0	156
> +#define PMXEVCNTR_EL0	157	/* Selected Event Count Register */
> +#define PMXEVTYPER_EL0	158	/* Selected Event Type Register */
> +#define PMCNTENSET_EL0	159	/* Count Enable Set Register */
> +#define PMCNTENCLR_EL0	160	/* Count Enable Clear Register */
> +#define PMINTENSET_EL1	161	/* Interrupt Enable Set Register */
> +#define PMINTENCLR_EL1	162	/* Interrupt Enable Clear Register */
> +#define PMUSERENR_EL0	163	/* User Enable Register */
> +#define PMCCFILTR_EL0	164	/* Cycle Count Filter Register */
> +#define PMSWINC_EL0	165	/* Software Increment Register */
> +
>  /* 32bit specific registers. Keep them at the end of the range */
> -#define	DACR32_EL2	88	/* Domain Access Control Register */
> -#define	IFSR32_EL2	89	/* Instruction Fault Status Register */
> -#define	FPEXC32_EL2	90	/* Floating-Point Exception Control Register */
> -#define	DBGVCR32_EL2	91	/* Debug Vector Catch Register */
> -#define	TEECR32_EL1	92	/* ThumbEE Configuration Register */
> -#define	TEEHBR32_EL1	93	/* ThumbEE Handler Base Register */
> -#define	NR_SYS_REGS	94
> +#define	DACR32_EL2	166	/* Domain Access Control Register */
> +#define	IFSR32_EL2	167	/* Instruction Fault Status Register */
> +#define	FPEXC32_EL2	168	/* Floating-Point Exception Control Register */
> +#define	DBGVCR32_EL2	169	/* Debug Vector Catch Register */
> +#define	TEECR32_EL1	170	/* ThumbEE Configuration Register */
> +#define	TEEHBR32_EL1	171	/* ThumbEE Handler Base Register */
> +#define	NR_SYS_REGS	172
>  
>  /* 32bit mapping */
>  #define c0_MPIDR	(MPIDR_EL1 * 2)	/* MultiProcessor ID Register */
> @@ -85,6 +107,24 @@
>  #define c6_IFAR		(c6_DFAR + 1)	/* Instruction Fault Address Register */
>  #define c7_PAR		(PAR_EL1 * 2)	/* Physical Address Register */
>  #define c7_PAR_high	(c7_PAR + 1)	/* PAR top 32 bits */
> +
> +/* Performance Monitors*/
> +#define c9_PMCR		(PMCR_EL0 * 2)
> +#define c9_PMOVSSET	(PMOVSSET_EL0 * 2)
> +#define c9_PMOVSCLR	(PMOVSCLR_EL0 * 2)
> +#define c9_PMCCNTR	(PMCCNTR_EL0 * 2)
> +#define c9_PMSELR	(PMSELR_EL0 * 2)
> +#define c9_PMCEID0	(PMCEID0_EL0 * 2)
> +#define c9_PMCEID1	(PMCEID1_EL0 * 2)
> +#define c9_PMXEVCNTR	(PMXEVCNTR_EL0 * 2)
> +#define c9_PMXEVTYPER	(PMXEVTYPER_EL0 * 2)
> +#define c9_PMCNTENSET	(PMCNTENSET_EL0 * 2)
> +#define c9_PMCNTENCLR	(PMCNTENCLR_EL0 * 2)
> +#define c9_PMINTENSET	(PMINTENSET_EL1 * 2)
> +#define c9_PMINTENCLR	(PMINTENCLR_EL1 * 2)
> +#define c9_PMUSERENR	(PMUSERENR_EL0 * 2)
> +#define c9_PMSWINC	(PMSWINC_EL0 * 2)
> +
>  #define c10_PRRR	(MAIR_EL1 * 2)	/* Primary Region Remap Register */
>  #define c10_NMRR	(c10_PRRR + 1)	/* Normal Memory Remap Register */
>  #define c12_VBAR	(VBAR_EL1 * 2)	/* Vector Base Address Register */
> @@ -96,6 +136,11 @@
>  #define c10_AMAIR1	(c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
>  #define c14_CNTKCTL	(CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
>  
> +/* Performance Monitors*/
> +#define c14_PMEVCNTR0  (PMEVCNTR0_EL0 * 2)
> +#define c14_PMEVTYPR0  (PMEVTYPER0_EL0 * 2)
> +#define c14_PMCCFILTR  (PMCCFILTR_EL0 * 2)
> +
>  #define cp14_DBGDSCRext	(MDSCR_EL1 * 2)
>  #define cp14_DBGBCR0	(DBGBCR0_EL1 * 2)
>  #define cp14_DBGBVR0	(DBGBVR0_EL1 * 2)
> -- 
> 2.1.0
> 

[PATCH 03/18] KVM: ARM64: Add offset defines for PMU registers

[Shannon Zhao]

On 2015/7/17 2:45, Christoffer Dall wrote:
> On Mon, Jul 06, 2015 at 10:17:33AM +0800, shannon.zhao at linaro.org wrote:
>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>
>> We are about to trap and emulate acccesses to each PMU register
>> individually. This adds the context offsets for the AArch64 PMU
>> registers and their AArch32 counterparts.
>>
>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>> ---
>>  arch/arm64/include/asm/kvm_asm.h | 59 +++++++++++++++++++++++++++++++++++-----
>>  1 file changed, 52 insertions(+), 7 deletions(-)
>>
>> diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
>> index 3c5fe68..21b5d3b 100644
>> --- a/arch/arm64/include/asm/kvm_asm.h
>> +++ b/arch/arm64/include/asm/kvm_asm.h
>> @@ -56,14 +56,36 @@
>>  #define DBGWVR15_EL1	86
>>  #define MDCCINT_EL1	87	/* Monitor Debug Comms Channel Interrupt Enable Reg */
>>  
>> +/* Performance Monitors Registers */
>> +#define PMCR_EL0	88	/* Control Register */
>> +#define PMOVSSET_EL0	89	/* Overflow Flag Status Set Register */
>> +#define PMOVSCLR_EL0	90	/* Overflow Flag Status Clear Register */
>> +#define PMCCNTR_EL0	91	/* Cycle Counter Register */
>> +#define PMSELR_EL0	92	/* Event Counter Selection Register */
>> +#define PMCEID0_EL0	93	/* Common Event Identification Register 0 */
>> +#define PMCEID1_EL0	94	/* Common Event Identification Register 1 */
>> +#define PMEVCNTR0_EL0	95	/* Event Counter Register (0-30) */
> 
> why do we need these when we trap-and-emulate and we have the kvm_pmc
> structs? 
This just makes the guest work when accessing these registers.

> Is that because the kvm_pmc structs are only used when we
> actually have an active counter running and registered with perf?
> 

Right, the kvm_pmc structs are used to store the status of perf evnets,
like the event type, count number of this perf event.

On the other hand, the kernel perf codes will not directly access to the
PMEVCNTRx_EL0 and PMEVTYPERx_EL0 registers. It will firstly write the
index of select counter to PMSELR_EL0 and access to PMXEVCNTR_EL0 or
PMXEVTYPER_EL0. Then this is architecturally mapped to PMEVCNTRx_EL0 and
PMEVTYPERx_EL0.

>> +#define PMEVTYPER0_EL0	96	/* Event Type Register (0-30) */
>> +#define PMEVCNTR30_EL0	155
>> +#define PMEVTYPER30_EL0	156
>> +#define PMXEVCNTR_EL0	157	/* Selected Event Count Register */
>> +#define PMXEVTYPER_EL0	158	/* Selected Event Type Register */
>> +#define PMCNTENSET_EL0	159	/* Count Enable Set Register */
>> +#define PMCNTENCLR_EL0	160	/* Count Enable Clear Register */
>> +#define PMINTENSET_EL1	161	/* Interrupt Enable Set Register */
>> +#define PMINTENCLR_EL1	162	/* Interrupt Enable Clear Register */
>> +#define PMUSERENR_EL0	163	/* User Enable Register */
>> +#define PMCCFILTR_EL0	164	/* Cycle Count Filter Register */
>> +#define PMSWINC_EL0	165	/* Software Increment Register */
>> +
>>  /* 32bit specific registers. Keep them at the end of the range */
>> -#define	DACR32_EL2	88	/* Domain Access Control Register */
>> -#define	IFSR32_EL2	89	/* Instruction Fault Status Register */
>> -#define	FPEXC32_EL2	90	/* Floating-Point Exception Control Register */
>> -#define	DBGVCR32_EL2	91	/* Debug Vector Catch Register */
>> -#define	TEECR32_EL1	92	/* ThumbEE Configuration Register */
>> -#define	TEEHBR32_EL1	93	/* ThumbEE Handler Base Register */
>> -#define	NR_SYS_REGS	94
>> +#define	DACR32_EL2	166	/* Domain Access Control Register */
>> +#define	IFSR32_EL2	167	/* Instruction Fault Status Register */
>> +#define	FPEXC32_EL2	168	/* Floating-Point Exception Control Register */
>> +#define	DBGVCR32_EL2	169	/* Debug Vector Catch Register */
>> +#define	TEECR32_EL1	170	/* ThumbEE Configuration Register */
>> +#define	TEEHBR32_EL1	171	/* ThumbEE Handler Base Register */
>> +#define	NR_SYS_REGS	172
>>  
>>  /* 32bit mapping */
>>  #define c0_MPIDR	(MPIDR_EL1 * 2)	/* MultiProcessor ID Register */
>> @@ -85,6 +107,24 @@
>>  #define c6_IFAR		(c6_DFAR + 1)	/* Instruction Fault Address Register */
>>  #define c7_PAR		(PAR_EL1 * 2)	/* Physical Address Register */
>>  #define c7_PAR_high	(c7_PAR + 1)	/* PAR top 32 bits */
>> +
>> +/* Performance Monitors*/
>> +#define c9_PMCR		(PMCR_EL0 * 2)
>> +#define c9_PMOVSSET	(PMOVSSET_EL0 * 2)
>> +#define c9_PMOVSCLR	(PMOVSCLR_EL0 * 2)
>> +#define c9_PMCCNTR	(PMCCNTR_EL0 * 2)
>> +#define c9_PMSELR	(PMSELR_EL0 * 2)
>> +#define c9_PMCEID0	(PMCEID0_EL0 * 2)
>> +#define c9_PMCEID1	(PMCEID1_EL0 * 2)
>> +#define c9_PMXEVCNTR	(PMXEVCNTR_EL0 * 2)
>> +#define c9_PMXEVTYPER	(PMXEVTYPER_EL0 * 2)
>> +#define c9_PMCNTENSET	(PMCNTENSET_EL0 * 2)
>> +#define c9_PMCNTENCLR	(PMCNTENCLR_EL0 * 2)
>> +#define c9_PMINTENSET	(PMINTENSET_EL1 * 2)
>> +#define c9_PMINTENCLR	(PMINTENCLR_EL1 * 2)
>> +#define c9_PMUSERENR	(PMUSERENR_EL0 * 2)
>> +#define c9_PMSWINC	(PMSWINC_EL0 * 2)
>> +
>>  #define c10_PRRR	(MAIR_EL1 * 2)	/* Primary Region Remap Register */
>>  #define c10_NMRR	(c10_PRRR + 1)	/* Normal Memory Remap Register */
>>  #define c12_VBAR	(VBAR_EL1 * 2)	/* Vector Base Address Register */
>> @@ -96,6 +136,11 @@
>>  #define c10_AMAIR1	(c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
>>  #define c14_CNTKCTL	(CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
>>  
>> +/* Performance Monitors*/
>> +#define c14_PMEVCNTR0  (PMEVCNTR0_EL0 * 2)
>> +#define c14_PMEVTYPR0  (PMEVTYPER0_EL0 * 2)
>> +#define c14_PMCCFILTR  (PMCCFILTR_EL0 * 2)
>> +
>>  #define cp14_DBGDSCRext	(MDSCR_EL1 * 2)
>>  #define cp14_DBGBCR0	(DBGBCR0_EL1 * 2)
>>  #define cp14_DBGBVR0	(DBGBVR0_EL1 * 2)
>> -- 
>> 2.1.0
>>

-- 
Shannon

[PATCH 03/18] KVM: ARM64: Add offset defines for PMU registers

[Christoffer Dall]

On Fri, Jul 17, 2015 at 04:25:06PM +0800, Shannon Zhao wrote:
> 
> 
> On 2015/7/17 2:45, Christoffer Dall wrote:
> > On Mon, Jul 06, 2015 at 10:17:33AM +0800, shannon.zhao at linaro.org wrote:
> >> From: Shannon Zhao <shannon.zhao@linaro.org>
> >>
> >> We are about to trap and emulate acccesses to each PMU register
> >> individually. This adds the context offsets for the AArch64 PMU
> >> registers and their AArch32 counterparts.
> >>
> >> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> >> ---
> >>  arch/arm64/include/asm/kvm_asm.h | 59 +++++++++++++++++++++++++++++++++++-----
> >>  1 file changed, 52 insertions(+), 7 deletions(-)
> >>
> >> diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
> >> index 3c5fe68..21b5d3b 100644
> >> --- a/arch/arm64/include/asm/kvm_asm.h
> >> +++ b/arch/arm64/include/asm/kvm_asm.h
> >> @@ -56,14 +56,36 @@
> >>  #define DBGWVR15_EL1	86
> >>  #define MDCCINT_EL1	87	/* Monitor Debug Comms Channel Interrupt Enable Reg */
> >>  
> >> +/* Performance Monitors Registers */
> >> +#define PMCR_EL0	88	/* Control Register */
> >> +#define PMOVSSET_EL0	89	/* Overflow Flag Status Set Register */
> >> +#define PMOVSCLR_EL0	90	/* Overflow Flag Status Clear Register */
> >> +#define PMCCNTR_EL0	91	/* Cycle Counter Register */
> >> +#define PMSELR_EL0	92	/* Event Counter Selection Register */
> >> +#define PMCEID0_EL0	93	/* Common Event Identification Register 0 */
> >> +#define PMCEID1_EL0	94	/* Common Event Identification Register 1 */
> >> +#define PMEVCNTR0_EL0	95	/* Event Counter Register (0-30) */
> > 
> > why do we need these when we trap-and-emulate and we have the kvm_pmc
> > structs? 
> This just makes the guest work when accessing these registers.
> 
> > Is that because the kvm_pmc structs are only used when we
> > actually have an active counter running and registered with perf?
> > 
> 
> Right, the kvm_pmc structs are used to store the status of perf evnets,
> like the event type, count number of this perf event.
> 
> On the other hand, the kernel perf codes will not directly access to the
> PMEVCNTRx_EL0 and PMEVTYPERx_EL0 registers. It will firstly write the
> index of select counter to PMSELR_EL0 and access to PMXEVCNTR_EL0 or
> PMXEVTYPER_EL0. Then this is architecturally mapped to PMEVCNTRx_EL0 and
> PMEVTYPERx_EL0.
> 

I'm just wondering if it makes sense to keep virtual state around for
all these registers, since we don't emulate the counter values, so why
do we need to preserve any virtual cpu state for all of them?

-Christoffer

[PATCH 03/18] KVM: ARM64: Add offset defines for PMU registers

[Shannon Zhao]

On 2015/7/17 18:17, Christoffer Dall wrote:
> On Fri, Jul 17, 2015 at 04:25:06PM +0800, Shannon Zhao wrote:
>>
>>
>> On 2015/7/17 2:45, Christoffer Dall wrote:
>>> On Mon, Jul 06, 2015 at 10:17:33AM +0800, shannon.zhao at linaro.org wrote:
>>>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>>>
>>>> We are about to trap and emulate acccesses to each PMU register
>>>> individually. This adds the context offsets for the AArch64 PMU
>>>> registers and their AArch32 counterparts.
>>>>
>>>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>>>> ---
>>>>   arch/arm64/include/asm/kvm_asm.h | 59 +++++++++++++++++++++++++++++++++++-----
>>>>   1 file changed, 52 insertions(+), 7 deletions(-)
>>>>
>>>> diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
>>>> index 3c5fe68..21b5d3b 100644
>>>> --- a/arch/arm64/include/asm/kvm_asm.h
>>>> +++ b/arch/arm64/include/asm/kvm_asm.h
>>>> @@ -56,14 +56,36 @@
>>>>   #define DBGWVR15_EL1	86
>>>>   #define MDCCINT_EL1	87	/* Monitor Debug Comms Channel Interrupt Enable Reg */
>>>>
>>>> +/* Performance Monitors Registers */
>>>> +#define PMCR_EL0	88	/* Control Register */
>>>> +#define PMOVSSET_EL0	89	/* Overflow Flag Status Set Register */
>>>> +#define PMOVSCLR_EL0	90	/* Overflow Flag Status Clear Register */
>>>> +#define PMCCNTR_EL0	91	/* Cycle Counter Register */
>>>> +#define PMSELR_EL0	92	/* Event Counter Selection Register */
>>>> +#define PMCEID0_EL0	93	/* Common Event Identification Register 0 */
>>>> +#define PMCEID1_EL0	94	/* Common Event Identification Register 1 */
>>>> +#define PMEVCNTR0_EL0	95	/* Event Counter Register (0-30) */
>>>
>>> why do we need these when we trap-and-emulate and we have the kvm_pmc
>>> structs?
>> This just makes the guest work when accessing these registers.
>>
>>> Is that because the kvm_pmc structs are only used when we
>>> actually have an active counter running and registered with perf?
>>>
>>
>> Right, the kvm_pmc structs are used to store the status of perf evnets,
>> like the event type, count number of this perf event.
>>
>> On the other hand, the kernel perf codes will not directly access to the
>> PMEVCNTRx_EL0 and PMEVTYPERx_EL0 registers. It will firstly write the
>> index of select counter to PMSELR_EL0 and access to PMXEVCNTR_EL0 or
>> PMXEVTYPER_EL0. Then this is architecturally mapped to PMEVCNTRx_EL0 and
>> PMEVTYPERx_EL0.
>>
>
> I'm just wondering if it makes sense to keep virtual state around for
> all these registers, since we don't emulate the counter values, so why
> do we need to preserve any virtual cpu state for all of them?
>

Good point. Will remove this :)

Thanks,
-- 
Shannon

[PATCH 04/18] KVM: ARM64: Add reset and access handlers for PMCR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Add reset handler which gets host value of PMCR_EL0 and make writable
bits architecturally UNKNOWN. Add access handler which emulates
writing and reading PMCR_EL0 register.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index c370b40..152ee17 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -33,6 +33,7 @@
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_host.h>
 #include <asm/kvm_mmu.h>
+#include <asm/pmu.h>
 
 #include <trace/events/kvm.h>
 
@@ -236,6 +237,44 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
 }
 
+static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
+{
+	u32 pmcr;
+
+	asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
+	vcpu_sys_reg(vcpu, PMCR_EL0) = (pmcr & ~ARMV8_PMCR_MASK)
+				       | (ARMV8_PMCR_MASK & 0xdecafbad);
+}
+
+/* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
+static bool access_pmcr(struct kvm_vcpu *vcpu,
+			const struct sys_reg_params *p,
+			const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		/* Only update writeable bits of PMCR */
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		val &= ~ARMV8_PMCR_MASK;
+		val |= *vcpu_reg(vcpu, p->Rt) & ARMV8_PMCR_MASK;
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) = val;
+		else
+			vcpu_cp15(vcpu, r->reg) = val;
+	} else {
+		if (!p->is_aarch32)
+			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
+		else
+			*vcpu_reg(vcpu, p->Rt) = vcpu_cp15(vcpu, r->reg);
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -427,7 +466,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 
 	/* PMCR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
-	  trap_raz_wi },
+	  access_pmcr, reset_pmcr_el0, PMCR_EL0, },
 	/* PMCNTENSET_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
 	  trap_raz_wi },
-- 
2.1.0

[PATCH 04/18] KVM: ARM64: Add reset and access handlers for PMCR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:34AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Add reset handler which gets host value of PMCR_EL0 and make writable
> bits architecturally UNKNOWN. Add access handler which emulates
> writing and reading PMCR_EL0 register.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 40 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index c370b40..152ee17 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -33,6 +33,7 @@
>  #include <asm/kvm_emulate.h>
>  #include <asm/kvm_host.h>
>  #include <asm/kvm_mmu.h>
> +#include <asm/pmu.h>
>  
>  #include <trace/events/kvm.h>
>  
> @@ -236,6 +237,44 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
>  	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
>  }
>  
> +static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
> +{
> +	u32 pmcr;
> +
> +	asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
> +	vcpu_sys_reg(vcpu, PMCR_EL0) = (pmcr & ~ARMV8_PMCR_MASK)
> +				       | (ARMV8_PMCR_MASK & 0xdecafbad);

You could add a comment that this resets to UNKNOWN as to not make
people confused about the pseudo-random hex value.

Have we thought about whether we want to tell the guest that it has the
same PMU as available on the real hardware, or does the virtualization
layer suggest to us that we should adjust this somehow?


> +}
> +
> +/* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
> +static bool access_pmcr(struct kvm_vcpu *vcpu,
> +			const struct sys_reg_params *p,
> +			const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		/* Only update writeable bits of PMCR */
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);

don't you need to add this function as the handler in the cp15_regs
array as well?

> +		val &= ~ARMV8_PMCR_MASK;
> +		val |= *vcpu_reg(vcpu, p->Rt) & ARMV8_PMCR_MASK;
> +		if (!p->is_aarch32)
> +			vcpu_sys_reg(vcpu, r->reg) = val;
> +		else
> +			vcpu_cp15(vcpu, r->reg) = val;
> +	} else {
> +		if (!p->is_aarch32)
> +			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			*vcpu_reg(vcpu, p->Rt) = vcpu_cp15(vcpu, r->reg);
> +	}
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -427,7 +466,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  
>  	/* PMCR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
> -	  trap_raz_wi },
> +	  access_pmcr, reset_pmcr_el0, PMCR_EL0, },
>  	/* PMCNTENSET_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
>  	  trap_raz_wi },
> -- 
> 2.1.0
> 
Thanks,
-Christoffer

[PATCH 04/18] KVM: ARM64: Add reset and access handlers for PMCR_EL0 register

[Shannon Zhao]

On 2015/7/17 3:55, Christoffer Dall wrote:
> On Mon, Jul 06, 2015 at 10:17:34AM +0800, shannon.zhao at linaro.org wrote:
>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>
>> Add reset handler which gets host value of PMCR_EL0 and make writable
>> bits architecturally UNKNOWN. Add access handler which emulates
>> writing and reading PMCR_EL0 register.
>>
>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>> ---
>>  arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
>>  1 file changed, 40 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
>> index c370b40..152ee17 100644
>> --- a/arch/arm64/kvm/sys_regs.c
>> +++ b/arch/arm64/kvm/sys_regs.c
>> @@ -33,6 +33,7 @@
>>  #include <asm/kvm_emulate.h>
>>  #include <asm/kvm_host.h>
>>  #include <asm/kvm_mmu.h>
>> +#include <asm/pmu.h>
>>  
>>  #include <trace/events/kvm.h>
>>  
>> @@ -236,6 +237,44 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
>>  	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
>>  }
>>  
>> +static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
>> +{
>> +	u32 pmcr;
>> +
>> +	asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
>> +	vcpu_sys_reg(vcpu, PMCR_EL0) = (pmcr & ~ARMV8_PMCR_MASK)
>> +				       | (ARMV8_PMCR_MASK & 0xdecafbad);
> 
> You could add a comment that this resets to UNKNOWN as to not make
> people confused about the pseudo-random hex value.
> 
Ok.

> Have we thought about whether we want to tell the guest that it has the
> same PMU as available on the real hardware, or does the virtualization
> layer suggest to us that we should adjust this somehow?
> 
I guess here the number of PMU counters is what we can adjust, right?
Are we worried about that the host will run out of counters when guest
and host register lots of events?
The PMU of cortex-a57 has 6 counters. IIUC, if one of the guest vcpu
process registers 6 events and some host process register 6 events too,
these events will be monitored in real hardware PMU counter when the
related process runs on the cpu. And when other processes are scheduled
to run, it will switch the contexts of PMU counters.

> 
>> +}
>> +
>> +/* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
>> +static bool access_pmcr(struct kvm_vcpu *vcpu,
>> +			const struct sys_reg_params *p,
>> +			const struct sys_reg_desc *r)
>> +{
>> +	unsigned long val;
>> +
>> +	if (p->is_write) {
>> +		/* Only update writeable bits of PMCR */
>> +		if (!p->is_aarch32)
>> +			val = vcpu_sys_reg(vcpu, r->reg);
>> +		else
>> +			val = vcpu_cp15(vcpu, r->reg);
> 
> don't you need to add this function as the handler in the cp15_regs
> array as well?
> 
Sorry, I'm not very clear about this. Will look at the codes and
understand the use of cp15_regs.

>> +		val &= ~ARMV8_PMCR_MASK;
>> +		val |= *vcpu_reg(vcpu, p->Rt) & ARMV8_PMCR_MASK;
>> +		if (!p->is_aarch32)
>> +			vcpu_sys_reg(vcpu, r->reg) = val;
>> +		else
>> +			vcpu_cp15(vcpu, r->reg) = val;
>> +	} else {
>> +		if (!p->is_aarch32)
>> +			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
>> +		else
>> +			*vcpu_reg(vcpu, p->Rt) = vcpu_cp15(vcpu, r->reg);
>> +	}
>> +
>> +	return true;
>> +}
>> +
>>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>>  	/* DBGBVRn_EL1 */						\
>> @@ -427,7 +466,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>>  
>>  	/* PMCR_EL0 */
>>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
>> -	  trap_raz_wi },
>> +	  access_pmcr, reset_pmcr_el0, PMCR_EL0, },
>>  	/* PMCNTENSET_EL0 */
>>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
>>  	  trap_raz_wi },
>> -- 
>> 2.1.0
>>
> Thanks,
> -Christoffer
> 

-- 
Shannon

[PATCH 04/18] KVM: ARM64: Add reset and access handlers for PMCR_EL0 register

[Christoffer Dall]

On Fri, Jul 17, 2015 at 04:45:44PM +0800, Shannon Zhao wrote:
> 
> 
> On 2015/7/17 3:55, Christoffer Dall wrote:
> > On Mon, Jul 06, 2015 at 10:17:34AM +0800, shannon.zhao at linaro.org wrote:
> >> From: Shannon Zhao <shannon.zhao@linaro.org>
> >>
> >> Add reset handler which gets host value of PMCR_EL0 and make writable
> >> bits architecturally UNKNOWN. Add access handler which emulates
> >> writing and reading PMCR_EL0 register.
> >>
> >> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> >> ---
> >>  arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
> >>  1 file changed, 40 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> >> index c370b40..152ee17 100644
> >> --- a/arch/arm64/kvm/sys_regs.c
> >> +++ b/arch/arm64/kvm/sys_regs.c
> >> @@ -33,6 +33,7 @@
> >>  #include <asm/kvm_emulate.h>
> >>  #include <asm/kvm_host.h>
> >>  #include <asm/kvm_mmu.h>
> >> +#include <asm/pmu.h>
> >>  
> >>  #include <trace/events/kvm.h>
> >>  
> >> @@ -236,6 +237,44 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
> >>  	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
> >>  }
> >>  
> >> +static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
> >> +{
> >> +	u32 pmcr;
> >> +
> >> +	asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
> >> +	vcpu_sys_reg(vcpu, PMCR_EL0) = (pmcr & ~ARMV8_PMCR_MASK)
> >> +				       | (ARMV8_PMCR_MASK & 0xdecafbad);
> > 
> > You could add a comment that this resets to UNKNOWN as to not make
> > people confused about the pseudo-random hex value.
> > 
> Ok.
> 
> > Have we thought about whether we want to tell the guest that it has the
> > same PMU as available on the real hardware, or does the virtualization
> > layer suggest to us that we should adjust this somehow?
> > 
> I guess here the number of PMU counters is what we can adjust, right?
> Are we worried about that the host will run out of counters when guest
> and host register lots of events?

that's what I wonder; if perf itself reserves a counter for example,
then we'll at best be able to measure with N-1 counters for the guest (N
being the number of counters on the physical CPU), so why tell the guest
we have N counters?

Of course, we can never even be guaranteed to have N-1 counters
avaialable either, so maybe the sane choice is just to tell the guest
what kind of hardware we have, and then fiddle the best we can with the
remaining counters?  After all, correct functionality of the guest
doesn't depend on this, it's a best-effort kind of thing...

Thoughts?

> The PMU of cortex-a57 has 6 counters. IIUC, if one of the guest vcpu
> process registers 6 events and some host process register 6 events too,
> these events will be monitored in real hardware PMU counter when the
> related process runs on the cpu. And when other processes are scheduled
> to run, it will switch the contexts of PMU counters.

That depends on the way the counters are used by perf I think.  What if
you have system wide events?  What if the KVM (vcpu) process itself is
being monitored for some events?

> 
> > 
> >> +}
> >> +
> >> +/* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
> >> +static bool access_pmcr(struct kvm_vcpu *vcpu,
> >> +			const struct sys_reg_params *p,
> >> +			const struct sys_reg_desc *r)
> >> +{
> >> +	unsigned long val;
> >> +
> >> +	if (p->is_write) {
> >> +		/* Only update writeable bits of PMCR */
> >> +		if (!p->is_aarch32)
> >> +			val = vcpu_sys_reg(vcpu, r->reg);
> >> +		else
> >> +			val = vcpu_cp15(vcpu, r->reg);
> > 
> > don't you need to add this function as the handler in the cp15_regs
> > array as well?
> > 
> Sorry, I'm not very clear about this. Will look at the codes and
> understand the use of cp15_regs.
> 

I think the point is that you cannot use the same value of r->reg to
index into both arrays, so the cp15 index must be passed from the
cp15_regs array first.

Thanks,
-Christoffer

[PATCH 04/18] KVM: ARM64: Add reset and access handlers for PMCR_EL0 register

[Shannon Zhao]

On 2015/7/17 18:21, Christoffer Dall wrote:
> On Fri, Jul 17, 2015 at 04:45:44PM +0800, Shannon Zhao wrote:
>>
>>
>> On 2015/7/17 3:55, Christoffer Dall wrote:
>>> On Mon, Jul 06, 2015 at 10:17:34AM +0800, shannon.zhao at linaro.org wrote:
>>>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>>>
>>>> Add reset handler which gets host value of PMCR_EL0 and make writable
>>>> bits architecturally UNKNOWN. Add access handler which emulates
>>>> writing and reading PMCR_EL0 register.
>>>>
>>>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>>>> ---
>>>>  arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
>>>>  1 file changed, 40 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
>>>> index c370b40..152ee17 100644
>>>> --- a/arch/arm64/kvm/sys_regs.c
>>>> +++ b/arch/arm64/kvm/sys_regs.c
>>>> @@ -33,6 +33,7 @@
>>>>  #include <asm/kvm_emulate.h>
>>>>  #include <asm/kvm_host.h>
>>>>  #include <asm/kvm_mmu.h>
>>>> +#include <asm/pmu.h>
>>>>  
>>>>  #include <trace/events/kvm.h>
>>>>  
>>>> @@ -236,6 +237,44 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
>>>>  	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
>>>>  }
>>>>  
>>>> +static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
>>>> +{
>>>> +	u32 pmcr;
>>>> +
>>>> +	asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
>>>> +	vcpu_sys_reg(vcpu, PMCR_EL0) = (pmcr & ~ARMV8_PMCR_MASK)
>>>> +				       | (ARMV8_PMCR_MASK & 0xdecafbad);
>>>
>>> You could add a comment that this resets to UNKNOWN as to not make
>>> people confused about the pseudo-random hex value.
>>>
>> Ok.
>>
>>> Have we thought about whether we want to tell the guest that it has the
>>> same PMU as available on the real hardware, or does the virtualization
>>> layer suggest to us that we should adjust this somehow?
>>>
>> I guess here the number of PMU counters is what we can adjust, right?
>> Are we worried about that the host will run out of counters when guest
>> and host register lots of events?
> 
> that's what I wonder; if perf itself reserves a counter for example,
> then we'll at best be able to measure with N-1 counters for the guest (N
> being the number of counters on the physical CPU), so why tell the guest
> we have N counters?
> 

I'm not sure whether perf itself reserves one counter. Here I just pass
the hardware information to guest.

> Of course, we can never even be guaranteed to have N-1 counters
> avaialable either, so maybe the sane choice is just to tell the guest
> what kind of hardware we have, and then fiddle the best we can with the
> remaining counters?  After all, correct functionality of the guest
> doesn't depend on this, it's a best-effort kind of thing...
> 
> Thoughts?
> 
>> The PMU of cortex-a57 has 6 counters. IIUC, if one of the guest vcpu
>> process registers 6 events and some host process register 6 events too,
>> these events will be monitored in real hardware PMU counter when the
>> related process runs on the cpu. And when other processes are scheduled
>> to run, it will switch the contexts of PMU counters.
> 
> That depends on the way the counters are used by perf I think.  What if
> you have system wide events?  What if the KVM (vcpu) process itself is
> being monitored for some events?
> 

Not sure what will happen when the number of monitored events is greater
than counters. I guess the perf layer may balance the events?

>>
>>>
>>>> +}
>>>> +
>>>> +/* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
>>>> +static bool access_pmcr(struct kvm_vcpu *vcpu,
>>>> +			const struct sys_reg_params *p,
>>>> +			const struct sys_reg_desc *r)
>>>> +{
>>>> +	unsigned long val;
>>>> +
>>>> +	if (p->is_write) {
>>>> +		/* Only update writeable bits of PMCR */
>>>> +		if (!p->is_aarch32)
>>>> +			val = vcpu_sys_reg(vcpu, r->reg);
>>>> +		else
>>>> +			val = vcpu_cp15(vcpu, r->reg);
>>>
>>> don't you need to add this function as the handler in the cp15_regs
>>> array as well?
>>>
>> Sorry, I'm not very clear about this. Will look at the codes and
>> understand the use of cp15_regs.
>>
> 
> I think the point is that you cannot use the same value of r->reg to
> index into both arrays, so the cp15 index must be passed from the
> cp15_regs array first.
> 
> Thanks,
> -Christoffer
> 

-- 
Shannon

[PATCH 04/18] KVM: ARM64: Add reset and access handlers for PMCR_EL0 register

[Christoffer Dall]

On Tue, Jul 21, 2015 at 09:16:57AM +0800, Shannon Zhao wrote:
> 
> 
> On 2015/7/17 18:21, Christoffer Dall wrote:
> > On Fri, Jul 17, 2015 at 04:45:44PM +0800, Shannon Zhao wrote:
> >>
> >>
> >> On 2015/7/17 3:55, Christoffer Dall wrote:
> >>> On Mon, Jul 06, 2015 at 10:17:34AM +0800, shannon.zhao at linaro.org wrote:
> >>>> From: Shannon Zhao <shannon.zhao@linaro.org>
> >>>>
> >>>> Add reset handler which gets host value of PMCR_EL0 and make writable
> >>>> bits architecturally UNKNOWN. Add access handler which emulates
> >>>> writing and reading PMCR_EL0 register.
> >>>>
> >>>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> >>>> ---
> >>>>  arch/arm64/kvm/sys_regs.c | 41 ++++++++++++++++++++++++++++++++++++++++-
> >>>>  1 file changed, 40 insertions(+), 1 deletion(-)
> >>>>
> >>>> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> >>>> index c370b40..152ee17 100644
> >>>> --- a/arch/arm64/kvm/sys_regs.c
> >>>> +++ b/arch/arm64/kvm/sys_regs.c
> >>>> @@ -33,6 +33,7 @@
> >>>>  #include <asm/kvm_emulate.h>
> >>>>  #include <asm/kvm_host.h>
> >>>>  #include <asm/kvm_mmu.h>
> >>>> +#include <asm/pmu.h>
> >>>>  
> >>>>  #include <trace/events/kvm.h>
> >>>>  
> >>>> @@ -236,6 +237,44 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
> >>>>  	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
> >>>>  }
> >>>>  
> >>>> +static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
> >>>> +{
> >>>> +	u32 pmcr;
> >>>> +
> >>>> +	asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
> >>>> +	vcpu_sys_reg(vcpu, PMCR_EL0) = (pmcr & ~ARMV8_PMCR_MASK)
> >>>> +				       | (ARMV8_PMCR_MASK & 0xdecafbad);
> >>>
> >>> You could add a comment that this resets to UNKNOWN as to not make
> >>> people confused about the pseudo-random hex value.
> >>>
> >> Ok.
> >>
> >>> Have we thought about whether we want to tell the guest that it has the
> >>> same PMU as available on the real hardware, or does the virtualization
> >>> layer suggest to us that we should adjust this somehow?
> >>>
> >> I guess here the number of PMU counters is what we can adjust, right?
> >> Are we worried about that the host will run out of counters when guest
> >> and host register lots of events?
> > 
> > that's what I wonder; if perf itself reserves a counter for example,
> > then we'll at best be able to measure with N-1 counters for the guest (N
> > being the number of counters on the physical CPU), so why tell the guest
> > we have N counters?
> > 
> 
> I'm not sure whether perf itself reserves one counter. Here I just pass
> the hardware information to guest.
> 
> > Of course, we can never even be guaranteed to have N-1 counters
> > avaialable either, so maybe the sane choice is just to tell the guest
> > what kind of hardware we have, and then fiddle the best we can with the
> > remaining counters?  After all, correct functionality of the guest
> > doesn't depend on this, it's a best-effort kind of thing...
> > 
> > Thoughts?
> > 
> >> The PMU of cortex-a57 has 6 counters. IIUC, if one of the guest vcpu
> >> process registers 6 events and some host process register 6 events too,
> >> these events will be monitored in real hardware PMU counter when the
> >> related process runs on the cpu. And when other processes are scheduled
> >> to run, it will switch the contexts of PMU counters.
> > 
> > That depends on the way the counters are used by perf I think.  What if
> > you have system wide events?  What if the KVM (vcpu) process itself is
> > being monitored for some events?
> > 
> 
> Not sure what will happen when the number of monitored events is greater
> than counters. I guess the perf layer may balance the events?
> 
I'm not sure either, but we need to have thought about these things, I
suppose.  You should probably look at the code or try it out.

Thanks,
-Christoffer

[PATCH 05/18] KVM: ARM64: Add reset and access handlers for PMSELR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMSELR_EL0 is UNKNOWN, use reset_unknown for
its reset handler. Add access handler which emulates writing and reading
PMSELR_EL0 register.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 152ee17..69c8c48 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -275,6 +275,30 @@ static bool access_pmcr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMSELR_EL0 accessor. */
+static bool access_pmselr(struct kvm_vcpu *vcpu,
+			  const struct sys_reg_params *p,
+			  const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) = val;
+		else
+			vcpu_cp15(vcpu, r->reg) = val & 0xffffffffUL;
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -481,7 +505,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  trap_raz_wi },
 	/* PMSELR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
-	  trap_raz_wi },
+	  access_pmselr, reset_unknown, PMSELR_EL0 },
 	/* PMCEID0_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
 	  trap_raz_wi },
-- 
2.1.0

[PATCH 06/18] KVM: ARM64: Add reset and access handlers for PMCEID0_EL0 and PMCEID1_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Add reset handler which gets host value of PMCEID0_EL0 or PMCEID1_EL0.
Add access handler which emulates writing and reading PMCEID0_EL0 or
PMCEID1_EL0 register.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 69c8c48..1df9ef3 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -246,6 +246,19 @@ static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 				       | (ARMV8_PMCR_MASK & 0xdecafbad);
 }
 
+static void reset_pmceid(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
+{
+	u32 pmceid;
+
+	if (r->reg == PMCEID0_EL0) {
+		asm volatile("mrs %0, pmceid0_el0\n" : "=r" (pmceid));
+		vcpu_sys_reg(vcpu, r->reg) = pmceid;
+	} else {
+		asm volatile("mrs %0, pmceid1_el0\n" : "=r" (pmceid));
+		vcpu_sys_reg(vcpu, r->reg) = pmceid;
+	}
+}
+
 /* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
 static bool access_pmcr(struct kvm_vcpu *vcpu,
 			const struct sys_reg_params *p,
@@ -299,6 +312,25 @@ static bool access_pmselr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMCEID0_EL0 and PMCEID1_EL0 accessor. */
+static bool access_pmceid(struct kvm_vcpu *vcpu,
+			  const struct sys_reg_params *p,
+			  const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		return ignore_write(vcpu, p);
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+		return true;
+	}
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -508,10 +540,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmselr, reset_unknown, PMSELR_EL0 },
 	/* PMCEID0_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
-	  trap_raz_wi },
+	  access_pmceid, reset_pmceid, PMCEID0_EL0, },
 	/* PMCEID1_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
-	  trap_raz_wi },
+	  access_pmceid, reset_pmceid, PMCEID1_EL0, },
 	/* PMCCNTR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
 	  trap_raz_wi },
-- 
2.1.0

[PATCH 06/18] KVM: ARM64: Add reset and access handlers for PMCEID0_EL0 and PMCEID1_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:36AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Add reset handler which gets host value of PMCEID0_EL0 or PMCEID1_EL0.
> Add access handler which emulates writing and reading PMCEID0_EL0 or
> PMCEID1_EL0 register.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 36 ++++++++++++++++++++++++++++++++++--
>  1 file changed, 34 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 69c8c48..1df9ef3 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -246,6 +246,19 @@ static void reset_pmcr_el0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
>  				       | (ARMV8_PMCR_MASK & 0xdecafbad);
>  }
>  
> +static void reset_pmceid(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
> +{
> +	u32 pmceid;
> +
> +	if (r->reg == PMCEID0_EL0) {
> +		asm volatile("mrs %0, pmceid0_el0\n" : "=r" (pmceid));
> +		vcpu_sys_reg(vcpu, r->reg) = pmceid;
> +	} else {

add /* PMCEID1_EL1 */
> +		asm volatile("mrs %0, pmceid1_el0\n" : "=r" (pmceid));
> +		vcpu_sys_reg(vcpu, r->reg) = pmceid;
> +	}
> +}
> +
>  /* PMCR_EL0 accessor. Only called as long as MDCR_EL2.TPMCR is set. */
>  static bool access_pmcr(struct kvm_vcpu *vcpu,
>  			const struct sys_reg_params *p,
> @@ -299,6 +312,25 @@ static bool access_pmselr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMCEID0_EL0 and PMCEID1_EL0 accessor. */
> +static bool access_pmceid(struct kvm_vcpu *vcpu,
> +			  const struct sys_reg_params *p,
> +			  const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		return ignore_write(vcpu, p);
> +	} else {
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +		return true;
> +	}
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -508,10 +540,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmselr, reset_unknown, PMSELR_EL0 },
>  	/* PMCEID0_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
> -	  trap_raz_wi },
> +	  access_pmceid, reset_pmceid, PMCEID0_EL0, },
>  	/* PMCEID1_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
> -	  trap_raz_wi },
> +	  access_pmceid, reset_pmceid, PMCEID1_EL0, },
>  	/* PMCCNTR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
>  	  trap_raz_wi },
> -- 
> 2.1.0
> 
same comment wrt. 32-bit table.

-Christoffer

[PATCH 07/18] KVM: ARM64: PMU: Add perf event map and introduce perf event creating function

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

When we use tools like perf on host, perf passes the event type and the
id in this type category to kernel, then kernel will map them to event
number and write this number to PMU PMEVTYPER<n>_EL0 register. While
we're trapping and emulating guest accesses to PMU registers, we get the
event number and map it to the event type and the id reversely.

Check whether the event type is same with the one to be set. If not,
stop counter to monitor current event and find the event type map id.
According to the bits of data to configure this perf_event attr and
set exclude_host to 1 for guest. Then call perf_event API to create the
corresponding event and save the event pointer.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 include/kvm/arm_pmu.h |   4 ++
 virt/kvm/arm/pmu.c    | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 177 insertions(+)

diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 27d14ca..1050b24 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -45,9 +45,13 @@ struct kvm_pmu {
 
 #ifdef CONFIG_KVM_ARM_PMU
 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
+void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
+				    unsigned long select_idx);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
 #else
 static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
+void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
+				    unsigned long select_idx) {}
 static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
 #endif
 
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index dc252d0..50a3c82 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -18,8 +18,68 @@
 #include <linux/cpu.h>
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
+#include <linux/perf_event.h>
 #include <kvm/arm_pmu.h>
 
+/* PMU HW events mapping. */
+static struct kvm_pmu_hw_event_map {
+	unsigned eventsel;
+	unsigned event_type;
+} kvm_pmu_hw_events[] = {
+	[0] = { 0x11, PERF_COUNT_HW_CPU_CYCLES },
+	[1] = { 0x08, PERF_COUNT_HW_INSTRUCTIONS },
+	[2] = { 0x04, PERF_COUNT_HW_CACHE_REFERENCES },
+	[3] = { 0x03, PERF_COUNT_HW_CACHE_MISSES },
+	[4] = { 0x10, PERF_COUNT_HW_BRANCH_MISSES },
+};
+
+/* PMU HW cache events mapping. */
+static struct kvm_pmu_hw_cache_event_map {
+	unsigned eventsel;
+	unsigned cache_type;
+	unsigned cache_op;
+	unsigned cache_result;
+} kvm_pmu_hw_cache_events[] = {
+	[0] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
+		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
+	[1] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
+		      PERF_COUNT_HW_CACHE_RESULT_MISS },
+	[2] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
+		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
+	[3] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
+		      PERF_COUNT_HW_CACHE_RESULT_MISS },
+	[4] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
+		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
+	[5] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
+		      PERF_COUNT_HW_CACHE_RESULT_MISS },
+	[6] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
+		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
+	[7] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
+		      PERF_COUNT_HW_CACHE_RESULT_MISS },
+};
+
+/**
+ * kvm_pmu_stop_counter - stop PMU counter for the selected counter
+ * @vcpu: The vcpu pointer
+ * @select_idx: The counter index
+ *
+ * If this counter has been configured to monitor some event, disable and
+ * release it.
+ */
+static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu,
+				 unsigned long select_idx)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	if (pmc->perf_event) {
+		perf_event_disable(pmc->perf_event);
+		perf_event_release_kernel(pmc->perf_event);
+	}
+	pmc->perf_event = NULL;
+	pmc->eventsel = 0xff;
+}
+
 /**
  * kvm_pmu_vcpu_reset - reset pmu state for cpu
  * @vcpu: The vcpu pointer
@@ -27,12 +87,125 @@
  */
 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
 {
+	int i;
 	struct kvm_pmu *pmu = &vcpu->arch.pmu;
 
+	for (i = 0; i < ARMV8_MAX_COUNTERS; i++)
+		kvm_pmu_stop_counter(vcpu, i);
+	pmu->overflow_status = 0;
 	pmu->irq_pending = false;
 }
 
 /**
+ * kvm_pmu_find_hw_event - find hardware event
+ * @pmu: The pmu pointer
+ * @event_select: The number of selected event type
+ *
+ * Based on the number of selected event type, find out whether it belongs to
+ * PERF_TYPE_HARDWARE. If so, return the corresponding event id.
+ */
+static unsigned kvm_pmu_find_hw_event(struct kvm_pmu *pmu,
+				      unsigned long event_select)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_events); i++)
+		if (kvm_pmu_hw_events[i].eventsel == event_select)
+			break;
+
+	if (i == ARRAY_SIZE(kvm_pmu_hw_events))
+		return PERF_COUNT_HW_MAX;
+
+	return kvm_pmu_hw_events[i].event_type;
+}
+
+/**
+ * kvm_pmu_find_hw_cache_event - find hardware cache event
+ * @pmu: The pmu pointer
+ * @event_select: The number of selected event type
+ *
+ * Based on the number of selected event type, find out whether it belongs to
+ * PERF_TYPE_HW_CACHE. If so, return the corresponding event id.
+ */
+static unsigned kvm_pmu_find_hw_cache_event(struct kvm_pmu *pmu,
+					    unsigned long event_select)
+{
+	int i;
+	unsigned config;
+
+	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_cache_events); i++)
+		if (kvm_pmu_hw_cache_events[i].eventsel == event_select)
+			break;
+
+	if (i == ARRAY_SIZE(kvm_pmu_hw_cache_events))
+		return PERF_COUNT_HW_CACHE_MAX;
+
+	config = (kvm_pmu_hw_cache_events[i].cache_type & 0xff)
+		 | ((kvm_pmu_hw_cache_events[i].cache_op & 0xff) << 8)
+		 | ((kvm_pmu_hw_cache_events[i].cache_result & 0xff) << 16);
+
+	return config;
+}
+
+/**
+ * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
+ * @vcpu: The vcpu pointer
+ * @data: The data guest writes to PMXEVTYPER_EL0
+ * @select_idx: The number of selected counter
+ *
+ * Firstly check whether the event type is same with the one to be set.
+ * If not, stop counter to monitor current event and find the event type map id.
+ * According to the bits of data to configure this perf_event attr and set
+ * exclude_host to 1 for guest. Then call perf_event API to create the
+ * corresponding event and save the event pointer.
+ */
+void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
+				    unsigned long select_idx)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+	struct perf_event *event;
+	struct perf_event_attr attr;
+	unsigned config, type = PERF_TYPE_RAW;
+
+	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
+		return;
+
+	kvm_pmu_stop_counter(vcpu, select_idx);
+	pmc->eventsel = data & ARMV8_EVTYPE_EVENT;
+
+	config = kvm_pmu_find_hw_event(pmu, pmc->eventsel);
+	if (config != PERF_COUNT_HW_MAX) {
+		type = PERF_TYPE_HARDWARE;
+	} else {
+		config = kvm_pmu_find_hw_cache_event(pmu, pmc->eventsel);
+		if (config != PERF_COUNT_HW_CACHE_MAX)
+			type = PERF_TYPE_HW_CACHE;
+	}
+
+	if (type == PERF_TYPE_RAW)
+		config = pmc->eventsel;
+
+	attr.type = type;
+	attr.size = sizeof(attr);
+	attr.pinned = true;
+	attr.exclude_user = data & ARMV8_EXCLUDE_EL0 ? 1 : 0;
+	attr.exclude_kernel = data & ARMV8_EXCLUDE_EL1 ? 1 : 0;
+	attr.exclude_hv = data & ARMV8_INCLUDE_EL2 ? 0 : 1;
+	attr.exclude_host = 1;
+	attr.config = config;
+	attr.sample_period = (-pmc->counter) & (((u64)1 << 32) - 1);
+
+	event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc);
+	if (IS_ERR(event)) {
+		kvm_err("kvm: pmu event creation failed %ld\n",
+			  PTR_ERR(event));
+		return;
+	}
+	pmc->perf_event = event;
+}
+
+/**
  * kvm_pmu_init - Initialize global PMU state for per vcpu
  * @vcpu: The vcpu pointer
  *
-- 
2.1.0

[PATCH 07/18] KVM: ARM64: PMU: Add perf event map and introduce perf event creating function

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:37AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> When we use tools like perf on host, perf passes the event type and the
> id in this type category to kernel, then kernel will map them to event
> number and write this number to PMU PMEVTYPER<n>_EL0 register. While
> we're trapping and emulating guest accesses to PMU registers, we get the
> event number and map it to the event type and the id reversely.

There's something with the nomenclature that makes this really hard to
read.

you mention here: "event type", "the id", and "event number".  The
former two I think are perf identifiers, and the latter is the hardware
event number, is this right?

> 
> Check whether the event type is same with the one to be set.

when?

> If not,
> stop counter to monitor current event and find the event type map id.
> According to the bits of data to configure this perf_event attr and
> set exclude_host to 1 for guest. Then call perf_event API to create the
> corresponding event and save the event pointer.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  include/kvm/arm_pmu.h |   4 ++
>  virt/kvm/arm/pmu.c    | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 177 insertions(+)
> 
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 27d14ca..1050b24 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -45,9 +45,13 @@ struct kvm_pmu {
>  
>  #ifdef CONFIG_KVM_ARM_PMU
>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
> +				    unsigned long select_idx);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
>  #else
>  static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
> +				    unsigned long select_idx) {}
>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
>  #endif
>  
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index dc252d0..50a3c82 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -18,8 +18,68 @@
>  #include <linux/cpu.h>
>  #include <linux/kvm.h>
>  #include <linux/kvm_host.h>
> +#include <linux/perf_event.h>
>  #include <kvm/arm_pmu.h>
>  
> +/* PMU HW events mapping. */
> +static struct kvm_pmu_hw_event_map {
> +	unsigned eventsel;
> +	unsigned event_type;
> +} kvm_pmu_hw_events[] = {
> +	[0] = { 0x11, PERF_COUNT_HW_CPU_CYCLES },
> +	[1] = { 0x08, PERF_COUNT_HW_INSTRUCTIONS },
> +	[2] = { 0x04, PERF_COUNT_HW_CACHE_REFERENCES },
> +	[3] = { 0x03, PERF_COUNT_HW_CACHE_MISSES },
> +	[4] = { 0x10, PERF_COUNT_HW_BRANCH_MISSES },
> +};
> +
> +/* PMU HW cache events mapping. */
> +static struct kvm_pmu_hw_cache_event_map {
> +	unsigned eventsel;
> +	unsigned cache_type;
> +	unsigned cache_op;
> +	unsigned cache_result;
> +} kvm_pmu_hw_cache_events[] = {
> +	[0] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> +	[1] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> +	[2] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> +	[3] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },

seems to me that the four entries above will never be used, because
you'll always match them in kvm_pmu_hw_events above?

Is this because perf map multiple generic perf events to the same
hardware event?  Does it matter if we register this with perf as one or
the other then?

> +	[4] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> +	[5] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> +	[6] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> +	[7] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> +};
> +
> +/**
> + * kvm_pmu_stop_counter - stop PMU counter for the selected counter
> + * @vcpu: The vcpu pointer
> + * @select_idx: The counter index
> + *
> + * If this counter has been configured to monitor some event, disable and
> + * release it.
> + */
> +static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu,
> +				 unsigned long select_idx)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	if (pmc->perf_event) {
> +		perf_event_disable(pmc->perf_event);
> +		perf_event_release_kernel(pmc->perf_event);
> +	}
> +	pmc->perf_event = NULL;
> +	pmc->eventsel = 0xff;

why is 0xff 'unused' or reserved?  If we're choosing this arbitrarily,
why is it not 0x3ff?  Should we create a define for this?

> +}
> +
>  /**
>   * kvm_pmu_vcpu_reset - reset pmu state for cpu
>   * @vcpu: The vcpu pointer
> @@ -27,12 +87,125 @@
>   */
>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
>  {
> +	int i;
>  	struct kvm_pmu *pmu = &vcpu->arch.pmu;
>  
> +	for (i = 0; i < ARMV8_MAX_COUNTERS; i++)
> +		kvm_pmu_stop_counter(vcpu, i);
> +	pmu->overflow_status = 0;
>  	pmu->irq_pending = false;
>  }
>  
>  /**
> + * kvm_pmu_find_hw_event - find hardware event
> + * @pmu: The pmu pointer
> + * @event_select: The number of selected event type
> + *
> + * Based on the number of selected event type, find out whether it belongs to
> + * PERF_TYPE_HARDWARE. If so, return the corresponding event id.
> + */
> +static unsigned kvm_pmu_find_hw_event(struct kvm_pmu *pmu,
> +				      unsigned long event_select)
> +{
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_events); i++)
> +		if (kvm_pmu_hw_events[i].eventsel == event_select)
> +			break;
> +
> +	if (i == ARRAY_SIZE(kvm_pmu_hw_events))
> +		return PERF_COUNT_HW_MAX;
> +
> +	return kvm_pmu_hw_events[i].event_type;

you can just return this directly in the loop if you have a match and
unconditionally return PERF_COUNT_HW_MAX outside the loop without having
to check the loop condition.

> +}
> +
> +/**
> + * kvm_pmu_find_hw_cache_event - find hardware cache event
> + * @pmu: The pmu pointer
> + * @event_select: The number of selected event type
> + *
> + * Based on the number of selected event type, find out whether it belongs to
> + * PERF_TYPE_HW_CACHE. If so, return the corresponding event id.
> + */
> +static unsigned kvm_pmu_find_hw_cache_event(struct kvm_pmu *pmu,
> +					    unsigned long event_select)
> +{
> +	int i;
> +	unsigned config;
> +
> +	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_cache_events); i++)
> +		if (kvm_pmu_hw_cache_events[i].eventsel == event_select)
> +			break;
> +
> +	if (i == ARRAY_SIZE(kvm_pmu_hw_cache_events))
> +		return PERF_COUNT_HW_CACHE_MAX;

I feel like I just read this code, can we reuse it with a pointer to the
array?

> +
> +	config = (kvm_pmu_hw_cache_events[i].cache_type & 0xff)
> +		 | ((kvm_pmu_hw_cache_events[i].cache_op & 0xff) << 8)
> +		 | ((kvm_pmu_hw_cache_events[i].cache_result & 0xff) << 16);
> +
> +	return config;
> +}
> +
> +/**
> + * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
> + * @vcpu: The vcpu pointer
> + * @data: The data guest writes to PMXEVTYPER_EL0
> + * @select_idx: The number of selected counter
> + *
> + * Firstly check whether the event type is same with the one to be set.
> + * If not, stop counter to monitor current event and find the event type map id.
> + * According to the bits of data to configure this perf_event attr and set
> + * exclude_host to 1 for guest. Then call perf_event API to create the
> + * corresponding event and save the event pointer.

This text seems to be describing more how the function does something,
as opposed to what it does and why.  I found it a little hard to read.

> + */
> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
> +				    unsigned long select_idx)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +	struct perf_event *event;
> +	struct perf_event_attr attr;
> +	unsigned config, type = PERF_TYPE_RAW;
> +
> +	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
> +		return;
> +
> +	kvm_pmu_stop_counter(vcpu, select_idx);
> +	pmc->eventsel = data & ARMV8_EVTYPE_EVENT;
> +
> +	config = kvm_pmu_find_hw_event(pmu, pmc->eventsel);
> +	if (config != PERF_COUNT_HW_MAX) {
> +		type = PERF_TYPE_HARDWARE;
> +	} else {
> +		config = kvm_pmu_find_hw_cache_event(pmu, pmc->eventsel);
> +		if (config != PERF_COUNT_HW_CACHE_MAX)
> +			type = PERF_TYPE_HW_CACHE;
> +	}
> +
> +	if (type == PERF_TYPE_RAW)
> +		config = pmc->eventsel;

don't you need to memset attr to 0 first?

otherwise, how do you ensure that for example exclude_guest is always
clear?

> +
> +	attr.type = type;
> +	attr.size = sizeof(attr);
> +	attr.pinned = true;
> +	attr.exclude_user = data & ARMV8_EXCLUDE_EL0 ? 1 : 0;
> +	attr.exclude_kernel = data & ARMV8_EXCLUDE_EL1 ? 1 : 0;
> +	attr.exclude_hv = data & ARMV8_INCLUDE_EL2 ? 0 : 1;

should the guest be able to see something counted in the hypervisor ever
or should that only be the host being able to see that?

my gut feeling is that the hypervisor should be hidden from the guest
and that exclude_hv = 0, is the right choice.  But this is a question
about the semantics of perf, I suppose.

> +	attr.exclude_host = 1;
> +	attr.config = config;
> +	attr.sample_period = (-pmc->counter) & (((u64)1 << 32) - 1);

whoa, what is this scary calculation?

definitely needs an explanation?

> +
> +	event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc);
> +	if (IS_ERR(event)) {
> +		kvm_err("kvm: pmu event creation failed %ld\n",
> +			  PTR_ERR(event));

doesn't this mean we'll spam the kernel log if the guest supplies
bogus/unsupported event numbers?

In that case it shoudl be kvm_debug and the guest should be able to see
this somehow (e.g. events don't count).

> +		return;
> +	}
> +	pmc->perf_event = event;
> +}
> +
> +/**
>   * kvm_pmu_init - Initialize global PMU state for per vcpu
>   * @vcpu: The vcpu pointer
>   *
> -- 
> 2.1.0
> 

[PATCH 07/18] KVM: ARM64: PMU: Add perf event map and introduce perf event creating function

[Shannon Zhao]

On 2015/7/17 22:30, Christoffer Dall wrote:
> On Mon, Jul 06, 2015 at 10:17:37AM +0800, shannon.zhao at linaro.org wrote:
>> From: Shannon Zhao <shannon.zhao@linaro.org>
>>
>> When we use tools like perf on host, perf passes the event type and the
>> id in this type category to kernel, then kernel will map them to event
>> number and write this number to PMU PMEVTYPER<n>_EL0 register. While
>> we're trapping and emulating guest accesses to PMU registers, we get the
>> event number and map it to the event type and the id reversely.
> 
> There's something with the nomenclature that makes this really hard to
> read.
> 
> you mention here: "event type", "the id", and "event number".  The
> former two I think are perf identifiers, and the latter is the hardware
> event number, is this right?
> 

Yeah, right.

>>
>> Check whether the event type is same with the one to be set.
> 
> when?
> 
In function kvm_pmu_set_counter_event_type before create a new perf event.

>> +	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
>> +		return;


>> If not,
>> stop counter to monitor current event and find the event type map id.
>> According to the bits of data to configure this perf_event attr and
>> set exclude_host to 1 for guest. Then call perf_event API to create the
>> corresponding event and save the event pointer.
>>
>> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
>> ---
>>  include/kvm/arm_pmu.h |   4 ++
>>  virt/kvm/arm/pmu.c    | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 177 insertions(+)
>>
>> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
>> index 27d14ca..1050b24 100644
>> --- a/include/kvm/arm_pmu.h
>> +++ b/include/kvm/arm_pmu.h
>> @@ -45,9 +45,13 @@ struct kvm_pmu {
>>  
>>  #ifdef CONFIG_KVM_ARM_PMU
>>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
>> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>> +				    unsigned long select_idx);
>>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
>>  #else
>>  static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
>> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>> +				    unsigned long select_idx) {}
>>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
>>  #endif
>>  
>> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
>> index dc252d0..50a3c82 100644
>> --- a/virt/kvm/arm/pmu.c
>> +++ b/virt/kvm/arm/pmu.c
>> @@ -18,8 +18,68 @@
>>  #include <linux/cpu.h>
>>  #include <linux/kvm.h>
>>  #include <linux/kvm_host.h>
>> +#include <linux/perf_event.h>
>>  #include <kvm/arm_pmu.h>
>>  
>> +/* PMU HW events mapping. */
>> +static struct kvm_pmu_hw_event_map {
>> +	unsigned eventsel;
>> +	unsigned event_type;
>> +} kvm_pmu_hw_events[] = {
>> +	[0] = { 0x11, PERF_COUNT_HW_CPU_CYCLES },
>> +	[1] = { 0x08, PERF_COUNT_HW_INSTRUCTIONS },
>> +	[2] = { 0x04, PERF_COUNT_HW_CACHE_REFERENCES },
>> +	[3] = { 0x03, PERF_COUNT_HW_CACHE_MISSES },
>> +	[4] = { 0x10, PERF_COUNT_HW_BRANCH_MISSES },
>> +};
>> +
>> +/* PMU HW cache events mapping. */
>> +static struct kvm_pmu_hw_cache_event_map {
>> +	unsigned eventsel;
>> +	unsigned cache_type;
>> +	unsigned cache_op;
>> +	unsigned cache_result;
>> +} kvm_pmu_hw_cache_events[] = {
>> +	[0] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
>> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
>> +	[1] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
>> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
>> +	[2] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
>> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
>> +	[3] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
>> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> 
> seems to me that the four entries above will never be used, because
> you'll always match them in kvm_pmu_hw_events above?
> 

Yes, I found this before, but for the completeness I list them.

> Is this because perf map multiple generic perf events to the same
> hardware event? 
I think so.

> Does it matter if we register this with perf as one or
> the other then?
> 
I think it's ok because the hardware event numbers are same.


>> +	[4] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
>> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
>> +	[5] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
>> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
>> +	[6] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
>> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
>> +	[7] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
>> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
>> +};
>> +
>> +/**
>> + * kvm_pmu_stop_counter - stop PMU counter for the selected counter
>> + * @vcpu: The vcpu pointer
>> + * @select_idx: The counter index
>> + *
>> + * If this counter has been configured to monitor some event, disable and
>> + * release it.
>> + */
>> +static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu,
>> +				 unsigned long select_idx)
>> +{
>> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
>> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
>> +
>> +	if (pmc->perf_event) {
>> +		perf_event_disable(pmc->perf_event);
>> +		perf_event_release_kernel(pmc->perf_event);
>> +	}
>> +	pmc->perf_event = NULL;
>> +	pmc->eventsel = 0xff;
> 
> why is 0xff 'unused' or reserved?  If we're choosing this arbitrarily,
> why is it not 0x3ff?  Should we create a define for this?
> 

Yeah, 0x3ff may be better.

>> +}
>> +
>>  /**
>>   * kvm_pmu_vcpu_reset - reset pmu state for cpu
>>   * @vcpu: The vcpu pointer
>> @@ -27,12 +87,125 @@
>>   */
>>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
>>  {
>> +	int i;
>>  	struct kvm_pmu *pmu = &vcpu->arch.pmu;
>>  
>> +	for (i = 0; i < ARMV8_MAX_COUNTERS; i++)
>> +		kvm_pmu_stop_counter(vcpu, i);
>> +	pmu->overflow_status = 0;
>>  	pmu->irq_pending = false;
>>  }
>>  
>>  /**
>> + * kvm_pmu_find_hw_event - find hardware event
>> + * @pmu: The pmu pointer
>> + * @event_select: The number of selected event type
>> + *
>> + * Based on the number of selected event type, find out whether it belongs to
>> + * PERF_TYPE_HARDWARE. If so, return the corresponding event id.
>> + */
>> +static unsigned kvm_pmu_find_hw_event(struct kvm_pmu *pmu,
>> +				      unsigned long event_select)
>> +{
>> +	int i;
>> +
>> +	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_events); i++)
>> +		if (kvm_pmu_hw_events[i].eventsel == event_select)
>> +			break;
>> +
>> +	if (i == ARRAY_SIZE(kvm_pmu_hw_events))
>> +		return PERF_COUNT_HW_MAX;
>> +
>> +	return kvm_pmu_hw_events[i].event_type;
> 
> you can just return this directly in the loop if you have a match and
> unconditionally return PERF_COUNT_HW_MAX outside the loop without having
> to check the loop condition.
> 
ok
>> +}
>> +
>> +/**
>> + * kvm_pmu_find_hw_cache_event - find hardware cache event
>> + * @pmu: The pmu pointer
>> + * @event_select: The number of selected event type
>> + *
>> + * Based on the number of selected event type, find out whether it belongs to
>> + * PERF_TYPE_HW_CACHE. If so, return the corresponding event id.
>> + */
>> +static unsigned kvm_pmu_find_hw_cache_event(struct kvm_pmu *pmu,
>> +					    unsigned long event_select)
>> +{
>> +	int i;
>> +	unsigned config;
>> +
>> +	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_cache_events); i++)
>> +		if (kvm_pmu_hw_cache_events[i].eventsel == event_select)
>> +			break;
>> +
>> +	if (i == ARRAY_SIZE(kvm_pmu_hw_cache_events))
>> +		return PERF_COUNT_HW_CACHE_MAX;
> 
> I feel like I just read this code, can we reuse it with a pointer to the
> array?
> 
>> +
>> +	config = (kvm_pmu_hw_cache_events[i].cache_type & 0xff)
>> +		 | ((kvm_pmu_hw_cache_events[i].cache_op & 0xff) << 8)
>> +		 | ((kvm_pmu_hw_cache_events[i].cache_result & 0xff) << 16);
>> +
>> +	return config;
>> +}
>> +
>> +/**
>> + * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
>> + * @vcpu: The vcpu pointer
>> + * @data: The data guest writes to PMXEVTYPER_EL0
>> + * @select_idx: The number of selected counter
>> + *
>> + * Firstly check whether the event type is same with the one to be set.
>> + * If not, stop counter to monitor current event and find the event type map id.
>> + * According to the bits of data to configure this perf_event attr and set
>> + * exclude_host to 1 for guest. Then call perf_event API to create the
>> + * corresponding event and save the event pointer.
> 
> This text seems to be describing more how the function does something,
> as opposed to what it does and why.  I found it a little hard to read.
> 
>> + */
>> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>> +				    unsigned long select_idx)
>> +{
>> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
>> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
>> +	struct perf_event *event;
>> +	struct perf_event_attr attr;
>> +	unsigned config, type = PERF_TYPE_RAW;
>> +
>> +	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
>> +		return;
>> +
>> +	kvm_pmu_stop_counter(vcpu, select_idx);
>> +	pmc->eventsel = data & ARMV8_EVTYPE_EVENT;
>> +
>> +	config = kvm_pmu_find_hw_event(pmu, pmc->eventsel);
>> +	if (config != PERF_COUNT_HW_MAX) {
>> +		type = PERF_TYPE_HARDWARE;
>> +	} else {
>> +		config = kvm_pmu_find_hw_cache_event(pmu, pmc->eventsel);
>> +		if (config != PERF_COUNT_HW_CACHE_MAX)
>> +			type = PERF_TYPE_HW_CACHE;
>> +	}
>> +
>> +	if (type == PERF_TYPE_RAW)
>> +		config = pmc->eventsel;
> 
> don't you need to memset attr to 0 first?
> 
ok
> otherwise, how do you ensure that for example exclude_guest is always
> clear?
> 
>> +
>> +	attr.type = type;
>> +	attr.size = sizeof(attr);
>> +	attr.pinned = true;
>> +	attr.exclude_user = data & ARMV8_EXCLUDE_EL0 ? 1 : 0;
>> +	attr.exclude_kernel = data & ARMV8_EXCLUDE_EL1 ? 1 : 0;
>> +	attr.exclude_hv = data & ARMV8_INCLUDE_EL2 ? 0 : 1;
> 
> should the guest be able to see something counted in the hypervisor ever
> or should that only be the host being able to see that?
> 
> my gut feeling is that the hypervisor should be hidden from the guest
> and that exclude_hv = 0, is the right choice.  But this is a question
> about the semantics of perf, I suppose.
> 

This is what I'm not sure. I just thought about if guest has complete
ELs(EL3-EL0) and how to deal with nested virtualization.

>> +	attr.exclude_host = 1;
>> +	attr.config = config;
>> +	attr.sample_period = (-pmc->counter) & (((u64)1 << 32) - 1);
> 
> whoa, what is this scary calculation?
> 
> definitely needs an explanation?
> 
>> +
>> +	event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc);
>> +	if (IS_ERR(event)) {
>> +		kvm_err("kvm: pmu event creation failed %ld\n",
>> +			  PTR_ERR(event));
> 
> doesn't this mean we'll spam the kernel log if the guest supplies
> bogus/unsupported event numbers?
> 

X86 uses printk_once, is that ok to us here?

> In that case it shoudl be kvm_debug and the guest should be able to see
> this somehow (e.g. events don't count).
> 

Yes, will think about this.

>> +		return;
>> +	}
>> +	pmc->perf_event = event;
>> +}
>> +
>> +/**
>>   * kvm_pmu_init - Initialize global PMU state for per vcpu
>>   * @vcpu: The vcpu pointer
>>   *
>> -- 
>> 2.1.0
>>

-- 
Shannon

[PATCH 07/18] KVM: ARM64: PMU: Add perf event map and introduce perf event creating function

[Christoffer Dall]

On Tue, Jul 21, 2015 at 09:35:03AM +0800, Shannon Zhao wrote:
> 
> 
> On 2015/7/17 22:30, Christoffer Dall wrote:
> > On Mon, Jul 06, 2015 at 10:17:37AM +0800, shannon.zhao at linaro.org wrote:
> >> From: Shannon Zhao <shannon.zhao@linaro.org>
> >>
> >> When we use tools like perf on host, perf passes the event type and the
> >> id in this type category to kernel, then kernel will map them to event
> >> number and write this number to PMU PMEVTYPER<n>_EL0 register. While
> >> we're trapping and emulating guest accesses to PMU registers, we get the
> >> event number and map it to the event type and the id reversely.
> > 
> > There's something with the nomenclature that makes this really hard to
> > read.
> > 
> > you mention here: "event type", "the id", and "event number".  The
> > former two I think are perf identifiers, and the latter is the hardware
> > event number, is this right?
> > 
> 
> Yeah, right.
> 

ok, if we can clarify our nomenclature throughout here I think that
would make the patches easier to understand/read.

> >>
> >> Check whether the event type is same with the one to be set.
> > 
> > when?
> > 
> In function kvm_pmu_set_counter_event_type before create a new perf event.

do you mean when the guest configures some counter and we we create a
perf event accordingly?

If so, I think this could be clarified in the commit text.

> 
> >> +	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
> >> +		return;
> 
> 
> >> If not,
> >> stop counter to monitor current event and find the event type map id.
> >> According to the bits of data to configure this perf_event attr and
> >> set exclude_host to 1 for guest. Then call perf_event API to create the
> >> corresponding event and save the event pointer.
> >>
> >> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> >> ---
> >>  include/kvm/arm_pmu.h |   4 ++
> >>  virt/kvm/arm/pmu.c    | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++
> >>  2 files changed, 177 insertions(+)
> >>
> >> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> >> index 27d14ca..1050b24 100644
> >> --- a/include/kvm/arm_pmu.h
> >> +++ b/include/kvm/arm_pmu.h
> >> @@ -45,9 +45,13 @@ struct kvm_pmu {
> >>  
> >>  #ifdef CONFIG_KVM_ARM_PMU
> >>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
> >> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
> >> +				    unsigned long select_idx);
> >>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
> >>  #else
> >>  static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
> >> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
> >> +				    unsigned long select_idx) {}
> >>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
> >>  #endif
> >>  
> >> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> >> index dc252d0..50a3c82 100644
> >> --- a/virt/kvm/arm/pmu.c
> >> +++ b/virt/kvm/arm/pmu.c
> >> @@ -18,8 +18,68 @@
> >>  #include <linux/cpu.h>
> >>  #include <linux/kvm.h>
> >>  #include <linux/kvm_host.h>
> >> +#include <linux/perf_event.h>
> >>  #include <kvm/arm_pmu.h>
> >>  
> >> +/* PMU HW events mapping. */
> >> +static struct kvm_pmu_hw_event_map {
> >> +	unsigned eventsel;
> >> +	unsigned event_type;
> >> +} kvm_pmu_hw_events[] = {
> >> +	[0] = { 0x11, PERF_COUNT_HW_CPU_CYCLES },
> >> +	[1] = { 0x08, PERF_COUNT_HW_INSTRUCTIONS },
> >> +	[2] = { 0x04, PERF_COUNT_HW_CACHE_REFERENCES },
> >> +	[3] = { 0x03, PERF_COUNT_HW_CACHE_MISSES },
> >> +	[4] = { 0x10, PERF_COUNT_HW_BRANCH_MISSES },
> >> +};
> >> +
> >> +/* PMU HW cache events mapping. */
> >> +static struct kvm_pmu_hw_cache_event_map {
> >> +	unsigned eventsel;
> >> +	unsigned cache_type;
> >> +	unsigned cache_op;
> >> +	unsigned cache_result;
> >> +} kvm_pmu_hw_cache_events[] = {
> >> +	[0] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> >> +	[1] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> >> +	[2] = { 0x04, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> >> +	[3] = { 0x03, PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_WRITE,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> > 
> > seems to me that the four entries above will never be used, because
> > you'll always match them in kvm_pmu_hw_events above?
> > 
> 
> Yes, I found this before, but for the completeness I list them.
> 

hmm, having unused entries for completeness is a bit weird, IMHO.
Either way, a comment explaining why they're missing or that some are
never used would be helpful.

> > Is this because perf map multiple generic perf events to the same
> > hardware event? 
> I think so.
> 
> > Does it matter if we register this with perf as one or
> > the other then?
> > 
> I think it's ok because the hardware event numbers are same.
> 

ok

> 
> >> +	[4] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> >> +	[5] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_READ,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> >> +	[6] = { 0x12, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_ACCESS },
> >> +	[7] = { 0x10, PERF_COUNT_HW_CACHE_BPU, PERF_COUNT_HW_CACHE_OP_WRITE,
> >> +		      PERF_COUNT_HW_CACHE_RESULT_MISS },
> >> +};
> >> +
> >> +/**
> >> + * kvm_pmu_stop_counter - stop PMU counter for the selected counter
> >> + * @vcpu: The vcpu pointer
> >> + * @select_idx: The counter index
> >> + *
> >> + * If this counter has been configured to monitor some event, disable and
> >> + * release it.
> >> + */
> >> +static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu,
> >> +				 unsigned long select_idx)
> >> +{
> >> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> >> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> >> +
> >> +	if (pmc->perf_event) {
> >> +		perf_event_disable(pmc->perf_event);
> >> +		perf_event_release_kernel(pmc->perf_event);
> >> +	}
> >> +	pmc->perf_event = NULL;
> >> +	pmc->eventsel = 0xff;
> > 
> > why is 0xff 'unused' or reserved?  If we're choosing this arbitrarily,
> > why is it not 0x3ff?  Should we create a define for this?
> > 
> 
> Yeah, 0x3ff may be better.
> 
> >> +}
> >> +
> >>  /**
> >>   * kvm_pmu_vcpu_reset - reset pmu state for cpu
> >>   * @vcpu: The vcpu pointer
> >> @@ -27,12 +87,125 @@
> >>   */
> >>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
> >>  {
> >> +	int i;
> >>  	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> >>  
> >> +	for (i = 0; i < ARMV8_MAX_COUNTERS; i++)
> >> +		kvm_pmu_stop_counter(vcpu, i);
> >> +	pmu->overflow_status = 0;
> >>  	pmu->irq_pending = false;
> >>  }
> >>  
> >>  /**
> >> + * kvm_pmu_find_hw_event - find hardware event
> >> + * @pmu: The pmu pointer
> >> + * @event_select: The number of selected event type
> >> + *
> >> + * Based on the number of selected event type, find out whether it belongs to
> >> + * PERF_TYPE_HARDWARE. If so, return the corresponding event id.
> >> + */
> >> +static unsigned kvm_pmu_find_hw_event(struct kvm_pmu *pmu,
> >> +				      unsigned long event_select)
> >> +{
> >> +	int i;
> >> +
> >> +	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_events); i++)
> >> +		if (kvm_pmu_hw_events[i].eventsel == event_select)
> >> +			break;
> >> +
> >> +	if (i == ARRAY_SIZE(kvm_pmu_hw_events))
> >> +		return PERF_COUNT_HW_MAX;
> >> +
> >> +	return kvm_pmu_hw_events[i].event_type;
> > 
> > you can just return this directly in the loop if you have a match and
> > unconditionally return PERF_COUNT_HW_MAX outside the loop without having
> > to check the loop condition.
> > 
> ok
> >> +}
> >> +
> >> +/**
> >> + * kvm_pmu_find_hw_cache_event - find hardware cache event
> >> + * @pmu: The pmu pointer
> >> + * @event_select: The number of selected event type
> >> + *
> >> + * Based on the number of selected event type, find out whether it belongs to
> >> + * PERF_TYPE_HW_CACHE. If so, return the corresponding event id.
> >> + */
> >> +static unsigned kvm_pmu_find_hw_cache_event(struct kvm_pmu *pmu,
> >> +					    unsigned long event_select)
> >> +{
> >> +	int i;
> >> +	unsigned config;
> >> +
> >> +	for (i = 0; i < ARRAY_SIZE(kvm_pmu_hw_cache_events); i++)
> >> +		if (kvm_pmu_hw_cache_events[i].eventsel == event_select)
> >> +			break;
> >> +
> >> +	if (i == ARRAY_SIZE(kvm_pmu_hw_cache_events))
> >> +		return PERF_COUNT_HW_CACHE_MAX;
> > 
> > I feel like I just read this code, can we reuse it with a pointer to the
> > array?
> > 
> >> +
> >> +	config = (kvm_pmu_hw_cache_events[i].cache_type & 0xff)
> >> +		 | ((kvm_pmu_hw_cache_events[i].cache_op & 0xff) << 8)
> >> +		 | ((kvm_pmu_hw_cache_events[i].cache_result & 0xff) << 16);
> >> +
> >> +	return config;
> >> +}
> >> +
> >> +/**
> >> + * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
> >> + * @vcpu: The vcpu pointer
> >> + * @data: The data guest writes to PMXEVTYPER_EL0
> >> + * @select_idx: The number of selected counter
> >> + *
> >> + * Firstly check whether the event type is same with the one to be set.
> >> + * If not, stop counter to monitor current event and find the event type map id.
> >> + * According to the bits of data to configure this perf_event attr and set
> >> + * exclude_host to 1 for guest. Then call perf_event API to create the
> >> + * corresponding event and save the event pointer.
> > 
> > This text seems to be describing more how the function does something,
> > as opposed to what it does and why.  I found it a little hard to read.
> > 
> >> + */
> >> +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
> >> +				    unsigned long select_idx)
> >> +{
> >> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> >> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> >> +	struct perf_event *event;
> >> +	struct perf_event_attr attr;
> >> +	unsigned config, type = PERF_TYPE_RAW;
> >> +
> >> +	if ((data & ARMV8_EVTYPE_EVENT) == pmc->eventsel)
> >> +		return;
> >> +
> >> +	kvm_pmu_stop_counter(vcpu, select_idx);
> >> +	pmc->eventsel = data & ARMV8_EVTYPE_EVENT;
> >> +
> >> +	config = kvm_pmu_find_hw_event(pmu, pmc->eventsel);
> >> +	if (config != PERF_COUNT_HW_MAX) {
> >> +		type = PERF_TYPE_HARDWARE;
> >> +	} else {
> >> +		config = kvm_pmu_find_hw_cache_event(pmu, pmc->eventsel);
> >> +		if (config != PERF_COUNT_HW_CACHE_MAX)
> >> +			type = PERF_TYPE_HW_CACHE;
> >> +	}
> >> +
> >> +	if (type == PERF_TYPE_RAW)
> >> +		config = pmc->eventsel;
> > 
> > don't you need to memset attr to 0 first?
> > 
> ok
> > otherwise, how do you ensure that for example exclude_guest is always
> > clear?
> > 
> >> +
> >> +	attr.type = type;
> >> +	attr.size = sizeof(attr);
> >> +	attr.pinned = true;
> >> +	attr.exclude_user = data & ARMV8_EXCLUDE_EL0 ? 1 : 0;
> >> +	attr.exclude_kernel = data & ARMV8_EXCLUDE_EL1 ? 1 : 0;
> >> +	attr.exclude_hv = data & ARMV8_INCLUDE_EL2 ? 0 : 1;
> > 
> > should the guest be able to see something counted in the hypervisor ever
> > or should that only be the host being able to see that?
> > 
> > my gut feeling is that the hypervisor should be hidden from the guest
> > and that exclude_hv = 0, is the right choice.  But this is a question
> > about the semantics of perf, I suppose.
> > 
> 
> This is what I'm not sure. I just thought about if guest has complete
> ELs(EL3-EL0) and how to deal with nested virtualization.
> 

nested virtualization is really not very likely to happen on arm64.  I
think the right way to look at this is that we're emulating a platform
without EL2 and therefore the guest should not see any events from EL2.

> >> +	attr.exclude_host = 1;
> >> +	attr.config = config;
> >> +	attr.sample_period = (-pmc->counter) & (((u64)1 << 32) - 1);
> > 
> > whoa, what is this scary calculation?
> > 
> > definitely needs an explanation?
> > 
> >> +
> >> +	event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc);
> >> +	if (IS_ERR(event)) {
> >> +		kvm_err("kvm: pmu event creation failed %ld\n",
> >> +			  PTR_ERR(event));
> > 
> > doesn't this mean we'll spam the kernel log if the guest supplies
> > bogus/unsupported event numbers?
> > 
> 
> X86 uses printk_once, is that ok to us here?

yeah, that should be fine.

> 
> > In that case it shoudl be kvm_debug and the guest should be able to see
> > this somehow (e.g. events don't count).
> > 
> 
> Yes, will think about this.
> 

printk_once of kvm_debug would be fine.

Thanks,
-Christoffer

[PATCH 08/18] KVM: ARM64: Add reset and access handlers for PMXEVTYPER_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMXEVTYPER_EL0 is UNKNOWN, use reset_unknown
for its reset handler. Add access handler which emulates writing and
reading PMXEVTYPER_EL0 register. When writing to PMXEVTYPER_EL0, call
kvm_pmu_set_counter_event_type create a perf event for the selected
event type.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 1df9ef3..b4f8dd9 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -331,6 +331,31 @@ static bool access_pmceid(struct kvm_vcpu *vcpu,
 	}
 }
 
+/* PMXEVTYPER_EL0 accessor. */
+static bool access_pmxevtyper(struct kvm_vcpu *vcpu,
+			      const struct sys_reg_params *p,
+			      const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
+		else
+			vcpu_cp15(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
+		kvm_pmu_set_counter_event_type(vcpu, *vcpu_reg(vcpu, p->Rt),
+					       vcpu_sys_reg(vcpu, PMSELR_EL0));
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -549,7 +574,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  trap_raz_wi },
 	/* PMXEVTYPER_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
-	  trap_raz_wi },
+	  access_pmxevtyper, reset_unknown, PMXEVTYPER_EL0 },
 	/* PMXEVCNTR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
 	  trap_raz_wi },
-- 
2.1.0

[PATCH 09/18] KVM: ARM64: Add reset and access handlers for PMXEVCNTR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMXEVTYPER_EL0 is UNKNOWN, use reset_unknown
for its reset handler. Add access handler which emulates writing and
reading PMXEVTYPER_EL0 register. When reading PMXEVCNTR_EL0, call
perf_event_read_value to get the count value of the perf event.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 21 ++++++++++++++++++++-
 include/kvm/arm_pmu.h     | 11 +++++++++++
 virt/kvm/arm/pmu.c        | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index b4f8dd9..2bcf1a0 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -356,6 +356,25 @@ static bool access_pmxevtyper(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+static bool access_pmxevcntr(struct kvm_vcpu *vcpu,
+			     const struct sys_reg_params *p,
+			     const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		kvm_pmu_set_counter_value(vcpu, vcpu_sys_reg(vcpu, PMSELR_EL0),
+					  val & 0xffffffffUL);
+	} else {
+		val = kvm_pmu_get_counter_value(vcpu,
+						vcpu_sys_reg(vcpu, PMSELR_EL0));
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -577,7 +596,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmxevtyper, reset_unknown, PMXEVTYPER_EL0 },
 	/* PMXEVCNTR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
-	  trap_raz_wi },
+	  access_pmxevcntr, reset_unknown, PMXEVCNTR_EL0 },
 	/* PMUSERENR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
 	  trap_raz_wi },
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 1050b24..40ab4a0 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -45,11 +45,22 @@ struct kvm_pmu {
 
 #ifdef CONFIG_KVM_ARM_PMU
 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
+void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
+			       unsigned long val);
+unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
+					unsigned long select_idx);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
 #else
 static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
+void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
+			       unsigned long val) {}
+unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
+					unsigned long select_idx)
+{
+	return 0;
+}
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx) {}
 static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index 50a3c82..361fa51 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -97,6 +97,43 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
 }
 
 /**
+ * kvm_pmu_set_counter_value - set PMU counter value
+ * @vcpu: The vcpu pointer
+ * @select_idx: The counter index
+ * @val: the value to be set
+ */
+void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
+			       unsigned long val)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	pmc->counter = val;
+}
+
+/**
+ * kvm_pmu_set_counter_value - set PMU counter value
+ * @vcpu: The vcpu pointer
+ * @select_idx: The counter index
+ *
+ * Call perf_event API to get the event count
+ */
+unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
+					unsigned long select_idx)
+{
+	u64 enabled, running;
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+	unsigned long counter = pmc->counter;
+
+	if (pmc->perf_event) {
+		counter += perf_event_read_value(pmc->perf_event,
+						&enabled, &running);
+	}
+	return counter;
+}
+
+/**
  * kvm_pmu_find_hw_event - find hardware event
  * @pmu: The pmu pointer
  * @event_select: The number of selected event type
-- 
2.1.0

[PATCH 09/18] KVM: ARM64: Add reset and access handlers for PMXEVCNTR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:39AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMXEVTYPER_EL0 is UNKNOWN, use reset_unknown
> for its reset handler. Add access handler which emulates writing and
> reading PMXEVTYPER_EL0 register. When reading PMXEVCNTR_EL0, call
> perf_event_read_value to get the count value of the perf event.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 21 ++++++++++++++++++++-
>  include/kvm/arm_pmu.h     | 11 +++++++++++
>  virt/kvm/arm/pmu.c        | 37 +++++++++++++++++++++++++++++++++++++
>  3 files changed, 68 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index b4f8dd9..2bcf1a0 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -356,6 +356,25 @@ static bool access_pmxevtyper(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +static bool access_pmxevcntr(struct kvm_vcpu *vcpu,
> +			     const struct sys_reg_params *p,
> +			     const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		kvm_pmu_set_counter_value(vcpu, vcpu_sys_reg(vcpu, PMSELR_EL0),
> +					  val & 0xffffffffUL);
> +	} else {
> +		val = kvm_pmu_get_counter_value(vcpu,
> +						vcpu_sys_reg(vcpu, PMSELR_EL0));
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -577,7 +596,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmxevtyper, reset_unknown, PMXEVTYPER_EL0 },
>  	/* PMXEVCNTR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
> -	  trap_raz_wi },
> +	  access_pmxevcntr, reset_unknown, PMXEVCNTR_EL0 },
>  	/* PMUSERENR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
>  	  trap_raz_wi },
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 1050b24..40ab4a0 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -45,11 +45,22 @@ struct kvm_pmu {
>  
>  #ifdef CONFIG_KVM_ARM_PMU
>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
> +void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
> +			       unsigned long val);
> +unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
> +					unsigned long select_idx);
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
>  #else
>  static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
> +void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
> +			       unsigned long val) {}
> +unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
> +					unsigned long select_idx)
> +{
> +	return 0;
> +}
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx) {}
>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index 50a3c82..361fa51 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -97,6 +97,43 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
>  }
>  
>  /**
> + * kvm_pmu_set_counter_value - set PMU counter value
> + * @vcpu: The vcpu pointer
> + * @select_idx: The counter index
> + * @val: the value to be set
> + */
> +void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
> +			       unsigned long val)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	pmc->counter = val;

how does this help generate an overflow event when expected?

> +}
> +
> +/**
> + * kvm_pmu_set_counter_value - set PMU counter value

s/set/get/

> + * @vcpu: The vcpu pointer
> + * @select_idx: The counter index
> + *
> + * Call perf_event API to get the event count
> + */
> +unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
> +					unsigned long select_idx)
> +{
> +	u64 enabled, running;
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +	unsigned long counter = pmc->counter;
> +
> +	if (pmc->perf_event) {
> +		counter += perf_event_read_value(pmc->perf_event,
> +						&enabled, &running);

I don't quite understand the summation here or feel convinced that
enabled and running should be simply thrown away.

Can you explain this, potentially in the kdocs above...

> +	}
> +	return counter;
> +}
> +
> +/**
>   * kvm_pmu_find_hw_event - find hardware event
>   * @pmu: The pmu pointer
>   * @event_select: The number of selected event type
> -- 
> 2.1.0
> 

[PATCH 10/18] KVM: ARM64: Add reset and access handlers for PMCCNTR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMCCNTR_EL0 is UNKNOWN, use reset_unknown for
its reset handler. Add access handler which emulates writing and reading
PMCCNTR_EL0 register.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 2bcf1a0..29883df 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -375,6 +375,23 @@ static bool access_pmxevcntr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+static bool access_pmccntr(struct kvm_vcpu *vcpu,
+			   const struct sys_reg_params *p,
+			   const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		kvm_pmu_set_counter_value(vcpu, 31, val);
+	} else {
+		val = kvm_pmu_get_counter_value(vcpu, 31);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -590,7 +607,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmceid, reset_pmceid, PMCEID1_EL0, },
 	/* PMCCNTR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
-	  trap_raz_wi },
+	  access_pmccntr, reset_unknown, PMCCNTR_EL0 },
 	/* PMXEVTYPER_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
 	  access_pmxevtyper, reset_unknown, PMXEVTYPER_EL0 },
-- 
2.1.0

[PATCH 10/18] KVM: ARM64: Add reset and access handlers for PMCCNTR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:40AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMCCNTR_EL0 is UNKNOWN, use reset_unknown for
> its reset handler. Add access handler which emulates writing and reading
> PMCCNTR_EL0 register.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 19 ++++++++++++++++++-
>  1 file changed, 18 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 2bcf1a0..29883df 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -375,6 +375,23 @@ static bool access_pmxevcntr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +static bool access_pmccntr(struct kvm_vcpu *vcpu,
> +			   const struct sys_reg_params *p,
> +			   const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		kvm_pmu_set_counter_value(vcpu, 31, val);

magic number?  do we have some existing define we can use or can we add
one?

> +	} else {
> +		val = kvm_pmu_get_counter_value(vcpu, 31);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -590,7 +607,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmceid, reset_pmceid, PMCEID1_EL0, },
>  	/* PMCCNTR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
> -	  trap_raz_wi },
> +	  access_pmccntr, reset_unknown, PMCCNTR_EL0 },
>  	/* PMXEVTYPER_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
>  	  access_pmxevtyper, reset_unknown, PMXEVTYPER_EL0 },
> -- 
> 2.1.0
> 

[PATCH 11/18] KVM: ARM64: Add reset and access handlers for PMCNTENSET_EL0 and PMCNTENCLR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMCNTENSET_EL0 and PMCNTENCLR_EL0 is UNKNOWN,
use reset_unknown for its reset handler. Add access handler which
emulates writing and reading PMCNTENSET_EL0 or PMCNTENCLR_EL0 register.
When writing to PMCNTENSET_EL0, call perf_event_enable to enable the
perf event. When writing to PMCNTENCLR_EL0, call perf_event_disable to
disable the perf event.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 56 +++++++++++++++++++++++++++++++++++++++++++++--
 include/kvm/arm_pmu.h     |  4 ++++
 virt/kvm/arm/pmu.c        | 41 ++++++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 29883df..c14ec8d 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -392,6 +392,58 @@ static bool access_pmccntr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMCNTENSET_EL0 accessor. */
+static bool access_pmcntenset(struct kvm_vcpu *vcpu,
+			      const struct sys_reg_params *p,
+			      const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) |= val;
+		else
+			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
+
+		kvm_pmu_enable_counter(vcpu, val);
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
+/* PMCNTENCLR_EL0 accessor. */
+static bool access_pmcntenclr(struct kvm_vcpu *vcpu,
+			      const struct sys_reg_params *p,
+			      const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) |= val;
+		else
+			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
+
+		kvm_pmu_disable_counter(vcpu, val);
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -586,10 +638,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmcr, reset_pmcr_el0, PMCR_EL0, },
 	/* PMCNTENSET_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
-	  trap_raz_wi },
+	  access_pmcntenset, reset_unknown, PMCNTENSET_EL0 },
 	/* PMCNTENCLR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
-	  trap_raz_wi },
+	  access_pmcntenclr, reset_unknown, PMCNTENCLR_EL0 },
 	/* PMOVSCLR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
 	  trap_raz_wi },
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 40ab4a0..2cfd9be 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -49,6 +49,8 @@ void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
 			       unsigned long val);
 unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
 					unsigned long select_idx);
+void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val);
+void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
@@ -61,6 +63,8 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
 {
 	return 0;
 }
+void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
+void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx) {}
 static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index 361fa51..cf59998 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -134,6 +134,47 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
 }
 
 /**
+ * kvm_pmu_enable_counter - enable selected PMU counter
+ * @vcpu: The vcpu pointer
+ * @val: the value guest writes to PMCNTENSET_EL0 register
+ *
+ * Call perf_event_enable to start counting the perf event
+ */
+void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	int select_idx = find_first_bit(&val, 32);
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	if (pmc->perf_event) {
+		local64_set(&pmc->perf_event->count, 0);
+		perf_event_enable(pmc->perf_event);
+		if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
+			printk("kvm: fail to enable event\n");
+	}
+	pmc->enable = true;
+}
+
+/**
+ * kvm_pmu_disable_counter - disable selected PMU counter
+ * @vcpu: The vcpu pointer
+ * @val: the value guest writes to PMCNTENCLR_EL0 register
+ *
+ * Call perf_event_disable to stop counting the perf event
+ */
+void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	int select_idx = find_first_bit(&val, 32);
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	if (pmc->perf_event)
+		perf_event_disable(pmc->perf_event);
+
+	pmc->enable = false;
+}
+
+/**
  * kvm_pmu_find_hw_event - find hardware event
  * @pmu: The pmu pointer
  * @event_select: The number of selected event type
-- 
2.1.0

[PATCH 11/18] KVM: ARM64: Add reset and access handlers for PMCNTENSET_EL0 and PMCNTENCLR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:41AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMCNTENSET_EL0 and PMCNTENCLR_EL0 is UNKNOWN,
> use reset_unknown for its reset handler. Add access handler which
> emulates writing and reading PMCNTENSET_EL0 or PMCNTENCLR_EL0 register.
> When writing to PMCNTENSET_EL0, call perf_event_enable to enable the
> perf event. When writing to PMCNTENCLR_EL0, call perf_event_disable to
> disable the perf event.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 56 +++++++++++++++++++++++++++++++++++++++++++++--
>  include/kvm/arm_pmu.h     |  4 ++++
>  virt/kvm/arm/pmu.c        | 41 ++++++++++++++++++++++++++++++++++
>  3 files changed, 99 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 29883df..c14ec8d 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -392,6 +392,58 @@ static bool access_pmccntr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMCNTENSET_EL0 accessor. */
> +static bool access_pmcntenset(struct kvm_vcpu *vcpu,
> +			      const struct sys_reg_params *p,
> +			      const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		if (!p->is_aarch32)
> +			vcpu_sys_reg(vcpu, r->reg) |= val;
> +		else
> +			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
> +
> +		kvm_pmu_enable_counter(vcpu, val);
> +	} else {
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}
> +
> +	return true;
> +}
> +
> +/* PMCNTENCLR_EL0 accessor. */
> +static bool access_pmcntenclr(struct kvm_vcpu *vcpu,
> +			      const struct sys_reg_params *p,
> +			      const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		if (!p->is_aarch32)
> +			vcpu_sys_reg(vcpu, r->reg) |= val;

huh, this is the clear register, don't you need to &= ~val ?

also, there's a lot of code duplication between these two functions, it
must be worthwhile having a single static function that they both call
if a bool differentiating between set/clear.


> +		else
> +			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
> +
> +		kvm_pmu_disable_counter(vcpu, val);
> +	} else {
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -586,10 +638,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmcr, reset_pmcr_el0, PMCR_EL0, },
>  	/* PMCNTENSET_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
> -	  trap_raz_wi },
> +	  access_pmcntenset, reset_unknown, PMCNTENSET_EL0 },
>  	/* PMCNTENCLR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
> -	  trap_raz_wi },
> +	  access_pmcntenclr, reset_unknown, PMCNTENCLR_EL0 },
>  	/* PMOVSCLR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
>  	  trap_raz_wi },
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 40ab4a0..2cfd9be 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -49,6 +49,8 @@ void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
>  			       unsigned long val);
>  unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
>  					unsigned long select_idx);
> +void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val);
> +void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
> @@ -61,6 +63,8 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
>  {
>  	return 0;
>  }
> +void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
> +void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx) {}
>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index 361fa51..cf59998 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -134,6 +134,47 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
>  }
>  
>  /**
> + * kvm_pmu_enable_counter - enable selected PMU counter
> + * @vcpu: The vcpu pointer
> + * @val: the value guest writes to PMCNTENSET_EL0 register
> + *
> + * Call perf_event_enable to start counting the perf event
> + */
> +void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val)
> +{
> +	int select_idx = find_first_bit(&val, 32);

Both of these functions should be able to handle more than one bit set
in val though, right?

Or perhaps you should make the parameter be select_idx and loop over the
bits in the caller, your choice.

> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	if (pmc->perf_event) {
> +		local64_set(&pmc->perf_event->count, 0);

why local64_set?  Is this local to this CPU somehow?

why do we clear the count?  Is it architecturally mandated?

> +		perf_event_enable(pmc->perf_event);
> +		if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
> +			printk("kvm: fail to enable event\n");

do you really want this print here?  What could cause this error,
anything the VM can provoke?

> +	}

what's the difference between having pmc->perf_event == NULL and having
pmc->perf_event->state == PERF_EVENT_STATE_INACTIVE ?



> +	pmc->enable = true;
> +}
> +
> +/**
> + * kvm_pmu_disable_counter - disable selected PMU counter
> + * @vcpu: The vcpu pointer
> + * @val: the value guest writes to PMCNTENCLR_EL0 register
> + *
> + * Call perf_event_disable to stop counting the perf event
> + */
> +void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val)
> +{
> +	int select_idx = find_first_bit(&val, 32);

ditto, see above.

> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	if (pmc->perf_event)
> +		perf_event_disable(pmc->perf_event);
> +
> +	pmc->enable = false;
> +}
> +
> +/**
>   * kvm_pmu_find_hw_event - find hardware event
>   * @pmu: The pmu pointer
>   * @event_select: The number of selected event type
> -- 
> 2.1.0
> 

Thanks,
-Christoffer

[PATCH 12/18] KVM: ARM64: Add reset and access handlers for PMINTENSET_EL1 and PMINTENCLR_EL1 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMINTENSET_EL1 and PMINTENCLR_EL1 is UNKNOWN,
use reset_unknown for its reset handler. Add access handler which
emulates writing and reading PMINTENSET_EL1 or PMINTENCLR_EL1 register.
When writing to PMINTENSET_EL1, set the interrupt flag true. While
writing to PMINTENCLR_EL1, set the interrupt flag false. This flag will
be useful for counter overflow interrupt.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 56 +++++++++++++++++++++++++++++++++++++++++++++--
 include/kvm/arm_pmu.h     |  4 ++++
 virt/kvm/arm/pmu.c        | 28 ++++++++++++++++++++++++
 3 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index c14ec8d..cbc07b8 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -444,6 +444,58 @@ static bool access_pmcntenclr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMINTENSET_EL1 accessor. */
+static bool access_pmintenset(struct kvm_vcpu *vcpu,
+			      const struct sys_reg_params *p,
+			      const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) |= val;
+		else
+			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
+
+		kvm_pmu_enable_interrupt(vcpu, val);
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
+/* PMINTENCLR_EL1 accessor. */
+static bool access_pmintenclr(struct kvm_vcpu *vcpu,
+			      const struct sys_reg_params *p,
+			      const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) |= val;
+		else
+			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
+
+		kvm_pmu_disable_interrupt(vcpu, val);
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -595,10 +647,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 
 	/* PMINTENSET_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
-	  trap_raz_wi },
+	  access_pmintenset, reset_unknown, PMINTENSET_EL1 },
 	/* PMINTENCLR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
-	  trap_raz_wi },
+	  access_pmintenclr, reset_unknown, PMINTENCLR_EL1 },
 
 	/* MAIR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 2cfd9be..dee8356 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -51,6 +51,8 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
 					unsigned long select_idx);
 void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val);
+void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
+void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
@@ -65,6 +67,8 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
 }
 void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
+void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
+void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx) {}
 static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index cf59998..7023ad5 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -175,6 +175,34 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val)
 }
 
 /**
+ * kvm_pmu_enable_interrupt - enable selected PMU counter interrupt
+ * @vcpu: The vcpu pointer
+ * @val: the value guest writes to PMINTENSET_EL0 register
+ */
+void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	int select_idx = find_first_bit(&val, 32);
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	pmc->interrupt = true;
+}
+
+/**
+ * kvm_pmu_disable_interrupt - disable selected PMU counter interrupt
+ * @vcpu: The vcpu pointer
+ * @val: the value guest writes to PMINTENCLR_EL0 register
+ */
+void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	int select_idx = find_first_bit(&val, 32);
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	pmc->interrupt = false;
+}
+
+/**
  * kvm_pmu_find_hw_event - find hardware event
  * @pmu: The pmu pointer
  * @event_select: The number of selected event type
-- 
2.1.0

[PATCH 12/18] KVM: ARM64: Add reset and access handlers for PMINTENSET_EL1 and PMINTENCLR_EL1 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:42AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMINTENSET_EL1 and PMINTENCLR_EL1 is UNKNOWN,
> use reset_unknown for its reset handler. Add access handler which
> emulates writing and reading PMINTENSET_EL1 or PMINTENCLR_EL1 register.
> When writing to PMINTENSET_EL1, set the interrupt flag true. While
> writing to PMINTENCLR_EL1, set the interrupt flag false. This flag will
> be useful for counter overflow interrupt.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 56 +++++++++++++++++++++++++++++++++++++++++++++--
>  include/kvm/arm_pmu.h     |  4 ++++
>  virt/kvm/arm/pmu.c        | 28 ++++++++++++++++++++++++
>  3 files changed, 86 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index c14ec8d..cbc07b8 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -444,6 +444,58 @@ static bool access_pmcntenclr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMINTENSET_EL1 accessor. */
> +static bool access_pmintenset(struct kvm_vcpu *vcpu,
> +			      const struct sys_reg_params *p,
> +			      const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		if (!p->is_aarch32)
> +			vcpu_sys_reg(vcpu, r->reg) |= val;
> +		else
> +			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
> +
> +		kvm_pmu_enable_interrupt(vcpu, val);
> +	} else {
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}
> +
> +	return true;
> +}
> +
> +/* PMINTENCLR_EL1 accessor. */
> +static bool access_pmintenclr(struct kvm_vcpu *vcpu,
> +			      const struct sys_reg_params *p,
> +			      const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		if (!p->is_aarch32)
> +			vcpu_sys_reg(vcpu, r->reg) |= val;
> +		else
> +			vcpu_cp15(vcpu, r->reg) |= val & 0xffffffffUL;
> +
> +		kvm_pmu_disable_interrupt(vcpu, val);
> +	} else {
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}

Same comments as in the previous patch, but now I realize what you're
doing with the set/clear registers.  These are not separate pieces of
state, but a single register, which are just accessed with two
interfaces, for set/clear patterns.

> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -595,10 +647,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  
>  	/* PMINTENSET_EL1 */
>  	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
> -	  trap_raz_wi },
> +	  access_pmintenset, reset_unknown, PMINTENSET_EL1 },
>  	/* PMINTENCLR_EL1 */
>  	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
> -	  trap_raz_wi },
> +	  access_pmintenclr, reset_unknown, PMINTENCLR_EL1 },
>  
>  	/* MAIR_EL1 */
>  	{ Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 2cfd9be..dee8356 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -51,6 +51,8 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
>  					unsigned long select_idx);
>  void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val);
> +void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
> +void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
> @@ -65,6 +67,8 @@ unsigned long kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu,
>  }
>  void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
> +void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
> +void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx) {}
>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index cf59998..7023ad5 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -175,6 +175,34 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val)
>  }
>  
>  /**
> + * kvm_pmu_enable_interrupt - enable selected PMU counter interrupt
> + * @vcpu: The vcpu pointer
> + * @val: the value guest writes to PMINTENSET_EL0 register
> + */
> +void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val)
> +{
> +	int select_idx = find_first_bit(&val, 32);

also same as before.

> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	pmc->interrupt = true;

ah, so this was interrupts enabled, I think this is better called
interrupt_enabled on the struct.


Thanks,
-Christoffer

> +}
> +
> +/**
> + * kvm_pmu_disable_interrupt - disable selected PMU counter interrupt
> + * @vcpu: The vcpu pointer
> + * @val: the value guest writes to PMINTENCLR_EL0 register
> + */
> +void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val)
> +{
> +	int select_idx = find_first_bit(&val, 32);
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	pmc->interrupt = false;
> +}
> +
> +/**
>   * kvm_pmu_find_hw_event - find hardware event
>   * @pmu: The pmu pointer
>   * @event_select: The number of selected event type
> -- 
> 2.1.0
> 

[PATCH 13/18] KVM: ARM64: Add reset and access handlers for PMOVSSET_EL0 and PMOVSCLR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMOVSSET_EL0 and PMOVSCLR_EL0 is UNKNOWN, use
reset_unknown for its reset handler. Add access handler which emulates
writing and reading PMOVSSET_EL0 or PMOVSCLR_EL0 register.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index cbc07b8..ec80937 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -496,6 +496,32 @@ static bool access_pmintenclr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMOVSSET_EL0 accessor. */
+static bool access_pmovsset(struct kvm_vcpu *vcpu,
+			    const struct sys_reg_params *p,
+			    const struct sys_reg_desc *r)
+{
+	if (p->is_write)
+		vcpu->arch.pmu.overflow_status |= *vcpu_reg(vcpu, p->Rt);
+	else
+		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.overflow_status;
+
+	return true;
+}
+
+/* PMOVSCLR_EL0 accessor. */
+static bool access_pmovsclr(struct kvm_vcpu *vcpu,
+			    const struct sys_reg_params *p,
+			    const struct sys_reg_desc *r)
+{
+	if (p->is_write)
+		vcpu->arch.pmu.overflow_status &= (~*vcpu_reg(vcpu, p->Rt));
+	else
+		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.overflow_status;
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -696,7 +722,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmcntenclr, reset_unknown, PMCNTENCLR_EL0 },
 	/* PMOVSCLR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
-	  trap_raz_wi },
+	  access_pmovsclr, reset_unknown, PMOVSCLR_EL0 },
 	/* PMSWINC_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
 	  trap_raz_wi },
@@ -723,7 +749,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  trap_raz_wi },
 	/* PMOVSSET_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
-	  trap_raz_wi },
+	  access_pmovsset, reset_unknown, PMOVSSET_EL0 },
 
 	/* TPIDR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
-- 
2.1.0

[PATCH 13/18] KVM: ARM64: Add reset and access handlers for PMOVSSET_EL0 and PMOVSCLR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:43AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMOVSSET_EL0 and PMOVSCLR_EL0 is UNKNOWN, use
> reset_unknown for its reset handler. Add access handler which emulates
> writing and reading PMOVSSET_EL0 or PMOVSCLR_EL0 register.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 30 ++++++++++++++++++++++++++++--
>  1 file changed, 28 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index cbc07b8..ec80937 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -496,6 +496,32 @@ static bool access_pmintenclr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMOVSSET_EL0 accessor. */
> +static bool access_pmovsset(struct kvm_vcpu *vcpu,
> +			    const struct sys_reg_params *p,
> +			    const struct sys_reg_desc *r)
> +{
> +	if (p->is_write)
> +		vcpu->arch.pmu.overflow_status |= *vcpu_reg(vcpu, p->Rt);
> +	else
> +		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.overflow_status;
> +
> +	return true;
> +}
> +
> +/* PMOVSCLR_EL0 accessor. */
> +static bool access_pmovsclr(struct kvm_vcpu *vcpu,
> +			    const struct sys_reg_params *p,
> +			    const struct sys_reg_desc *r)
> +{
> +	if (p->is_write)
> +		vcpu->arch.pmu.overflow_status &= (~*vcpu_reg(vcpu, p->Rt));

why the extra parenthesis here?  If anything, I would think the bitwise
not should be on the outside of the parenthesis.

-Christoffer


> +	else
> +		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.overflow_status;
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -696,7 +722,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmcntenclr, reset_unknown, PMCNTENCLR_EL0 },
>  	/* PMOVSCLR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
> -	  trap_raz_wi },
> +	  access_pmovsclr, reset_unknown, PMOVSCLR_EL0 },
>  	/* PMSWINC_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
>  	  trap_raz_wi },
> @@ -723,7 +749,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  trap_raz_wi },
>  	/* PMOVSSET_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
> -	  trap_raz_wi },
> +	  access_pmovsset, reset_unknown, PMOVSSET_EL0 },
>  
>  	/* TPIDR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
> -- 
> 2.1.0
> 

[PATCH 13/18] KVM: ARM64: Add reset and access handlers for PMOVSSET_EL0 and PMOVSCLR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:43AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMOVSSET_EL0 and PMOVSCLR_EL0 is UNKNOWN, use
> reset_unknown for its reset handler. Add access handler which emulates
> writing and reading PMOVSSET_EL0 or PMOVSCLR_EL0 register.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 30 ++++++++++++++++++++++++++++--
>  1 file changed, 28 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index cbc07b8..ec80937 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -496,6 +496,32 @@ static bool access_pmintenclr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMOVSSET_EL0 accessor. */
> +static bool access_pmovsset(struct kvm_vcpu *vcpu,
> +			    const struct sys_reg_params *p,
> +			    const struct sys_reg_desc *r)
> +{
> +	if (p->is_write)
> +		vcpu->arch.pmu.overflow_status |= *vcpu_reg(vcpu, p->Rt);
> +	else
> +		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.overflow_status;

if you store this state in PMOVSSET_EL0 you get VM migration
save/restore support for (almost) free.

-Christoffer

> +
> +	return true;
> +}
> +
> +/* PMOVSCLR_EL0 accessor. */
> +static bool access_pmovsclr(struct kvm_vcpu *vcpu,
> +			    const struct sys_reg_params *p,
> +			    const struct sys_reg_desc *r)
> +{
> +	if (p->is_write)
> +		vcpu->arch.pmu.overflow_status &= (~*vcpu_reg(vcpu, p->Rt));
> +	else
> +		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.overflow_status;
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -696,7 +722,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmcntenclr, reset_unknown, PMCNTENCLR_EL0 },
>  	/* PMOVSCLR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
> -	  trap_raz_wi },
> +	  access_pmovsclr, reset_unknown, PMOVSCLR_EL0 },
>  	/* PMSWINC_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
>  	  trap_raz_wi },
> @@ -723,7 +749,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  trap_raz_wi },
>  	/* PMOVSSET_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
> -	  trap_raz_wi },
> +	  access_pmovsset, reset_unknown, PMOVSSET_EL0 },
>  
>  	/* TPIDR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
> -- 
> 2.1.0
> 

[PATCH 14/18] KVM: ARM64: Add reset and access handlers for PMUSERENR_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Since the reset value of PMUSERENR_EL0 is UNKNOWN, use reset_unknown for
its reset handler. Add access handler which emulates writing and reading
PMUSERENR_EL0 register.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 15 ++++++++++++++-
 include/kvm/arm_pmu.h     |  1 +
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index ec80937..d5984d0 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -522,6 +522,19 @@ static bool access_pmovsclr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMUSERENR_EL0 accessor. */
+static bool access_pmuserenr(struct kvm_vcpu *vcpu,
+			     const struct sys_reg_params *p,
+			     const struct sys_reg_desc *r)
+{
+	if (p->is_write)
+		vcpu->arch.pmu.user_enable |= *vcpu_reg(vcpu, p->Rt);
+	else
+		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.user_enable;
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -746,7 +759,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmxevcntr, reset_unknown, PMXEVCNTR_EL0 },
 	/* PMUSERENR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
-	  trap_raz_wi },
+	  access_pmuserenr, reset_unknown, PMUSERENR_EL0 },
 	/* PMOVSSET_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
 	  access_pmovsset, reset_unknown, PMOVSSET_EL0 },
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index dee8356..4f3d8a6 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -39,6 +39,7 @@ struct kvm_pmu {
 	/* IRQ pending flag */
 	bool irq_pending;
 	struct irq_work irq_work;
+	u32 user_enable;
 	struct kvm_pmc pmc[ARMV8_MAX_COUNTERS];
 #endif
 };
-- 
2.1.0

[PATCH 14/18] KVM: ARM64: Add reset and access handlers for PMUSERENR_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:44AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Since the reset value of PMUSERENR_EL0 is UNKNOWN, use reset_unknown for
> its reset handler. Add access handler which emulates writing and reading
> PMUSERENR_EL0 register.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 15 ++++++++++++++-
>  include/kvm/arm_pmu.h     |  1 +
>  2 files changed, 15 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index ec80937..d5984d0 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -522,6 +522,19 @@ static bool access_pmovsclr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMUSERENR_EL0 accessor. */
> +static bool access_pmuserenr(struct kvm_vcpu *vcpu,
> +			     const struct sys_reg_params *p,
> +			     const struct sys_reg_desc *r)
> +{
> +	if (p->is_write)
> +		vcpu->arch.pmu.user_enable |= *vcpu_reg(vcpu, p->Rt);
> +	else
> +		*vcpu_reg(vcpu, p->Rt) = vcpu->arch.pmu.user_enable;
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -746,7 +759,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmxevcntr, reset_unknown, PMXEVCNTR_EL0 },
>  	/* PMUSERENR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
> -	  trap_raz_wi },
> +	  access_pmuserenr, reset_unknown, PMUSERENR_EL0 },
>  	/* PMOVSSET_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
>  	  access_pmovsset, reset_unknown, PMOVSSET_EL0 },
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index dee8356..4f3d8a6 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -39,6 +39,7 @@ struct kvm_pmu {
>  	/* IRQ pending flag */
>  	bool irq_pending;
>  	struct irq_work irq_work;
> +	u32 user_enable;

why not store this in PMUSERENR_EL0 and get VM migration of this state
included for free?

Also, I assume the functionality to respect these flags are implemented
in a later patch or simply not supported?  It would have been good to
note this in the cover letter.

-Christoffer

>  	struct kvm_pmc pmc[ARMV8_MAX_COUNTERS];
>  #endif
>  };
> -- 
> 2.1.0
> 

[PATCH 15/18] KVM: ARM64: Add reset and access handlers for PMSWINC_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Add access handler which emulates writing and reading PMSWINC_EL0
register and add support for creating software increment event.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 15 ++++++++++++++-
 include/kvm/arm_pmu.h     |  2 ++
 virt/kvm/arm/pmu.c        | 20 ++++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index d5984d0..70afcba 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -535,6 +535,19 @@ static bool access_pmuserenr(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMSWINC_EL0 accessor. */
+static bool access_pmswinc(struct kvm_vcpu *vcpu,
+			   const struct sys_reg_params *p,
+			   const struct sys_reg_desc *r)
+{
+	if (p->is_write)
+		kvm_pmu_software_increment(vcpu, *vcpu_reg(vcpu, p->Rt));
+	else
+		return read_zero(vcpu, p);
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -738,7 +751,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	  access_pmovsclr, reset_unknown, PMOVSCLR_EL0 },
 	/* PMSWINC_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
-	  trap_raz_wi },
+	  access_pmswinc, reset_unknown, PMSWINC_EL0 },
 	/* PMSELR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
 	  access_pmselr, reset_unknown, PMSELR_EL0 },
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 4f3d8a6..6985809 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -54,6 +54,7 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
+void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
@@ -70,6 +71,7 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
+void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx) {}
 static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index 7023ad5..e655426 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -203,6 +203,22 @@ void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val)
 }
 
 /**
+ * kvm_pmu_software_increment - do software increment
+ * @vcpu: The vcpu pointer
+ * @val: the value guest writes to PMSWINC_EL0 register
+ */
+void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	int select_idx = find_first_bit(&val, 31);
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
+
+	if (pmu->user_enable & 0x3)
+		if ((pmc->eventsel == 0) && (pmc->enable == true))
+			pmc->counter++;
+}
+
+/**
  * kvm_pmu_find_hw_event - find hardware event
  * @pmu: The pmu pointer
  * @event_select: The number of selected event type
@@ -280,6 +296,10 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 	kvm_pmu_stop_counter(vcpu, select_idx);
 	pmc->eventsel = data & ARMV8_EVTYPE_EVENT;
 
+	/* For software increment event we don't need to create perf event */
+	if (pmc->eventsel == 0)
+		return;
+
 	config = kvm_pmu_find_hw_event(pmu, pmc->eventsel);
 	if (config != PERF_COUNT_HW_MAX) {
 		type = PERF_TYPE_HARDWARE;
-- 
2.1.0

[PATCH 15/18] KVM: ARM64: Add reset and access handlers for PMSWINC_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:45AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Add access handler which emulates writing and reading PMSWINC_EL0
> register and add support for creating software increment event.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 15 ++++++++++++++-
>  include/kvm/arm_pmu.h     |  2 ++
>  virt/kvm/arm/pmu.c        | 20 ++++++++++++++++++++
>  3 files changed, 36 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index d5984d0..70afcba 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -535,6 +535,19 @@ static bool access_pmuserenr(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMSWINC_EL0 accessor. */
> +static bool access_pmswinc(struct kvm_vcpu *vcpu,
> +			   const struct sys_reg_params *p,
> +			   const struct sys_reg_desc *r)
> +{
> +	if (p->is_write)
> +		kvm_pmu_software_increment(vcpu, *vcpu_reg(vcpu, p->Rt));
> +	else
> +		return read_zero(vcpu, p);
> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -738,7 +751,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	  access_pmovsclr, reset_unknown, PMOVSCLR_EL0 },
>  	/* PMSWINC_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
> -	  trap_raz_wi },
> +	  access_pmswinc, reset_unknown, PMSWINC_EL0 },
>  	/* PMSELR_EL0 */
>  	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
>  	  access_pmselr, reset_unknown, PMSELR_EL0 },
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 4f3d8a6..6985809 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -54,6 +54,7 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
> +void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
> @@ -70,6 +71,7 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
> +void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx) {}
>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index 7023ad5..e655426 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -203,6 +203,22 @@ void kvm_pmu_disable_interrupt(struct kvm_vcpu *vcpu, unsigned long val)
>  }
>  
>  /**
> + * kvm_pmu_software_increment - do software increment
> + * @vcpu: The vcpu pointer
> + * @val: the value guest writes to PMSWINC_EL0 register
> + */
> +void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val)
> +{
> +	int select_idx = find_first_bit(&val, 31);

can you not also increment multiple counters with a single write here?

or is it an error to configure multiple counters to the same event?  And
if it is, do we enforce that somehow?  If not, should they not reflect
the same underlying value instead of a separate pmc->counter value?


> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
> +
> +	if (pmu->user_enable & 0x3)

shouldn't this be:
	if (vcpu_mode_priv(vcpu) || pmu->user_enable & 0x3 == 0x3)

?


> +		if ((pmc->eventsel == 0) && (pmc->enable == true))
> +			pmc->counter++;

how do we migrate this state?  do we care?

-Christoffer

> +}
> +
> +/**
>   * kvm_pmu_find_hw_event - find hardware event
>   * @pmu: The pmu pointer
>   * @event_select: The number of selected event type
> @@ -280,6 +296,10 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  	kvm_pmu_stop_counter(vcpu, select_idx);
>  	pmc->eventsel = data & ARMV8_EVTYPE_EVENT;
>  
> +	/* For software increment event we don't need to create perf event */
> +	if (pmc->eventsel == 0)
> +		return;
> +
>  	config = kvm_pmu_find_hw_event(pmu, pmc->eventsel);
>  	if (config != PERF_COUNT_HW_MAX) {
>  		type = PERF_TYPE_HARDWARE;
> -- 
> 2.1.0
> 

[PATCH 16/18] KVM: ARM64: Add access handlers for PMEVCNTRn_EL0 and PMEVTYPERn_EL0 register

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Add access handler which emulates writing and reading PMEVCNTRn_EL0 and
PMEVTYPERn_EL0.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm64/kvm/sys_regs.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 70afcba..5663d83 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -548,6 +548,30 @@ static bool access_pmswinc(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+/* PMU reg accessor. */
+static bool access_pmu_reg(struct kvm_vcpu *vcpu,
+			   const struct sys_reg_params *p,
+			   const struct sys_reg_desc *r)
+{
+	unsigned long val;
+
+	if (p->is_write) {
+		val = *vcpu_reg(vcpu, p->Rt);
+		if (!p->is_aarch32)
+			vcpu_sys_reg(vcpu, r->reg) = val;
+		else
+			vcpu_cp15(vcpu, r->reg) = val & 0xffffffffUL;
+	} else {
+		if (!p->is_aarch32)
+			val = vcpu_sys_reg(vcpu, r->reg);
+		else
+			val = vcpu_cp15(vcpu, r->reg);
+		*vcpu_reg(vcpu, p->Rt) = val;
+	}
+
+	return true;
+}
+
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
@@ -563,6 +587,20 @@ static bool access_pmswinc(struct kvm_vcpu *vcpu,
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111),	\
 	  trap_debug_regs, reset_val, (DBGWCR0_EL1 + (n)), 0 }
 
+/* Macro to expand the PMEVCNTRn_EL0 register */
+#define PMU_PMEVCNTR_EL0(n)						\
+	/* PMEVCNTRn_EL0 */						\
+	{ Op0(0b11), Op1(0b011), CRn(0b1110),				\
+	  CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),		\
+	  access_pmu_reg, reset_val, (PMEVCNTR0_EL0 + (n)*2), 0 }
+
+/* Macro to expand the PMEVTYPERn_EL0 register */
+#define PMU_PMEVTYPER_EL0(n)						\
+	/* PMEVTYPERn_EL0 */						\
+	{ Op0(0b11), Op1(0b011), CRn(0b1110),				\
+	  CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),		\
+	  access_pmu_reg, reset_val, (PMEVTYPER0_EL0 + (n)*2), 0 }
+
 /*
  * Architected system registers.
  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
@@ -784,6 +822,74 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
 	  NULL, reset_unknown, TPIDRRO_EL0 },
 
+	/* PMEVCNTRn_EL0 */
+	PMU_PMEVCNTR_EL0(0),
+	PMU_PMEVCNTR_EL0(1),
+	PMU_PMEVCNTR_EL0(2),
+	PMU_PMEVCNTR_EL0(3),
+	PMU_PMEVCNTR_EL0(4),
+	PMU_PMEVCNTR_EL0(5),
+	PMU_PMEVCNTR_EL0(6),
+	PMU_PMEVCNTR_EL0(7),
+	PMU_PMEVCNTR_EL0(8),
+	PMU_PMEVCNTR_EL0(9),
+	PMU_PMEVCNTR_EL0(10),
+	PMU_PMEVCNTR_EL0(11),
+	PMU_PMEVCNTR_EL0(12),
+	PMU_PMEVCNTR_EL0(13),
+	PMU_PMEVCNTR_EL0(14),
+	PMU_PMEVCNTR_EL0(15),
+	PMU_PMEVCNTR_EL0(16),
+	PMU_PMEVCNTR_EL0(17),
+	PMU_PMEVCNTR_EL0(18),
+	PMU_PMEVCNTR_EL0(19),
+	PMU_PMEVCNTR_EL0(20),
+	PMU_PMEVCNTR_EL0(21),
+	PMU_PMEVCNTR_EL0(22),
+	PMU_PMEVCNTR_EL0(23),
+	PMU_PMEVCNTR_EL0(24),
+	PMU_PMEVCNTR_EL0(25),
+	PMU_PMEVCNTR_EL0(26),
+	PMU_PMEVCNTR_EL0(27),
+	PMU_PMEVCNTR_EL0(28),
+	PMU_PMEVCNTR_EL0(29),
+	PMU_PMEVCNTR_EL0(30),
+	/* PMEVTYPERn_EL0 */
+	PMU_PMEVTYPER_EL0(0),
+	PMU_PMEVTYPER_EL0(1),
+	PMU_PMEVTYPER_EL0(2),
+	PMU_PMEVTYPER_EL0(3),
+	PMU_PMEVTYPER_EL0(4),
+	PMU_PMEVTYPER_EL0(5),
+	PMU_PMEVTYPER_EL0(6),
+	PMU_PMEVTYPER_EL0(7),
+	PMU_PMEVTYPER_EL0(8),
+	PMU_PMEVTYPER_EL0(9),
+	PMU_PMEVTYPER_EL0(10),
+	PMU_PMEVTYPER_EL0(11),
+	PMU_PMEVTYPER_EL0(12),
+	PMU_PMEVTYPER_EL0(13),
+	PMU_PMEVTYPER_EL0(14),
+	PMU_PMEVTYPER_EL0(15),
+	PMU_PMEVTYPER_EL0(16),
+	PMU_PMEVTYPER_EL0(17),
+	PMU_PMEVTYPER_EL0(18),
+	PMU_PMEVTYPER_EL0(19),
+	PMU_PMEVTYPER_EL0(20),
+	PMU_PMEVTYPER_EL0(21),
+	PMU_PMEVTYPER_EL0(22),
+	PMU_PMEVTYPER_EL0(23),
+	PMU_PMEVTYPER_EL0(24),
+	PMU_PMEVTYPER_EL0(25),
+	PMU_PMEVTYPER_EL0(26),
+	PMU_PMEVTYPER_EL0(27),
+	PMU_PMEVTYPER_EL0(28),
+	PMU_PMEVTYPER_EL0(29),
+	PMU_PMEVTYPER_EL0(30),
+	/* PMCCFILTR_EL0 */
+	{ Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b1111), Op2(0b111),
+	  access_pmu_reg, reset_val, PMCCFILTR_EL0, 0 },
+
 	/* DACR32_EL2 */
 	{ Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
 	  NULL, reset_unknown, DACR32_EL2 },
-- 
2.1.0

[PATCH 16/18] KVM: ARM64: Add access handlers for PMEVCNTRn_EL0 and PMEVTYPERn_EL0 register

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:46AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Add access handler which emulates writing and reading PMEVCNTRn_EL0 and
> PMEVTYPERn_EL0.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm64/kvm/sys_regs.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 106 insertions(+)
> 
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index 70afcba..5663d83 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -548,6 +548,30 @@ static bool access_pmswinc(struct kvm_vcpu *vcpu,
>  	return true;
>  }
>  
> +/* PMU reg accessor. */
> +static bool access_pmu_reg(struct kvm_vcpu *vcpu,
> +			   const struct sys_reg_params *p,
> +			   const struct sys_reg_desc *r)
> +{
> +	unsigned long val;
> +
> +	if (p->is_write) {
> +		val = *vcpu_reg(vcpu, p->Rt);
> +		if (!p->is_aarch32)
> +			vcpu_sys_reg(vcpu, r->reg) = val;
> +		else
> +			vcpu_cp15(vcpu, r->reg) = val & 0xffffffffUL;
> +	} else {
> +		if (!p->is_aarch32)
> +			val = vcpu_sys_reg(vcpu, r->reg);
> +		else
> +			val = vcpu_cp15(vcpu, r->reg);
> +		*vcpu_reg(vcpu, p->Rt) = val;
> +	}

shouldn't these functions act completely analogously to access_pmxevcntr
(introduced in patch 09/18), only instead of using the valur of
PMSELR_EL0 for the index, this should be some offset calculation or
r->reg?

I think you also need a 32-bit mapping with the right offset for the
p->is_aarch32 check to make sense here (I may have forgotten this in a
few patches, please check all of them for this).

> +
> +	return true;
> +}
> +
>  /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
>  #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
>  	/* DBGBVRn_EL1 */						\
> @@ -563,6 +587,20 @@ static bool access_pmswinc(struct kvm_vcpu *vcpu,
>  	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111),	\
>  	  trap_debug_regs, reset_val, (DBGWCR0_EL1 + (n)), 0 }
>  
> +/* Macro to expand the PMEVCNTRn_EL0 register */
> +#define PMU_PMEVCNTR_EL0(n)						\
> +	/* PMEVCNTRn_EL0 */						\
> +	{ Op0(0b11), Op1(0b011), CRn(0b1110),				\
> +	  CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),		\
> +	  access_pmu_reg, reset_val, (PMEVCNTR0_EL0 + (n)*2), 0 }
> +
> +/* Macro to expand the PMEVTYPERn_EL0 register */
> +#define PMU_PMEVTYPER_EL0(n)						\
> +	/* PMEVTYPERn_EL0 */						\
> +	{ Op0(0b11), Op1(0b011), CRn(0b1110),				\
> +	  CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),		\
> +	  access_pmu_reg, reset_val, (PMEVTYPER0_EL0 + (n)*2), 0 }
> +
>  /*
>   * Architected system registers.
>   * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
> @@ -784,6 +822,74 @@ static const struct sys_reg_desc sys_reg_descs[] = {
>  	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
>  	  NULL, reset_unknown, TPIDRRO_EL0 },
>  
> +	/* PMEVCNTRn_EL0 */
> +	PMU_PMEVCNTR_EL0(0),
> +	PMU_PMEVCNTR_EL0(1),
> +	PMU_PMEVCNTR_EL0(2),
> +	PMU_PMEVCNTR_EL0(3),
> +	PMU_PMEVCNTR_EL0(4),
> +	PMU_PMEVCNTR_EL0(5),
> +	PMU_PMEVCNTR_EL0(6),
> +	PMU_PMEVCNTR_EL0(7),
> +	PMU_PMEVCNTR_EL0(8),
> +	PMU_PMEVCNTR_EL0(9),
> +	PMU_PMEVCNTR_EL0(10),
> +	PMU_PMEVCNTR_EL0(11),
> +	PMU_PMEVCNTR_EL0(12),
> +	PMU_PMEVCNTR_EL0(13),
> +	PMU_PMEVCNTR_EL0(14),
> +	PMU_PMEVCNTR_EL0(15),
> +	PMU_PMEVCNTR_EL0(16),
> +	PMU_PMEVCNTR_EL0(17),
> +	PMU_PMEVCNTR_EL0(18),
> +	PMU_PMEVCNTR_EL0(19),
> +	PMU_PMEVCNTR_EL0(20),
> +	PMU_PMEVCNTR_EL0(21),
> +	PMU_PMEVCNTR_EL0(22),
> +	PMU_PMEVCNTR_EL0(23),
> +	PMU_PMEVCNTR_EL0(24),
> +	PMU_PMEVCNTR_EL0(25),
> +	PMU_PMEVCNTR_EL0(26),
> +	PMU_PMEVCNTR_EL0(27),
> +	PMU_PMEVCNTR_EL0(28),
> +	PMU_PMEVCNTR_EL0(29),
> +	PMU_PMEVCNTR_EL0(30),
> +	/* PMEVTYPERn_EL0 */
> +	PMU_PMEVTYPER_EL0(0),
> +	PMU_PMEVTYPER_EL0(1),
> +	PMU_PMEVTYPER_EL0(2),
> +	PMU_PMEVTYPER_EL0(3),
> +	PMU_PMEVTYPER_EL0(4),
> +	PMU_PMEVTYPER_EL0(5),
> +	PMU_PMEVTYPER_EL0(6),
> +	PMU_PMEVTYPER_EL0(7),
> +	PMU_PMEVTYPER_EL0(8),
> +	PMU_PMEVTYPER_EL0(9),
> +	PMU_PMEVTYPER_EL0(10),
> +	PMU_PMEVTYPER_EL0(11),
> +	PMU_PMEVTYPER_EL0(12),
> +	PMU_PMEVTYPER_EL0(13),
> +	PMU_PMEVTYPER_EL0(14),
> +	PMU_PMEVTYPER_EL0(15),
> +	PMU_PMEVTYPER_EL0(16),
> +	PMU_PMEVTYPER_EL0(17),
> +	PMU_PMEVTYPER_EL0(18),
> +	PMU_PMEVTYPER_EL0(19),
> +	PMU_PMEVTYPER_EL0(20),
> +	PMU_PMEVTYPER_EL0(21),
> +	PMU_PMEVTYPER_EL0(22),
> +	PMU_PMEVTYPER_EL0(23),
> +	PMU_PMEVTYPER_EL0(24),
> +	PMU_PMEVTYPER_EL0(25),
> +	PMU_PMEVTYPER_EL0(26),
> +	PMU_PMEVTYPER_EL0(27),
> +	PMU_PMEVTYPER_EL0(28),
> +	PMU_PMEVTYPER_EL0(29),
> +	PMU_PMEVTYPER_EL0(30),
> +	/* PMCCFILTR_EL0 */
> +	{ Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b1111), Op2(0b111),
> +	  access_pmu_reg, reset_val, PMCCFILTR_EL0, 0 },
> +

why is PMCCFULTR just accessing state on the VCPU, shouldn't this have
the same behavior as accesses to PMXEVTYPER_EL0, just for the cycle
counter event?

>  	/* DACR32_EL2 */
>  	{ Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
>  	  NULL, reset_unknown, DACR32_EL2 },
> -- 
> 2.1.0
> 

Thanks,
-Christoffer

[PATCH 17/18] KVM: ARM64: Add PMU overflow interrupt routing

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

When calling perf_event_create_kernel_counter to create perf_event,
assign a overflow handler. Then when perf event overflows, if vcpu
doesn't run, call irq_work_queue to wake up vcpu. Otherwise call
kvm_vgic_inject_irq to inject the interrupt.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm/kvm/arm.c    |  2 ++
 include/kvm/arm_pmu.h |  2 ++
 virt/kvm/arm/pmu.c    | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 4e82625..41eb063 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -551,6 +551,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 			preempt_enable();
 			kvm_timer_sync_hwstate(vcpu);
 			kvm_vgic_sync_hwstate(vcpu);
+			kvm_pmu_sync_hwstate(vcpu);
 			continue;
 		}
 
@@ -595,6 +596,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 		kvm_timer_sync_hwstate(vcpu);
 		kvm_vgic_sync_hwstate(vcpu);
+		kvm_pmu_sync_hwstate(vcpu);
 
 		ret = handle_exit(vcpu, run, ret);
 	}
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 6985809..5bcf27b 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -45,6 +45,7 @@ struct kvm_pmu {
 };
 
 #ifdef CONFIG_KVM_ARM_PMU
+void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
 			       unsigned long val);
@@ -59,6 +60,7 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
 #else
+void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
 static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
 			       unsigned long val) {}
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index e655426..f957b85 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -20,6 +20,7 @@
 #include <linux/kvm_host.h>
 #include <linux/perf_event.h>
 #include <kvm/arm_pmu.h>
+#include <kvm/arm_vgic.h>
 
 /* PMU HW events mapping. */
 static struct kvm_pmu_hw_event_map {
@@ -81,6 +82,23 @@ static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu,
 }
 
 /**
+ * kvm_pmu_sync_hwstate - sync pmu state for cpu
+ * @vcpu: The vcpu pointer
+ *
+ * Inject virtual PMU IRQ if IRQ is pending for this cpu.
+ */
+void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+
+	if (pmu->irq_pending) {
+		kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, pmu->irq_num, 1);
+		pmu->irq_pending = 0;
+		return;
+	}
+}
+
+/**
  * kvm_pmu_vcpu_reset - reset pmu state for cpu
  * @vcpu: The vcpu pointer
  *
@@ -96,6 +114,37 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
 	pmu->irq_pending = false;
 }
 
+static void kvm_pum_trigger_pmi(struct irq_work *irq_work)
+{
+	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
+	struct kvm_vcpu *vcpu = container_of(pmu, struct kvm_vcpu, arch.pmu);
+
+	kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, pmu->irq_num, 1);
+}
+
+/**
+ * When perf event overflows, if vcpu doesn't run, call irq_work_queue to wake
+ * up vcpu. Otherwise call kvm_vgic_inject_irq to inject the interrupt.
+ */
+static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
+				  struct perf_sample_data *data,
+				  struct pt_regs *regs)
+{
+	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
+	struct kvm_vcpu *vcpu = pmc->vcpu;
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+
+	if (pmc->interrupt == true) {
+		__set_bit(pmc->idx, (unsigned long *)&pmu->overflow_status);
+		pmu->irq_pending = 1;
+		if (vcpu->mode != IN_GUEST_MODE)
+			irq_work_queue(&pmu->irq_work);
+		else
+			kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
+					    pmu->irq_num, 1);
+	}
+}
+
 /**
  * kvm_pmu_set_counter_value - set PMU counter value
  * @vcpu: The vcpu pointer
@@ -322,7 +371,8 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 	attr.config = config;
 	attr.sample_period = (-pmc->counter) & (((u64)1 << 32) - 1);
 
-	event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc);
+	event = perf_event_create_kernel_counter(&attr, -1, current,
+						 kvm_pmu_perf_overflow, pmc);
 	if (IS_ERR(event)) {
 		kvm_err("kvm: pmu event creation failed %ld\n",
 			  PTR_ERR(event));
@@ -351,4 +401,5 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
 	}
 
 	pmu->irq_num = -1;
+	init_irq_work(&pmu->irq_work, kvm_pum_trigger_pmi);
 }
-- 
2.1.0

[PATCH 17/18] KVM: ARM64: Add PMU overflow interrupt routing

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:47AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> When calling perf_event_create_kernel_counter to create perf_event,
> assign a overflow handler. Then when perf event overflows, if vcpu
> doesn't run, call irq_work_queue to wake up vcpu. Otherwise call
> kvm_vgic_inject_irq to inject the interrupt.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm/kvm/arm.c    |  2 ++
>  include/kvm/arm_pmu.h |  2 ++
>  virt/kvm/arm/pmu.c    | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++-
>  3 files changed, 56 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 4e82625..41eb063 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -551,6 +551,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  			preempt_enable();
>  			kvm_timer_sync_hwstate(vcpu);
>  			kvm_vgic_sync_hwstate(vcpu);
> +			kvm_pmu_sync_hwstate(vcpu);
>  			continue;
>  		}
>  
> @@ -595,6 +596,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  
>  		kvm_timer_sync_hwstate(vcpu);
>  		kvm_vgic_sync_hwstate(vcpu);
> +		kvm_pmu_sync_hwstate(vcpu);
>  
>  		ret = handle_exit(vcpu, run, ret);
>  	}
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 6985809..5bcf27b 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -45,6 +45,7 @@ struct kvm_pmu {
>  };
>  
>  #ifdef CONFIG_KVM_ARM_PMU
> +void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
>  void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu);
>  void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
>  			       unsigned long val);
> @@ -59,6 +60,7 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
>  #else
> +void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
>  static inline void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) {}
>  void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, unsigned long select_idx,
>  			       unsigned long val) {}
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index e655426..f957b85 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -20,6 +20,7 @@
>  #include <linux/kvm_host.h>
>  #include <linux/perf_event.h>
>  #include <kvm/arm_pmu.h>
> +#include <kvm/arm_vgic.h>
>  
>  /* PMU HW events mapping. */
>  static struct kvm_pmu_hw_event_map {
> @@ -81,6 +82,23 @@ static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu,
>  }
>  
>  /**
> + * kvm_pmu_sync_hwstate - sync pmu state for cpu
> + * @vcpu: The vcpu pointer
> + *
> + * Inject virtual PMU IRQ if IRQ is pending for this cpu.
> + */
> +void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +
> +	if (pmu->irq_pending) {
> +		kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, pmu->irq_num, 1);

don't you need to check if irq_num is set all these places where you use
it, somehow?

> +		pmu->irq_pending = 0;
> +		return;
> +	}
> +}

I'm not sure I understand why this function is needed at this point in
the first place.  Why don't we just inject the interrupt when the
overflow happens?

I'm also not completely clear on the relationship between when the
physical counter overflows and when the virtual one does - do we always
keep that in sync somehow?  (this may relate to one of my questions
in one of the previous patches).


> +
> +/**
>   * kvm_pmu_vcpu_reset - reset pmu state for cpu
>   * @vcpu: The vcpu pointer
>   *
> @@ -96,6 +114,37 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
>  	pmu->irq_pending = false;
>  }
>  
> +static void kvm_pum_trigger_pmi(struct irq_work *irq_work)
> +{
> +	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
> +	struct kvm_vcpu *vcpu = container_of(pmu, struct kvm_vcpu, arch.pmu);
> +
> +	kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, pmu->irq_num, 1);
> +}
> +
> +/**
> + * When perf event overflows, if vcpu doesn't run, call irq_work_queue to wake
> + * up vcpu. Otherwise call kvm_vgic_inject_irq to inject the interrupt.
> + */
> +static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
> +				  struct perf_sample_data *data,
> +				  struct pt_regs *regs)
> +{
> +	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
> +	struct kvm_vcpu *vcpu = pmc->vcpu;
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +
> +	if (pmc->interrupt == true) {
> +		__set_bit(pmc->idx, (unsigned long *)&pmu->overflow_status);

why not declare overflow_status as an unsigned long instead?

> +		pmu->irq_pending = 1;
> +		if (vcpu->mode != IN_GUEST_MODE)
> +			irq_work_queue(&pmu->irq_work);

why do you need to do this only when the vcpu is in guest mode?

also, aren't all the counters per-cpu, so how can the cpu ever be in
guest mode while this is happening?

> +		else
> +			kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
> +					    pmu->irq_num, 1);

what context is this overflow handler function?  kvm_vgic_inject_irq
grabs a mutex, so it can sleep...

from a quick glance at the perf core code, it looks like this is in
interrupt context, so that call to kvm_vgic_inject_irq looks bad.


> +	}
> +}
> +
>  /**
>   * kvm_pmu_set_counter_value - set PMU counter value
>   * @vcpu: The vcpu pointer
> @@ -322,7 +371,8 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  	attr.config = config;
>  	attr.sample_period = (-pmc->counter) & (((u64)1 << 32) - 1);
>  
> -	event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc);
> +	event = perf_event_create_kernel_counter(&attr, -1, current,
> +						 kvm_pmu_perf_overflow, pmc);

do we properly tear this down when the vcpu dies so that we never start
dereferencing the vcpu in kvm_pmu_perf_overflow if the vcpu goes away?

>  	if (IS_ERR(event)) {
>  		kvm_err("kvm: pmu event creation failed %ld\n",
>  			  PTR_ERR(event));
> @@ -351,4 +401,5 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu)
>  	}
>  
>  	pmu->irq_num = -1;
> +	init_irq_work(&pmu->irq_work, kvm_pum_trigger_pmi);
>  }
> -- 
> 2.1.0
> 

Thanks,
-Christoffer

[PATCH 18/18] KVM: ARM64: Add KVM_CAP_ARM_PMU and KVM_ARM_PMU_SET_IRQ

[shannon.zhao at linaro.org]

From: Shannon Zhao <shannon.zhao@linaro.org>

Add KVM_CAP_ARM_PMU for userspace to check whether KVM supports PMU. Add
KVM_ARM_PMU_SET_IRQ for userspace to set PMU IRQ number.

Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
---
 arch/arm/kvm/arm.c       | 8 ++++++++
 include/kvm/arm_pmu.h    | 5 +++++
 include/uapi/linux/kvm.h | 4 ++++
 virt/kvm/arm/pmu.c       | 9 +++++++++
 4 files changed, 26 insertions(+)

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 41eb063..350866e 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -182,6 +182,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_ARM_PSCI_0_2:
 	case KVM_CAP_READONLY_MEM:
 	case KVM_CAP_MP_STATE:
+	case KVM_CAP_ARM_PMU:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
@@ -816,6 +817,13 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 			return -E2BIG;
 		return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
 	}
+	case KVM_ARM_PMU_SET_IRQ: {
+		uint32_t irq;
+
+		if (copy_from_user(&irq, argp, sizeof(irq)))
+			return -EFAULT;
+		return kvm_pmu_set_irq_num(vcpu, irq);
+	}
 	default:
 		return -EINVAL;
 	}
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 5bcf27b..1a93f53 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -58,6 +58,7 @@ void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx);
+int kvm_pmu_set_irq_num(struct kvm_vcpu *vcpu, u32 irq);
 void kvm_pmu_init(struct kvm_vcpu *vcpu);
 #else
 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
@@ -76,6 +77,10 @@ void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val) {}
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 				    unsigned long select_idx) {}
+int kvm_pmu_set_irq_num(struct kvm_vcpu *vcpu, u32 irq)
+{
+	return -ENXIO;
+}
 static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
 #endif
 
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 716ad4a..90f5e73 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -817,6 +817,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_DISABLE_QUIRKS 116
 #define KVM_CAP_X86_SMM 117
 #define KVM_CAP_MULTI_ADDRESS_SPACE 118
+#define KVM_CAP_ARM_PMU 119
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1205,6 +1206,9 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_X86_SMM */
 #define KVM_SMI                   _IO(KVMIO,   0xb7)
 
+/* Available with KVM_CAP_ARM_PMU */
+#define KVM_ARM_PMU_SET_IRQ	  _IOW(KVMIO, 0xb8, __u32)
+
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
 #define KVM_DEV_ASSIGN_MASK_INTX	(1 << 2)
diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
index f957b85..57585e1 100644
--- a/virt/kvm/arm/pmu.c
+++ b/virt/kvm/arm/pmu.c
@@ -381,6 +381,15 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
 	pmc->perf_event = event;
 }
 
+int kvm_pmu_set_irq_num(struct kvm_vcpu *vcpu, u32 irq)
+{
+	struct kvm_pmu *pmu = &vcpu->arch.pmu;
+
+	kvm_info("kvm_arm_set_pmu_irq: irq: %u\n", irq);
+	pmu->irq_num = irq;
+	return 0;
+}
+
 /**
  * kvm_pmu_init - Initialize global PMU state for per vcpu
  * @vcpu: The vcpu pointer
-- 
2.1.0

[PATCH 18/18] KVM: ARM64: Add KVM_CAP_ARM_PMU and KVM_ARM_PMU_SET_IRQ

[Christoffer Dall]

On Mon, Jul 06, 2015 at 10:17:48AM +0800, shannon.zhao at linaro.org wrote:
> From: Shannon Zhao <shannon.zhao@linaro.org>
> 
> Add KVM_CAP_ARM_PMU for userspace to check whether KVM supports PMU. Add
> KVM_ARM_PMU_SET_IRQ for userspace to set PMU IRQ number.
> 
> Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
> ---
>  arch/arm/kvm/arm.c       | 8 ++++++++
>  include/kvm/arm_pmu.h    | 5 +++++
>  include/uapi/linux/kvm.h | 4 ++++
>  virt/kvm/arm/pmu.c       | 9 +++++++++
>  4 files changed, 26 insertions(+)
> 
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 41eb063..350866e 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -182,6 +182,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>  	case KVM_CAP_ARM_PSCI_0_2:
>  	case KVM_CAP_READONLY_MEM:
>  	case KVM_CAP_MP_STATE:
> +	case KVM_CAP_ARM_PMU:
>  		r = 1;
>  		break;
>  	case KVM_CAP_COALESCED_MMIO:
> @@ -816,6 +817,13 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
>  			return -E2BIG;
>  		return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
>  	}
> +	case KVM_ARM_PMU_SET_IRQ: {
> +		uint32_t irq;
> +
> +		if (copy_from_user(&irq, argp, sizeof(irq)))
> +			return -EFAULT;
> +		return kvm_pmu_set_irq_num(vcpu, irq);
> +	}
>  	default:
>  		return -EINVAL;
>  	}
> diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
> index 5bcf27b..1a93f53 100644
> --- a/include/kvm/arm_pmu.h
> +++ b/include/kvm/arm_pmu.h
> @@ -58,6 +58,7 @@ void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val);
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx);
> +int kvm_pmu_set_irq_num(struct kvm_vcpu *vcpu, u32 irq);
>  void kvm_pmu_init(struct kvm_vcpu *vcpu);
>  #else
>  void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
> @@ -76,6 +77,10 @@ void kvm_pmu_enable_interrupt(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, unsigned long val) {}
>  void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  				    unsigned long select_idx) {}
> +int kvm_pmu_set_irq_num(struct kvm_vcpu *vcpu, u32 irq)
> +{
> +	return -ENXIO;
> +}
>  static inline void kvm_pmu_init(struct kvm_vcpu *vcpu) {}
>  #endif
>  
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 716ad4a..90f5e73 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -817,6 +817,7 @@ struct kvm_ppc_smmu_info {
>  #define KVM_CAP_DISABLE_QUIRKS 116
>  #define KVM_CAP_X86_SMM 117
>  #define KVM_CAP_MULTI_ADDRESS_SPACE 118
> +#define KVM_CAP_ARM_PMU 119
>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  
> @@ -1205,6 +1206,9 @@ struct kvm_s390_ucas_mapping {
>  /* Available with KVM_CAP_X86_SMM */
>  #define KVM_SMI                   _IO(KVMIO,   0xb7)
>  
> +/* Available with KVM_CAP_ARM_PMU */
> +#define KVM_ARM_PMU_SET_IRQ	  _IOW(KVMIO, 0xb8, __u32)
> +

does this really warrant a completely new ioctl?  Perhaps a new kvm
device that you create with a PMU would be better, in that way you could
choose whether you want the PMU for the guest or not.

Alternatively, we have some room to spare in the features array of
KVM_ARM_VCPU_INIT.

Alternatively to the alternative, does the IRQ number appear in any
register somewhere?  If so, we could use the ONE_REG interface to set
the irq number.

In any case, the new ABI should be documented in
Documentation/virtual/kvm/api.txt.


>  #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
>  #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
>  #define KVM_DEV_ASSIGN_MASK_INTX	(1 << 2)
> diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c
> index f957b85..57585e1 100644
> --- a/virt/kvm/arm/pmu.c
> +++ b/virt/kvm/arm/pmu.c
> @@ -381,6 +381,15 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, unsigned long data,
>  	pmc->perf_event = event;
>  }
>  
> +int kvm_pmu_set_irq_num(struct kvm_vcpu *vcpu, u32 irq)
> +{
> +	struct kvm_pmu *pmu = &vcpu->arch.pmu;
> +
> +	kvm_info("kvm_arm_set_pmu_irq: irq: %u\n", irq);

kvm_debug instead?

> +	pmu->irq_num = irq;
> +	return 0;
> +}
> +
>  /**
>   * kvm_pmu_init - Initialize global PMU state for per vcpu
>   * @vcpu: The vcpu pointer
> -- 
> 2.1.0
> 

Thanks,
-Christoffer