[PATCH v2 6/6] xen/PMU: PMU emulation code

[Boris Ostrovsky <boris.ostrovsky@oracle.com>]

Add PMU emulation code that runs when we are processing a PMU interrupt. This code
will allow us not to trap to hypervisor on each MSR/LVTPC access (of which there
may be quite a few in the handler).

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/pmu.c             | 209 ++++++++++++++++++++++++++++++++++++-----
 include/xen/interface/xenpmu.h |   1 +
 2 files changed, 187 insertions(+), 23 deletions(-)

diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
index f92d406..e543629 100644
--- a/arch/x86/xen/pmu.c
+++ b/arch/x86/xen/pmu.c
@@ -13,11 +13,22 @@
 /* x86_pmu.handle_irq definition */
 #include <../kernel/cpu/perf_event.h>
 
+#define XENPMU_IRQ_PROCESSING    1
+struct xenpmu {
+	/* Shared page between hypervisor and domain */
+	struct xen_pmu_data *xenpmu_data;
 
-/* Shared page between hypervisor and domain */
-DEFINE_PER_CPU(struct xen_pmu_data *, xenpmu_shared);
-#define get_xenpmu_data()    per_cpu(xenpmu_shared, smp_processor_id());
+	uint8_t flags;
+};
+DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
+#define get_xenpmu_data()    ((per_cpu(xenpmu_shared, \
+				       smp_processor_id())).xenpmu_data)
+#define get_xenpmu_flags()   ((per_cpu(xenpmu_shared, \
+				       smp_processor_id())).flags)
 
+/* Macro for computing address of a PMU MSR bank */
+#define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
+					    (uintptr_t)ctxt->field))
 
 /* AMD PMU */
 #define F15H_NUM_COUNTERS   6
@@ -160,18 +171,124 @@ static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index)
 	return false;
 }
 
-bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
+static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type,
+				  int index, bool is_read)
+{
+	uint64_t *reg = NULL;
+	struct xen_pmu_intel_ctxt *ctxt;
+	uint64_t *fix_counters;
+	struct xen_pmu_cntr_pair *arch_cntr_pair;
+	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
+	uint8_t xenpmu_flags = get_xenpmu_flags();
+
+
+	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
+		return false;
+
+	ctxt = &xenpmu_data->pmu.intel;
+
+	switch (msr) {
+	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
+		reg = &ctxt->global_ovf_ctrl;
+		break;
+	case MSR_CORE_PERF_GLOBAL_STATUS:
+		reg = &ctxt->global_status;
+		break;
+	case MSR_CORE_PERF_GLOBAL_CTRL:
+		reg = &ctxt->global_ctrl;
+		break;
+	case MSR_CORE_PERF_FIXED_CTR_CTRL:
+		reg = &ctxt->fixed_ctrl;
+		break;
+	default:
+		switch (type) {
+		case MSR_TYPE_COUNTER:
+			fix_counters = field_offset(ctxt, fixed_counters);
+			reg = &fix_counters[index];
+			break;
+		case MSR_TYPE_ARCH_COUNTER:
+			arch_cntr_pair = field_offset(ctxt, arch_counters);
+			reg = &arch_cntr_pair[index].counter;
+			break;
+		case MSR_TYPE_ARCH_CTRL:
+			arch_cntr_pair = field_offset(ctxt, arch_counters);
+			reg = &arch_cntr_pair[index].control;
+			break;
+		default:
+			return false;
+		}
+	}
+
+	if (reg) {
+		if (is_read)
+			*val = *reg;
+		else {
+			*reg = *val;
+
+			if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
+				ctxt->global_status &= (~(*val));
+		}
+		return true;
+	}
+
+	return false;
+}
+
+static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read)
 {
+	uint64_t *reg = NULL;
+	int i, off = 0;
+	struct xen_pmu_amd_ctxt *ctxt;
+	uint64_t *counter_regs, *ctrl_regs;
+	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
+	uint8_t xenpmu_flags = get_xenpmu_flags();
+
+	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
+		return false;
+
+	if (k7_counters_mirrored &&
+	    ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
+		msr = get_fam15h_addr(msr);
 
+	ctxt = &xenpmu_data->pmu.amd;
+	for (i = 0; i < amd_num_counters; i++) {
+		if (msr == amd_ctrls_base + off) {
+			ctrl_regs = field_offset(ctxt, ctrls);
+			reg = &ctrl_regs[i];
+			break;
+		} else if (msr == amd_counters_base + off) {
+			counter_regs = field_offset(ctxt, counters);
+			reg = &counter_regs[i];
+			break;
+		}
+		off += amd_msr_step;
+	}
+
+	if (reg) {
+		if (is_read)
+			*val = *reg;
+		else
+			*reg = *val;
+
+		return true;
+	}
+	return false;
+}
+
+bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
+{
 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 		if (is_amd_pmu_msr(msr)) {
-			*val = native_read_msr_safe(msr, err);
+			if (!xen_amd_pmu_emulate(msr, val, 1))
+				*val = native_read_msr_safe(msr, err);
 			return true;
 		}
 	} else {
 		int type, index;
+
 		if (is_intel_pmu_msr(msr, &type, &index)) {
-			*val = native_read_msr_safe(msr, err);
+			if (!xen_intel_pmu_emulate(msr, val, type, index, 1))
+				*val = native_read_msr_safe(msr, err);
 			return true;
 		}
 	}
@@ -181,16 +298,20 @@ bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
 
 bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
 {
+	uint64_t val = ((uint64_t)high << 32) | low;
+
 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 		if (is_amd_pmu_msr(msr)) {
-			*err = native_write_msr_safe(msr, low, high);
+			if (!xen_amd_pmu_emulate(msr, &val, 0))
+				*err = native_write_msr_safe(msr, low, high);
 			return true;
 		}
 	} else {
 		int type, index;
 
 		if (is_intel_pmu_msr(msr, &type, &index)) {
-			*err = native_write_msr_safe(msr, low, high);
+			if (!xen_intel_pmu_emulate(msr, &val, type, index, 0))
+				*err = native_write_msr_safe(msr, low, high);
 			return true;
 		}
 	}
@@ -200,24 +321,52 @@ bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
 
 unsigned long long xen_amd_read_pmc(int counter)
 {
-	uint32_t msr;
-	int err;
+	struct xen_pmu_amd_ctxt *ctxt;
+	uint64_t *counter_regs;
+	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
+	uint8_t xenpmu_flags = get_xenpmu_flags();
+
+	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
+		uint32_t msr;
+		int err;
+
+		msr = amd_counters_base + (counter * amd_msr_step);
+		return native_read_msr_safe(msr, &err);
+	}
 
-	msr = amd_counters_base + (counter * amd_msr_step);
-	return native_read_msr_safe(msr, &err);
+	ctxt = &xenpmu_data->pmu.amd;
+	counter_regs = field_offset(ctxt, counters);
+	return counter_regs[counter];
 }
 
 unsigned long long xen_intel_read_pmc(int counter)
 {
-	int err;
-	uint32_t msr;
+	struct xen_pmu_intel_ctxt *ctxt;
+	uint64_t *fixed_counters;
+	struct xen_pmu_cntr_pair *arch_cntr_pair;
+	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
+	uint8_t xenpmu_flags = get_xenpmu_flags();
 
-	if (counter & (1<<30))
-		msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
-	else
-		msr = MSR_IA32_PERFCTR0 + counter;
+	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
+		uint32_t msr;
+		int err;
+
+		if (counter & (1<<30))
+			msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
+		else
+			msr = MSR_IA32_PERFCTR0 + counter;
 
-	return native_read_msr_safe(msr, &err);
+		return native_read_msr_safe(msr, &err);
+	}
+
+	ctxt = &xenpmu_data->pmu.intel;
+	if (counter & (1<<30)) {
+		fixed_counters = field_offset(ctxt, fixed_counters);
+		return fixed_counters[counter & 0xffff];
+	} else {
+		arch_cntr_pair = field_offset(ctxt, arch_counters);
+		return arch_cntr_pair[counter].counter;
+	}
 }
 
 unsigned long long xen_read_pmc(int counter)
@@ -302,24 +451,37 @@ static void xen_convert_regs(struct cpu_user_regs *xen_regs,
 irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
 {
 	int ret = IRQ_NONE;
+	int err;
 	struct pt_regs regs;
 	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
+	uint8_t xenpmu_flags = get_xenpmu_flags();
 
 	if (!xenpmu_data) {
 		WARN_ONCE(1, "%s: pmudata not initialized\n", __func__);
 		return ret;
 	}
 
+	per_cpu(xenpmu_shared, smp_processor_id()).flags =
+		xenpmu_flags | XENPMU_IRQ_PROCESSING;
+
 	xen_convert_regs(&xenpmu_data->pmu.regs, &regs);
 	if (x86_pmu.handle_irq(&regs))
 		ret = IRQ_HANDLED;
 
+	/* Write out cached context to HW */
+	err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
+	per_cpu(xenpmu_shared, smp_processor_id()).flags = xenpmu_flags;
+	if (err) {
+		WARN_ONCE(1, "%s failed hypercall, err: %d\n", __func__, err);
+		return IRQ_NONE;
+	}
+
 	return ret;
 }
 
 bool is_xen_pmu(int cpu)
 {
-	return (per_cpu(xenpmu_shared, cpu) != NULL);
+	return (get_xenpmu_data() != NULL);
 }
 
 int xen_pmu_init(int cpu)
@@ -345,7 +507,8 @@ int xen_pmu_init(int cpu)
 	if (ret)
 		goto fail;
 
-	per_cpu(xenpmu_shared, cpu) = xenpmu_data;
+	per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
+	per_cpu(xenpmu_shared, cpu).flags = 0;
 
 	if (cpu == 0) {
 		perf_register_guest_info_callbacks(&xen_guest_cbs);
@@ -369,6 +532,6 @@ void xen_pmu_finish(int cpu)
 
 	(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
 
-	vfree(per_cpu(xenpmu_shared, cpu));
-	per_cpu(xenpmu_shared, cpu) = NULL;
+	vfree(per_cpu(xenpmu_shared, cpu).xenpmu_data);
+	per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
 }
diff --git a/include/xen/interface/xenpmu.h b/include/xen/interface/xenpmu.h
index 79384e4..ec90673 100644
--- a/include/xen/interface/xenpmu.h
+++ b/include/xen/interface/xenpmu.h
@@ -14,6 +14,7 @@
 #define XENPMU_init            4
 #define XENPMU_finish          5
 #define XENPMU_lvtpc_set       6
+#define XENPMU_flush           7 /* Write cached MSR values to HW     */
 
 /* Parameter structure for HYPERVISOR_xenpmu_op call */
 struct xen_pmu_params {
-- 
1.8.1.4