[PATCH 00/15] KVM: x86/pmu: Add mediated vPMU PerfMon v5 support

[PATCH 00/15] KVM: x86/pmu: Add mediated vPMU PerfMon v5 support

[Zide Chen]

KVM currently caps the guest Intel architectural PMU version at 2. This
series bumps it to PerfMon v5 and adds support for some features that
are introduced in PerfMon v3-v5.

- CPUID.0AH:ECX fixed-counter bitmap (v5): supersedes EDX[4:0] and
  lets KVM/guests represent non-contiguous fixed counters (e.g. some
  E-core server parts skip fixed counter 3).
- Add support for IA32_PERF_GLOBAL_STATUS_SET (v4): lets software set
  individual bits in the global status MSR.
- Add support for IA32_PERF_GLOBAL_INUSE (v4): reports which counters
  and PMI are currently claimed.
- Add support for streamlined Freeze_LBRs_On_PMI.
- Implement ANYTHREAD_DEPRECATION capability.

Intel PerfMon v5 doesn't support non-contiguous GP counters; however,
this series lays the groundwork for GP counter bitmap support anyway,
for consistency with the host perf API interface, and prepares for
perfmon_mask support.

Freeze_PerfMon_On_PMI virtualization is not implemented because as
commit 3daa96d67274 ("perf/intel: Remove Perfmon-v4 counter_freezing
support") states, the Freeze-on-PMI mechanism violates the perf counter
independence.

Patch series summary:
patch 1-3: Code cleanup: rename and remove redundant definitions.
patch 4-9: Implement PMC bitmap support.
patch 10-13: Implement PerfMon v3-v5 new features.
patch 14: Advertise Perfmon version up to v5.
patch 15: Fix pmu_counters_test selftests fixed counter test case.

This patchset is on top of the Topdown metrics patchset:
https://lore.kernel.org/kvm/20260629231938.15129-1-zide.chen@intel.com/T/#t

Dapeng Mi (6):
  KVM: x86/pmu: Drop nr_arch_{gp,fixed}_counters from kvm_pmu
  perf/x86: Plumb counter bitmap from x86_pmu to x86_pmu_cap
  KVM: x86/pmu: Switch to bitmask-based KVM PMU capabilities
  perf/x86: Remove num_counters_{gp,fixed} from x86_pmu_capability
  KVM: x86/pmu: Advertise PerfMon version 5 on Intel hosts
  KVM: selftests: Support fixed counters bitmap in pmu_counters_test

Zide Chen (9):
  KVM: x86/pmu: Remove redundant Perf Global Status MSR bit definitions
  KVM: x86/pmu: Rename all_valid_pmc_idx to all_valid_pmc_mask
  KVM: x86/pmu: Rename reserved_bits to eventsel_rsvd in kvm_pmu
  KVM: x86/pmu: Add PMC bitmap accessor helpers
  KVM: x86/pmu: Expose kvm_host_pmu to vendor modules
  KVM: x86/pmu: Emulate the GLOBAL_STATUS_SET and GLOBAL_INUSE MSRs
  KVM: x86/pmu: Emulate streamlined Freeze-LBR-on-PMI
  KVM: x86/pmu: Populate CPUID.0AH:ECX fixed-counter bitmap
  KVM: x86/pmu: Ignore AnyThread bit if CPUID.0AH:EDX[15] is not set

 arch/x86/events/core.c                        |   8 +-
 arch/x86/include/asm/kvm_host.h               |   9 +-
 arch/x86/include/asm/msr-index.h              |  12 +-
 arch/x86/include/asm/perf_event.h             |  10 +-
 arch/x86/kvm/cpuid.c                          |  19 +-
 arch/x86/kvm/msrs.c                           |  12 +-
 arch/x86/kvm/pmu.c                            |  86 +++++----
 arch/x86/kvm/pmu.h                            |  56 +++++-
 arch/x86/kvm/svm/pmu.c                        |  31 ++--
 arch/x86/kvm/svm/svm.c                        |  14 +-
 arch/x86/kvm/vmx/nested.c                     |   6 +-
 arch/x86/kvm/vmx/pmu_intel.c                  | 167 +++++++++++++-----
 arch/x86/kvm/vmx/vmx.c                        |  19 +-
 .../selftests/kvm/x86/pmu_counters_test.c     |  25 ++-
 14 files changed, 319 insertions(+), 155 deletions(-)

-- 
2.54.0

[PATCH 01/15] KVM: x86/pmu: Remove redundant Perf Global Status MSR bit definitions

[Zide Chen]

IA32_PERF_GLOBAL_STATUS and its SET/RESET counterparts share bit
definitions with perf_event.h, which is the canonical home for Intel
PMU bit definitions (e.g., PERFEVTSEL, FIXED_CTR_CTRL, GLOBAL_STATUS).
Drop the duplicate definitions from msr-index.h and update KVM code to
use the macros from perf_event.h.

Drop the comments as well, since the macros now match the MSR
naming.

No functional change intended.

Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/include/asm/msr-index.h |  8 --------
 arch/x86/kvm/vmx/pmu_intel.c     | 11 ++---------
 arch/x86/kvm/vmx/vmx.c           |  2 +-
 3 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index fdcaeb6c8352..feee92aab504 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -1244,14 +1244,6 @@
 
 #define MSR_PERF_METRICS		0x00000329
 
-/* PERF_GLOBAL_OVF_CTL bits */
-#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT	55
-#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI		(1ULL << MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT)
-#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF_BIT		62
-#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF			(1ULL <<  MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF_BIT)
-#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD_BIT		63
-#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD			(1ULL << MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD_BIT)
-
 /* Geode defined MSRs */
 #define MSR_GEODE_BUSCONT_CONF0		0x00001900
 
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 93b5a8360377..d0e16a3211ca 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -604,17 +604,10 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 			 ((BIT_ULL(pmu->nr_arch_fixed_counters) - 1) << KVM_FIXED_PMC_BASE_IDX));
 	pmu->global_ctrl_rsvd = counter_rsvd;
 
-	/*
-	 * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET)
-	 * share reserved bit definitions.  The kernel just happens to use
-	 * OVF_CTRL for the names.
-	 */
 	pmu->global_status_rsvd = pmu->global_ctrl_rsvd
-			& ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF |
-			    MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD);
+			& ~(GLOBAL_STATUS_BUFFER_OVF | GLOBAL_STATUS_COND_CHG);
 	if (vmx_pt_mode_is_host_guest())
-		pmu->global_status_rsvd &=
-				~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI;
+		pmu->global_status_rsvd &= ~GLOBAL_STATUS_TRACE_TOPAPMI;
 
 	if (perf_capabilities & PERF_CAP_PERF_METRICS) {
 		pmu->global_ctrl_rsvd &= ~GLOBAL_CTRL_EN_PERF_METRICS;
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 21eb4b339fa6..e54b45d9bace 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8550,7 +8550,7 @@ static unsigned int vmx_handle_intel_pt_intr(void)
 		return 0;
 
 	kvm_make_request(KVM_REQ_PMI, vcpu);
-	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
+	__set_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT,
 		  (unsigned long *)&vcpu->arch.pmu.global_status);
 	return 1;
 }
-- 
2.54.0

[PATCH 02/15] KVM: x86/pmu: Rename all_valid_pmc_idx to all_valid_pmc_mask

[Zide Chen]

all_valid_pmc_idx is a bitmap of valid PMC indices, not an index
itself; rename it to all_valid_pmc_mask to better reflect what it
holds.

No functional change intended.

Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/pmu.c              | 10 +++++-----
 arch/x86/kvm/vmx/pmu_intel.c    |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 96376d8a5199..4677773cfa30 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -643,7 +643,7 @@ struct kvm_pmu {
 		DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX);
 		atomic64_t __reprogram_pmi;
 	};
-	DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX);
+	DECLARE_BITMAP(all_valid_pmc_mask, X86_PMC_IDX_MAX);
 	DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX);
 
 	DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX);
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 04b9c840f218..9db12c54814d 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -967,7 +967,7 @@ static void kvm_pmu_reset(struct kvm_vcpu *vcpu)
 	bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX);
 	bitmap_zero(pmu->pmc_has_mode_specific_enables, X86_PMC_IDX_MAX);
 
-	kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) {
+	kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_mask) {
 		pmc_stop_counter(pmc);
 		pmc->counter = 0;
 		pmc->emulated_counter = 0;
@@ -1014,7 +1014,7 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 	pmu->fixed_ctr_ctrl_rsvd = ~0ull;
 	pmu->pebs_enable_rsvd = ~0ull;
 	pmu->pebs_data_cfg_rsvd = ~0ull;
-	bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX);
+	bitmap_zero(pmu->all_valid_pmc_mask, X86_PMC_IDX_MAX);
 
 	if (!vcpu->kvm->arch.enable_pmu)
 		return;
@@ -1035,8 +1035,8 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 	if (kvm_vcpu_has_mediated_pmu(vcpu))
 		kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl);
 
-	bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters);
-	bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX,
+	bitmap_set(pmu->all_valid_pmc_mask, 0, pmu->nr_arch_gp_counters);
+	bitmap_set(pmu->all_valid_pmc_mask, KVM_FIXED_PMC_BASE_IDX,
 		   pmu->nr_arch_fixed_counters);
 }
 
@@ -1058,7 +1058,7 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
 
 	pmu->need_cleanup = false;
 
-	bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
+	bitmap_andnot(bitmask, pmu->all_valid_pmc_mask,
 		      pmu->pmc_in_use, X86_PMC_IDX_MAX);
 
 	kvm_for_each_pmc(pmu, pmc, i, bitmask) {
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index d0e16a3211ca..5950445ebc69 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -585,7 +585,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 		lbr_desc->records.nr = 0;
 
 	if (lbr_desc->records.nr)
-		bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1);
+		bitmap_set(pmu->all_valid_pmc_mask, INTEL_PMC_IDX_FIXED_VLBR, 1);
 
 	if (pmu->version == 1)
 		return;
-- 
2.54.0

[PATCH 03/15] KVM: x86/pmu: Rename reserved_bits to eventsel_rsvd in kvm_pmu

[Zide Chen]

This field stores the bits that are reserved in guest IA32_PERFEVTSELx
MSRs. Rename it to eventsel_rsvd to better reflect its purpose and to
align with other field names in struct kvm_pmu.

Opportunistically, replace the magic number 0xffffffff00200000ull with
macros.

No functional change intended.

Suggested-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/pmu.c              |  3 ++-
 arch/x86/kvm/svm/pmu.c          |  6 +++---
 arch/x86/kvm/vmx/pmu_intel.c    | 12 ++++++------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 4677773cfa30..395b6f20e9ac 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -628,7 +628,7 @@ struct kvm_pmu {
 	u64 counter_bitmask[2];
 	u64 global_ctrl_rsvd;
 	u64 global_status_rsvd;
-	u64 reserved_bits;
+	u64 eventsel_rsvd;
 	u64 raw_event_mask;
 	u64 perf_metrics;
 	struct kvm_pmc gp_counters[KVM_MAX_NR_GP_COUNTERS];
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 9db12c54814d..cef22fed6c53 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -1007,7 +1007,8 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 	pmu->nr_arch_fixed_counters = 0;
 	pmu->counter_bitmask[KVM_PMC_GP] = 0;
 	pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
-	pmu->reserved_bits = 0xffffffff00200000ull;
+	/* KVM is not able to emulate the AnyThread bit */
+	pmu->eventsel_rsvd = GENMASK_ULL(63, 32) | ARCH_PERFMON_EVENTSEL_ANY;
 	pmu->raw_event_mask = X86_RAW_EVENT_MASK;
 	pmu->global_ctrl_rsvd = ~0ull;
 	pmu->global_status_rsvd = ~0ull;
diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c
index 0517fd4bbcd7..90832160fa34 100644
--- a/arch/x86/kvm/svm/pmu.c
+++ b/arch/x86/kvm/svm/pmu.c
@@ -168,7 +168,7 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	/* MSR_EVNTSELn */
 	pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_EVNTSEL);
 	if (pmc) {
-		data &= ~pmu->reserved_bits;
+		data &= ~pmu->eventsel_rsvd;
 		if (data != pmc->eventsel) {
 			pmc->eventsel = data;
 			pmc->eventsel_hw = (data & ~AMD64_EVENTSEL_HOSTONLY) |
@@ -219,9 +219,9 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu)
 
 	pmu->counter_bitmask[KVM_PMC_GP] = BIT_ULL(48) - 1;
 
-	pmu->reserved_bits = 0xfffffff000280000ull;
+	pmu->eventsel_rsvd = 0xfffffff000280000ull;
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_SVM) && kvm_vcpu_has_mediated_pmu(vcpu))
-		pmu->reserved_bits &= ~AMD64_EVENTSEL_HOST_GUEST_MASK;
+		pmu->eventsel_rsvd &= ~AMD64_EVENTSEL_HOST_GUEST_MASK;
 
 	pmu->raw_event_mask = AMD64_RAW_EVENT_MASK;
 	/* not applicable to AMD; but clean them to prevent any fall out */
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 5950445ebc69..f42b2972cb7b 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -402,7 +402,7 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	struct kvm_pmc *pmc;
 	u32 msr = msr_info->index;
 	u64 data = msr_info->data;
-	u64 reserved_bits, diff;
+	u64 eventsel_rsvd, diff;
 
 	switch (msr) {
 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
@@ -459,11 +459,11 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			pmc_write_counter(pmc, data);
 			break;
 		} else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
-			reserved_bits = pmu->reserved_bits;
+			eventsel_rsvd = pmu->eventsel_rsvd;
 			if ((pmc->idx == 2) &&
 			    (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED))
-				reserved_bits ^= HSW_IN_TX_CHECKPOINTED;
-			if (data & reserved_bits)
+				eventsel_rsvd ^= HSW_IN_TX_CHECKPOINTED;
+			if (data & eventsel_rsvd)
 				return 1;
 
 			if (data != pmc->eventsel) {
@@ -573,7 +573,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	if (entry &&
 	    (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
 	    (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) {
-		pmu->reserved_bits ^= HSW_IN_TX;
+		pmu->eventsel_rsvd ^= HSW_IN_TX;
 		pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
 	}
 
@@ -617,7 +617,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	if (perf_capabilities & PERF_CAP_PEBS_FORMAT) {
 		if (perf_capabilities & PERF_CAP_PEBS_BASELINE) {
 			pmu->pebs_enable_rsvd = counter_rsvd;
-			pmu->reserved_bits &= ~ICL_EVENTSEL_ADAPTIVE;
+			pmu->eventsel_rsvd &= ~ICL_EVENTSEL_ADAPTIVE;
 			pmu->pebs_data_cfg_rsvd = ~0xff00000full;
 			intel_pmu_enable_fixed_counter_bits(pmu, ICL_FIXED_0_ADAPTIVE);
 		} else {
-- 
2.54.0

[PATCH 04/15] KVM: x86/pmu: Add PMC bitmap accessor helpers

[Zide Chen]

pmu->nr_arch_{gp,fixed}_counters is not able to represent that a PMU
may include non-contiguous GP or fixed counters.

pmu->all_valid_pmc_mask already holds a bitmap indicating both fixed
and general-purpose counters, and loops over valid counters can be
done via pmu->all_valid_pmc_mask alone. Extend it to a union so that
the u64 alias is available for convenient mask arithmetic operations.

Add the necessary helpers to prepare for bitmap-based PMC counter
implementation.

No functional change intended.

Co-developed-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/include/asm/kvm_host.h |  5 ++-
 arch/x86/kvm/pmu.h              | 55 +++++++++++++++++++++++++++++----
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 395b6f20e9ac..f648dc168685 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -643,7 +643,10 @@ struct kvm_pmu {
 		DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX);
 		atomic64_t __reprogram_pmi;
 	};
-	DECLARE_BITMAP(all_valid_pmc_mask, X86_PMC_IDX_MAX);
+	union {
+		DECLARE_BITMAP(all_valid_pmc_mask, X86_PMC_IDX_MAX);
+		u64 all_valid_pmc_mask64;
+	};
 	DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX);
 
 	DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX);
diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index cdbefda844b9..95dc95a9ae37 100644
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -88,6 +88,32 @@ static inline bool kvm_vcpu_has_mediated_pmu(struct kvm_vcpu *vcpu)
 	return enable_mediated_pmu && vcpu_to_pmu(vcpu)->version;
 }
 
+static inline unsigned long kvm_gp_pmc_mask(struct kvm_pmu *pmu)
+{
+	return pmu->all_valid_pmc_mask64 &
+	       GENMASK_ULL(KVM_MAX_NR_GP_COUNTERS - 1, 0);
+}
+
+static inline unsigned long kvm_fixed_pmc_mask(struct kvm_pmu *pmu)
+{
+	return (pmu->all_valid_pmc_mask64 >> KVM_FIXED_PMC_BASE_IDX) &
+		GENMASK_ULL(KVM_MAX_NR_FIXED_COUNTERS - 1, 0);
+}
+
+static inline bool kvm_gp_pmc_supported(struct kvm_pmu *pmu, unsigned int idx)
+{
+	unsigned long bitmap = kvm_gp_pmc_mask(pmu);
+
+	return idx < KVM_MAX_NR_GP_COUNTERS && test_bit(idx, &bitmap);
+}
+
+static inline bool kvm_fixed_pmc_supported(struct kvm_pmu *pmu, unsigned int idx)
+{
+	unsigned long bitmap = kvm_fixed_pmc_mask(pmu);
+
+	return idx < KVM_MAX_NR_FIXED_COUNTERS && test_bit(idx, &bitmap);
+}
+
 /*
  * KVM tracks all counters in 64-bit bitmaps, with general purpose counters
  * mapped to bits 31:0 and fixed counters mapped to 63:32, e.g. fixed counter 0
@@ -104,11 +130,11 @@ static inline bool kvm_vcpu_has_mediated_pmu(struct kvm_vcpu *vcpu)
  */
 static inline struct kvm_pmc *kvm_pmc_idx_to_pmc(struct kvm_pmu *pmu, int idx)
 {
-	if (idx < pmu->nr_arch_gp_counters)
+	if (kvm_gp_pmc_supported(pmu, idx))
 		return &pmu->gp_counters[idx];
 
 	idx -= KVM_FIXED_PMC_BASE_IDX;
-	if (idx >= 0 && idx < pmu->nr_arch_fixed_counters)
+	if (kvm_fixed_pmc_supported(pmu, idx))
 		return &pmu->fixed_counters[idx];
 
 	return NULL;
@@ -120,6 +146,17 @@ static inline struct kvm_pmc *kvm_pmc_idx_to_pmc(struct kvm_pmu *pmu, int idx)
 			continue;				\
 		else						\
 
+/*
+ * @mask must be an lvalue of type unsigned long because for_each_set_bit()
+ * takes its address.
+ *
+ * @type is token-pasted into KVM_MAX_NR_##type##_COUNTERS to match one of the
+ * counter defines, e.g. GP, FIXED, AMD_GP, INTEL_GP, or INTEL_FIXED. This
+ * reflects what KVM supports, not the underlying host's PMU capabilities.
+ */
+#define kvm_for_each_set_pmc_idx(i, mask, type)		\
+	for_each_set_bit((i), &(mask), KVM_MAX_NR_##type##_COUNTERS)
+
 static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
 {
 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
@@ -168,9 +205,12 @@ static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu,
 static inline struct kvm_pmc *get_gp_pmc(struct kvm_pmu *pmu, u32 msr,
 					 u32 base)
 {
-	if (msr >= base && msr < base + pmu->nr_arch_gp_counters) {
+	if (msr >= base && msr < base + KVM_MAX_NR_GP_COUNTERS) {
 		u32 index = array_index_nospec(msr - base,
-					       pmu->nr_arch_gp_counters);
+					       KVM_MAX_NR_GP_COUNTERS);
+
+		if (!kvm_gp_pmc_supported(pmu, index))
+			return NULL;
 
 		return &pmu->gp_counters[index];
 	}
@@ -183,9 +223,12 @@ static inline struct kvm_pmc *get_fixed_pmc(struct kvm_pmu *pmu, u32 msr)
 {
 	int base = MSR_CORE_PERF_FIXED_CTR0;
 
-	if (msr >= base && msr < base + pmu->nr_arch_fixed_counters) {
+	if (msr >= base && msr < base + KVM_MAX_NR_FIXED_COUNTERS) {
 		u32 index = array_index_nospec(msr - base,
-					       pmu->nr_arch_fixed_counters);
+					       KVM_MAX_NR_FIXED_COUNTERS);
+
+		if (!kvm_fixed_pmc_supported(pmu, index))
+			return NULL;
 
 		return &pmu->fixed_counters[index];
 	}
-- 
2.54.0

[PATCH 05/15] KVM: x86/pmu: Drop nr_arch_{gp,fixed}_counters from kvm_pmu

[Zide Chen]

From: Dapeng Mi <dapeng1.mi@linux.intel.com>

This is a preparatory step toward fully bitmap-based PMU capability
handling. Later patches switch KVM's view of host PMU capabilities from
counter counts to counter bitmaps.

Populate and use all_valid_pmc_mask directly when walking guest-visible
PMCs, instead of relying on pmu->nr_arch_{gp,fixed}_counters as
intermediate state.

Iterate counters via the newly added all_valid_pmc_mask based helpers,
and remove the now-redundant nr_arch_{gp,fixed}_counters fields from
struct kvm_pmu.

Note: prior to a later patch in this series, the bitmap-based iteration in
{vmx,svm}_recalc_pmu_msr_intercepts() may leave stale intercepts on
trailing counter MSRs after CPUID narrowing. This affects only a narrow
corner case and is resolved by a subsequent patch ("KVM: x86/pmu: Switch
to bitmask-based KVM PMU").

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Co-developed-by: Zide Chen <zide.chen@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/include/asm/kvm_host.h |  2 --
 arch/x86/kvm/pmu.c              | 26 +++++++++-----------
 arch/x86/kvm/svm/pmu.c          | 25 +++++++++----------
 arch/x86/kvm/svm/svm.c          |  5 ++--
 arch/x86/kvm/vmx/nested.c       |  6 +++--
 arch/x86/kvm/vmx/pmu_intel.c    | 43 ++++++++++++++++++++++-----------
 arch/x86/kvm/vmx/vmx.c          |  6 +++--
 7 files changed, 63 insertions(+), 50 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f648dc168685..3dfb2c5ac62d 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -617,8 +617,6 @@ struct kvm_pmc {
 
 struct kvm_pmu {
 	u8 version;
-	unsigned nr_arch_gp_counters;
-	unsigned nr_arch_fixed_counters;
 	unsigned available_event_types;
 	u64 fixed_ctr_ctrl;
 	u64 fixed_ctr_ctrl_hw;
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index cef22fed6c53..0b66e7756ecc 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -795,8 +795,8 @@ static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu)
 	 * KVM's capabilities are constrained based on KVM support, i.e. KVM's
 	 * capabilities themselves may be a subset of hardware capabilities.
 	 */
-	return pmu->nr_arch_gp_counters != kvm_host_pmu.num_counters_gp ||
-	       pmu->nr_arch_fixed_counters != kvm_host_pmu.num_counters_fixed;
+	return kvm_gp_pmc_mask(pmu) != BIT_ULL(kvm_host_pmu.num_counters_gp) - 1 ||
+	       kvm_fixed_pmc_mask(pmu) != BIT_ULL(kvm_host_pmu.num_counters_fixed) - 1;
 }
 
 bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu)
@@ -1003,8 +1003,6 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 	kvm_pmu_reset(vcpu);
 
 	pmu->version = 0;
-	pmu->nr_arch_gp_counters = 0;
-	pmu->nr_arch_fixed_counters = 0;
 	pmu->counter_bitmask[KVM_PMC_GP] = 0;
 	pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
 	/* KVM is not able to emulate the AnyThread bit */
@@ -1029,16 +1027,12 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 	 * in the global controls).  Emulate that behavior when refreshing the
 	 * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL.
 	 */
-	if (pmu->nr_arch_gp_counters &&
+	if (kvm_gp_pmc_mask(pmu) &&
 	    (kvm_pmu_has_perf_global_ctrl(pmu) || kvm_vcpu_has_mediated_pmu(vcpu)))
-		pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0);
+		pmu->global_ctrl = kvm_gp_pmc_mask(pmu);
 
 	if (kvm_vcpu_has_mediated_pmu(vcpu))
 		kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl);
-
-	bitmap_set(pmu->all_valid_pmc_mask, 0, pmu->nr_arch_gp_counters);
-	bitmap_set(pmu->all_valid_pmc_mask, KVM_FIXED_PMC_BASE_IDX,
-		   pmu->nr_arch_fixed_counters);
 }
 
 void kvm_pmu_init(struct kvm_vcpu *vcpu)
@@ -1347,6 +1341,8 @@ static __always_inline u32 gp_eventsel_msr(u32 idx)
 static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
+	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
 	struct kvm_pmc *pmc;
 	u32 i;
 
@@ -1355,14 +1351,14 @@ static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu)
 	 * is intercepted if hardware has counters that aren't visible to the
 	 * guest (KVM will inject #GP as appropriate).
 	 */
-	for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, gp_mask, GP) {
 		pmc = &pmu->gp_counters[i];
 
 		if (pmc->counter != rdpmc(i))
 			wrmsrl(gp_counter_msr(i), pmc->counter);
 		wrmsrl(gp_eventsel_msr(i), pmc->eventsel_hw);
 	}
-	for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, fixed_mask, FIXED) {
 		pmc = &pmu->fixed_counters[i];
 
 		if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i))
@@ -1405,6 +1401,8 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu)
 static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
+	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
 	struct kvm_pmc *pmc;
 	u32 i;
 
@@ -1412,7 +1410,7 @@ static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu)
 	 * Clear selectors and counters to ensure hardware doesn't count using
 	 * guest controls when the host (perf) restores its state.
 	 */
-	for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, gp_mask, GP) {
 		pmc = &pmu->gp_counters[i];
 
 		pmc->counter = rdpmc(i);
@@ -1422,7 +1420,7 @@ static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu)
 			wrmsrq(gp_eventsel_msr(i), 0);
 	}
 
-	for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, fixed_mask, FIXED) {
 		pmc = &pmu->fixed_counters[i];
 
 		pmc->counter = rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i);
diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c
index 90832160fa34..02cf960a215a 100644
--- a/arch/x86/kvm/svm/pmu.c
+++ b/arch/x86/kvm/svm/pmu.c
@@ -27,12 +27,11 @@ enum pmu_type {
 
 static struct kvm_pmc *amd_pmu_get_pmc(struct kvm_pmu *pmu, int pmc_idx)
 {
-	unsigned int num_counters = pmu->nr_arch_gp_counters;
-
-	if (pmc_idx >= num_counters)
+	if (!kvm_gp_pmc_supported(pmu, pmc_idx))
 		return NULL;
 
-	return &pmu->gp_counters[array_index_nospec(pmc_idx, num_counters)];
+	pmc_idx = array_index_nospec(pmc_idx, KVM_MAX_NR_AMD_GP_COUNTERS);
+	return &pmu->gp_counters[pmc_idx];
 }
 
 static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
@@ -77,7 +76,7 @@ static int amd_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
-	if (idx >= pmu->nr_arch_gp_counters)
+	if (!kvm_gp_pmc_supported(pmu, idx))
 		return -EINVAL;
 
 	return 0;
@@ -122,7 +121,7 @@ static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 		return pmu->version > 1;
 	default:
 		if (msr > MSR_F15H_PERF_CTR5 &&
-		    msr < MSR_F15H_PERF_CTL0 + 2 * pmu->nr_arch_gp_counters)
+		    msr < MSR_F15H_PERF_CTL0 + 2 * hweight_long(kvm_gp_pmc_mask(pmu)))
 			return pmu->version > 1;
 		break;
 	}
@@ -189,6 +188,7 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 
 static void amd_pmu_refresh(struct kvm_vcpu *vcpu)
 {
+	unsigned int nr_gp_counters = AMD64_NUM_COUNTERS;
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	union cpuid_0x80000022_ebx ebx;
 
@@ -202,18 +202,16 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu)
 		BUILD_BUG_ON(x86_feature_cpuid(X86_FEATURE_PERFMON_V2).function != 0x80000022 ||
 			     x86_feature_cpuid(X86_FEATURE_PERFMON_V2).index);
 		ebx.full = kvm_find_cpuid_entry_index(vcpu, 0x80000022, 0)->ebx;
-		pmu->nr_arch_gp_counters = ebx.split.num_core_pmc;
+		nr_gp_counters = ebx.split.num_core_pmc;
 	} else if (guest_cpu_cap_has(vcpu, X86_FEATURE_PERFCTR_CORE)) {
-		pmu->nr_arch_gp_counters = AMD64_NUM_COUNTERS_CORE;
-	} else {
-		pmu->nr_arch_gp_counters = AMD64_NUM_COUNTERS;
+		nr_gp_counters = AMD64_NUM_COUNTERS_CORE;
 	}
 
-	pmu->nr_arch_gp_counters = min_t(unsigned int, pmu->nr_arch_gp_counters,
-					 kvm_pmu_cap.num_counters_gp);
+	pmu->all_valid_pmc_mask64 = (BIT_ULL(nr_gp_counters) - 1) &
+				    (BIT_ULL(kvm_pmu_cap.num_counters_gp) - 1);
 
 	if (pmu->version > 1) {
-		pmu->global_ctrl_rsvd = ~(BIT_ULL(pmu->nr_arch_gp_counters) - 1);
+		pmu->global_ctrl_rsvd = ~pmu->all_valid_pmc_mask64;
 		pmu->global_status_rsvd = pmu->global_ctrl_rsvd;
 	}
 
@@ -226,7 +224,6 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu)
 	pmu->raw_event_mask = AMD64_RAW_EVENT_MASK;
 	/* not applicable to AMD; but clean them to prevent any fall out */
 	pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
-	pmu->nr_arch_fixed_counters = 0;
 }
 
 static void amd_pmu_init(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index ef69a51ab27f..3b3e98b6abb6 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -753,18 +753,19 @@ static void svm_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 {
 	bool intercept = !kvm_vcpu_has_mediated_pmu(vcpu);
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
 	int i;
 
 	if (!enable_mediated_pmu)
 		return;
 
 	/* Legacy counters are always available for AMD CPUs with a PMU. */
-	for (i = 0; i < min(pmu->nr_arch_gp_counters, AMD64_NUM_COUNTERS); i++)
+	for_each_set_bit(i, &gp_mask, AMD64_NUM_COUNTERS)
 		svm_set_intercept_for_msr(vcpu, MSR_K7_PERFCTR0 + i,
 					  MSR_TYPE_RW, intercept);
 
 	intercept |= !guest_cpu_cap_has(vcpu, X86_FEATURE_PERFCTR_CORE);
-	for (i = 0; i < pmu->nr_arch_gp_counters; i++)
+	kvm_for_each_set_pmc_idx(i, gp_mask, AMD_GP)
 		svm_set_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i,
 					  MSR_TYPE_RW, intercept);
 
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 23def2157bc5..f888b2de76ac 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -670,6 +670,8 @@ static void nested_vmx_merge_pmu_msr_bitmaps(struct kvm_vcpu *vcpu,
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
 	int i;
 
 	/*
@@ -679,12 +681,12 @@ static void nested_vmx_merge_pmu_msr_bitmaps(struct kvm_vcpu *vcpu,
 	if (!kvm_vcpu_has_mediated_pmu(vcpu))
 		return;
 
-	for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, gp_mask, INTEL_GP) {
 		nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PERFCTR0 + i);
 		nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PMC0 + i);
 	}
 
-	for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
+	kvm_for_each_set_pmc_idx(i, fixed_mask, INTEL_FIXED)
 		nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_FIXED_CTR0 + i);
 
 	nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_GLOBAL_CTRL);
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index f42b2972cb7b..31422bd20d96 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -66,12 +66,13 @@ static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data)
 	 * hardware, e.g. to ensure the event filter is enforced.
 	 */
 	u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw;
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
 	struct kvm_pmc *pmc;
 	int i;
 
 	pmu->fixed_ctr_ctrl = data;
 	pmu->fixed_ctr_ctrl_hw = data;
-	for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, fixed_mask, INTEL_FIXED) {
 		u8 new_ctrl = fixed_ctrl_field(data, i);
 		u8 old_ctrl = fixed_ctrl_field(old_fixed_ctr_ctrl, i);
 
@@ -118,12 +119,18 @@ static int intel_emulate_rdpmc(struct kvm_vcpu *vcpu, unsigned int idx,
 	 */
 	switch (type) {
 	case INTEL_RDPMC_FIXED:
+		if (!kvm_fixed_pmc_supported(pmu, idx))
+			return 1;
+
 		counters = pmu->fixed_counters;
-		num_counters = pmu->nr_arch_fixed_counters;
+		num_counters = KVM_MAX_NR_INTEL_FIXED_COUNTERS;
 		break;
 	case INTEL_RDPMC_GP:
+		if (!kvm_gp_pmc_supported(pmu, idx))
+			return 1;
+
 		counters = pmu->gp_counters;
-		num_counters = pmu->nr_arch_gp_counters;
+		num_counters = KVM_MAX_NR_INTEL_GP_COUNTERS;
 		break;
 	case INTEL_RDPMC_METRICS:
 		if (!kvm_vcpu_has_perf_metrics(vcpu))
@@ -142,9 +149,6 @@ static int intel_emulate_rdpmc(struct kvm_vcpu *vcpu, unsigned int idx,
 		return 1;
 	}
 
-	if (idx >= num_counters)
-		return 1;
-
 	pmc = &counters[array_index_nospec(idx, num_counters)];
 	*data = pmc_read_counter(pmc);
 	return 0;
@@ -520,9 +524,10 @@ static u64 intel_get_fixed_pmc_eventsel(unsigned int index)
 
 static void intel_pmu_enable_fixed_counter_bits(struct kvm_pmu *pmu, u64 bits)
 {
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
 	int i;
 
-	for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
+	kvm_for_each_set_pmc_idx(i, fixed_mask, INTEL_FIXED)
 		pmu->fixed_ctr_ctrl_rsvd &= ~intel_fixed_bits_by_idx(i, bits);
 }
 
@@ -534,6 +539,8 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	union cpuid10_eax eax;
 	union cpuid10_edx edx;
 	u64 perf_capabilities;
+	u64 fixed_cntr_mask;
+	int nr_gp_counters;
 	u64 counter_rsvd;
 
 	if (!lbr_desc)
@@ -560,8 +567,6 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	if (!pmu->version)
 		return;
 
-	pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters,
-					 kvm_pmu_cap.num_counters_gp);
 	eax.split.bit_width = min_t(int, eax.split.bit_width,
 				    kvm_pmu_cap.bit_width_gp);
 	pmu->counter_bitmask[KVM_PMC_GP] = BIT_ULL(eax.split.bit_width) - 1;
@@ -569,6 +574,17 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 				      kvm_pmu_cap.events_mask_len);
 	pmu->available_event_types = ~entry->ebx & (BIT_ULL(eax.split.mask_length) - 1);
 
+	fixed_cntr_mask = BIT_ULL(edx.split.num_counters_fixed) - 1;
+	fixed_cntr_mask &= BIT_ULL(kvm_pmu_cap.num_counters_fixed) - 1;
+
+	/*
+	 * The number of counters comes from guest CPUID data. Clamp the value
+	 * to avoid a shift-by-64 in BIT_ULL().
+	 */
+	nr_gp_counters = min_t(int, eax.split.num_counters, X86_PMC_IDX_MAX - 1);
+	pmu->all_valid_pmc_mask64 = (BIT_ULL(nr_gp_counters) - 1) &
+				    (BIT_ULL(kvm_pmu_cap.num_counters_gp) - 1);
+
 	entry = kvm_find_cpuid_entry_index(vcpu, 7, 0);
 	if (entry &&
 	    (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
@@ -590,8 +606,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	if (pmu->version == 1)
 		return;
 
-	pmu->nr_arch_fixed_counters = min_t(int, edx.split.num_counters_fixed,
-					    kvm_pmu_cap.num_counters_fixed);
+	pmu->all_valid_pmc_mask64 |= fixed_cntr_mask << INTEL_PMC_IDX_FIXED;
 	edx.split.bit_width_fixed = min_t(int, edx.split.bit_width_fixed,
 					  kvm_pmu_cap.bit_width_fixed);
 	pmu->counter_bitmask[KVM_PMC_FIXED] = BIT_ULL(edx.split.bit_width_fixed) - 1;
@@ -600,8 +615,8 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 						 INTEL_FIXED_0_USER |
 						 INTEL_FIXED_0_ENABLE_PMI);
 
-	counter_rsvd = ~((BIT_ULL(pmu->nr_arch_gp_counters) - 1) |
-			 ((BIT_ULL(pmu->nr_arch_fixed_counters) - 1) << KVM_FIXED_PMC_BASE_IDX));
+	counter_rsvd = ~(kvm_gp_pmc_mask(pmu) |
+			(kvm_fixed_pmc_mask(pmu) << KVM_FIXED_PMC_BASE_IDX));
 	pmu->global_ctrl_rsvd = counter_rsvd;
 
 	pmu->global_status_rsvd = pmu->global_ctrl_rsvd
@@ -621,7 +636,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 			pmu->pebs_data_cfg_rsvd = ~0xff00000full;
 			intel_pmu_enable_fixed_counter_bits(pmu, ICL_FIXED_0_ADAPTIVE);
 		} else {
-			pmu->pebs_enable_rsvd = ~(BIT_ULL(pmu->nr_arch_gp_counters) - 1);
+			pmu->pebs_enable_rsvd = ~kvm_gp_pmc_mask(pmu);
 		}
 	}
 }
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index e54b45d9bace..0a6880bb7ebe 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -4226,6 +4226,8 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 	bool has_mediated_pmu = kvm_vcpu_has_mediated_pmu(vcpu);
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
+	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
 	bool intercept = !has_mediated_pmu;
 	int i;
 
@@ -4246,7 +4248,7 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 
 	vm_exit_controls_changebit(vmx, vm_exit_controls_bits, has_mediated_pmu);
 
-	for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+	kvm_for_each_set_pmc_idx(i, gp_mask, INTEL_GP) {
 		vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i,
 					  MSR_TYPE_RW, intercept);
 		vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i, MSR_TYPE_RW,
@@ -4259,7 +4261,7 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 					  MSR_TYPE_RW, true);
 	}
 
-	for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
+	kvm_for_each_set_pmc_idx(i, fixed_mask, INTEL_FIXED)
 		vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i,
 					  MSR_TYPE_RW, intercept);
 	for ( ; i < kvm_pmu_cap.num_counters_fixed; i++)
-- 
2.54.0

[PATCH 06/15] KVM: x86/pmu: Expose kvm_host_pmu to vendor modules

[Zide Chen]

kvm_host_pmu holds the unadulterated host PMU capabilities, which
can be used to compare against guest capabilities to determine whether
certain MSRs should be intercepted by KVM.

Exposing it directly avoids the need for multiple per-field accessors,
which adds boilerplate without hiding any implementation detail worth
encapsulating.

Opportunistically, fix typo "Unadultered" in the comment.

Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/pmu.c | 5 +++--
 arch/x86/kvm/pmu.h | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 0b66e7756ecc..3647ce3f0e3f 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -27,8 +27,9 @@
 /* This is enough to filter the vast majority of currently defined events. */
 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
 
-/* Unadultered PMU capabilities of the host, i.e. of hardware. */
-static struct x86_pmu_capability __read_mostly kvm_host_pmu;
+/* Unadulterated PMU capabilities of the host, i.e. of hardware. */
+struct x86_pmu_capability __read_mostly kvm_host_pmu;
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host_pmu);
 
 /* KVM's PMU capabilities, i.e. the intersection of KVM and hardware support. */
 struct x86_pmu_capability __read_mostly kvm_pmu_cap;
diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index 95dc95a9ae37..95b73aac72db 100644
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -254,6 +254,7 @@ static inline bool pmc_is_locally_enabled(struct kvm_pmc *pmc)
 	return !kvm_pmu_call(pmc_is_disabled_in_current_mode)(pmc);
 }
 
+extern struct x86_pmu_capability kvm_host_pmu;
 extern struct x86_pmu_capability kvm_pmu_cap;
 
 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops);
-- 
2.54.0

[PATCH 07/15] perf/x86: Plumb counter bitmap from x86_pmu to x86_pmu_cap

[Zide Chen]

From: Dapeng Mi <dapeng1.mi@linux.intel.com>

Intel PerfMon v5 introduced CPUID.0AH:ECX to support non-contiguous
fixed counters and Architectural PerfMon Extension leaf (0x23) further
supports non-contiguous general-purpose counters.

num_counters_{gp,fixed} indicates the total number of GP or fixed
counters, but cannot represent non-contiguous counters.

Add cntr_mask and fixed_cntr_mask union so that KVM can get the
accurate counter availability directly from x86_pmu_cap. The u64
alias is convenient for mask arithmetic, while the bitmap form works
with for_each_set_bit() and friends.

num_counters_{gp,fixed} in x86_pmu_capability will be removed once
callers have been converted to the use of {,fixed_}cntr_mask.

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/events/core.c            | 6 ++++--
 arch/x86/include/asm/perf_event.h | 8 ++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 4b9e105309c6..65349819ba43 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -3134,8 +3134,10 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
 	cap->version		= x86_pmu.version;
 	cap->num_counters_gp	= x86_pmu_num_counters(NULL);
 	cap->num_counters_fixed	= x86_pmu_num_counters_fixed(NULL);
-	cap->bit_width_gp	= cap->num_counters_gp ? x86_pmu.cntval_bits : 0;
-	cap->bit_width_fixed	= cap->num_counters_fixed ? x86_pmu.cntval_bits : 0;
+	cap->cntr_mask64	= x86_pmu.cntr_mask64;
+	cap->fixed_cntr_mask64	= x86_pmu.fixed_cntr_mask64;
+	cap->bit_width_gp	= cap->cntr_mask64 ? x86_pmu.cntval_bits : 0;
+	cap->bit_width_fixed	= cap->fixed_cntr_mask64 ? x86_pmu.cntval_bits : 0;
 	cap->events_mask	= (unsigned int)x86_pmu.events_maskl;
 	cap->events_mask_len	= x86_pmu.events_mask_len;
 	cap->pebs_ept		= x86_pmu.pebs_ept;
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index bc2e1cbcd9b9..f59a3d466195 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -302,6 +302,14 @@ struct x86_pmu_capability {
 	int		version;
 	int		num_counters_gp;
 	int		num_counters_fixed;
+	union {
+		u64	cntr_mask64;
+		DECLARE_BITMAP(cntr_mask, X86_PMC_IDX_MAX);
+	};
+	union {
+		u64	fixed_cntr_mask64;
+		DECLARE_BITMAP(fixed_cntr_mask, X86_PMC_IDX_MAX);
+	};
 	int		bit_width_gp;
 	int		bit_width_fixed;
 	unsigned int	events_mask;
-- 
2.54.0

[PATCH 08/15] KVM: x86/pmu: Switch to bitmask-based KVM PMU capabilities

[Zide Chen]

From: Dapeng Mi <dapeng1.mi@linux.intel.com>

Intel platforms support non-contiguous fixed counters via CPUID.0AH:ECX
starting with PerfMon v5, and support non-contiguous GP counters
through the Architectural PerfMon Extension (CPUID leaf 23H).

struct x86_pmu_capability now exposes {,fixed_}cntr_mask64 bitmaps,
which may contain sparse bits representing non-contiguous counters.
Switch KVM's kvm_host_pmu and kvm_pmu_cap consumers over to the new
bitmask fields.

CPUID.0AH:EAX[15:8] and CPUID.0AH:EDX[4:0] enumerate only contiguous
counters. Derive these values from kvm_pmu_cap.{,fixed_}cntr_mask64 as
the number of consecutive counters starting at index 0.

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Co-developed-by: Zide Chen <zide.chen@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/cpuid.c         | 14 +++++++++++---
 arch/x86/kvm/msrs.c          | 12 ++++++------
 arch/x86/kvm/pmu.c           | 34 +++++++++++-----------------------
 arch/x86/kvm/svm/pmu.c       |  2 +-
 arch/x86/kvm/svm/svm.c       |  9 +++++----
 arch/x86/kvm/vmx/pmu_intel.c |  7 ++++---
 arch/x86/kvm/vmx/vmx.c       |  7 +++++--
 7 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 2698fa42cd97..151a4794f834 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -1514,10 +1514,18 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 		}
 
 		eax.split.version_id = kvm_pmu_cap.version;
-		eax.split.num_counters = kvm_pmu_cap.num_counters_gp;
+
+		/* Contiguous GP counters only. */
+		eax.split.num_counters =
+			find_first_zero_bit(kvm_pmu_cap.cntr_mask,
+					    KVM_MAX_NR_GP_COUNTERS);
 		eax.split.bit_width = kvm_pmu_cap.bit_width_gp;
 		eax.split.mask_length = kvm_pmu_cap.events_mask_len;
-		edx.split.num_counters_fixed = kvm_pmu_cap.num_counters_fixed;
+
+		/* Contiguous fixed counters only. */
+		edx.split.num_counters_fixed =
+			find_first_zero_bit(kvm_pmu_cap.fixed_cntr_mask,
+					    KVM_MAX_NR_FIXED_COUNTERS);
 		edx.split.bit_width_fixed = kvm_pmu_cap.bit_width_fixed;
 
 		if (kvm_pmu_cap.version)
@@ -1882,7 +1890,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 
 		cpuid_entry_override(entry, CPUID_8000_0022_EAX);
 
-		ebx.split.num_core_pmc = kvm_pmu_cap.num_counters_gp;
+		ebx.split.num_core_pmc = hweight64(kvm_pmu_cap.cntr_mask64);
 		entry->ebx = ebx.full;
 		break;
 	}
diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c
index c751a8dbd45d..7524d019f1be 100644
--- a/arch/x86/kvm/msrs.c
+++ b/arch/x86/kvm/msrs.c
@@ -2631,20 +2631,20 @@ static void kvm_probe_msr_to_save(u32 msr_index)
 		break;
 	case MSR_ARCH_PERFMON_PERFCTR0 ...
 	     MSR_ARCH_PERFMON_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1:
-		if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >=
-		    kvm_pmu_cap.num_counters_gp)
+		if (!(BIT_ULL(msr_index - MSR_ARCH_PERFMON_PERFCTR0) &
+		      kvm_pmu_cap.cntr_mask64))
 			return;
 		break;
 	case MSR_ARCH_PERFMON_EVENTSEL0 ...
 	     MSR_ARCH_PERFMON_EVENTSEL0 + KVM_MAX_NR_GP_COUNTERS - 1:
-		if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >=
-		    kvm_pmu_cap.num_counters_gp)
+		if (!(BIT_ULL(msr_index - MSR_ARCH_PERFMON_EVENTSEL0) &
+		      kvm_pmu_cap.cntr_mask64))
 			return;
 		break;
 	case MSR_ARCH_PERFMON_FIXED_CTR0 ...
 	     MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1:
-		if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >=
-		    kvm_pmu_cap.num_counters_fixed)
+		if (!(BIT_ULL(msr_index - MSR_ARCH_PERFMON_FIXED_CTR0) &
+		      kvm_pmu_cap.fixed_cntr_mask64))
 			return;
 		break;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 3647ce3f0e3f..bc2ca60114e9 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -20,7 +20,6 @@
 #include <asm/perf_event.h>
 #include <asm/cpu_device_id.h>
 #include "x86.h"
-#include "cpuid.h"
 #include "lapic.h"
 #include "pmu.h"
 
@@ -136,8 +135,6 @@ void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops)
 {
 	bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL;
 	int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS;
-	union cpuid10_edx edx;
-	u32 eax, ebx, ecx;
 
 	/*
 	 * Hybrid PMUs don't play nice with virtualization without careful
@@ -159,8 +156,8 @@ void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops)
 		 * there are a non-zero number of counters, but fewer than what
 		 * is architecturally required.
 		 */
-		if (!kvm_host_pmu.num_counters_gp ||
-		    WARN_ON_ONCE(kvm_host_pmu.num_counters_gp < min_nr_gp_ctrs))
+		if (!kvm_host_pmu.cntr_mask64 ||
+		    WARN_ON_ONCE(hweight64(kvm_host_pmu.cntr_mask64) < min_nr_gp_ctrs))
 			enable_pmu = false;
 		else if (is_intel && !kvm_host_pmu.version)
 			enable_pmu = false;
@@ -180,23 +177,14 @@ void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops)
 
 	memcpy(&kvm_pmu_cap, &kvm_host_pmu, sizeof(kvm_host_pmu));
 	kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2);
-	kvm_pmu_cap.num_counters_gp = min(kvm_pmu_cap.num_counters_gp,
-					  pmu_ops->MAX_NR_GP_COUNTERS);
-	kvm_pmu_cap.num_counters_fixed = min(kvm_pmu_cap.num_counters_fixed,
-					     KVM_MAX_NR_FIXED_COUNTERS);
+	kvm_pmu_cap.cntr_mask64 &=
+		GENMASK_ULL(pmu_ops->MAX_NR_GP_COUNTERS - 1, 0);
+	kvm_pmu_cap.fixed_cntr_mask64 &=
+		GENMASK_ULL(KVM_MAX_NR_FIXED_COUNTERS - 1, 0);
 
-	/*
-	 * Currently, KVM doesn't support non-contiguous fixed counters; make
-	 * sure only contiguous ones are retained in kvm_pmu_cap.
-	 */
-	if (kvm_host_pmu.version >= 5) {
-		cpuid(0xa, &eax, &ebx, &ecx, &edx.full);
-		if (kvm_pmu_cap.num_counters_fixed > edx.split.num_counters_fixed)
-			kvm_pmu_cap.num_counters_fixed = edx.split.num_counters_fixed;
-	}
-
-	if (!enable_mediated_pmu && kvm_pmu_cap.num_counters_fixed > 3)
-		kvm_pmu_cap.num_counters_fixed = 3;
+	/* Legacy vPMU exposes at most 3 fixed counters. */
+	if (!enable_mediated_pmu)
+		kvm_pmu_cap.fixed_cntr_mask64 &= GENMASK_ULL(2, 0);
 
 	kvm_pmu_eventsel.INSTRUCTIONS_RETIRED =
 		perf_get_hw_event_config(PERF_COUNT_HW_INSTRUCTIONS);
@@ -796,8 +784,8 @@ static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu)
 	 * KVM's capabilities are constrained based on KVM support, i.e. KVM's
 	 * capabilities themselves may be a subset of hardware capabilities.
 	 */
-	return kvm_gp_pmc_mask(pmu) != BIT_ULL(kvm_host_pmu.num_counters_gp) - 1 ||
-	       kvm_fixed_pmc_mask(pmu) != BIT_ULL(kvm_host_pmu.num_counters_fixed) - 1;
+	return kvm_gp_pmc_mask(pmu) != kvm_host_pmu.cntr_mask64 ||
+	       kvm_fixed_pmc_mask(pmu) != kvm_host_pmu.fixed_cntr_mask64;
 }
 
 bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c
index 02cf960a215a..d519eba518bf 100644
--- a/arch/x86/kvm/svm/pmu.c
+++ b/arch/x86/kvm/svm/pmu.c
@@ -208,7 +208,7 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu)
 	}
 
 	pmu->all_valid_pmc_mask64 = (BIT_ULL(nr_gp_counters) - 1) &
-				    (BIT_ULL(kvm_pmu_cap.num_counters_gp) - 1);
+				    kvm_pmu_cap.cntr_mask64;
 
 	if (pmu->version > 1) {
 		pmu->global_ctrl_rsvd = ~pmu->all_valid_pmc_mask64;
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 3b3e98b6abb6..002cdd074fd7 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -754,6 +754,7 @@ static void svm_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 	bool intercept = !kvm_vcpu_has_mediated_pmu(vcpu);
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
+	unsigned long host_only_gp_mask;
 	int i;
 
 	if (!enable_mediated_pmu)
@@ -769,7 +770,8 @@ static void svm_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 		svm_set_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i,
 					  MSR_TYPE_RW, intercept);
 
-	for ( ; i < kvm_pmu_cap.num_counters_gp; i++)
+	host_only_gp_mask = kvm_pmu_cap.cntr_mask64 & ~gp_mask;
+	kvm_for_each_set_pmc_idx(i, host_only_gp_mask, AMD_GP)
 		svm_enable_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i,
 					     MSR_TYPE_RW);
 
@@ -5574,9 +5576,8 @@ static __init void svm_set_cpu_caps(void)
 		 * access to enough counters to virtualize "core" support,
 		 * otherwise limit vPMU support to the legacy number of counters.
 		 */
-		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
-			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
-							  kvm_pmu_cap.num_counters_gp);
+		if (hweight64(kvm_pmu_cap.cntr_mask64) < AMD64_NUM_COUNTERS_CORE)
+			kvm_pmu_cap.cntr_mask64 &= GENMASK_ULL(AMD64_NUM_COUNTERS - 1, 0);
 		else
 			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
 
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 31422bd20d96..425f17aa9be2 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -518,7 +518,8 @@ static u64 intel_get_fixed_pmc_eventsel(unsigned int index)
 	 * have a known encoding for the associated general purpose event.
 	 */
 	eventsel = perf_get_hw_event_config(fixed_pmc_perf_ids[index]);
-	WARN_ON_ONCE(!eventsel && index < kvm_pmu_cap.num_counters_fixed);
+	WARN_ON_ONCE(!eventsel &&
+		     (kvm_pmu_cap.fixed_cntr_mask64 & BIT_ULL(index)));
 	return eventsel;
 }
 
@@ -575,7 +576,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	pmu->available_event_types = ~entry->ebx & (BIT_ULL(eax.split.mask_length) - 1);
 
 	fixed_cntr_mask = BIT_ULL(edx.split.num_counters_fixed) - 1;
-	fixed_cntr_mask &= BIT_ULL(kvm_pmu_cap.num_counters_fixed) - 1;
+	fixed_cntr_mask &= kvm_pmu_cap.fixed_cntr_mask64;
 
 	/*
 	 * The number of counters comes from guest CPUID data. Clamp the value
@@ -583,7 +584,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	 */
 	nr_gp_counters = min_t(int, eax.split.num_counters, X86_PMC_IDX_MAX - 1);
 	pmu->all_valid_pmc_mask64 = (BIT_ULL(nr_gp_counters) - 1) &
-				    (BIT_ULL(kvm_pmu_cap.num_counters_gp) - 1);
+				    kvm_pmu_cap.cntr_mask64;
 
 	entry = kvm_find_cpuid_entry_index(vcpu, 7, 0);
 	if (entry &&
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 0a6880bb7ebe..2a59bbe52bd8 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -4226,6 +4226,7 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 	bool has_mediated_pmu = kvm_vcpu_has_mediated_pmu(vcpu);
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	unsigned long host_only_gp_mask, host_only_fixed_mask;
 	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
 	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
 	bool intercept = !has_mediated_pmu;
@@ -4248,23 +4249,25 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 
 	vm_exit_controls_changebit(vmx, vm_exit_controls_bits, has_mediated_pmu);
 
+	host_only_gp_mask = kvm_host_pmu.cntr_mask64 & ~gp_mask;
 	kvm_for_each_set_pmc_idx(i, gp_mask, INTEL_GP) {
 		vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i,
 					  MSR_TYPE_RW, intercept);
 		vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i, MSR_TYPE_RW,
 					  intercept || !fw_writes_is_enabled(vcpu));
 	}
-	for ( ; i < kvm_pmu_cap.num_counters_gp; i++) {
+	for_each_set_bit(i, &host_only_gp_mask, INTEL_PMC_MAX_GENERIC) {
 		vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i,
 					  MSR_TYPE_RW, true);
 		vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i,
 					  MSR_TYPE_RW, true);
 	}
 
+	host_only_fixed_mask = kvm_host_pmu.fixed_cntr_mask64 & ~fixed_mask;
 	kvm_for_each_set_pmc_idx(i, fixed_mask, INTEL_FIXED)
 		vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i,
 					  MSR_TYPE_RW, intercept);
-	for ( ; i < kvm_pmu_cap.num_counters_fixed; i++)
+	for_each_set_bit(i, &host_only_fixed_mask, INTEL_PMC_MAX_FIXED)
 		vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i,
 					  MSR_TYPE_RW, true);
 
-- 
2.54.0

[PATCH 09/15] perf/x86: Remove num_counters_{gp,fixed} from x86_pmu_capability

[Zide Chen]

From: Dapeng Mi <dapeng1.mi@linux.intel.com>

Now that KVM has switched to bitmap-based PMU capabilities,
num_counters_{gp,fixed} can be removed from x86_pmu_capability.

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/events/core.c            | 2 --
 arch/x86/include/asm/perf_event.h | 2 --
 2 files changed, 4 deletions(-)

diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 65349819ba43..5bcd2a48d2b3 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -3132,8 +3132,6 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
 	 * base PMU holds the correct number of counters for P-cores.
 	 */
 	cap->version		= x86_pmu.version;
-	cap->num_counters_gp	= x86_pmu_num_counters(NULL);
-	cap->num_counters_fixed	= x86_pmu_num_counters_fixed(NULL);
 	cap->cntr_mask64	= x86_pmu.cntr_mask64;
 	cap->fixed_cntr_mask64	= x86_pmu.fixed_cntr_mask64;
 	cap->bit_width_gp	= cap->cntr_mask64 ? x86_pmu.cntval_bits : 0;
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index f59a3d466195..4b035629903c 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -300,8 +300,6 @@ union cpuid_0x80000022_ebx {
 
 struct x86_pmu_capability {
 	int		version;
-	int		num_counters_gp;
-	int		num_counters_fixed;
 	union {
 		u64	cntr_mask64;
 		DECLARE_BITMAP(cntr_mask, X86_PMC_IDX_MAX);
-- 
2.54.0

[PATCH 10/15] KVM: x86/pmu: Emulate the GLOBAL_STATUS_SET and GLOBAL_INUSE MSRs

[Zide Chen]

Intel PerfMon v4 introduces IA32_PERF_GLOBAL_STATUS_SET (0x391) to
allow software to set individual bits in the global status MSR. Reads
of IA32_PERF_GLOBAL_STATUS_SET always return zero.

IA32_PERF_GLOBAL_INUSE (0x392) is also introduced in v4, to track
which counters and the PMI are currently claimed by other agents,
allowing independent software agents to check counter availability
without a shared scheduler arbitrating between them.

IA32_PERF_GLOBAL_INUSE is an R/O MSR, and any write attempt results
in a #GP.

Neither MSR is part of the VM state, so they don't need to be
advertised to userspace, nor saved and restored during live
migration.

Originally-by: Yang Weijiang <weijiang.yang@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/include/asm/msr-index.h |  4 ++++
 arch/x86/kvm/pmu.c               |  9 ++++++++
 arch/x86/kvm/vmx/pmu_intel.c     | 38 ++++++++++++++++++++++++++++++++
 arch/x86/kvm/vmx/vmx.c           |  4 ++++
 4 files changed, 55 insertions(+)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index feee92aab504..46ca6b56628b 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -1241,6 +1241,10 @@
 #define MSR_CORE_PERF_GLOBAL_CTRL	0x0000038f
 #define MSR_CORE_PERF_GLOBAL_OVF_CTRL	0x00000390
 #define MSR_CORE_PERF_GLOBAL_STATUS_SET	0x00000391
+#define MSR_CORE_PERF_GLOBAL_INUSE	0x00000392
+
+/* Intel IA32_PERF_GLOBAL_INUSE MSR */
+#define PERF_GLOBAL_INUSE_PMI_INUSE	BIT_ULL(63)
 
 #define MSR_PERF_METRICS		0x00000329
 
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index bc2ca60114e9..7d58f7a2a2db 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -834,6 +834,8 @@ bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 	case MSR_CORE_PERF_GLOBAL_CTRL:
 	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
 		return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
+	case MSR_CORE_PERF_GLOBAL_STATUS_SET:
+		return vcpu_to_pmu(vcpu)->version > 3;
 	default:
 		break;
 	}
@@ -867,6 +869,7 @@ int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET:
 	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
+	case MSR_CORE_PERF_GLOBAL_STATUS_SET:
 		msr_info->data = 0;
 		break;
 	default:
@@ -933,6 +936,12 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		if (!msr_info->host_initiated)
 			pmu->global_status &= ~data;
 		break;
+	case MSR_CORE_PERF_GLOBAL_STATUS_SET:
+		if (data & pmu->global_status_rsvd)
+			return 1;
+		if (!msr_info->host_initiated)
+			pmu->global_status |= data;
+		break;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET:
 		if (!msr_info->host_initiated)
 			pmu->global_status |= data & ~pmu->global_status_rsvd;
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 425f17aa9be2..cb6f9c272e03 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -205,6 +205,8 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 	switch (msr) {
 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
 		return kvm_pmu_has_perf_global_ctrl(pmu);
+	case MSR_CORE_PERF_GLOBAL_INUSE:
+		return pmu->version > 3;
 	case MSR_PERF_METRICS:
 		return kvm_vcpu_has_perf_metrics(vcpu);
 	case MSR_IA32_PEBS_ENABLE:
@@ -354,6 +356,40 @@ static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+static int intel_pmu_get_global_inuse(struct kvm_vcpu *vcpu,
+				      struct msr_data *msr_info)
+{
+	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+	unsigned long fixed_mask = kvm_fixed_pmc_mask(pmu);
+	unsigned long gp_mask = kvm_gp_pmc_mask(pmu);
+	bool pmi_inuse = false;
+	u32 fixed_ctrl;
+	u64 eventsel;
+	int i;
+
+	msr_info->data = 0;
+	kvm_for_each_set_pmc_idx(i, gp_mask, INTEL_GP) {
+		eventsel = pmu->gp_counters[i].eventsel;
+
+		if (eventsel & ARCH_PERFMON_EVENTSEL_EVENT)
+			msr_info->data |= BIT_ULL(i);
+		pmi_inuse |= eventsel & ARCH_PERFMON_EVENTSEL_INT;
+	}
+	kvm_for_each_set_pmc_idx(i, fixed_mask, INTEL_FIXED) {
+		fixed_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, i);
+
+		if (fixed_ctrl & (INTEL_FIXED_0_KERNEL | INTEL_FIXED_0_USER))
+			msr_info->data |= BIT_ULL(KVM_FIXED_PMC_BASE_IDX + i);
+		pmi_inuse |= fixed_ctrl & INTEL_FIXED_0_ENABLE_PMI;
+	}
+	pmi_inuse |= pmu->pebs_enable;
+
+	if (pmi_inuse)
+		msr_info->data |= PERF_GLOBAL_INUSE_PMI_INUSE;
+
+	return 0;
+}
+
 static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -364,6 +400,8 @@ static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
 		msr_info->data = pmu->fixed_ctr_ctrl;
 		break;
+	case MSR_CORE_PERF_GLOBAL_INUSE:
+		return intel_pmu_get_global_inuse(vcpu, msr_info);
 	case MSR_PERF_METRICS:
 		msr_info->data = pmu->perf_metrics;
 		break;
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 2a59bbe52bd8..c69cda5bb898 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -4278,6 +4278,10 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
 				  MSR_TYPE_RW, intercept);
 	vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
 				  MSR_TYPE_RW, intercept);
+	vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_STATUS_SET,
+				  MSR_TYPE_RW, intercept || pmu->version < 4);
+	vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_INUSE,
+				  MSR_TYPE_RW, intercept || pmu->version < 4);
 
 	intercept = !has_mediated_pmu || !kvm_vcpu_has_perf_metrics(vcpu);
 	vmx_set_intercept_for_msr(vcpu, MSR_PERF_METRICS,
-- 
2.54.0

[PATCH 11/15] KVM: x86/pmu: Emulate streamlined Freeze-LBR-on-PMI

[Zide Chen]

PerfMon v4 streamlined the Freeze-LBR-on-PMI mechanism. When
DEBUGCTLMSR_FREEZE_LBRS_ON_PMI is set and a PMI fires, hardware sets
IA32_PERF_GLOBAL_STATUS.LBR_Frz instead of clearing DEBUGCTLMSR_LBR.

Guest PerfMon v4+ is supported under mediated vPMU only. When KVM
relays a guest-induced PMI, the mediated vPMU framework already
propagates IA32_PERF_GLOBAL_STATUS.LBR_Frz, and no additional handling
is required.

For PMIs generated by KVM-emulated PMU events, however, KVM must
emulate IA32_PERF_GLOBAL_STATUS.LBR_Frz so that guest LBR recording
is frozen as required by the architectural behavior.

Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/vmx/pmu_intel.c | 47 ++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index cb6f9c272e03..556c119d5e91 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -660,6 +660,8 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 
 	pmu->global_status_rsvd = pmu->global_ctrl_rsvd
 			& ~(GLOBAL_STATUS_BUFFER_OVF | GLOBAL_STATUS_COND_CHG);
+	if (pmu->version > 3)
+		pmu->global_status_rsvd &= ~GLOBAL_STATUS_LBRS_FROZEN;
 	if (vmx_pt_mode_is_host_guest())
 		pmu->global_status_rsvd &= ~GLOBAL_STATUS_TRACE_TOPAPMI;
 
@@ -718,32 +720,41 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu)
 }
 
 /*
- * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4.
- *
- * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and
- * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL.
- *
- * Guest needs to re-enable LBR to resume branches recording.
+ * Emulate legacy and streamlined Freeze_LBR_On_PMI behavior.
+ * In either case, guest needs to re-enable LBR to resume branches recording.
  */
-static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu)
+static void intel_pmu_freeze_lbr_on_pmi(struct kvm_vcpu *vcpu)
 {
-	u64 data = vmx_guest_debugctl_read();
+	u8 version;
+	u64 data;
 
-	if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) {
-		data &= ~DEBUGCTLMSR_LBR;
-		vmx_guest_debugctl_write(vcpu, data);
+	if (!intel_pmu_lbr_is_enabled(vcpu))
+		return;
+
+	data = vmx_guest_debugctl_read();
+	version = vcpu_to_pmu(vcpu)->version;
+
+	if (!(data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI))
+		return;
+
+	if (version > 1 && version < 4) {
+		if (data & DEBUGCTLMSR_LBR) {
+			data &= ~DEBUGCTLMSR_LBR;
+			vmx_guest_debugctl_write(vcpu, data);
+		}
+	} else if (vcpu_to_lbr_desc(vcpu)->msr_passthrough &&
+		   kvm_vcpu_has_mediated_pmu(vcpu)) {
+		/*
+		 * This will be restored to guest before VM-Entry, setting
+		 * LBR_Frz to freeze LBR recording until the guest clears it.
+		 */
+		vcpu_to_pmu(vcpu)->global_status |= GLOBAL_STATUS_LBRS_FROZEN;
 	}
 }
 
 static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
 {
-	u8 version = vcpu_to_pmu(vcpu)->version;
-
-	if (!intel_pmu_lbr_is_enabled(vcpu))
-		return;
-
-	if (version > 1 && version < 4)
-		intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu);
+	intel_pmu_freeze_lbr_on_pmi(vcpu);
 }
 
 static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set)
-- 
2.54.0

[PATCH 12/15] KVM: x86/pmu: Populate CPUID.0AH:ECX fixed-counter bitmap

[Zide Chen]

PerfMon v5 adds a fixed-counter support bitmap in CPUID.0AH:ECX.
This is the superset of EDX[4:0] which indicates the contiguous
counters starting from 0. For backward compatibility, it's recommended
to use the following to determine if a fixed counter is supported:

FxCtr[i]_is_supported := ECX[i] || (EDX[4:0] > i);

Originally-by: Yang Weijiang <weijiang.yang@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/cpuid.c         | 5 ++++-
 arch/x86/kvm/vmx/pmu_intel.c | 3 +++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 151a4794f834..ae6176fb0a93 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -1533,7 +1533,10 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 
 		entry->eax = eax.full;
 		entry->ebx = kvm_pmu_cap.events_mask;
-		entry->ecx = 0;
+		if (kvm_pmu_cap.version > 4)
+			entry->ecx &= (u32)kvm_pmu_cap.fixed_cntr_mask64;
+		else
+			entry->ecx = 0;
 		entry->edx = edx.full;
 		break;
 	}
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 556c119d5e91..3f41e4916986 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -613,7 +613,10 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 				      kvm_pmu_cap.events_mask_len);
 	pmu->available_event_types = ~entry->ebx & (BIT_ULL(eax.split.mask_length) - 1);
 
+	/* FxCtr[i]_is_supported := ECX[i] || (EDX[4:0] > i) */
 	fixed_cntr_mask = BIT_ULL(edx.split.num_counters_fixed) - 1;
+	if (pmu->version > 4)
+		fixed_cntr_mask |= entry->ecx;
 	fixed_cntr_mask &= kvm_pmu_cap.fixed_cntr_mask64;
 
 	/*
-- 
2.54.0

[PATCH 13/15] KVM: x86/pmu: Ignore AnyThread bit if CPUID.0AH:EDX[15] is not set

[Zide Chen]

Intel PerfMon v5 introduces the ANYTHREAD_DEPRECATION capability
(CPUID.0AH:EDX[15]) to indicate that AnyThread counting is deprecated
and that writes to the AnyThread bit in IA32_PERFEVTSELx are ignored.

When ANYTHREAD_DEPRECATION is present in the guest CPUID, emulate the
architectural behavior and silently ignore writes to the AnyThread bit
instead of injecting #GP.

Continue to inject #GP when ANYTHREAD_DEPRECATION is not present in the
guest CPUID, e.g. for PerfMon v3/v4 guests or when the capability is
not configured for a PerfMon v5+ guest.

Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/vmx/pmu_intel.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 3f41e4916986..2c2a54ac55ad 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -508,6 +508,11 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			if (data & eventsel_rsvd)
 				return 1;
 
+			/*
+			 * On PerfMon v5+, KVM may allow writes to the AnyThread
+			 * bit and silently discard them.
+			 */
+			data &= ~ARCH_PERFMON_EVENTSEL_ANY;
 			if (data != pmc->eventsel) {
 				pmc->eventsel = data;
 				pmc->eventsel_hw = data;
@@ -627,6 +632,9 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
 	pmu->all_valid_pmc_mask64 = (BIT_ULL(nr_gp_counters) - 1) &
 				    kvm_pmu_cap.cntr_mask64;
 
+	if (pmu->version >= 5 && edx.split.anythread_deprecated)
+		pmu->eventsel_rsvd &= ~ARCH_PERFMON_EVENTSEL_ANY;
+
 	entry = kvm_find_cpuid_entry_index(vcpu, 7, 0);
 	if (entry &&
 	    (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
-- 
2.54.0

[PATCH 14/15] KVM: x86/pmu: Advertise PerfMon version 5 on Intel hosts

[Zide Chen]

From: Dapeng Mi <dapeng1.mi@linux.intel.com>

KVM currently caps the guest PerfMon version at 2 for all Intel
platforms. Now that KVM emulates the basic architectural PMU features
introduced in PerfMon versions 3 through 5, raise the guest PerfMon
version to 5.  Features that require additional emulation support, e.g.
architectural LBR will be enabled separately.

When enable_mediated_pmu is disabled or in the non-Intel paths, KVM
retains the existing cap of version 2.

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/pmu.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 7d58f7a2a2db..83332e8ed1e0 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -176,7 +176,18 @@ void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops)
 	}
 
 	memcpy(&kvm_pmu_cap, &kvm_host_pmu, sizeof(kvm_host_pmu));
-	kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2);
+
+	/*
+	 * AnyThread counting is not supported by KVM due to cross-VM
+	 * information leakage concerns on SMT cores. Therefore, AnyThread
+	 * remains unavailable for PerfMon v3/v4 guests, where AnyThread
+	 * deprecation is not enumerated.
+	 */
+	if (is_intel && enable_mediated_pmu)
+		kvm_pmu_cap.version = min(kvm_pmu_cap.version, 5);
+	else
+		kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2);
+
 	kvm_pmu_cap.cntr_mask64 &=
 		GENMASK_ULL(pmu_ops->MAX_NR_GP_COUNTERS - 1, 0);
 	kvm_pmu_cap.fixed_cntr_mask64 &=
-- 
2.54.0

[PATCH 15/15] KVM: selftests: Support fixed counters bitmap in pmu_counters_test

[Zide Chen]

From: Dapeng Mi <dapeng1.mi@linux.intel.com>

On PerfMon v5, CPUID.0AH:ECX represents the fixed counter bitmask,
while EDX[4:0] represents the number of contiguous fixed counters
starting from 0.

pmu_counters_test does not correctly set up these two fixed counter
fields. For example, it may set EDX[4:0] to 0x0 while setting ECX to
0x1. It assumes that fixed counter 0 is available by checking:

FxCtr[i]_is_supported := ECX[i] || (EDX[4:0] > i).

However, this is an invalid setup because EDX[4:0] should not be zero
when the number of contiguous fixed counters is 1.

Correct the setting of EDX[4:0] when the fixed counter bitmask is
present, and also handle the non-contiguous fixed counter case.

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Co-developed-by: Zide Chen <zide.chen@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 .../selftests/kvm/x86/pmu_counters_test.c     | 25 ++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/tools/testing/selftests/kvm/x86/pmu_counters_test.c b/tools/testing/selftests/kvm/x86/pmu_counters_test.c
index 38057754e024..6f38a35a4ea7 100644
--- a/tools/testing/selftests/kvm/x86/pmu_counters_test.c
+++ b/tools/testing/selftests/kvm/x86/pmu_counters_test.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2023, Tencent, Inc.
  */
 #include <x86intrin.h>
+#include <linux/bitmap.h>
 
 #include "pmu.h"
 #include "processor.h"
@@ -659,6 +660,8 @@ static void test_intel_counters(void)
 	u8 nr_gp_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
 	u8 pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION);
 	u64 advertised_perf_caps = kvm_get_feature_msr(MSR_IA32_PERF_CAPABILITIES);
+	unsigned long fixed_subset;
+	u32 fixed_bitmap = 0;
 	unsigned int i;
 	u8 v, j;
 	u32 k;
@@ -696,6 +699,12 @@ static void test_intel_counters(void)
 	 */
 	u8 max_pmu_version = max_t(typeof(pmu_version), pmu_version, 5);
 
+	if (nr_fixed_counters)
+		fixed_bitmap = GENMASK(nr_fixed_counters - 1, 0);
+
+	if (pmu_version > 4)
+		fixed_bitmap |= kvm_cpu_property(X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK);
+
 	/*
 	 * Detect the existence of events that aren't supported by selftests.
 	 * This will (obviously) fail any time hardware adds support for a new
@@ -750,9 +759,19 @@ static void test_intel_counters(void)
 
 			pr_info("Testing fixed counters, PMU version %u, perf_caps = %lx\n",
 				v, perf_caps[i]);
-			for (j = 0; j <= nr_fixed_counters; j++) {
-				for (k = 0; k <= (BIT(nr_fixed_counters) - 1); k++)
-					test_fixed_counters(v, perf_caps[i], j, k);
+			for (fixed_subset = 0; fixed_subset <= fixed_bitmap; fixed_subset++) {
+				u32 nr_contiguous;
+
+				/*
+				 * The loop walks all values from 0 to fixed_bitmap, so skip any
+				 * value that is not a true subset of fixed_bitmap.
+				 */
+				if (fixed_subset & ~fixed_bitmap)
+					continue;
+
+				nr_contiguous = find_first_zero_bit(&fixed_subset,
+								    MAX_NR_FIXED_COUNTERS);
+				test_fixed_counters(v, perf_caps[i], nr_contiguous, fixed_subset);
 			}
 
 			pr_info("Testing Perf Metrics, PMU version %u, perf_caps = %lx\n",
-- 
2.54.0