[PATCH 0/3] Enable x86 mediated vPMU

[PATCH 0/3] Enable x86 mediated vPMU

[Dapeng Mi]

This small patch series enables the newly introduced KVM x86 mediated
vPMU solution. As KVM maintainer's suggestion, KVM mediated vPMU is
disabled by default unless user explicitly calls KVM_CAP_PMU_CAPABILITY
ioctl to enable it.

As for mediated vPMU, it's a new pass-through vPMU solution which is
designed to replace the legacy perf-based vPMU which has several
drawbacks, such as high performance overhead, hard to support new PMU
features, etc. Most of PMU MSRs except EVENTSELx are passed through to
guest in mediated vPMU. Currently the latest mediated vPMU patch series
is v3[1], the v4 patchset would be sent soon.

In this series, patch 1/3 introduces a helper
kvm_arch_pre_create_vcpu() which would be called before creating vCPU.
This patch comes from Xiaoyao's "QEMU TDX support" patchset[2]. Patch
2/3 leverages the patch 1/3 introduced helper to call
KVM_CAP_PMU_CAPABILITY ioctl to enable/disable KVM vPMU (mediated vPMU).
This patch is similar with patch 4/10 of Dongli's
"target/i386/kvm/pmu: PMU Enhancement, Bugfix and Cleanup" patchset[3],
but can be considered as an enhanced version. Patch 3/3 provides support
for newly introduced VMCS bit SAVE_IA32_PERF_GLOBAL_CTRL.

Tests:
  * Tests on Sapphire Rapids platform, both mediated vPMU and legacy
    perf-based vPMU can be enabled/disabled with "+/-pmu" option.

Ref:
[1] https://lore.kernel.org/all/20240801045907.4010984-1-mizhang@google.com/
[2] https://lore.kernel.org/all/20250124132048.3229049-8-xiaoyao.li@intel.com/
[3] https://lore.kernel.org/all/20250302220112.17653-5-dongli.zhang@oracle.com/  

Dapeng Mi (2):
  target/i386: Call KVM_CAP_PMU_CAPABILITY iotcl to enable/disable PMU
  target/i386: Support VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL

Xiaoyao Li (1):
  kvm: Introduce kvm_arch_pre_create_vcpu()

 accel/kvm/kvm-all.c        |  5 +++++
 include/system/kvm.h       |  1 +
 target/arm/kvm.c           |  5 +++++
 target/i386/cpu.c          | 12 ++++++++----
 target/i386/cpu.h          |  1 +
 target/i386/kvm/kvm.c      | 22 ++++++++++++++++++++++
 target/loongarch/kvm/kvm.c |  5 +++++
 target/mips/kvm.c          |  5 +++++
 target/ppc/kvm.c           |  5 +++++
 target/riscv/kvm/kvm-cpu.c |  5 +++++
 target/s390x/kvm/kvm.c     |  5 +++++
 11 files changed, 67 insertions(+), 4 deletions(-)


base-commit: 71119ed3651622e1c531d1294839e9f3341adaf5
-- 
2.40.1

[PATCH 1/3] kvm: Introduce kvm_arch_pre_create_vcpu()

[Dapeng Mi]

From: Xiaoyao Li <xiaoyao.li@intel.com>

Introduce kvm_arch_pre_create_vcpu(), to perform arch-dependent
work prior to create any vcpu. This is for i386 TDX because it needs
call TDX_INIT_VM before creating any vcpu.

The specific implemnet of i386 will be added in the future patch.

Signed-off-by: Xiaoyao Li <xiaoyao.li@intel.com>
Acked-by: Gerd Hoffmann <kraxel@redhat.com>
---
 accel/kvm/kvm-all.c        | 5 +++++
 include/system/kvm.h       | 1 +
 target/arm/kvm.c           | 5 +++++
 target/i386/kvm/kvm.c      | 5 +++++
 target/loongarch/kvm/kvm.c | 5 +++++
 target/mips/kvm.c          | 5 +++++
 target/ppc/kvm.c           | 5 +++++
 target/riscv/kvm/kvm-cpu.c | 5 +++++
 target/s390x/kvm/kvm.c     | 5 +++++
 9 files changed, 41 insertions(+)

diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index f89568bfa3..df9840e53a 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -540,6 +540,11 @@ int kvm_init_vcpu(CPUState *cpu, Error **errp)
 
     trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
 
+    ret = kvm_arch_pre_create_vcpu(cpu, errp);
+    if (ret < 0) {
+        goto err;
+    }
+
     ret = kvm_create_vcpu(cpu);
     if (ret < 0) {
         error_setg_errno(errp, -ret,
diff --git a/include/system/kvm.h b/include/system/kvm.h
index ab17c09a55..d7dfa25493 100644
--- a/include/system/kvm.h
+++ b/include/system/kvm.h
@@ -374,6 +374,7 @@ int kvm_arch_get_default_type(MachineState *ms);
 
 int kvm_arch_init(MachineState *ms, KVMState *s);
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp);
 int kvm_arch_init_vcpu(CPUState *cpu);
 int kvm_arch_destroy_vcpu(CPUState *cpu);
 
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index da30bdbb23..93f1a7245b 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -1874,6 +1874,11 @@ static int kvm_arm_sve_set_vls(ARMCPU *cpu)
 
 #define ARM_CPU_ID_MPIDR       3, 0, 0, 0, 5
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     int ret;
diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
index 6c749d4ee8..f41e190fb8 100644
--- a/target/i386/kvm/kvm.c
+++ b/target/i386/kvm/kvm.c
@@ -2051,6 +2051,11 @@ full:
     abort();
 }
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     struct {
diff --git a/target/loongarch/kvm/kvm.c b/target/loongarch/kvm/kvm.c
index 7f63e7c8fe..ed0ddf1cbf 100644
--- a/target/loongarch/kvm/kvm.c
+++ b/target/loongarch/kvm/kvm.c
@@ -1075,6 +1075,11 @@ static int kvm_cpu_check_pv_features(CPUState *cs, Error **errp)
     return 0;
 }
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     uint64_t val;
diff --git a/target/mips/kvm.c b/target/mips/kvm.c
index d67b7c1a8e..ec53acb51a 100644
--- a/target/mips/kvm.c
+++ b/target/mips/kvm.c
@@ -61,6 +61,11 @@ int kvm_arch_irqchip_create(KVMState *s)
     return 0;
 }
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     CPUMIPSState *env = cpu_env(cs);
diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c
index 992356cb75..20fabccecd 100644
--- a/target/ppc/kvm.c
+++ b/target/ppc/kvm.c
@@ -479,6 +479,11 @@ static void kvmppc_hw_debug_points_init(CPUPPCState *cenv)
     }
 }
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
diff --git a/target/riscv/kvm/kvm-cpu.c b/target/riscv/kvm/kvm-cpu.c
index 4ffeeaa1c9..451c00f17c 100644
--- a/target/riscv/kvm/kvm-cpu.c
+++ b/target/riscv/kvm/kvm-cpu.c
@@ -1389,6 +1389,11 @@ static int kvm_vcpu_enable_sbi_dbcn(RISCVCPU *cpu, CPUState *cs)
     return kvm_set_one_reg(cs, kvm_sbi_dbcn.kvm_reg_id, &reg);
 }
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     int ret = 0;
diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c
index 4d56e653dd..1f592733f4 100644
--- a/target/s390x/kvm/kvm.c
+++ b/target/s390x/kvm/kvm.c
@@ -404,6 +404,11 @@ unsigned long kvm_arch_vcpu_id(CPUState *cpu)
     return cpu->cpu_index;
 }
 
+int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
+{
+    return 0;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     unsigned int max_cpus = MACHINE(qdev_get_machine())->smp.max_cpus;
-- 
2.40.1

[PATCH 2/3] target/i386: Call KVM_CAP_PMU_CAPABILITY iotcl to enable/disable PMU

[Dapeng Mi]

After introducing mediated vPMU, mediated vPMU must be enabled by
explicitly calling KVM_CAP_PMU_CAPABILITY to enable. Thus call
KVM_CAP_PMU_CAPABILITY to enable/disable PMU base on user configuration.

Suggested-by: Zhao Liu <zhao1.liu@intel.com>
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
---
 target/i386/kvm/kvm.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
index f41e190fb8..d3e6984844 100644
--- a/target/i386/kvm/kvm.c
+++ b/target/i386/kvm/kvm.c
@@ -2051,8 +2051,25 @@ full:
     abort();
 }
 
+static bool pmu_cap_set = false;
 int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
 {
+    KVMState *s = kvm_state;
+    X86CPU *x86_cpu = X86_CPU(cpu);
+
+    if (!pmu_cap_set && kvm_check_extension(s, KVM_CAP_PMU_CAPABILITY)) {
+        int r = kvm_vm_enable_cap(s, KVM_CAP_PMU_CAPABILITY, 0,
+                                  KVM_PMU_CAP_DISABLE & !x86_cpu->enable_pmu);
+        if (r < 0) {
+            error_report("kvm: Failed to %s pmu cap: %s",
+                         x86_cpu->enable_pmu ? "enable" : "disable",
+                         strerror(-r));
+            return r;
+        }
+
+        pmu_cap_set = true;
+    }
+
     return 0;
 }
 
-- 
2.40.1

Re: [PATCH 2/3] target/i386: Call KVM_CAP_PMU_CAPABILITY iotcl to enable/disable PMU

[Dongli Zhang]

Hi Dapeng,

PATCH 1-4 from the below patchset are already reviewed. (PATCH 5-10 are for PMU
registers reset).

https://lore.kernel.org/all/20250302220112.17653-1-dongli.zhang@oracle.com/

They require only trivial modification. i.e.:

https://github.com/finallyjustice/patchset/tree/master/qemu-amd-pmu-mid/v03

Therefore, since PATCH 5-10 are for another topic, any chance if I re-send 1-4
as a prerequisite for the patch to explicitly call KVM_CAP_PMU_CAPABILITY?

In addition, I have a silly question. Can mediated vPMU coexist with legacy
perf-based vPMU, that is, something like tdp and tdp_mmu? Or the legacy
perf-based vPMU is going to be purged from the most recent kernel?

If they can coexist, how about add property to QEMU control between
legacy/modern? i.e. by default use legacy and change to modern as default in the
future once the feature is stable.

Thank you very much!

Dongli Zhang

On 3/24/25 5:37 AM, Dapeng Mi wrote:
> After introducing mediated vPMU, mediated vPMU must be enabled by
> explicitly calling KVM_CAP_PMU_CAPABILITY to enable. Thus call
> KVM_CAP_PMU_CAPABILITY to enable/disable PMU base on user configuration.
> 
> Suggested-by: Zhao Liu <zhao1.liu@intel.com>
> Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
> ---
>  target/i386/kvm/kvm.c | 17 +++++++++++++++++
>  1 file changed, 17 insertions(+)
> 
> diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
> index f41e190fb8..d3e6984844 100644
> --- a/target/i386/kvm/kvm.c
> +++ b/target/i386/kvm/kvm.c
> @@ -2051,8 +2051,25 @@ full:
>      abort();
>  }
>  
> +static bool pmu_cap_set = false;
>  int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
>  {
> +    KVMState *s = kvm_state;
> +    X86CPU *x86_cpu = X86_CPU(cpu);
> +
> +    if (!pmu_cap_set && kvm_check_extension(s, KVM_CAP_PMU_CAPABILITY)) {
> +        int r = kvm_vm_enable_cap(s, KVM_CAP_PMU_CAPABILITY, 0,
> +                                  KVM_PMU_CAP_DISABLE & !x86_cpu->enable_pmu);
> +        if (r < 0) {
> +            error_report("kvm: Failed to %s pmu cap: %s",
> +                         x86_cpu->enable_pmu ? "enable" : "disable",
> +                         strerror(-r));
> +            return r;
> +        }
> +
> +        pmu_cap_set = true;
> +    }
> +
>      return 0;
>  }
>  

Re: [PATCH 2/3] target/i386: Call KVM_CAP_PMU_CAPABILITY iotcl to enable/disable PMU

[Mi, Dapeng]

On 3/26/2025 2:46 PM, Dongli Zhang wrote:
> Hi Dapeng,
>
> PATCH 1-4 from the below patchset are already reviewed. (PATCH 5-10 are for PMU
> registers reset).
>
> https://lore.kernel.org/all/20250302220112.17653-1-dongli.zhang@oracle.com/
>
> They require only trivial modification. i.e.:
>
> https://github.com/finallyjustice/patchset/tree/master/qemu-amd-pmu-mid/v03
>
> Therefore, since PATCH 5-10 are for another topic, any chance if I re-send 1-4
> as a prerequisite for the patch to explicitly call KVM_CAP_PMU_CAPABILITY?

any option is fine for me, spiting it into two separated ones or still keep
in a whole patch series. I would rebase this this patchset on top of your
v3 patches.


>
> In addition, I have a silly question. Can mediated vPMU coexist with legacy
> perf-based vPMU, that is, something like tdp and tdp_mmu? Or the legacy
> perf-based vPMU is going to be purged from the most recent kernel?

No, they can't. As long as mediated vPMU is enabled, it would totally
replace the legacy perf-based vPMU. The legacy perf-based vPMU code would
still be kept in the kernel in near future, but the long-term target is to
totally remove the perf-based vPMU once mediated vPMU is mature.


>
> If they can coexist, how about add property to QEMU control between
> legacy/modern? i.e. by default use legacy and change to modern as default in the
> future once the feature is stable.
>
> Thank you very much!
>
> Dongli Zhang
>
> On 3/24/25 5:37 AM, Dapeng Mi wrote:
>> After introducing mediated vPMU, mediated vPMU must be enabled by
>> explicitly calling KVM_CAP_PMU_CAPABILITY to enable. Thus call
>> KVM_CAP_PMU_CAPABILITY to enable/disable PMU base on user configuration.
>>
>> Suggested-by: Zhao Liu <zhao1.liu@intel.com>
>> Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
>> ---
>>  target/i386/kvm/kvm.c | 17 +++++++++++++++++
>>  1 file changed, 17 insertions(+)
>>
>> diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
>> index f41e190fb8..d3e6984844 100644
>> --- a/target/i386/kvm/kvm.c
>> +++ b/target/i386/kvm/kvm.c
>> @@ -2051,8 +2051,25 @@ full:
>>      abort();
>>  }
>>  
>> +static bool pmu_cap_set = false;
>>  int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
>>  {
>> +    KVMState *s = kvm_state;
>> +    X86CPU *x86_cpu = X86_CPU(cpu);
>> +
>> +    if (!pmu_cap_set && kvm_check_extension(s, KVM_CAP_PMU_CAPABILITY)) {
>> +        int r = kvm_vm_enable_cap(s, KVM_CAP_PMU_CAPABILITY, 0,
>> +                                  KVM_PMU_CAP_DISABLE & !x86_cpu->enable_pmu);
>> +        if (r < 0) {
>> +            error_report("kvm: Failed to %s pmu cap: %s",
>> +                         x86_cpu->enable_pmu ? "enable" : "disable",
>> +                         strerror(-r));
>> +            return r;
>> +        }
>> +
>> +        pmu_cap_set = true;
>> +    }
>> +
>>      return 0;
>>  }
>>  

Re: [PATCH 2/3] target/i386: Call KVM_CAP_PMU_CAPABILITY iotcl to enable/disable PMU

[Mingwei Zhang]

On Wed, Mar 26, 2025 at 5:44 PM Mi, Dapeng <dapeng1.mi@linux.intel.com> wrote:
>
>
> On 3/26/2025 2:46 PM, Dongli Zhang wrote:
> > Hi Dapeng,
> >
> > PATCH 1-4 from the below patchset are already reviewed. (PATCH 5-10 are for PMU
> > registers reset).
> >
> > https://lore.kernel.org/all/20250302220112.17653-1-dongli.zhang@oracle.com/
> >
> > They require only trivial modification. i.e.:
> >
> > https://github.com/finallyjustice/patchset/tree/master/qemu-amd-pmu-mid/v03
> >
> > Therefore, since PATCH 5-10 are for another topic, any chance if I re-send 1-4
> > as a prerequisite for the patch to explicitly call KVM_CAP_PMU_CAPABILITY?
>
> any option is fine for me, spiting it into two separated ones or still keep
> in a whole patch series. I would rebase this this patchset on top of your
> v3 patches.
>
>
> >
> > In addition, I have a silly question. Can mediated vPMU coexist with legacy
> > perf-based vPMU, that is, something like tdp and tdp_mmu? Or the legacy
> > perf-based vPMU is going to be purged from the most recent kernel?
>
> No, they can't. As long as mediated vPMU is enabled, it would totally
> replace the legacy perf-based vPMU. The legacy perf-based vPMU code would
> still be kept in the kernel in near future, but the long-term target is to
> totally remove the perf-based vPMU once mediated vPMU is mature.

mediated vPMU will co-exist with legacy vPMU right? Mediated vPMU
currently was constrained to SPR+ on Intel and Genoa+ on AMD. So
legacy CPUs will have no choice but legacy vPMU.

In the future, to fully replace legacy vPMU we need to solve the
performance issue due to PMU context switching being located at VM
enter/exit boundary. Once those limitations are resolved, and older
x86 CPUs fade away, mediated vPMU can fully take over.

Thanks.
-Mingwei
>
>
> >
> > If they can coexist, how about add property to QEMU control between
> > legacy/modern? i.e. by default use legacy and change to modern as default in the
> > future once the feature is stable.
> >
> > Thank you very much!
> >
> > Dongli Zhang
> >
> > On 3/24/25 5:37 AM, Dapeng Mi wrote:
> >> After introducing mediated vPMU, mediated vPMU must be enabled by
> >> explicitly calling KVM_CAP_PMU_CAPABILITY to enable. Thus call
> >> KVM_CAP_PMU_CAPABILITY to enable/disable PMU base on user configuration.
> >>
> >> Suggested-by: Zhao Liu <zhao1.liu@intel.com>
> >> Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
> >> ---
> >>  target/i386/kvm/kvm.c | 17 +++++++++++++++++
> >>  1 file changed, 17 insertions(+)
> >>
> >> diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
> >> index f41e190fb8..d3e6984844 100644
> >> --- a/target/i386/kvm/kvm.c
> >> +++ b/target/i386/kvm/kvm.c
> >> @@ -2051,8 +2051,25 @@ full:
> >>      abort();
> >>  }
> >>
> >> +static bool pmu_cap_set = false;
> >>  int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
> >>  {
> >> +    KVMState *s = kvm_state;
> >> +    X86CPU *x86_cpu = X86_CPU(cpu);
> >> +
> >> +    if (!pmu_cap_set && kvm_check_extension(s, KVM_CAP_PMU_CAPABILITY)) {
> >> +        int r = kvm_vm_enable_cap(s, KVM_CAP_PMU_CAPABILITY, 0,
> >> +                                  KVM_PMU_CAP_DISABLE & !x86_cpu->enable_pmu);
> >> +        if (r < 0) {
> >> +            error_report("kvm: Failed to %s pmu cap: %s",
> >> +                         x86_cpu->enable_pmu ? "enable" : "disable",
> >> +                         strerror(-r));
> >> +            return r;
> >> +        }
> >> +
> >> +        pmu_cap_set = true;
> >> +    }
> >> +
> >>      return 0;
> >>  }
> >>

Re: [PATCH 2/3] target/i386: Call KVM_CAP_PMU_CAPABILITY iotcl to enable/disable PMU

[Mi, Dapeng]

On 3/27/2025 10:15 AM, Mingwei Zhang wrote:
> On Wed, Mar 26, 2025 at 5:44 PM Mi, Dapeng <dapeng1.mi@linux.intel.com> wrote:
>>
>> On 3/26/2025 2:46 PM, Dongli Zhang wrote:
>>> Hi Dapeng,
>>>
>>> PATCH 1-4 from the below patchset are already reviewed. (PATCH 5-10 are for PMU
>>> registers reset).
>>>
>>> https://lore.kernel.org/all/20250302220112.17653-1-dongli.zhang@oracle.com/
>>>
>>> They require only trivial modification. i.e.:
>>>
>>> https://github.com/finallyjustice/patchset/tree/master/qemu-amd-pmu-mid/v03
>>>
>>> Therefore, since PATCH 5-10 are for another topic, any chance if I re-send 1-4
>>> as a prerequisite for the patch to explicitly call KVM_CAP_PMU_CAPABILITY?
>> any option is fine for me, spiting it into two separated ones or still keep
>> in a whole patch series. I would rebase this this patchset on top of your
>> v3 patches.
>>
>>
>>> In addition, I have a silly question. Can mediated vPMU coexist with legacy
>>> perf-based vPMU, that is, something like tdp and tdp_mmu? Or the legacy
>>> perf-based vPMU is going to be purged from the most recent kernel?
>> No, they can't. As long as mediated vPMU is enabled, it would totally
>> replace the legacy perf-based vPMU. The legacy perf-based vPMU code would
>> still be kept in the kernel in near future, but the long-term target is to
>> totally remove the perf-based vPMU once mediated vPMU is mature.
> mediated vPMU will co-exist with legacy vPMU right? Mediated vPMU
> currently was constrained to SPR+ on Intel and Genoa+ on AMD. So
> legacy CPUs will have no choice but legacy vPMU.
>
> In the future, to fully replace legacy vPMU we need to solve the
> performance issue due to PMU context switching being located at VM
> enter/exit boundary. Once those limitations are resolved, and older
> x86 CPUs fade away, mediated vPMU can fully take over.

yeah, the code would co-exist in near feature or maybe longer, but mediated
vPMU and the legacy vPMU won't work concurrently, once mediated vPMU is
enabled, it would preempt the legacy vPMU.


>
> Thanks.
> -Mingwei
>>
>>> If they can coexist, how about add property to QEMU control between
>>> legacy/modern? i.e. by default use legacy and change to modern as default in the
>>> future once the feature is stable.

I don't prefer to add such property in Qemu. Whether KVM selects to enable
mediated vPMU or the legacy perf-based vPMU, it should be transparent for
Qemu. Qemu is unnecessary to know it.

Currently Qemu already has too much PMU related options, such as pmu,
lbr/arch_lbr. It's too complicated, it introduces too much dependency on
the code and user needs to take time to learn how to configure these
options correctly as well. 

We don't need more options, on the contrary we need to simplify the PMU
options in Qemu. The ideal situation is to keep only one "pmu" option which
is used to manage all these PMU features, like basic perfmon, LBR and PEBS
etc,. But it may cause back-compatible issues if remove these additional
options...


>>>
>>> Thank you very much!
>>>
>>> Dongli Zhang
>>>
>>> On 3/24/25 5:37 AM, Dapeng Mi wrote:
>>>> After introducing mediated vPMU, mediated vPMU must be enabled by
>>>> explicitly calling KVM_CAP_PMU_CAPABILITY to enable. Thus call
>>>> KVM_CAP_PMU_CAPABILITY to enable/disable PMU base on user configuration.
>>>>
>>>> Suggested-by: Zhao Liu <zhao1.liu@intel.com>
>>>> Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
>>>> ---
>>>>  target/i386/kvm/kvm.c | 17 +++++++++++++++++
>>>>  1 file changed, 17 insertions(+)
>>>>
>>>> diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
>>>> index f41e190fb8..d3e6984844 100644
>>>> --- a/target/i386/kvm/kvm.c
>>>> +++ b/target/i386/kvm/kvm.c
>>>> @@ -2051,8 +2051,25 @@ full:
>>>>      abort();
>>>>  }
>>>>
>>>> +static bool pmu_cap_set = false;
>>>>  int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
>>>>  {
>>>> +    KVMState *s = kvm_state;
>>>> +    X86CPU *x86_cpu = X86_CPU(cpu);
>>>> +
>>>> +    if (!pmu_cap_set && kvm_check_extension(s, KVM_CAP_PMU_CAPABILITY)) {
>>>> +        int r = kvm_vm_enable_cap(s, KVM_CAP_PMU_CAPABILITY, 0,
>>>> +                                  KVM_PMU_CAP_DISABLE & !x86_cpu->enable_pmu);
>>>> +        if (r < 0) {
>>>> +            error_report("kvm: Failed to %s pmu cap: %s",
>>>> +                         x86_cpu->enable_pmu ? "enable" : "disable",
>>>> +                         strerror(-r));
>>>> +            return r;
>>>> +        }
>>>> +
>>>> +        pmu_cap_set = true;
>>>> +    }
>>>> +
>>>>      return 0;
>>>>  }
>>>>

[PATCH 3/3] target/i386: Support VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL

[Dapeng Mi]

Since Sapphire Rapids starts, VMX instrocude a new bit
SAVE_IA32_PERF_GLOBAL_CTRL in VMCS VM-EXIT control field to manage if
vmx can save guest PERF_GLOBAL_CTRL MSR.

This patch enables this feature.

Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
---
 target/i386/cpu.c | 12 ++++++++----
 target/i386/cpu.h |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 1b64ceaaba..317ccc8b0a 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -1481,7 +1481,8 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
             "vmx-exit-save-efer", "vmx-exit-load-efer",
                 "vmx-exit-save-preemption-timer", "vmx-exit-clear-bndcfgs",
             NULL, "vmx-exit-clear-rtit-ctl", NULL, NULL,
-            NULL, "vmx-exit-load-pkrs", NULL, "vmx-exit-secondary-ctls",
+            NULL, "vmx-exit-load-pkrs", "vmx-exit-save-perf-global-ctrl",
+            "vmx-exit-secondary-ctls",
         },
         .msr = {
             .index = MSR_IA32_VMX_TRUE_EXIT_CTLS,
@@ -4212,7 +4213,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
             VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
             VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
             VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
-            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
+            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
+            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
         .features[FEAT_VMX_MISC] =
             MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
             MSR_VMX_MISC_VMWRITE_VMEXIT,
@@ -4368,7 +4370,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
             VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
             VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
             VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
-            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
+            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
+            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
         .features[FEAT_VMX_MISC] =
             MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
             MSR_VMX_MISC_VMWRITE_VMEXIT,
@@ -4511,7 +4514,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
             VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
             VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
             VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
-            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
+            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
+            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
         .features[FEAT_VMX_MISC] =
             MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
             MSR_VMX_MISC_VMWRITE_VMEXIT,
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 76f24446a5..ad387e6ee7 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1312,6 +1312,7 @@ uint64_t x86_cpu_get_supported_feature_word(X86CPU *cpu, FeatureWord w);
 #define VMX_VM_EXIT_PT_CONCEAL_PIP                  0x01000000
 #define VMX_VM_EXIT_CLEAR_IA32_RTIT_CTL             0x02000000
 #define VMX_VM_EXIT_LOAD_IA32_PKRS                  0x20000000
+#define VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL      0x40000000
 #define VMX_VM_EXIT_ACTIVATE_SECONDARY_CONTROLS     0x80000000
 
 #define VMX_VM_ENTRY_LOAD_DEBUG_CONTROLS            0x00000004
-- 
2.40.1

Re: [PATCH 3/3] target/i386: Support VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL

[Zhao Liu]

> @@ -4212,7 +4213,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
>              VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
>              VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
>              VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
> -            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
> +            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
> +            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
>          .features[FEAT_VMX_MISC] =
>              MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
>              MSR_VMX_MISC_VMWRITE_VMEXIT,
> @@ -4368,7 +4370,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
>              VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
>              VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
>              VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
> -            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
> +            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
> +            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
>          .features[FEAT_VMX_MISC] =
>              MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
>              MSR_VMX_MISC_VMWRITE_VMEXIT,
> @@ -4511,7 +4514,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
>              VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
>              VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
>              VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
> -            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
> +            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
> +            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
>          .features[FEAT_VMX_MISC] =
>              MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
>              MSR_VMX_MISC_VMWRITE_VMEXIT,

Instead of modifying SPR's CPU model directly, we should introduce a new
version (SapphireRapids-v4), like Skylake-Server-v4 enables
"vmx-eptp-switching".

Re: [PATCH 3/3] target/i386: Support VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL

[Mi, Dapeng]

On 4/27/2025 4:54 PM, Zhao Liu wrote:
>> @@ -4212,7 +4213,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
>>              VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
>>              VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
>>              VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
>> -            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
>> +            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
>> +            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
>>          .features[FEAT_VMX_MISC] =
>>              MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
>>              MSR_VMX_MISC_VMWRITE_VMEXIT,
>> @@ -4368,7 +4370,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
>>              VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
>>              VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
>>              VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
>> -            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
>> +            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
>> +            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
>>          .features[FEAT_VMX_MISC] =
>>              MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
>>              MSR_VMX_MISC_VMWRITE_VMEXIT,
>> @@ -4511,7 +4514,8 @@ static const X86CPUDefinition builtin_x86_defs[] = {
>>              VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
>>              VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT |
>>              VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER |
>> -            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER,
>> +            VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER |
>> +            VMX_VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL,
>>          .features[FEAT_VMX_MISC] =
>>              MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT |
>>              MSR_VMX_MISC_VMWRITE_VMEXIT,
> Instead of modifying SPR's CPU model directly, we should introduce a new
> version (SapphireRapids-v4), like Skylake-Server-v4 enables
> "vmx-eptp-switching".

Sure. Let me have a look this.


>