[PATCH v2 0/3] ppc/kvm: Handle CPU compatibility mode correctly for nested guests

[PATCH v2 0/3] ppc/kvm: Handle CPU compatibility mode correctly for nested guests

[Amit Machhiwal]

On POWER systems, newer processor generations can operate in compatibility
modes corresponding to earlier generations (e.g., a Power11 system running
in Power10 compatibility mode). In such cases, the effective CPU level
exposed to guests differs from the physical processor generation.

This creates issues for nested virtualization. When booting a nested KVM
guest, QEMU may derive the CPU model from the raw hardware PVR and attempt
to configure the guest accordingly. However, the host is constrained by the
compatibility level negotiated with the hypervisor, and requests exceeding
that level are rejected by KVM, leading to guest boot failures such as:

  KVM-NESTEDv2: couldn't set guest wide elements

This series addresses the issue in two ways:

1. Do not silently fall back to raw mode when KVM rejects a requested
   compatibility level during CAS. Instead, propagate the error so invalid
   configurations are visible and fail early.

2. Query the effective CPU compatibility modes supported by the host via
   KVM and use this information to select an appropriate CPU model for
   nested guests.

With these changes, QEMU avoids masking KVM errors and ensures that nested
guests are configured with CPU models consistent with the host
compatibility mode, allowing them to boot correctly.

Patch summary:
  [1/3] hw/ppc/spapr: Do not fallback to raw mode when KVM rejects compat
  [2/3] [DO_NOT_MERGE] linux-headers: Add uapi header changes
  [3/3] target/ppc/kvm: Use host compatibility mode for nested guests

Changes in v2:
- Patch 3: Guard compatibility mode code with #if defined(TARGET_PPC64)
  to fix compilation for ppc32 targets. The POWER9/10/11 PVR constants
  are only defined for 64-bit builds, and compatibility modes are only
  relevant for 64-bit systems.

Tested on:
  - Power11 pSeries LPAR in Power10 compatibility mode
  - Power10 PowerNV and QEMU PowerNV 11 TCG L0 host

CI test results: https://gitlab.com/amachhiw/qemu/-/pipelines/2494987253

Note: Patch 2 is marked DO_NOT_MERGE as it contains linux-headers updates
that will be synced separately once the corresponding kernel patches are
merged.

The corresponding Linux patches have been posted [1]

[1] https://lore.kernel.org/all/20260430054906.94431-1-amachhiw@linux.ibm.com/

Amit Machhiwal (3):
  hw/ppc/spapr: Do not fallback to raw mode when KVM rejects compat
  [DO_NOT_MERGE] linux-headers: Add uapi header changes
  target/ppc/kvm: Use host compatibility mode for nested guests

 hw/ppc/spapr_hcall.c            |  9 ++++++
 linux-headers/asm-powerpc/kvm.h |  7 ++++
 linux-headers/linux/kvm.h       |  3 ++
 target/ppc/kvm.c                | 57 +++++++++++++++++++++++++++++++++
 4 files changed, 76 insertions(+)


base-commit: 3d626609ccae61a2e552bccd59c7a0931bab8261
-- 
2.50.1 (Apple Git-155)

[PATCH v2 2/3] [DO_NOT_MERGE] linux-headers: Add uapi header changes

[Amit Machhiwal]

This is a temporary patch intended for review and testing purposes only.
It defines the new ioctls introduced for checking the compatibility
modes. The actual header changes are to be done once the kernel changes
[1] are merged.

[1] https://lore.kernel.org/all/20260430054906.94431-1-amachhiw@linux.ibm.com/

Signed-off-by: Amit Machhiwal <amachhiw@linux.ibm.com>
---
 linux-headers/asm-powerpc/kvm.h | 7 +++++++
 linux-headers/linux/kvm.h       | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/linux-headers/asm-powerpc/kvm.h b/linux-headers/asm-powerpc/kvm.h
index 077c5437f521..df91b1f40c5b 100644
--- a/linux-headers/asm-powerpc/kvm.h
+++ b/linux-headers/asm-powerpc/kvm.h
@@ -437,6 +437,13 @@ struct kvm_ppc_cpu_char {
 	__u64	behaviour_mask;		/* valid bits in behaviour */
 };
 
+/* For KVM_PPC_GET_COMPAT_CAPS */
+struct kvm_ppc_compat_caps {
+       __u32   flags;
+       __u64   compat_capabilities;    /* Capabilities supported by the host */
+};
+
+
 /*
  * Values for character and character_mask.
  * These are identical to the values used by H_GET_CPU_CHARACTERISTICS.
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index a4ab42dcba97..b0c0014a0a0a 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -966,6 +966,7 @@ struct kvm_enable_cap {
 #define KVM_CAP_GUEST_MEMFD_FLAGS 244
 #define KVM_CAP_ARM_SEA_TO_USER 245
 #define KVM_CAP_S390_USER_OPEREXEC 246
+#define KVM_CAP_PPC_COMPAT_CAPS 249
 
 struct kvm_irq_routing_irqchip {
 	__u32 irqchip;
@@ -1295,6 +1296,8 @@ struct kvm_vfio_spapr_tce {
 /* Available with KVM_CAP_COUNTER_OFFSET */
 #define KVM_ARM_SET_COUNTER_OFFSET _IOW(KVMIO,  0xb5, struct kvm_arm_counter_offset)
 #define KVM_ARM_GET_REG_WRITABLE_MASKS _IOR(KVMIO,  0xb6, struct reg_mask_range)
+/* Available with KVM_CAP_PPC_COMPAT_CAPS */
+#define KVM_PPC_GET_COMPAT_CAPS          _IOR(KVMIO,  0xe4, struct kvm_ppc_compat_caps)
 
 /* ioctl for vm fd */
 #define KVM_CREATE_DEVICE	  _IOWR(KVMIO,  0xe0, struct kvm_create_device)
-- 
2.50.1 (Apple Git-155)

[PATCH v2 3/3] target/ppc/kvm: Use host compatibility mode for nested guests

[Amit Machhiwal]

On POWER systems, the host CPU may run in a compatibility mode (e.g.,
a Power11 processor operating in Power10 compatibility mode). In such
cases, the effective CPU level exposed to guests differs from the
physical processor generation.

When running nested KVM guests, QEMU currently derives the host CPU type
using mfpvr(), which reflects the physical processor version. This can
result in a mismatch between the CPU model used by QEMU and the
compatibility mode enforced by the host, leading to guest boot failures.

In particular, booting a nested guest on a Power11 LPAR configured in
Power10 compatibility mode fails with errors such as:

  KVM-NESTEDv2: couldn't set guest wide elements

This occurs because QEMU selects a CPU model based on the physical
processor version, while the host operates in a lower compatibility
mode. As a result, KVM rejects the requested compatibility level during
guest initialization.

Add support for querying host compatibility capabilities via the
KVM_PPC_GET_COMPAT_CAPS ioctl and derive the effective PVR based on the
compatibility mode reported by KVM. When available, use this
compatibility PVR instead of the raw hardware PVR when selecting
the CPU model.

If the capability is not supported or the query fails, fall back to the
existing behavior.

This ensures that QEMU selects a CPU model consistent with the host
compatibility mode, allowing nested guests to boot correctly.

Signed-off-by: Amit Machhiwal <amachhiw@linux.ibm.com>
---
 target/ppc/kvm.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c
index 25c28ad089c6..7a00092c4208 100644
--- a/target/ppc/kvm.c
+++ b/target/ppc/kvm.c
@@ -2602,11 +2602,68 @@ bool kvmppc_supports_ail_3(void)
     return cap_ail_mode_3;
 }
 
+#if defined(TARGET_PPC64)
+static target_ulong kvmppc_get_compat_caps(void)
+{
+    struct kvm_ppc_compat_caps host_compat;
+    target_ulong host_caps;
+    int ret;
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_PPC_COMPAT_CAPS)) {
+        return 0;
+    }
+
+    ret = kvm_vm_ioctl(kvm_state, KVM_PPC_GET_COMPAT_CAPS, &host_compat);
+    if (ret < 0) {
+        fprintf(stderr, "KVM: failed to get host capabilities\n");
+        return 0;
+    }
+
+    host_caps = host_compat.compat_capabilities;
+    return host_caps;
+}
+
+static uint32_t kvm_ppc_host_compat_pvr(void)
+{
+    uint32_t compat_host_pvr = 0;
+    int cap_idx = 0;
+    target_ulong host_caps = kvmppc_get_compat_caps();
+
+    if (host_caps) {
+        cap_idx = 63 - __builtin_ctzll(host_caps);
+        switch (cap_idx) {
+        case H_GUEST_CAP_P9_MODE_BMAP:
+            compat_host_pvr = CPU_POWERPC_POWER9_DD22;
+            break;
+        case H_GUEST_CAP_P10_MODE_BMAP:
+            compat_host_pvr = CPU_POWERPC_POWER10_DD20;
+            break;
+        case H_GUEST_CAP_P11_MODE_BMAP:
+            compat_host_pvr = CPU_POWERPC_POWER11_DD20;
+            break;
+        default:
+            break;
+        }
+    }
+
+    return compat_host_pvr;
+}
+#endif /* TARGET_PPC64 */
+
 PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void)
 {
     uint32_t host_pvr = mfpvr();
     PowerPCCPUClass *pvr_pcc;
 
+#if defined(TARGET_PPC64)
+    uint32_t compat_host_pvr;
+
+    compat_host_pvr = kvm_ppc_host_compat_pvr();
+    if (compat_host_pvr) {
+        host_pvr = compat_host_pvr;
+    }
+#endif /* TARGET_PPC64 */
+
     pvr_pcc = ppc_cpu_class_by_pvr(host_pvr);
     if (pvr_pcc == NULL) {
         pvr_pcc = ppc_cpu_class_by_pvr_mask(host_pvr);
-- 
2.50.1 (Apple Git-155)