[Qemu-devel] [PATCH 1/3] target-i386: disable pv eoi to fix migration across QEMU versions

[Qemu-devel] [PATCH 1/3] target-i386: disable pv eoi to fix migration across QEMU versions

[Anthony Liguori]

We have a problem with how we handle migration with KVM paravirt features.
We unconditionally enable paravirt features regardless of whether we know how
to migrate them.

We also don't tie paravirt features to specific machine types so an old QEMU on
a new kernel would expose features that never existed.

The 1.2 cycle is over and as things stand, migration is broken.  Michael has
another series that adds support for migrating PV EOI and attempts to make it
work correctly for different machine types.

After speaking with Michael on IRC, we agreed to take this patch plus 1 & 4
from his series.  This makes sure QEMU can migrate PV EOI if it's enabled, but
does not enable it by default.

This also means that we won't unconditionally enable new features for guests
future proofing us from this happening again in the future.

Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
---
 target-i386/cpu.c |   13 ++++++++++++-
 1 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 120a2e3..f3cac49 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -33,6 +33,7 @@
 #include "hyperv.h"
 
 #include "hw/hw.h"
+#include <linux/kvm_para.h>
 
 /* feature flags taken from "Intel Processor Identification and the CPUID
  * Instruction" and AMD's "CPUID Specification".  In cases of disagreement
@@ -887,7 +888,17 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
         memcpy(x86_cpu_def, def, sizeof(*def));
     }
 
-    plus_kvm_features = ~0; /* not supported bits will be filtered out later */
+#if defined(CONFIG_KVM)
+    plus_kvm_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
+        (1 << KVM_FEATURE_NOP_IO_DELAY) | 
+        (1 << KVM_FEATURE_MMU_OP) |
+        (1 << KVM_FEATURE_CLOCKSOURCE2) |
+        (1 << KVM_FEATURE_ASYNC_PF) | 
+        (1 << KVM_FEATURE_STEAL_TIME) |
+        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
+#else
+    plus_kvm_features = 0;
+#endif
 
     add_flagname_to_bitmaps("hypervisor", &plus_features,
         &plus_ext_features, &plus_ext2_features, &plus_ext3_features,
-- 
1.7.5.4

[Qemu-devel] [PATCH 2/3] linux-headers: update to 3.6-rc3

[Anthony Liguori]

From: "Michael S. Tsirkin" <mst@redhat.com>

Update linux-headers to version present in Linux 3.6-rc3.
Header asm-x96_64/kvm_para.h update is needed for the new PV EOI
feature.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
---
 linux-headers/asm-s390/kvm.h      |    2 +-
 linux-headers/asm-s390/kvm_para.h |    2 +-
 linux-headers/asm-x86/kvm.h       |    1 +
 linux-headers/asm-x86/kvm_para.h  |    7 +++++++
 linux-headers/linux/kvm.h         |    3 +++
 5 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index bdcbe0f..d25da59 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -1,7 +1,7 @@
 #ifndef __LINUX_KVM_S390_H
 #define __LINUX_KVM_S390_H
 /*
- * asm-s390/kvm.h - KVM s390 specific structures and definitions
+ * KVM s390 specific structures and definitions
  *
  * Copyright IBM Corp. 2008
  *
diff --git a/linux-headers/asm-s390/kvm_para.h b/linux-headers/asm-s390/kvm_para.h
index 8e2dd67..870051f 100644
--- a/linux-headers/asm-s390/kvm_para.h
+++ b/linux-headers/asm-s390/kvm_para.h
@@ -1,5 +1,5 @@
 /*
- * asm-s390/kvm_para.h - definition for paravirtual devices on s390
+ * definition for paravirtual devices on s390
  *
  * Copyright IBM Corp. 2008
  *
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index e7d1c19..246617e 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -12,6 +12,7 @@
 /* Select x86 specific features in <linux/kvm.h> */
 #define __KVM_HAVE_PIT
 #define __KVM_HAVE_IOAPIC
+#define __KVM_HAVE_IRQ_LINE
 #define __KVM_HAVE_DEVICE_ASSIGNMENT
 #define __KVM_HAVE_MSI
 #define __KVM_HAVE_USER_NMI
diff --git a/linux-headers/asm-x86/kvm_para.h b/linux-headers/asm-x86/kvm_para.h
index f2ac46a..a1c3d72 100644
--- a/linux-headers/asm-x86/kvm_para.h
+++ b/linux-headers/asm-x86/kvm_para.h
@@ -22,6 +22,7 @@
 #define KVM_FEATURE_CLOCKSOURCE2        3
 #define KVM_FEATURE_ASYNC_PF		4
 #define KVM_FEATURE_STEAL_TIME		5
+#define KVM_FEATURE_PV_EOI		6
 
 /* The last 8 bits are used to indicate how to interpret the flags field
  * in pvclock structure. If no bits are set, all flags are ignored.
@@ -37,6 +38,7 @@
 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01
 #define MSR_KVM_ASYNC_PF_EN 0x4b564d02
 #define MSR_KVM_STEAL_TIME  0x4b564d03
+#define MSR_KVM_PV_EOI_EN      0x4b564d04
 
 struct kvm_steal_time {
 	__u64 steal;
@@ -89,5 +91,10 @@ struct kvm_vcpu_pv_apf_data {
 	__u32 enabled;
 };
 
+#define KVM_PV_EOI_BIT 0
+#define KVM_PV_EOI_MASK (0x1 << KVM_PV_EOI_BIT)
+#define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
+#define KVM_PV_EOI_DISABLED 0x0
+
 
 #endif /* _ASM_X86_KVM_PARA_H */
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 5a9d4e3..4b9e575 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -617,6 +617,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_SIGNAL_MSI 77
 #define KVM_CAP_PPC_GET_SMMU_INFO 78
 #define KVM_CAP_S390_COW 79
+#define KVM_CAP_PPC_ALLOC_HTAB 80
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -828,6 +829,8 @@ struct kvm_s390_ucas_mapping {
 #define KVM_SIGNAL_MSI            _IOW(KVMIO,  0xa5, struct kvm_msi)
 /* Available with KVM_CAP_PPC_GET_SMMU_INFO */
 #define KVM_PPC_GET_SMMU_INFO	  _IOR(KVMIO,  0xa6, struct kvm_ppc_smmu_info)
+/* Available with KVM_CAP_PPC_ALLOC_HTAB */
+#define KVM_PPC_ALLOCATE_HTAB	  _IOWR(KVMIO, 0xa7, __u32)
 
 /*
  * ioctls for vcpu fds
-- 
1.7.5.4

[Qemu-devel] [PATCH 3/3] kvm: get/set PV EOI MSR

[Anthony Liguori]

From: "Michael S. Tsirkin" <mst@redhat.com>

Support get/set of new PV EOI MSR, for migration.
Add an optional section for MSR value - send it
out in case MSR was changed from the default value (0).

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
---
 target-i386/cpu.h     |    1 +
 target-i386/kvm.c     |   13 +++++++++++++
 target-i386/machine.c |   21 +++++++++++++++++++++
 3 files changed, 35 insertions(+), 0 deletions(-)

diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 60f9e97..0677502 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -699,6 +699,7 @@ typedef struct CPUX86State {
     uint64_t system_time_msr;
     uint64_t wall_clock_msr;
     uint64_t async_pf_en_msr;
+    uint64_t pv_eoi_en_msr;
 
     uint64_t tsc;
     uint64_t tsc_deadline;
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 696b14a..ffc294e 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -63,6 +63,7 @@ static bool has_msr_star;
 static bool has_msr_hsave_pa;
 static bool has_msr_tsc_deadline;
 static bool has_msr_async_pf_en;
+static bool has_msr_pv_eoi_en;
 static bool has_msr_misc_enable;
 static int lm_capable_kernel;
 
@@ -455,6 +456,8 @@ int kvm_arch_init_vcpu(CPUX86State *env)
 
     has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
 
+    has_msr_pv_eoi_en = c->eax & (1 << KVM_FEATURE_PV_EOI);
+
     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0; i <= limit; i++) {
@@ -1017,6 +1020,10 @@ static int kvm_put_msrs(CPUX86State *env, int level)
             kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
                               env->async_pf_en_msr);
         }
+        if (has_msr_pv_eoi_en) {
+            kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_EOI_EN,
+                              env->pv_eoi_en_msr);
+        }
         if (hyperv_hypercall_available()) {
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_GUEST_OS_ID, 0);
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_HYPERCALL, 0);
@@ -1259,6 +1266,9 @@ static int kvm_get_msrs(CPUX86State *env)
     if (has_msr_async_pf_en) {
         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
     }
+    if (has_msr_pv_eoi_en) {
+        msrs[n++].index = MSR_KVM_PV_EOI_EN;
+    }
 
     if (env->mcg_cap) {
         msrs[n++].index = MSR_MCG_STATUS;
@@ -1338,6 +1348,9 @@ static int kvm_get_msrs(CPUX86State *env)
         case MSR_KVM_ASYNC_PF_EN:
             env->async_pf_en_msr = msrs[i].data;
             break;
+        case MSR_KVM_PV_EOI_EN:
+            env->pv_eoi_en_msr = msrs[i].data;
+            break;
         }
     }
 
diff --git a/target-i386/machine.c b/target-i386/machine.c
index a8be058..4771508 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -279,6 +279,13 @@ static bool async_pf_msr_needed(void *opaque)
     return cpu->async_pf_en_msr != 0;
 }
 
+static bool pv_eoi_msr_needed(void *opaque)
+{
+    CPUX86State *cpu = opaque;
+
+    return cpu->pv_eoi_en_msr != 0;
+}
+
 static const VMStateDescription vmstate_async_pf_msr = {
     .name = "cpu/async_pf_msr",
     .version_id = 1,
@@ -290,6 +297,17 @@ static const VMStateDescription vmstate_async_pf_msr = {
     }
 };
 
+static const VMStateDescription vmstate_pv_eoi_msr = {
+    .name = "cpu/async_pv_eoi_msr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_UINT64(pv_eoi_en_msr, CPUX86State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool fpop_ip_dp_needed(void *opaque)
 {
     CPUX86State *env = opaque;
@@ -454,6 +472,9 @@ static const VMStateDescription vmstate_cpu = {
             .vmsd = &vmstate_async_pf_msr,
             .needed = async_pf_msr_needed,
         } , {
+            .vmsd = &vmstate_pv_eoi_msr,
+            .needed = pv_eoi_msr_needed,
+        } , {
             .vmsd = &vmstate_fpop_ip_dp,
             .needed = fpop_ip_dp_needed,
         }, {
-- 
1.7.5.4

Re: [Qemu-devel] [PATCH 1/3] target-i386: disable pv eoi to fix migration across QEMU versions

[Eduardo Habkost]

On Wed, Aug 29, 2012 at 10:52:07AM -0500, Anthony Liguori wrote:
> We have a problem with how we handle migration with KVM paravirt features.
> We unconditionally enable paravirt features regardless of whether we know how
> to migrate them.
> 
> We also don't tie paravirt features to specific machine types so an old QEMU on
> a new kernel would expose features that never existed.
> 
> The 1.2 cycle is over and as things stand, migration is broken.  Michael has
> another series that adds support for migrating PV EOI and attempts to make it
> work correctly for different machine types.
> 
> After speaking with Michael on IRC, we agreed to take this patch plus 1 & 4
> from his series.  This makes sure QEMU can migrate PV EOI if it's enabled, but
> does not enable it by default.
> 
> This also means that we won't unconditionally enable new features for guests
> future proofing us from this happening again in the future.
> 
> Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

Lots of trailing whitespaces below, but they can be fixed before pushing
the patch, so:

Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
Tested-by: Eduardo Habkost <ehabkost@redhat.com>

(tested on a 3.5.2-1.fc17.x86_64 host. all features were enabled on
guest, except for PV_EOI and MMU_OP, as expected)


> ---
>  target-i386/cpu.c |   13 ++++++++++++-
>  1 files changed, 12 insertions(+), 1 deletions(-)
> 
> diff --git a/target-i386/cpu.c b/target-i386/cpu.c
> index 120a2e3..f3cac49 100644
> --- a/target-i386/cpu.c
> +++ b/target-i386/cpu.c
> @@ -33,6 +33,7 @@
>  #include "hyperv.h"
>  
>  #include "hw/hw.h"
> +#include <linux/kvm_para.h>
>  
>  /* feature flags taken from "Intel Processor Identification and the CPUID
>   * Instruction" and AMD's "CPUID Specification".  In cases of disagreement
> @@ -887,7 +888,17 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
>          memcpy(x86_cpu_def, def, sizeof(*def));
>      }
>  
> -    plus_kvm_features = ~0; /* not supported bits will be filtered out later */
> +#if defined(CONFIG_KVM)
> +    plus_kvm_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
> +        (1 << KVM_FEATURE_NOP_IO_DELAY) | 
> +        (1 << KVM_FEATURE_MMU_OP) |
> +        (1 << KVM_FEATURE_CLOCKSOURCE2) |
> +        (1 << KVM_FEATURE_ASYNC_PF) | 
> +        (1 << KVM_FEATURE_STEAL_TIME) |
> +        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
> +#else
> +    plus_kvm_features = 0;
> +#endif
>  
>      add_flagname_to_bitmaps("hypervisor", &plus_features,
>          &plus_ext_features, &plus_ext2_features, &plus_ext3_features,
> -- 
> 1.7.5.4
> 
> 

-- 
Eduardo