[Qemu-devel] [RESEND 0/2] Optimze the env->tsc update operation

[Qemu-devel] [RESEND 0/2] Optimze the env->tsc update operation

[Liang Li]

This patch aims for reducing the live migration downtime. It updates
the env->tsc value when stopping kvmclock with a ligth weight interface,
rather than using 'cpu_synchronize_all_states', which takes more time.

For a VM with 4 CPUs, this path can help to reduce the VM downtime about
100us.

Liang Li (2):
  kvmclock: use a light weight interface to update env->tsc.
  Revert "Introduce cpu_clean_all_dirty"

 cpus.c                 |  9 ---------
 hw/i386/kvm/clock.c    | 26 +++++++++-----------------
 include/sysemu/cpus.h  |  1 -
 include/sysemu/kvm.h   |  8 --------
 kvm-all.c              |  5 -----
 target-i386/kvm.c      | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 target-i386/kvm_i386.h |  1 +
 7 files changed, 56 insertions(+), 40 deletions(-)

-- 
1.9.1

[Qemu-devel] [RESEND 1/2] kvmclock: use a light weight interface to update env->tsc.

[Liang Li]

The commit 317b0a6d8 fixed an issue which caused by the outdated env->tsc
value, but the fix lead to cpu_synchronize_all_states called twice during
live migration. The cpu_synchronize_all_states takes about 130us for a VM
which has 4 vcpus, it's a bit expensive.

This patch use a light weight interface to update the env->tsc value, it
only takes about 20us to update the env->tsc.

Signed-off-by: Liang Li <liang.z.li@intel.com>
---
 hw/i386/kvm/clock.c    | 26 +++++++++-----------------
 target-i386/kvm.c      | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 target-i386/kvm_i386.h |  1 +
 3 files changed, 56 insertions(+), 17 deletions(-)

diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c
index efdf165..2e69cdb 100644
--- a/hw/i386/kvm/clock.c
+++ b/hw/i386/kvm/clock.c
@@ -17,7 +17,7 @@
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
-#include "sysemu/cpus.h"
+#include "kvm_i386.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -125,22 +125,14 @@ static void kvmclock_vm_state_change(void *opaque, int running,
             return;
         }
 
-        cpu_synchronize_all_states();
-        /* In theory, the cpu_synchronize_all_states() call above wouldn't
-         * affect the rest of the code, as the VCPU state inside CPUState
-         * is supposed to always match the VCPU state on the kernel side.
-         *
-         * In practice, calling cpu_synchronize_state() too soon will load the
-         * kernel-side APIC state into X86CPU.apic_state too early, APIC state
-         * won't be reloaded later because CPUState.vcpu_dirty==true, and
-         * outdated APIC state may be migrated to another host.
-         *
-         * The real fix would be to make sure outdated APIC state is read
-         * from the kernel again when necessary. While this is not fixed, we
-         * need the cpu_clean_all_dirty() call below.
-         */
-        cpu_clean_all_dirty();
-
+        CPU_FOREACH(cpu) {
+            ret = kvm_get_tsc(cpu);
+            if (ret < 0) {
+                fprintf(stderr, "KVM_GET_MSRS failed: %s\n", strerror(ret));
+                abort();
+                return;
+            }
+        }
         ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
         if (ret < 0) {
             fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 64046cb..eae90e1 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -111,6 +111,52 @@ bool kvm_allows_irq0_override(void)
     return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
 }
 
+int kvm_get_tsc(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    struct {
+        struct kvm_msrs info;
+        struct kvm_msr_entry entries[1];
+    } msr_data;
+    struct kvm_msr_entry *msrs = msr_data.entries;
+    int ret, i, n;
+
+    n = 0;
+
+    if (!env->tsc_valid) {
+        msrs[n++].index = MSR_IA32_TSC;
+        env->tsc_valid = !runstate_is_running();
+    }
+
+    if (n == 0) {
+        return 0;
+    }
+
+    msr_data.info = (struct kvm_msrs) {
+        .nmsrs = n,
+    };
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    for (i = 0; i < ret; i++) {
+        uint32_t index = msrs[i].index;
+        switch (index) {
+        case MSR_IA32_TSC:
+            env->tsc = msrs[i].data;
+            break;
+        default:
+            break;
+        }
+    }
+
+    return 0;
+}
+
+
 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
 {
     struct kvm_cpuid2 *cpuid;
diff --git a/target-i386/kvm_i386.h b/target-i386/kvm_i386.h
index e557e94..141fab5 100644
--- a/target-i386/kvm_i386.h
+++ b/target-i386/kvm_i386.h
@@ -17,6 +17,7 @@ bool kvm_allows_irq0_override(void);
 bool kvm_has_smm(void);
 void kvm_arch_reset_vcpu(X86CPU *cs);
 void kvm_arch_do_init_vcpu(X86CPU *cs);
+int  kvm_get_tsc(CPUState *cs);
 
 int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
                           uint32_t flags, uint32_t *dev_id);
-- 
1.9.1

[Qemu-devel] [RESEND 2/2] Revert "Introduce cpu_clean_all_dirty"

[Liang Li]

This reverts commit de9d61e83d43be9069e6646fa9d57a3f47779d28.

Now 'cpu_clean_all_dirty' is useless, we can revert the related code.

Conflicts:
	include/sysemu/kvm.h
---
 cpus.c                | 9 ---------
 include/sysemu/cpus.h | 1 -
 include/sysemu/kvm.h  | 8 --------
 kvm-all.c             | 5 -----
 4 files changed, 23 deletions(-)

diff --git a/cpus.c b/cpus.c
index d2e9e4f..c6a5d0e 100644
--- a/cpus.c
+++ b/cpus.c
@@ -694,15 +694,6 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
-void cpu_clean_all_dirty(void)
-{
-    CPUState *cpu;
-
-    CPU_FOREACH(cpu) {
-        cpu_clean_state(cpu);
-    }
-}
-
 static int do_vm_stop(RunState state)
 {
     int ret = 0;
diff --git a/include/sysemu/cpus.h b/include/sysemu/cpus.h
index 30ddd12..3d1e5ba 100644
--- a/include/sysemu/cpus.h
+++ b/include/sysemu/cpus.h
@@ -11,7 +11,6 @@ void cpu_stop_current(void);
 void cpu_synchronize_all_states(void);
 void cpu_synchronize_all_post_reset(void);
 void cpu_synchronize_all_post_init(void);
-void cpu_clean_all_dirty(void);
 
 void qtest_clock_warp(int64_t dest);
 
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 461ef65..4ac6176 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -417,7 +417,6 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);
-void kvm_cpu_clean_state(CPUState *cpu);
 
 /* generic hooks - to be moved/refactored once there are more users */
 
@@ -442,13 +441,6 @@ static inline void cpu_synchronize_post_init(CPUState *cpu)
     }
 }
 
-static inline void cpu_clean_state(CPUState *cpu)
-{
-    if (kvm_enabled()) {
-        kvm_cpu_clean_state(cpu);
-    }
-}
-
 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev);
 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
                                  PCIDevice *dev);
diff --git a/kvm-all.c b/kvm-all.c
index c442838..1bc1273 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -1766,11 +1766,6 @@ void kvm_cpu_synchronize_post_init(CPUState *cpu)
     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
 }
 
-void kvm_cpu_clean_state(CPUState *cpu)
-{
-    cpu->kvm_vcpu_dirty = false;
-}
-
 int kvm_cpu_exec(CPUState *cpu)
 {
     struct kvm_run *run = cpu->kvm_run;
-- 
1.9.1

Re: [Qemu-devel] [RESEND 1/2] kvmclock: use a light weight interface to update env->tsc.

[Paolo Bonzini]

On 29/10/2015 09:22, Liang Li wrote:
> +int kvm_get_tsc(CPUState *cs)
> +{
> +    X86CPU *cpu = X86_CPU(cs);
> +    CPUX86State *env = &cpu->env;
> +    struct {
> +        struct kvm_msrs info;
> +        struct kvm_msr_entry entries[1];
> +    } msr_data;
> +    struct kvm_msr_entry *msrs = msr_data.entries;
> +    int ret, i, n;
> +
> +    n = 0;
> +
> +    if (!env->tsc_valid) {
> +        msrs[n++].index = MSR_IA32_TSC;
> +        env->tsc_valid = !runstate_is_running();
> +    }
> +
> +    if (n == 0) {
> +        return 0;
> +    }
> +
> +    msr_data.info = (struct kvm_msrs) {
> +        .nmsrs = n,
> +    };
> +
> +    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
> +    if (ret < 0) {
> +        return ret;
> +    }
> +
> +    for (i = 0; i < ret; i++) {
> +        uint32_t index = msrs[i].index;
> +        switch (index) {
> +        case MSR_IA32_TSC:
> +            env->tsc = msrs[i].data;
> +            break;
> +        default:
> +            break;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +

This can be simplified a bit:

int kvm_get_tsc(CPUState *cs)
{
    X86CPU *cpu = X86_CPU(cs);
    CPUX86State *env = &cpu->env;
    struct {
        struct kvm_msrs info;
        struct kvm_msr_entry entries[1];
    } msr_data;
    int ret;

    if (env->tsc_valid) {
        return 0;
    }

    msr_data.info.nmsrs = 1;
    msr_data.entries[0].index = MSR_IA32_TSC;
    env->tsc_valid = !runstate_is_running();

    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
    if (ret < 0) {
        return ret;
    }

    env->tsc = msr_data.entries[0].data;
    return 0;
}

> 
> +        CPU_FOREACH(cpu) {
> +            ret = kvm_get_tsc(cpu);
> +            if (ret < 0) {
> +                fprintf(stderr, "KVM_GET_MSRS failed: %s\n", strerror(ret));
> +                abort();
> +                return;
> +            }
> +        }


This should be run in the appropriate thread using run_on_cpu.  VCPU
ioctls should only be invoked from the VCPU thread.  So you should
introduce a new function kvm_synchronize_all_tsc() or something like that.

Otherwise, the idea behind the patches is fine.  Thanks!

Paolo

Re: [Qemu-devel] [RESEND 1/2] kvmclock: use a light weight interface to update env->tsc.

[Li, Liang Z]

> This can be simplified a bit:
> 
> int kvm_get_tsc(CPUState *cs)
> {
>     X86CPU *cpu = X86_CPU(cs);
>     CPUX86State *env = &cpu->env;
>     struct {
>         struct kvm_msrs info;
>         struct kvm_msr_entry entries[1];
>     } msr_data;
>     int ret;
> 
>     if (env->tsc_valid) {
>         return 0;
>     }
> 
>     msr_data.info.nmsrs = 1;
>     msr_data.entries[0].index = MSR_IA32_TSC;
>     env->tsc_valid = !runstate_is_running();
> 
>     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
>     if (ret < 0) {
>         return ret;
>     }
> 
>     env->tsc = msr_data.entries[0].data;
>     return 0;
> }
> 
> >
> > +        CPU_FOREACH(cpu) {
> > +            ret = kvm_get_tsc(cpu);
> > +            if (ret < 0) {
> > +                fprintf(stderr, "KVM_GET_MSRS failed: %s\n", strerror(ret));
> > +                abort();
> > +                return;
> > +            }
> > +        }
> 
> 
> This should be run in the appropriate thread using run_on_cpu.  VCPU ioctls
> should only be invoked from the VCPU thread.  So you should introduce a new
> function kvm_synchronize_all_tsc() or something like that.
> 
> Otherwise, the idea behind the patches is fine.  Thanks!
> 
> Paolo

Thanks for your comments, I will send the v3.

Liang