[Qemu-devel] [PATCH 0/7] icount migration and clock.

[Qemu-devel] [PATCH 0/7] icount migration and clock.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

Those are some icount patches required for reverse execution.

It introduces an icount clock which is only growing with icount.
It allows QEMU to migrate icount so virtual clock is kept if the VM is migrated
in icount mode (which is mandatory for migration based snapshot).

They are rebased on uq/master of
git://git.kernel.org/pub/scm/virt/kvm/qemu-kvm.git

And can be cloned here:
git://git.greensocs.com/qemu_cexe.git:cexe_1_3_v6_rebased

Thanks,
Fred

KONRAD Frederic (7):
  icount: put icount variables into TimerState.
  migration: migrate icount fields.
  migration: make qemu_savevm_state public.
  icount: introduce icount timer.
  icount: check for icount clock deadline when cpu loop exits.
  icount: make icount extra computed on icount clock as well.
  timer: add cpu_icount_to_ns function.

 cpus.c                  | 95 ++++++++++++++++++++++++++++++++++++++-----------
 include/qemu/timer.h    | 10 +++++-
 include/sysemu/sysemu.h |  1 +
 qemu-timer.c            |  8 ++++-
 savevm.c                |  2 +-
 stubs/cpu-get-icount.c  |  2 +-
 6 files changed, 94 insertions(+), 24 deletions(-)

-- 
1.9.0

[Qemu-devel] [PATCH 1/7] icount: put icount variables into TimerState.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

This puts qemu_icount and qemu_icount_bias into TimerState structure to allow
them to be migrated.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
---
 cpus.c | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/cpus.c b/cpus.c
index 9cf3063..ebfd4fb 100644
--- a/cpus.c
+++ b/cpus.c
@@ -104,8 +104,6 @@ static bool all_cpu_threads_idle(void)
 
 /* Protected by TimersState seqlock */
 
-/* Compensate for varying guest execution speed.  */
-static int64_t qemu_icount_bias;
 static int64_t vm_clock_warp_start = -1;
 /* Conversion factor from emulated instructions to virtual clock ticks.
  * icount_time_shift is defined as extern in include/qemu-common.h because
@@ -116,9 +114,6 @@ int icount_time_shift;
 /* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
 #define MAX_ICOUNT_SHIFT 10
 
-/* Only written by TCG thread */
-static int64_t qemu_icount;
-
 static QEMUTimer *icount_rt_timer;
 static QEMUTimer *icount_vm_timer;
 static QEMUTimer *icount_warp_timer;
@@ -135,6 +130,11 @@ typedef struct TimersState {
     int64_t cpu_clock_offset;
     int32_t cpu_ticks_enabled;
     int64_t dummy;
+
+    /* Compensate for varying guest execution speed.  */
+    int64_t qemu_icount_bias;
+    /* Only written by TCG thread */
+    int64_t qemu_icount;
 } TimersState;
 
 static TimersState timers_state;
@@ -145,14 +145,14 @@ static int64_t cpu_get_icount_locked(void)
     int64_t icount;
     CPUState *cpu = current_cpu;
 
-    icount = qemu_icount;
+    icount = timers_state.qemu_icount;
     if (cpu) {
         if (!cpu_can_do_io(cpu)) {
             fprintf(stderr, "Bad clock read\n");
         }
         icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
     }
-    return qemu_icount_bias + (icount << icount_time_shift);
+    return timers_state.qemu_icount_bias + (icount << icount_time_shift);
 }
 
 int64_t cpu_get_icount(void)
@@ -307,7 +307,8 @@ static void icount_adjust(void)
         icount_time_shift++;
     }
     last_delta = delta;
-    qemu_icount_bias = cur_icount - (qemu_icount << icount_time_shift);
+    timers_state.qemu_icount_bias = cur_icount
+                              - (timers_state.qemu_icount << icount_time_shift);
     seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 }
 
@@ -356,7 +357,7 @@ static void icount_warp_rt(void *opaque)
             int64_t delta = cur_time - cur_icount;
             warp_delta = MIN(warp_delta, delta);
         }
-        qemu_icount_bias += warp_delta;
+        timers_state.qemu_icount_bias += warp_delta;
     }
     vm_clock_warp_start = -1;
     seqlock_write_unlock(&timers_state.vm_clock_seqlock);
@@ -374,7 +375,7 @@ void qtest_clock_warp(int64_t dest)
         int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
         int64_t warp = qemu_soonest_timeout(dest - clock, deadline);
         seqlock_write_lock(&timers_state.vm_clock_seqlock);
-        qemu_icount_bias += warp;
+        timers_state.qemu_icount_bias += warp;
         seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 
         qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
@@ -1286,7 +1287,8 @@ static int tcg_cpu_exec(CPUArchState *env)
         int64_t count;
         int64_t deadline;
         int decr;
-        qemu_icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
+        timers_state.qemu_icount -= (cpu->icount_decr.u16.low
+                                    + cpu->icount_extra);
         cpu->icount_decr.u16.low = 0;
         cpu->icount_extra = 0;
         deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
@@ -1301,7 +1303,7 @@ static int tcg_cpu_exec(CPUArchState *env)
         }
 
         count = qemu_icount_round(deadline);
-        qemu_icount += count;
+        timers_state.qemu_icount += count;
         decr = (count > 0xffff) ? 0xffff : count;
         count -= decr;
         cpu->icount_decr.u16.low = decr;
@@ -1314,7 +1316,8 @@ static int tcg_cpu_exec(CPUArchState *env)
     if (use_icount) {
         /* Fold pending instructions back into the
            instruction counter, and clear the interrupt flag.  */
-        qemu_icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
+        timers_state.qemu_icount -= (cpu->icount_decr.u16.low
+                        + cpu->icount_extra);
         cpu->icount_decr.u32 = 0;
         cpu->icount_extra = 0;
     }
-- 
1.9.0

[Qemu-devel] [PATCH 2/7] migration: migrate icount fields.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

This fixes a bug where qemu_icount and qemu_icount_bias are not migrated.
It adds a subsection "timer/icount" to vmstate_timers so icount is migrated only
when needed.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
Reviewed-by: Amit Shah <amit.shah@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
---
 cpus.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/cpus.c b/cpus.c
index ebfd4fb..98e1203 100644
--- a/cpus.c
+++ b/cpus.c
@@ -452,6 +452,25 @@ void qemu_clock_warp(QEMUClockType type)
     }
 }
 
+static bool icount_state_needed(void *opaque)
+{
+    return use_icount;
+}
+
+/*
+ * This is a subsection for icount migration.
+ */
+static const VMStateDescription icount_vmstate_timers = {
+    .name = "timer/icount",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT64(qemu_icount_bias, TimersState),
+        VMSTATE_INT64(qemu_icount, TimersState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_timers = {
     .name = "timer",
     .version_id = 2,
@@ -461,6 +480,14 @@ static const VMStateDescription vmstate_timers = {
         VMSTATE_INT64(dummy, TimersState),
         VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &icount_vmstate_timers,
+            .needed = icount_state_needed,
+        }, {
+            /* empty */
+        }
     }
 };
 
-- 
1.9.0

[Qemu-devel] [PATCH 3/7] migration: make qemu_savevm_state public.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

This makes qemu_savevm_state public for reverse-execution.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
Reviewed-by: Amit Shah <amit.shah@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
---
 include/sysemu/sysemu.h | 1 +
 savevm.c                | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index d8539fd..bc52525 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -81,6 +81,7 @@ void do_info_snapshots(Monitor *mon, const QDict *qdict);
 
 void qemu_announce_self(void);
 
+int qemu_savevm_state(QEMUFile *f);
 bool qemu_savevm_state_blocked(Error **errp);
 void qemu_savevm_state_begin(QEMUFile *f,
                              const MigrationParams *params);
diff --git a/savevm.c b/savevm.c
index e19ae0a..0286f9b 100644
--- a/savevm.c
+++ b/savevm.c
@@ -779,7 +779,7 @@ void qemu_savevm_state_cancel(void)
     }
 }
 
-static int qemu_savevm_state(QEMUFile *f)
+int qemu_savevm_state(QEMUFile *f)
 {
     int ret;
     MigrationParams params = {
-- 
1.9.0

[Qemu-devel] [PATCH 4/7] icount: introduce icount timer.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

This introduces a new timer based only on instruction counter and without any
compensation.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
---
 cpus.c                 | 21 +++++++++++++--------
 include/qemu/timer.h   |  9 ++++++++-
 qemu-timer.c           |  8 +++++++-
 stubs/cpu-get-icount.c |  2 +-
 4 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/cpus.c b/cpus.c
index 98e1203..e8971e3 100644
--- a/cpus.c
+++ b/cpus.c
@@ -140,7 +140,7 @@ typedef struct TimersState {
 static TimersState timers_state;
 
 /* Return the virtual CPU time, based on the instruction counter.  */
-static int64_t cpu_get_icount_locked(void)
+static int64_t cpu_get_icount_locked(int with_bias)
 {
     int64_t icount;
     CPUState *cpu = current_cpu;
@@ -152,17 +152,22 @@ static int64_t cpu_get_icount_locked(void)
         }
         icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
     }
-    return timers_state.qemu_icount_bias + (icount << icount_time_shift);
+
+    if (with_bias) {
+        return timers_state.qemu_icount_bias + (icount << icount_time_shift);
+    } else {
+        return icount << icount_time_shift;
+    }
 }
 
-int64_t cpu_get_icount(void)
+int64_t cpu_get_icount(int with_bias)
 {
     int64_t icount;
     unsigned start;
 
     do {
         start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
-        icount = cpu_get_icount_locked();
+        icount = cpu_get_icount_locked(with_bias);
     } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
 
     return icount;
@@ -175,7 +180,7 @@ int64_t cpu_get_ticks(void)
     int64_t ticks;
 
     if (use_icount) {
-        return cpu_get_icount();
+        return cpu_get_icount(true);
     }
 
     ticks = timers_state.cpu_ticks_offset;
@@ -290,7 +295,7 @@ static void icount_adjust(void)
 
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     cur_time = cpu_get_clock_locked();
-    cur_icount = cpu_get_icount_locked();
+    cur_icount = cpu_get_icount_locked(true);
 
     delta = cur_icount - cur_time;
     /* FIXME: This is a very crude algorithm, somewhat prone to oscillation.  */
@@ -353,7 +358,7 @@ static void icount_warp_rt(void *opaque)
              * far ahead of real time.
              */
             int64_t cur_time = cpu_get_clock_locked();
-            int64_t cur_icount = cpu_get_icount_locked();
+            int64_t cur_icount = cpu_get_icount_locked(true);
             int64_t delta = cur_time - cur_icount;
             warp_delta = MIN(warp_delta, delta);
         }
@@ -1557,7 +1562,7 @@ void qmp_inject_nmi(Error **errp)
 void dump_drift_info(FILE *f, fprintf_function cpu_fprintf)
 {
     cpu_fprintf(f, "Host - Guest clock  %ld(ms)\n",
-                (cpu_get_clock() - cpu_get_icount())/SCALE_MS);
+                (cpu_get_clock() - cpu_get_icount(true))/SCALE_MS);
     if (icount_align_option) {
         cpu_fprintf(f, "Max guest delay     %ld(ms)\n", -max_delay/SCALE_MS);
         cpu_fprintf(f, "Max guest advance   %ld(ms)\n", max_advance/SCALE_MS);
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index 102f442..1ad0f87 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -36,12 +36,19 @@
  * is suspended, and it will reflect system time changes the host may
  * undergo (e.g. due to NTP). The host clock has the same precision as
  * the virtual clock.
+ *
+ * @QEMU_CLOCK_ICOUNT: icount clock
+ *
+ * The icount clock is based on instruction counter without any compensation for
+ * speed. It will run only when instruction are executed and make only sense in
+ * icount mode.
  */
 
 typedef enum {
     QEMU_CLOCK_REALTIME = 0,
     QEMU_CLOCK_VIRTUAL = 1,
     QEMU_CLOCK_HOST = 2,
+    QEMU_CLOCK_ICOUNT = 3,
     QEMU_CLOCK_MAX
 } QEMUClockType;
 
@@ -743,7 +750,7 @@ static inline int64_t get_clock(void)
 #endif
 
 /* icount */
-int64_t cpu_get_icount(void);
+int64_t cpu_get_icount(int with_bias);
 int64_t cpu_get_clock(void);
 int64_t cpu_get_clock_offset(void);
 
diff --git a/qemu-timer.c b/qemu-timer.c
index 00a5d35..3be80b1 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -554,7 +554,7 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
     default:
     case QEMU_CLOCK_VIRTUAL:
         if (use_icount) {
-            return cpu_get_icount();
+            return cpu_get_icount(true);
         } else {
             return cpu_get_clock();
         }
@@ -566,6 +566,12 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
             notifier_list_notify(&clock->reset_notifiers, &now);
         }
         return now;
+    case QEMU_CLOCK_ICOUNT:
+        if (use_icount) {
+            return cpu_get_icount(false);
+        } else {
+            return -1;
+        }
     }
 }
 
diff --git a/stubs/cpu-get-icount.c b/stubs/cpu-get-icount.c
index d685859..1968de7 100644
--- a/stubs/cpu-get-icount.c
+++ b/stubs/cpu-get-icount.c
@@ -3,7 +3,7 @@
 
 int use_icount;
 
-int64_t cpu_get_icount(void)
+int64_t cpu_get_icount(int with_bias)
 {
     abort();
 }
-- 
1.9.0

[Qemu-devel] [PATCH 5/7] icount: check for icount clock deadline when cpu loop exits.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

Notify events on icount clock when CPU loop exits.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
---
 cpus.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cpus.c b/cpus.c
index e8971e3..8bd3423 100644
--- a/cpus.c
+++ b/cpus.c
@@ -1028,6 +1028,11 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
             if (deadline == 0) {
                 qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
             }
+
+            deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_ICOUNT);
+            if (deadline == 0) {
+                qemu_clock_notify(QEMU_CLOCK_ICOUNT);
+            }
         }
         qemu_tcg_wait_io_event();
     }
-- 
1.9.0

[Qemu-devel] [PATCH 6/7] icount: make icount extra computed on icount clock as well.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

This takes icount clock in account for icount extra computation so icount
clock's timers will be triggered at the exact time.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
---
 cpus.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/cpus.c b/cpus.c
index 8bd3423..f63872b 100644
--- a/cpus.c
+++ b/cpus.c
@@ -1323,6 +1323,7 @@ static int tcg_cpu_exec(CPUArchState *env)
     if (use_icount) {
         int64_t count;
         int64_t deadline;
+        int64_t icount_deadline;
         int decr;
         timers_state.qemu_icount -= (cpu->icount_decr.u16.low
                                     + cpu->icount_extra);
@@ -1339,6 +1340,15 @@ static int tcg_cpu_exec(CPUArchState *env)
             deadline = INT32_MAX;
         }
 
+        /*
+         * Take icount clock deadline in account too, and keep the nearest
+         * deadline.
+         */
+        icount_deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_ICOUNT);
+        if ((icount_deadline >= 0) && (icount_deadline < deadline)) {
+            deadline = icount_deadline;
+        }
+
         count = qemu_icount_round(deadline);
         timers_state.qemu_icount += count;
         decr = (count > 0xffff) ? 0xffff : count;
-- 
1.9.0

[Qemu-devel] [PATCH 7/7] timer: add cpu_icount_to_ns function.

[fred.konrad]

From: KONRAD Frederic <fred.konrad@greensocs.com>

This adds cpu_icount_to_ns function which is needed for reverse execution.

It returns the time for a specific instruction.

Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
---
 cpus.c               | 9 +++++++--
 include/qemu/timer.h | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cpus.c b/cpus.c
index f63872b..580c754 100644
--- a/cpus.c
+++ b/cpus.c
@@ -154,9 +154,9 @@ static int64_t cpu_get_icount_locked(int with_bias)
     }
 
     if (with_bias) {
-        return timers_state.qemu_icount_bias + (icount << icount_time_shift);
+        return timers_state.qemu_icount_bias + cpu_icount_to_ns(icount);
     } else {
-        return icount << icount_time_shift;
+        return cpu_icount_to_ns(icount);
     }
 }
 
@@ -173,6 +173,11 @@ int64_t cpu_get_icount(int with_bias)
     return icount;
 }
 
+int64_t cpu_icount_to_ns(int64_t icount)
+{
+    return icount << icount_time_shift;
+}
+
 /* return the host CPU cycle counter and handle stop/restart */
 /* Caller must hold the BQL */
 int64_t cpu_get_ticks(void)
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index 1ad0f87..16bafde 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -753,6 +753,7 @@ static inline int64_t get_clock(void)
 int64_t cpu_get_icount(int with_bias);
 int64_t cpu_get_clock(void);
 int64_t cpu_get_clock_offset(void);
+int64_t cpu_icount_to_ns(int64_t icount);
 
 /*******************************************/
 /* host CPU ticks (if available) */
-- 
1.9.0

Re: [Qemu-devel] [PATCH 0/7] icount migration and clock.

[Paolo Bonzini]

Il 01/08/2014 01:37, fred.konrad@greensocs.com ha scritto:
> From: KONRAD Frederic <fred.konrad@greensocs.com>
> 
> Those are some icount patches required for reverse execution.
> 
> It introduces an icount clock which is only growing with icount.
> It allows QEMU to migrate icount so virtual clock is kept if the VM is migrated
> in icount mode (which is mandatory for migration based snapshot).
> 
> They are rebased on uq/master of
> git://git.kernel.org/pub/scm/virt/kvm/qemu-kvm.git
> 
> And can be cloned here:
> git://git.greensocs.com/qemu_cexe.git:cexe_1_3_v6_rebased
> 
> Thanks,
> Fred
> 
> KONRAD Frederic (7):
>   icount: put icount variables into TimerState.
>   migration: migrate icount fields.
>   migration: make qemu_savevm_state public.
>   icount: introduce icount timer.
>   icount: check for icount clock deadline when cpu loop exits.
>   icount: make icount extra computed on icount clock as well.
>   timer: add cpu_icount_to_ns function.
> 
>  cpus.c                  | 95 ++++++++++++++++++++++++++++++++++++++-----------
>  include/qemu/timer.h    | 10 +++++-
>  include/sysemu/sysemu.h |  1 
>  qemu-timer.c            |  8 ++++-
>  savevm.c                |  2 +-
>  stubs/cpu-get-icount.c  |  2 +-
>  6 files changed, 94 insertions(+), 24 deletions(-)
> 

Thanks, I applied all patches and pushed everything to icount on
git://github.com/bonzini/qemu.git.  I've also dropped Sebastian's
"icount: Make icount_time_shift available everywhere" and adjusted his
stuff to use cpu_icount_to_ns instead.

I'm planning to send out patches 1/2/7 in my first 2.2 pull request,
together with Sebastian's host clock alignment feature.  The others will
have to wait for the reverse execution patches which actually use it.

Paolo