Re: [Qemu-devel] [RFC PATCH v8 11/21] replay: recording and replaying clock ticks

[Paolo Bonzini <pbonzini@redhat.com>]

On 22/01/2015 09:52, Pavel Dovgalyuk wrote:
> Clock ticks are considered as the sources of non-deterministic data for
> virtual machine. This patch implements saving the clock values when they
> are acquired (virtual, host clock, rdtsc, and some other timers).
> When replaying the execution corresponding values are read from log and
> transfered to the module, which wants to read the values.
> Such a design required the clock polling to be synchronized. Sometimes
> it is not true - e.g. when timeouts for timer lists are checked. In this case
> we use a cached value of the clock, passing it to the client code.
> 
> Signed-off-by: Pavel Dovgalyuk <pavel.dovgaluk@ispras.ru>
> ---
>  cpus.c                   |    3 +-
>  include/qemu/timer.h     |   10 +++++
>  qemu-timer.c             |    7 ++--
>  replay/Makefile.objs     |    1 +
>  replay/replay-internal.h |   13 +++++++
>  replay/replay-time.c     |   84 ++++++++++++++++++++++++++++++++++++++++++++++
>  replay/replay.h          |   25 ++++++++++++++
>  stubs/replay.c           |    9 +++++
>  8 files changed, 147 insertions(+), 5 deletions(-)
>  create mode 100755 replay/replay-time.c
> 
> diff --git a/cpus.c b/cpus.c
> index 8787277..01d89aa 100644
> --- a/cpus.c
> +++ b/cpus.c
> @@ -353,7 +353,8 @@ static void icount_warp_rt(void *opaque)
>  
>      seqlock_write_lock(&timers_state.vm_clock_seqlock);
>      if (runstate_is_running()) {
> -        int64_t clock = cpu_get_clock_locked();
> +        int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
> +                                     cpu_get_clock_locked());
>          int64_t warp_delta;
>  
>          warp_delta = clock - vm_clock_warp_start;
> diff --git a/include/qemu/timer.h b/include/qemu/timer.h
> index 0666920..0c2472c 100644
> --- a/include/qemu/timer.h
> +++ b/include/qemu/timer.h
> @@ -4,6 +4,7 @@
>  #include "qemu/typedefs.h"
>  #include "qemu-common.h"
>  #include "qemu/notify.h"
> +#include "replay/replay.h"
>  
>  /* timers */
>  
> @@ -760,6 +761,8 @@ int64_t cpu_icount_to_ns(int64_t icount);
>  /*******************************************/
>  /* host CPU ticks (if available) */
>  
> +#define cpu_get_real_ticks cpu_get_real_ticks_impl
> +
>  #if defined(_ARCH_PPC)
>  
>  static inline int64_t cpu_get_real_ticks(void)
> @@ -913,6 +916,13 @@ static inline int64_t cpu_get_real_ticks (void)
>  }
>  #endif
>  
> +#undef cpu_get_real_ticks
> +
> +static inline int64_t cpu_get_real_ticks(void)

cpu_get_real_ticks should never be used.  Please instead wrap
cpu_get_ticks() with REPLAY_CLOCK.

> +{
> +    return REPLAY_CLOCK(REPLAY_CLOCK_REAL_TICKS, cpu_get_real_ticks_impl());
> +}
> +
>  #ifdef CONFIG_PROFILER
>  static inline int64_t profile_getclock(void)
>  {
> diff --git a/qemu-timer.c b/qemu-timer.c
> index 98d9d1b..bc981a2 100644
> --- a/qemu-timer.c
> +++ b/qemu-timer.c
> @@ -25,6 +25,7 @@
>  #include "sysemu/sysemu.h"
>  #include "monitor/monitor.h"
>  #include "ui/console.h"
> +#include "replay/replay.h"
>  
>  #include "hw/hw.h"
>  
> @@ -566,15 +567,15 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
>              return cpu_get_clock();
>          }
>      case QEMU_CLOCK_HOST:
> -        now = get_clock_realtime();
> +        now = REPLAY_CLOCK(REPLAY_CLOCK_HOST, get_clock_realtime());
>          last = clock->last;
>          clock->last = now;
> -        if (now < last) {
> +        if (now < last && replay_mode == REPLAY_MODE_NONE) {
>              notifier_list_notify(&clock->reset_notifiers, &now);
>          }
>          return now;
>      case QEMU_CLOCK_VIRTUAL_RT:
> -        return cpu_get_clock();
> +        return REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT, cpu_get_clock());
>      }
>  }
>  
> diff --git a/replay/Makefile.objs b/replay/Makefile.objs
> index 56da09c..257c320 100755
> --- a/replay/Makefile.objs
> +++ b/replay/Makefile.objs
> @@ -1,3 +1,4 @@
>  obj-y += replay.o
>  obj-y += replay-internal.o
>  obj-y += replay-events.o
> +obj-y += replay-time.o
> diff --git a/replay/replay-internal.h b/replay/replay-internal.h
> index 1666d6e..e906ec3 100755
> --- a/replay/replay-internal.h
> +++ b/replay/replay-internal.h
> @@ -22,7 +22,10 @@ enum ReplayEvents {
>      /* for emulated exceptions */
>      EVENT_EXCEPTION,
>      /* for async events */
> -    EVENT_ASYNC
> +    EVENT_ASYNC,
> +    /* for clock read/writes */
> +    /* some of grteater codes are reserved for clocks */
> +    EVENT_CLOCK
>  };
>  
>  /* Asynchronous events IDs */
> @@ -34,6 +37,8 @@ enum ReplayAsyncEventKind {
>  typedef enum ReplayAsyncEventKind ReplayAsyncEventKind;
>  
>  typedef struct ReplayState {
> +    /*! Cached clock values. */
> +    int64_t cached_clock[REPLAY_CLOCK_COUNT];
>      /*! Current step - number of processed instructions and timer events. */
>      uint64_t current_step;
>      /*! Number of instructions to be executed before other events happen. */
> @@ -88,6 +93,12 @@ bool skip_async_events(int stop_event);
>      reports an error and stops the execution. */
>  void skip_async_events_until(unsigned int kind);
>  
> +/*! Reads next clock value from the file.
> +    If clock kind read from the file is different from the parameter,
> +    the value is not used.
> +    If the parameter is -1, the clock value is read to the cache anyway. */

In what case could the clock kind not match?

Paolo


> +void replay_read_next_clock(unsigned int kind);
> +
>  /* Asynchronous events queue */
>  
>  /*! Initializes events' processing internals */
> diff --git a/replay/replay-time.c b/replay/replay-time.c
> new file mode 100755
> index 0000000..5f5bc6a
> --- /dev/null
> +++ b/replay/replay-time.c
> @@ -0,0 +1,84 @@
> +/*
> + * replay-time.c
> + *
> + * Copyright (c) 2010-2015 Institute for System Programming
> + *                         of the Russian Academy of Sciences.
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#include "qemu-common.h"
> +#include "replay.h"
> +#include "replay-internal.h"
> +
> +
> +int64_t replay_save_clock(ReplayClockKind kind, int64_t clock)
> +{
> +    replay_save_instructions();
> +
> +    if (kind >= REPLAY_CLOCK_COUNT) {
> +        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
> +        exit(1);
> +    }
> +
> +    if (replay_file) {
> +        replay_mutex_lock();
> +        replay_put_event(EVENT_CLOCK + kind);
> +        replay_put_qword(clock);
> +        replay_mutex_unlock();
> +    }
> +
> +    return clock;
> +}
> +
> +void replay_read_next_clock(ReplayClockKind kind)
> +{
> +    replay_fetch_data_kind();
> +    if (replay_file) {
> +        unsigned int read_kind = replay_data_kind - EVENT_CLOCK;
> +
> +        if (kind != -1 && read_kind != kind) {
> +            return;
> +        }
> +        if (read_kind >= REPLAY_CLOCK_COUNT) {
> +            fprintf(stderr,
> +                    "invalid clock ID %d was read from replay\n", read_kind);
> +            exit(1);
> +        }
> +
> +        int64_t clock = replay_get_qword();
> +
> +        replay_check_error();
> +        replay_has_unread_data = 0;
> +
> +        replay_state.cached_clock[read_kind] = clock;
> +    }
> +}
> +
> +/*! Reads next clock event from the input. */
> +int64_t replay_read_clock(ReplayClockKind kind)
> +{
> +    if (kind >= REPLAY_CLOCK_COUNT) {
> +        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
> +        exit(1);
> +    }
> +
> +    replay_exec_instructions();
> +
> +    if (replay_file) {
> +        int64_t ret;
> +        replay_mutex_lock();
> +        if (skip_async_events(EVENT_CLOCK + kind)) {
> +            replay_read_next_clock(kind);
> +        }
> +        ret = replay_state.cached_clock[kind];
> +        replay_mutex_unlock();
> +
> +        return ret;
> +    }
> +
> +    fprintf(stderr, "REPLAY INTERNAL ERROR %d\n", __LINE__);
> +    exit(1);
> +}
> diff --git a/replay/replay.h b/replay/replay.h
> index 31ca3b9..8a8e7e0 100755
> --- a/replay/replay.h
> +++ b/replay/replay.h
> @@ -16,6 +16,18 @@
>  #include <stdint.h>
>  #include "qapi-types.h"
>  
> +/* replay clock kinds */
> +enum ReplayClockKind {
> +    /* rdtsc */
> +    REPLAY_CLOCK_REAL_TICKS,
> +    /* host_clock */
> +    REPLAY_CLOCK_HOST,
> +    /* virtual_rt_clock */
> +    REPLAY_CLOCK_VIRTUAL_RT,
> +    REPLAY_CLOCK_COUNT
> +};
> +typedef enum ReplayClockKind ReplayClockKind;
> +
>  extern ReplayMode replay_mode;
>  
>  /* Processing the instructions */
> @@ -43,6 +55,19 @@ bool replay_interrupt(void);
>      Returns true, when interrupt request is pending */
>  bool replay_has_interrupt(void);
>  
> +/* Processing clocks and other time sources */
> +
> +/*! Save the specified clock */
> +int64_t replay_save_clock(ReplayClockKind kind, int64_t clock);
> +/*! Read the specified clock from the log or return cached data */
> +int64_t replay_read_clock(ReplayClockKind kind);
> +/*! Saves or reads the clock depending on the current replay mode. */
> +#define REPLAY_CLOCK(clock, value)                                      \
> +    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
> +        : replay_mode == REPLAY_MODE_RECORD                             \
> +            ? replay_save_clock((clock), (value))                       \
> +        : (value))
> +
>  /* Asynchronous events queue */
>  
>  /*! Disables storing events in the queue */
> diff --git a/stubs/replay.c b/stubs/replay.c
> index 563c777..121bca6 100755
> --- a/stubs/replay.c
> +++ b/stubs/replay.c
> @@ -1,3 +1,12 @@
>  #include "replay/replay.h"
>  
>  ReplayMode replay_mode;
> +
> +int64_t replay_save_clock(unsigned int kind, int64_t clock)
> +{
> +}
> +
> +int64_t replay_read_clock(unsigned int kind)
> +{
> +    return 0;
> +}
>