Re: [Qemu-devel] [PATCH v5 1/7] replay: character devices

[Paolo Bonzini <pbonzini@redhat.com>]

On 14/03/2016 08:44, Pavel Dovgalyuk wrote:
> This patch implements record and replay of character devices.
> It records chardevs communication in replay mode. Recorded information
> include data read from backend and counter of bytes written
> from frontend to backend to preserve frontend internal state.
> If character device was configured through the command line in record mode,
> then in replay mode it should be also added to command line. Backend of
> the character device could be changed in replay mode.
> Replaying of devices that perform ioctl and get_msgfd operations is not
> supported.
> gdbstub which also acts as a backend is not recorded to allow controlling
> the replaying through gdb. Monitor backends are also not recorded.
> 
> Signed-off-by: Pavel Dovgalyuk <pavel.dovgaluk@ispras.ru>

Thanks, this looks good.

Paolo

> ---
>  gdbstub.c                |    2 -
>  include/sysemu/char.h    |   26 +++++++
>  include/sysemu/replay.h  |   17 +++++
>  qemu-char.c              |  138 +++++++++++++++++++++++++++++++-------
>  replay/Makefile.objs     |    1 
>  replay/replay-char.c     |  167 ++++++++++++++++++++++++++++++++++++++++++++++
>  replay/replay-events.c   |   17 ++++-
>  replay/replay-internal.h |   18 +++++
>  replay/replay.c          |    2 -
>  9 files changed, 358 insertions(+), 30 deletions(-)
>  create mode 100755 replay/replay-char.c
> 
> diff --git a/gdbstub.c b/gdbstub.c
> index 61c12b1..fdcb0ee 100644
> --- a/gdbstub.c
> +++ b/gdbstub.c
> @@ -1752,7 +1752,7 @@ int gdbserver_start(const char *device)
>              sigaction(SIGINT, &act, NULL);
>          }
>  #endif
> -        chr = qemu_chr_new("gdb", device, NULL);
> +        chr = qemu_chr_new_noreplay("gdb", device, NULL);
>          if (!chr)
>              return -1;
>  
> diff --git a/include/sysemu/char.h b/include/sysemu/char.h
> index e46884f..4c2f777 100644
> --- a/include/sysemu/char.h
> +++ b/include/sysemu/char.h
> @@ -86,6 +86,7 @@ struct CharDriverState {
>      int is_mux;
>      guint fd_in_tag;
>      QemuOpts *opts;
> +    bool replay;
>      QTAILQ_ENTRY(CharDriverState) next;
>  };
>  
> @@ -139,6 +140,22 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename,
>                                void (*init)(struct CharDriverState *s));
>  
>  /**
> + * @qemu_chr_new_noreplay:
> + *
> + * Create a new character backend from a URI.
> + * Character device communications are not written
> + * into the replay log.
> + *
> + * @label the name of the backend
> + * @filename the URI
> + * @init not sure..
> + *
> + * Returns: a new character backend
> + */
> +CharDriverState *qemu_chr_new_noreplay(const char *label, const char *filename,
> +                                       void (*init)(struct CharDriverState *s));
> +
> +/**
>   * @qemu_chr_delete:
>   *
>   * Destroy a character backend and remove it from the list of
> @@ -341,6 +358,15 @@ int qemu_chr_be_can_write(CharDriverState *s);
>   */
>  void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len);
>  
> +/**
> + * @qemu_chr_be_write_impl:
> + *
> + * Implementation of back end writing. Used by replay module.
> + *
> + * @buf a buffer to receive data from the front end
> + * @len the number of bytes to receive from the front end
> + */
> +void qemu_chr_be_write_impl(CharDriverState *s, uint8_t *buf, int len);
>  
>  /**
>   * @qemu_chr_be_event:
> diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h
> index e4108e8..d24d502 100644
> --- a/include/sysemu/replay.h
> +++ b/include/sysemu/replay.h
> @@ -114,4 +114,21 @@ void replay_input_event(QemuConsole *src, InputEvent *evt);
>  /*! Adds input sync event to the queue */
>  void replay_input_sync_event(void);
>  
> +/* Character device */
> +
> +/*! Registers char driver to save it's events */
> +void replay_register_char_driver(struct CharDriverState *chr);
> +/*! Saves write to char device event to the log */
> +void replay_chr_be_write(struct CharDriverState *s, uint8_t *buf, int len);
> +/*! Writes char write return value to the replay log. */
> +void replay_char_write_event_save(int res, int offset);
> +/*! Reads char write return value from the replay log. */
> +void replay_char_write_event_load(int *res, int *offset);
> +/*! Reads information about read_all character event. */
> +int replay_char_read_all_load(uint8_t *buf);
> +/*! Writes character read_all error code into the replay log. */
> +void replay_char_read_all_save_error(int res);
> +/*! Writes character read_all execution result into the replay log. */
> +void replay_char_read_all_save_buf(uint8_t *buf, int offset);
> +
>  #endif
> diff --git a/qemu-char.c b/qemu-char.c
> index e0147f3..b418307 100644
> --- a/qemu-char.c
> +++ b/qemu-char.c
> @@ -37,6 +37,7 @@
>  #include "io/channel-socket.h"
>  #include "io/channel-file.h"
>  #include "io/channel-tls.h"
> +#include "sysemu/replay.h"
>  
>  #include <zlib.h>
>  
> @@ -234,10 +235,46 @@ static void qemu_chr_fe_write_log(CharDriverState *s,
>      }
>  }
>  
> +static int qemu_chr_fe_write_buffer(CharDriverState *s, const uint8_t *buf, int len, int *offset)
> +{
> +    int res = 0;
> +    *offset = 0;
> +
> +    qemu_mutex_lock(&s->chr_write_lock);
> +    while (*offset < len) {
> +        do {
> +            res = s->chr_write(s, buf + *offset, len - *offset);
> +            if (res == -1 && errno == EAGAIN) {
> +                g_usleep(100);
> +            }
> +        } while (res == -1 && errno == EAGAIN);
> +
> +        if (res <= 0) {
> +            break;
> +        }
> +
> +        *offset += res;
> +    }
> +    if (*offset > 0) {
> +        qemu_chr_fe_write_log(s, buf, *offset);
> +    }
> +    qemu_mutex_unlock(&s->chr_write_lock);
> +
> +    return res;
> +}
> +
>  int qemu_chr_fe_write(CharDriverState *s, const uint8_t *buf, int len)
>  {
>      int ret;
>  
> +    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
> +        int offset;
> +        replay_char_write_event_load(&ret, &offset);
> +        assert(offset <= len);
> +        qemu_chr_fe_write_buffer(s, buf, offset, &offset);
> +        return ret;
> +    }
> +
>      qemu_mutex_lock(&s->chr_write_lock);
>      ret = s->chr_write(s, buf, len);
>  
> @@ -246,34 +283,31 @@ int qemu_chr_fe_write(CharDriverState *s, const uint8_t *buf, int len)
>      }
>  
>      qemu_mutex_unlock(&s->chr_write_lock);
> +    
> +    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
> +        replay_char_write_event_save(ret, ret < 0 ? 0 : ret);
> +    }
> +    
>      return ret;
>  }
>  
>  int qemu_chr_fe_write_all(CharDriverState *s, const uint8_t *buf, int len)
>  {
> -    int offset = 0;
> -    int res = 0;
> +    int offset;
> +    int res;
>  
> -    qemu_mutex_lock(&s->chr_write_lock);
> -    while (offset < len) {
> -        do {
> -            res = s->chr_write(s, buf + offset, len - offset);
> -            if (res == -1 && errno == EAGAIN) {
> -                g_usleep(100);
> -            }
> -        } while (res == -1 && errno == EAGAIN);
> +    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
> +        replay_char_write_event_load(&res, &offset);
> +        assert(offset <= len);
> +        qemu_chr_fe_write_buffer(s, buf, offset, &offset);
> +        return res;
> +    }
>  
> -        if (res <= 0) {
> -            break;
> -        }
> +    res = qemu_chr_fe_write_buffer(s, buf, len, &offset);
>  
> -        offset += res;
> +    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
> +        replay_char_write_event_save(res, offset);
>      }
> -    if (offset > 0) {
> -        qemu_chr_fe_write_log(s, buf, offset);
> -    }
> -
> -    qemu_mutex_unlock(&s->chr_write_lock);
>  
>      if (res < 0) {
>          return res;
> @@ -289,6 +323,10 @@ int qemu_chr_fe_read_all(CharDriverState *s, uint8_t *buf, int len)
>      if (!s->chr_sync_read) {
>          return 0;
>      }
> +    
> +    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
> +        return replay_char_read_all_load(buf);
> +    }
>  
>      while (offset < len) {
>          do {
> @@ -303,6 +341,9 @@ int qemu_chr_fe_read_all(CharDriverState *s, uint8_t *buf, int len)
>          }
>  
>          if (res < 0) {
> +            if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
> +                replay_char_read_all_save_error(res);
> +            }
>              return res;
>          }
>  
> @@ -313,14 +354,22 @@ int qemu_chr_fe_read_all(CharDriverState *s, uint8_t *buf, int len)
>          }
>      }
>  
> +    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
> +        replay_char_read_all_save_buf(buf, offset);
> +    }
>      return offset;
>  }
>  
>  int qemu_chr_fe_ioctl(CharDriverState *s, int cmd, void *arg)
>  {
> -    if (!s->chr_ioctl)
> -        return -ENOTSUP;
> -    return s->chr_ioctl(s, cmd, arg);
> +    int res;
> +    if (!s->chr_ioctl || s->replay) {
> +        res = -ENOTSUP;
> +    } else {
> +        res = s->chr_ioctl(s, cmd, arg);
> +    }
> +
> +    return res;
>  }
>  
>  int qemu_chr_be_can_write(CharDriverState *s)
> @@ -330,17 +379,35 @@ int qemu_chr_be_can_write(CharDriverState *s)
>      return s->chr_can_read(s->handler_opaque);
>  }
>  
> -void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
> +void qemu_chr_be_write_impl(CharDriverState *s, uint8_t *buf, int len)
>  {
>      if (s->chr_read) {
>          s->chr_read(s->handler_opaque, buf, len);
>      }
>  }
>  
> +void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
> +{
> +    if (s->replay) {
> +        if (replay_mode == REPLAY_MODE_PLAY) {
> +            return;
> +        }
> +        replay_chr_be_write(s, buf, len);
> +    } else {
> +        qemu_chr_be_write_impl(s, buf, len);
> +    }
> +}
> +
>  int qemu_chr_fe_get_msgfd(CharDriverState *s)
>  {
>      int fd;
> -    return (qemu_chr_fe_get_msgfds(s, &fd, 1) == 1) ? fd : -1;
> +    int res = (qemu_chr_fe_get_msgfds(s, &fd, 1) == 1) ? fd : -1;
> +    if (s->replay) {
> +        fprintf(stderr,
> +                "Replay: get msgfd is not supported for serial devices yet\n");
> +        exit(1);
> +    }
> +    return res;
>  }
>  
>  int qemu_chr_fe_get_msgfds(CharDriverState *s, int *fds, int len)
> @@ -3855,7 +3922,8 @@ err:
>      return NULL;
>  }
>  
> -CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
> +CharDriverState *qemu_chr_new_noreplay(const char *label, const char *filename,
> +                                       void (*init)(struct CharDriverState *s))
>  {
>      const char *p;
>      CharDriverState *chr;
> @@ -3881,6 +3949,21 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*in
>      return chr;
>  }
>  
> +CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
> +{
> +    CharDriverState *chr;
> +    chr = qemu_chr_new_noreplay(label, filename, init);
> +    if (chr) {
> +        chr->replay = replay_mode != REPLAY_MODE_NONE;
> +        if (chr->replay && chr->chr_ioctl) {
> +            fprintf(stderr,
> +                    "Replay: ioctl is not supported for serial devices yet\n");
> +        }
> +        replay_register_char_driver(chr);
> +    }
> +    return chr;
> +}
> +
>  void qemu_chr_fe_set_echo(struct CharDriverState *chr, bool echo)
>  {
>      if (chr->chr_set_echo) {
> @@ -4475,6 +4558,11 @@ void qmp_chardev_remove(const char *id, Error **errp)
>          error_setg(errp, "Chardev '%s' is busy", id);
>          return;
>      }
> +    if (chr->replay) {
> +        error_setg(errp,
> +            "Chardev '%s' cannot be unplugged in record/replay mode", id);
> +        return;
> +    }
>      qemu_chr_delete(chr);
>  }
>  
> diff --git a/replay/Makefile.objs b/replay/Makefile.objs
> index 232193a..70e5572 100644
> --- a/replay/Makefile.objs
> +++ b/replay/Makefile.objs
> @@ -3,3 +3,4 @@ common-obj-y += replay-internal.o
>  common-obj-y += replay-events.o
>  common-obj-y += replay-time.o
>  common-obj-y += replay-input.o
> +common-obj-y += replay-char.o
> diff --git a/replay/replay-char.c b/replay/replay-char.c
> new file mode 100755
> index 0000000..bfbaf2e
> --- /dev/null
> +++ b/replay/replay-char.c
> @@ -0,0 +1,167 @@
> +/*
> + * replay-char.c
> + *
> + * Copyright (c) 2010-2016 Institute for System Programming
> + *                         of the Russian Academy of Sciences.
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +
> +#include "qemu/osdep.h"
> +#include "sysemu/replay.h"
> +#include "replay-internal.h"
> +#include "sysemu/sysemu.h"
> +#include "sysemu/char.h"
> +
> +/* Char drivers that generate qemu_chr_be_write events
> +   that should be saved into the log. */
> +static CharDriverState **char_drivers;
> +static int drivers_count;
> +
> +/* Char event attributes. */
> +typedef struct CharEvent {
> +    int id;
> +    uint8_t *buf;
> +    size_t len;
> +} CharEvent;
> +
> +static int find_char_driver(CharDriverState *chr)
> +{
> +    int i = 0;
> +    for ( ; i < drivers_count ; ++i) {
> +        if (char_drivers[i] == chr) {
> +            return i;
> +        }
> +    }
> +    return -1;
> +}
> +
> +void replay_register_char_driver(CharDriverState *chr)
> +{
> +    if (replay_mode == REPLAY_MODE_NONE) {
> +        return;
> +    }
> +    char_drivers = g_realloc(char_drivers,
> +                             sizeof(*char_drivers) * (drivers_count + 1));
> +    char_drivers[drivers_count++] = chr;
> +}
> +
> +void replay_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
> +{
> +    CharEvent *event = g_malloc0(sizeof(CharEvent));
> +
> +    event->id = find_char_driver(s);
> +    if (event->id < 0) {
> +        fprintf(stderr, "Replay: cannot find char driver\n");
> +        exit(1);
> +    }
> +    event->buf = g_malloc(len);
> +    memcpy(event->buf, buf, len);
> +    event->len = len;
> +
> +    replay_add_event(REPLAY_ASYNC_EVENT_CHAR_READ, event, NULL, 0);
> +}
> +
> +void replay_event_char_read_run(void *opaque)
> +{
> +    CharEvent *event = (CharEvent *)opaque;
> +
> +    qemu_chr_be_write_impl(char_drivers[event->id], event->buf,
> +                           (int)event->len);
> +
> +    g_free(event->buf);
> +    g_free(event);
> +}
> +
> +void replay_event_char_read_save(void *opaque)
> +{
> +    CharEvent *event = (CharEvent *)opaque;
> +
> +    replay_put_byte(event->id);
> +    replay_put_array(event->buf, event->len);
> +}
> +
> +void *replay_event_char_read_load(void)
> +{
> +    CharEvent *event = g_malloc0(sizeof(CharEvent));
> +
> +    event->id = replay_get_byte();
> +    replay_get_array_alloc(&event->buf, &event->len);
> +
> +    return event;
> +}
> +
> +void replay_char_write_event_save(int res, int offset)
> +{
> +    replay_save_instructions();
> +    replay_mutex_lock();
> +    replay_put_event(EVENT_CHAR_WRITE);
> +    replay_put_dword(res);
> +    replay_put_dword(offset);
> +    replay_mutex_unlock();
> +}
> +
> +void replay_char_write_event_load(int *res, int *offset)
> +{
> +    replay_account_executed_instructions();
> +    replay_mutex_lock();
> +    if (replay_next_event_is(EVENT_CHAR_WRITE)) {
> +        *res = replay_get_dword();
> +        *offset = replay_get_dword();
> +        replay_finish_event();
> +        replay_mutex_unlock();
> +    } else {
> +        replay_mutex_unlock();
> +        error_report("Missing character write event in the replay log");
> +        exit(1);
> +    }
> +}
> +
> +int replay_char_read_all_load(uint8_t *buf)
> +{
> +    replay_mutex_lock();
> +    if (replay_next_event_is(EVENT_CHAR_READ_ALL)) {
> +        size_t size;
> +        int res;
> +        replay_get_array(buf, &size);
> +        replay_finish_event();
> +        replay_mutex_unlock();
> +        res = (int)size;
> +        assert(res >= 0);
> +        return res;
> +    } else if (replay_next_event_is(EVENT_CHAR_READ_ALL_ERROR)) {
> +        int res = replay_get_dword();
> +        replay_finish_event();
> +        replay_mutex_unlock();
> +        return res;
> +    } else {
> +        replay_mutex_unlock();
> +        error_report("Missing character read all event in the replay log");
> +        exit(1);
> +    }
> +}
> +
> +void replay_char_read_all_save_error(int res)
> +{
> +    assert(res < 0);
> +    replay_save_instructions();
> +    replay_mutex_lock();
> +    replay_put_event(EVENT_CHAR_READ_ALL_ERROR);
> +    replay_put_dword(res);
> +    replay_mutex_unlock();
> +}
> +
> +void replay_char_read_all_save_buf(uint8_t *buf, int offset)
> +{
> +    replay_save_instructions();
> +    replay_mutex_lock();
> +    replay_put_event(EVENT_CHAR_READ_ALL);
> +    replay_put_array(buf, offset);
> +    replay_mutex_unlock();
> +}
> diff --git a/replay/replay-events.c b/replay/replay-events.c
> index 2628109..ca940f7 100644
> --- a/replay/replay-events.c
> +++ b/replay/replay-events.c
> @@ -48,6 +48,9 @@ static void replay_run_event(Event *event)
>      case REPLAY_ASYNC_EVENT_INPUT_SYNC:
>          qemu_input_event_sync_impl();
>          break;
> +    case REPLAY_ASYNC_EVENT_CHAR_READ:
> +        replay_event_char_read_run(event->opaque);
> +        break;
>      default:
>          error_report("Replay: invalid async event ID (%d) in the queue",
>                      event->event_kind);
> @@ -102,9 +105,9 @@ void replay_clear_events(void)
>  }
>  
>  /*! Adds specified async event to the queue */
> -static void replay_add_event(ReplayAsyncEventKind event_kind,
> -                             void *opaque,
> -                             void *opaque2, uint64_t id)
> +void replay_add_event(ReplayAsyncEventKind event_kind,
> +                      void *opaque,
> +                      void *opaque2, uint64_t id)
>  {
>      assert(event_kind < REPLAY_ASYNC_COUNT);
>  
> @@ -168,6 +171,9 @@ static void replay_save_event(Event *event, int checkpoint)
>              break;
>          case REPLAY_ASYNC_EVENT_INPUT_SYNC:
>              break;
> +        case REPLAY_ASYNC_EVENT_CHAR_READ:
> +            replay_event_char_read_save(event->opaque);
> +            break;
>          default:
>              error_report("Unknown ID %d of replay event", read_event_kind);
>              exit(1);
> @@ -221,6 +227,11 @@ static Event *replay_read_event(int checkpoint)
>          event->event_kind = read_event_kind;
>          event->opaque = 0;
>          return event;
> +    case REPLAY_ASYNC_EVENT_CHAR_READ:
> +        event = g_malloc0(sizeof(Event));
> +        event->event_kind = read_event_kind;
> +        event->opaque = replay_event_char_read_load();
> +        return event;
>      default:
>          error_report("Unknown ID %d of replay event", read_event_kind);
>          exit(1);
> diff --git a/replay/replay-internal.h b/replay/replay-internal.h
> index 5438ebd..11f9a85 100644
> --- a/replay/replay-internal.h
> +++ b/replay/replay-internal.h
> @@ -24,6 +24,11 @@ enum ReplayEvents {
>      EVENT_ASYNC,
>      /* for shutdown request */
>      EVENT_SHUTDOWN,
> +    /* for character device write event */
> +    EVENT_CHAR_WRITE,
> +    /* for character device read all event */
> +    EVENT_CHAR_READ_ALL,
> +    EVENT_CHAR_READ_ALL_ERROR,
>      /* for clock read/writes */
>      /* some of greater codes are reserved for clocks */
>      EVENT_CLOCK,
> @@ -43,6 +48,7 @@ enum ReplayAsyncEventKind {
>      REPLAY_ASYNC_EVENT_BH,
>      REPLAY_ASYNC_EVENT_INPUT,
>      REPLAY_ASYNC_EVENT_INPUT_SYNC,
> +    REPLAY_ASYNC_EVENT_CHAR_READ,
>      REPLAY_ASYNC_COUNT
>  };
>  
> @@ -124,6 +130,9 @@ bool replay_has_events(void);
>  void replay_save_events(int checkpoint);
>  /*! Read events from the file into the input queue */
>  void replay_read_events(int checkpoint);
> +/*! Adds specified async event to the queue */
> +void replay_add_event(ReplayAsyncEventKind event_kind, void *opaque,
> +                      void *opaque2, uint64_t id);
>  
>  /* Input events */
>  
> @@ -136,4 +145,13 @@ void replay_add_input_event(struct InputEvent *event);
>  /*! Adds input sync event to the queue */
>  void replay_add_input_sync_event(void);
>  
> +/* Character devices */
> +
> +/*! Called to run char device read event. */
> +void replay_event_char_read_run(void *opaque);
> +/*! Writes char read event to the file. */
> +void replay_event_char_read_save(void *opaque);
> +/*! Reads char event read from the file. */
> +void *replay_event_char_read_load(void);
> +
>  #endif
> diff --git a/replay/replay.c b/replay/replay.c
> index f8739c2..fcfde4f 100644
> --- a/replay/replay.c
> +++ b/replay/replay.c
> @@ -20,7 +20,7 @@
>  
>  /* Current version of the replay mechanism.
>     Increase it when file format changes. */
> -#define REPLAY_VERSION              0xe02002
> +#define REPLAY_VERSION              0xe02003
>  /* Size of replay log header */
>  #define HEADER_SIZE                 (sizeof(uint32_t) + sizeof(uint64_t))
>  
>