Re: [Qemu-devel] [PATCH] replay: add record/replay for audio passthrough

["Pavel Dovgalyuk" <dovgaluk@ispras.ru>]

Ping?

Pavel Dovgalyuk


> -----Original Message-----
> From: Pavel Dovgalyuk [mailto:pavel.dovgaluk@ispras.ru]
> Sent: Tuesday, January 31, 2017 2:59 PM
> To: qemu-devel@nongnu.org
> Cc: pbonzini@redhat.com; dovgaluk@ispras.ru; kraxel@redhat.com
> Subject: [PATCH] replay: add record/replay for audio passthrough
> 
> This patch adds recording and replaying audio data. Is saves synchronization
> information for audio out and inputs from the microphone.
> 
> Signed-off-by: Pavel Dovgalyuk <pavel.dovgaluk@ispras.ru>
> ---
>  audio/audio.c            |   11 +++++-
>  audio/audio.h            |    5 +++
>  audio/mixeng.c           |   31 ++++++++++++++++++
>  docs/replay.txt          |    7 ++++
>  include/sysemu/replay.h  |    7 ++++
>  replay/Makefile.objs     |    1 +
>  replay/replay-audio.c    |   79 ++++++++++++++++++++++++++++++++++++++++++++++
>  replay/replay-internal.h |    4 ++
>  8 files changed, 142 insertions(+), 3 deletions(-)
>  create mode 100644 replay/replay-audio.c
> 
> diff --git a/audio/audio.c b/audio/audio.c
> index 1ee95a5..79c0788 100644
> --- a/audio/audio.c
> +++ b/audio/audio.c
> @@ -28,6 +28,7 @@
>  #include "qemu/timer.h"
>  #include "sysemu/sysemu.h"
>  #include "qemu/cutils.h"
> +#include "sysemu/replay.h"
> 
>  #define AUDIO_CAP "audio"
>  #include "audio_int.h"
> @@ -1113,7 +1114,7 @@ static void audio_reset_timer (AudioState *s)
>  {
>      if (audio_is_timer_needed ()) {
>          if (!timer_pending(s->ts)) {
> -            timer_mod (s->ts,
> +            timer_mod(s->ts,
>                  qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + conf.period.ticks);
>          }
>      }
> @@ -1389,6 +1390,7 @@ static void audio_run_out (AudioState *s)
> 
>          prev_rpos = hw->rpos;
>          played = hw->pcm_ops->run_out (hw, live);
> +        replay_audio_out(&played);
>          if (audio_bug (AUDIO_FUNC, hw->rpos >= hw->samples)) {
>              dolog ("hw->rpos=%d hw->samples=%d played=%d\n",
>                     hw->rpos, hw->samples, played);
> @@ -1452,9 +1454,12 @@ static void audio_run_in (AudioState *s)
> 
>      while ((hw = audio_pcm_hw_find_any_enabled_in (hw))) {
>          SWVoiceIn *sw;
> -        int captured, min;
> +        int captured = 0, min;
> 
> -        captured = hw->pcm_ops->run_in (hw);
> +        if (replay_mode != REPLAY_MODE_PLAY) {
> +            captured = hw->pcm_ops->run_in(hw);
> +        }
> +        replay_audio_in(&captured, hw->conv_buf, &hw->wpos, hw->samples);
> 
>          min = audio_pcm_hw_find_min_in (hw);
>          hw->total_samples_captured += captured - min;
> diff --git a/audio/audio.h b/audio/audio.h
> index c3c5198..f4339a1 100644
> --- a/audio/audio.h
> +++ b/audio/audio.h
> @@ -166,4 +166,9 @@ int wav_start_capture (CaptureState *s, const char *path, int freq,
>  bool audio_is_cleaning_up(void);
>  void audio_cleanup(void);
> 
> +void audio_sample_to_uint64(void *samples, int pos,
> +                            uint64_t *left, uint64_t *right);
> +void audio_sample_from_uint64(void *samples, int pos,
> +                            uint64_t left, uint64_t right);
> +
>  #endif /* QEMU_AUDIO_H */
> diff --git a/audio/mixeng.c b/audio/mixeng.c
> index 66c0328..c23508e 100644
> --- a/audio/mixeng.c
> +++ b/audio/mixeng.c
> @@ -267,6 +267,37 @@ f_sample *mixeng_clip[2][2][2][3] = {
>      }
>  };
> 
> +
> +void audio_sample_to_uint64(void *samples, int pos,
> +                            uint64_t *left, uint64_t *right)
> +{
> +    struct st_sample *sample = samples;
> +    sample += pos;
> +#ifdef FLOAT_MIXENG
> +    error_report(
> +        "Coreaudio and floating point samples are not supported by replay yet");
> +    abort();
> +#else
> +    *left = sample->l;
> +    *right = sample->r;
> +#endif
> +}
> +
> +void audio_sample_from_uint64(void *samples, int pos,
> +                            uint64_t left, uint64_t right)
> +{
> +    struct st_sample *sample = samples;
> +    sample += pos;
> +#ifdef FLOAT_MIXENG
> +    error_report(
> +        "Coreaudio and floating point samples are not supported by replay yet");
> +    abort();
> +#else
> +    sample->l = left;
> +    sample->r = right;
> +#endif
> +}
> +
>  /*
>   * August 21, 1998
>   * Copyright 1998 Fabrice Bellard.
> diff --git a/docs/replay.txt b/docs/replay.txt
> index 03e1931..486c1e0 100644
> --- a/docs/replay.txt
> +++ b/docs/replay.txt
> @@ -225,3 +225,10 @@ recording the virtual machine this filter puts all packets coming from
>  the outer world into the log. In replay mode packets from the log are
>  injected into the network device. All interactions with network backend
>  in replay mode are disabled.
> +
> +Audio devices
> +-------------
> +
> +Audio data is recorded and replay automatically. The command line for recording
> +and replaying must contain identical specifications of audio hardware, e.g.:
> + -soundhw ac97
> diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h
> index 7aad20b..f1c0712 100644
> --- a/include/sysemu/replay.h
> +++ b/include/sysemu/replay.h
> @@ -152,6 +152,13 @@ void replay_unregister_net(ReplayNetState *rns);
>  void replay_net_packet_event(ReplayNetState *rns, unsigned flags,
>                               const struct iovec *iov, int iovcnt);
> 
> +/* Audio */
> +
> +/*! Saves/restores number of played samples of audio out operation. */
> +void replay_audio_out(int *played);
> +/*! Saves/restores recorded samples of audio in operation. */
> +void replay_audio_in(int *recorded, void *samples, int *wpos, int size);
> +
>  /* VM state operations */
> 
>  /*! Called at the start of execution.
> diff --git a/replay/Makefile.objs b/replay/Makefile.objs
> index b2afd40..cee6539 100644
> --- a/replay/Makefile.objs
> +++ b/replay/Makefile.objs
> @@ -6,3 +6,4 @@ common-obj-y += replay-input.o
>  common-obj-y += replay-char.o
>  common-obj-y += replay-snapshot.o
>  common-obj-y += replay-net.o
> +common-obj-y += replay-audio.o
> \ No newline at end of file
> diff --git a/replay/replay-audio.c b/replay/replay-audio.c
> new file mode 100644
> index 0000000..3d83743
> --- /dev/null
> +++ b/replay/replay-audio.c
> @@ -0,0 +1,79 @@
> +/*
> + * replay-audio.c
> + *
> + * Copyright (c) 2010-2017 Institute for System Programming
> + *                         of the Russian Academy of Sciences.
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#include "qemu/osdep.h"
> +#include "qemu/error-report.h"
> +#include "sysemu/replay.h"
> +#include "replay-internal.h"
> +#include "sysemu/sysemu.h"
> +#include "audio/audio.h"
> +
> +void replay_audio_out(int *played)
> +{
> +    if (replay_mode == REPLAY_MODE_RECORD) {
> +        replay_save_instructions();
> +        replay_mutex_lock();
> +        replay_put_event(EVENT_AUDIO_OUT);
> +        replay_put_dword(*played);
> +        replay_mutex_unlock();
> +    } else if (replay_mode == REPLAY_MODE_PLAY) {
> +        replay_account_executed_instructions();
> +        replay_mutex_lock();
> +        if (replay_next_event_is(EVENT_AUDIO_OUT)) {
> +            *played = replay_get_dword();
> +            replay_finish_event();
> +            replay_mutex_unlock();
> +        } else {
> +            replay_mutex_unlock();
> +            error_report("Missing audio out event in the replay log");
> +            abort();
> +        }
> +    }
> +}
> +
> +void replay_audio_in(int *recorded, void *samples, int *wpos, int size)
> +{
> +    int pos;
> +    uint64_t left, right;
> +    if (replay_mode == REPLAY_MODE_RECORD) {
> +        replay_save_instructions();
> +        replay_mutex_lock();
> +        replay_put_event(EVENT_AUDIO_IN);
> +        replay_put_dword(*recorded);
> +        replay_put_dword(*wpos);
> +        for (pos = (*wpos - *recorded + size) % size ; pos != *wpos
> +             ; pos = (pos + 1) % size) {
> +            audio_sample_to_uint64(samples, pos, &left, &right);
> +            replay_put_qword(left);
> +            replay_put_qword(right);
> +        }
> +        replay_mutex_unlock();
> +    } else if (replay_mode == REPLAY_MODE_PLAY) {
> +        replay_account_executed_instructions();
> +        replay_mutex_lock();
> +        if (replay_next_event_is(EVENT_AUDIO_IN)) {
> +            *recorded = replay_get_dword();
> +            *wpos = replay_get_dword();
> +            for (pos = (*wpos - *recorded + size) % size ; pos != *wpos
> +                 ; pos = (pos + 1) % size) {
> +                left = replay_get_qword();
> +                right = replay_get_qword();
> +                audio_sample_from_uint64(samples, pos, left, right);
> +            }
> +            replay_finish_event();
> +            replay_mutex_unlock();
> +        } else {
> +            replay_mutex_unlock();
> +            error_report("Missing audio in event in the replay log");
> +            abort();
> +        }
> +    }
> +}
> diff --git a/replay/replay-internal.h b/replay/replay-internal.h
> index c26d079..ed66ed8 100644
> --- a/replay/replay-internal.h
> +++ b/replay/replay-internal.h
> @@ -29,6 +29,10 @@ enum ReplayEvents {
>      /* for character device read all event */
>      EVENT_CHAR_READ_ALL,
>      EVENT_CHAR_READ_ALL_ERROR,
> +    /* for audio out event */
> +    EVENT_AUDIO_OUT,
> +    /* for audio in event */
> +    EVENT_AUDIO_IN,
>      /* for clock read/writes */
>      /* some of greater codes are reserved for clocks */
>      EVENT_CLOCK,