Re: [PATCH v3 COLOPre 10/26] migration/save: pass checkpointed_stream from libxl to libxc

[Ian Campbell <ian.campbell@citrix.com>]

On Thu, 2015-06-25 at 14:25 +0800, Yang Hongyang wrote:
> Pass checkpointed_stream from libxl to libxc.

Hrm, so now we seem to have a purely libxl internal usage of this new
enum, but still nothing crossing the libxl.h boundary. I suspect I've
missed something!

> It won't affact legacy migration because legacy migration
> won't use this param.
> 
> Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
> CC: Ian Campbell <Ian.Campbell@citrix.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> CC: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
>  tools/libxc/include/xenguest.h   |  9 ++++++---
>  tools/libxc/xc_domain_save.c     |  6 ++++--
>  tools/libxc/xc_nomigrate.c       |  3 ++-
>  tools/libxc/xc_sr_common.h       |  2 +-
>  tools/libxc/xc_sr_save.c         |  5 +++--
>  tools/libxl/libxl.c              |  2 ++
>  tools/libxl/libxl_dom_save.c     | 12 +++++++++---
>  tools/libxl/libxl_internal.h     |  1 +
>  tools/libxl/libxl_save_callout.c |  2 +-
>  tools/libxl/libxl_save_helper.c  |  3 ++-
>  10 files changed, 31 insertions(+), 14 deletions(-)
> 
> diff --git a/tools/libxc/include/xenguest.h b/tools/libxc/include/xenguest.h
> index b0d27ed..d7c8fe4 100644
> --- a/tools/libxc/include/xenguest.h
> +++ b/tools/libxc/include/xenguest.h
> @@ -30,7 +30,6 @@
>  #define XCFLAGS_HVM       (1 << 2)
>  #define XCFLAGS_STDVGA    (1 << 3)
>  #define XCFLAGS_CHECKPOINT_COMPRESS    (1 << 4)
> -#define XCFLAGS_CHECKPOINTED    (1 << 5)
>  
>  #define X86_64_B_SIZE   64 
>  #define X86_32_B_SIZE   32
> @@ -85,16 +84,20 @@ struct save_callbacks {
>   * @parm xch a handle to an open hypervisor interface
>   * @parm fd the file descriptor to save a domain to
>   * @parm dom the id of the domain
> + * @parm checkpointed_stream non-zero if the far end of the stream is using
> + *       checkpointing
>   * @return 0 on success, -1 on failure
>   */
>  int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
>                     uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */,
> -                   struct save_callbacks* callbacks, int hvm);
> +                   struct save_callbacks* callbacks, int hvm,
> +                   int checkpointed_stream);
>  
>  /* Domain Save v2 */
>  int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
>                      uint32_t max_factor, uint32_t flags,
> -                    struct save_callbacks* callbacks, int hvm);
> +                    struct save_callbacks* callbacks, int hvm,
> +                    int checkpointed_stream);
>  
>  /* callbacks provided by xc_domain_restore */
>  struct restore_callbacks {
> diff --git a/tools/libxc/xc_domain_save.c b/tools/libxc/xc_domain_save.c
> index 301e770..c76980e 100644
> --- a/tools/libxc/xc_domain_save.c
> +++ b/tools/libxc/xc_domain_save.c
> @@ -802,7 +802,8 @@ static int save_tsc_info(xc_interface *xch, uint32_t dom, int io_fd)
>  
>  int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
>                     uint32_t max_factor, uint32_t flags,
> -                   struct save_callbacks* callbacks, int hvm)
> +                   struct save_callbacks* callbacks, int hvm,
> +                   int checkpointed_stream)
>  {
>      xc_dominfo_t info;
>      DECLARE_DOMCTL;
> @@ -897,7 +898,8 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iter
>      if ( getenv("XG_MIGRATION_V2") )
>      {
>          return xc_domain_save2(xch, io_fd, dom, max_iters,
> -                               max_factor, flags, callbacks, hvm);
> +                               max_factor, flags, callbacks, hvm,
> +                               checkpointed_stream);
>      }
>  
>      DPRINTF("%s: starting save of domid %u", __func__, dom);
> diff --git a/tools/libxc/xc_nomigrate.c b/tools/libxc/xc_nomigrate.c
> index 76978a0..374d5bf 100644
> --- a/tools/libxc/xc_nomigrate.c
> +++ b/tools/libxc/xc_nomigrate.c
> @@ -23,7 +23,8 @@
>  
>  int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
>                     uint32_t max_factor, uint32_t flags,
> -                   struct save_callbacks* callbacks, int hvm)
> +                   struct save_callbacks* callbacks, int hvm,
> +                   int checkpointed_stream)
>  {
>      errno = ENOSYS;
>      return -1;
> diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
> index 42af55b..04f32f5 100644
> --- a/tools/libxc/xc_sr_common.h
> +++ b/tools/libxc/xc_sr_common.h
> @@ -175,7 +175,7 @@ struct xc_sr_context
>              bool live;
>  
>              /* Plain VM, or checkpoints over time. */
> -            bool checkpointed;
> +            int checkpointed;
>  
>              /* Further debugging information in the stream. */
>              bool debug;
> diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
> index d63b783..6102b66 100644
> --- a/tools/libxc/xc_sr_save.c
> +++ b/tools/libxc/xc_sr_save.c
> @@ -820,7 +820,8 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
>  
>  int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
>                      uint32_t max_iters, uint32_t max_factor, uint32_t flags,
> -                    struct save_callbacks* callbacks, int hvm)
> +                    struct save_callbacks* callbacks, int hvm,
> +                    int checkpointed_stream)
>  {
>      xen_pfn_t nr_pfns;
>      struct xc_sr_context ctx =
> @@ -833,7 +834,7 @@ int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom,
>      ctx.save.callbacks = callbacks;
>      ctx.save.live  = !!(flags & XCFLAGS_LIVE);
>      ctx.save.debug = !!(flags & XCFLAGS_DEBUG);
> -    ctx.save.checkpointed = !!(flags & XCFLAGS_CHECKPOINTED);
> +    ctx.save.checkpointed = checkpointed_stream;
>  
>      /*
>       * TODO: Find some time to better tweak the live migration algorithm.
> diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
> index 8b63eff..2ca59ea 100644
> --- a/tools/libxl/libxl.c
> +++ b/tools/libxl/libxl.c
> @@ -837,6 +837,7 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
>      dss->live = 1;
>      dss->debug = 0;
>      dss->remus = info;
> +    dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_REMUS;
>  
>      assert(info);
>  
> @@ -891,6 +892,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
>      dss->type = type;
>      dss->live = flags & LIBXL_SUSPEND_LIVE;
>      dss->debug = flags & LIBXL_SUSPEND_DEBUG;
> +    dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
>  
>      libxl__domain_save(egc, dss);
>      return AO_INPROGRESS;
> diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
> index 93061c7..8fe1625 100644
> --- a/tools/libxl/libxl_dom_save.c
> +++ b/tools/libxl/libxl_dom_save.c
> @@ -357,6 +357,13 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>          &dss->shs.callbacks.save.a;
>      libxl__domain_suspend_state *dsps = &dss->dsps;
>  
> +    if (dss->checkpointed_stream && !r_info) {
> +        LOG(ERROR, "Migration stream is checkpointed, but there's no "
> +                   "checkpoint info!");
> +        rc = ERROR_FAIL;
> +        goto out;
> +    }
> +
>      logdirty_init(&dss->logdirty);
>      libxl__xswait_init(&dsps->pvcontrol);
>      libxl__ev_evtchn_init(&dsps->guest_evtchn);
> @@ -388,9 +395,8 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>      dsps->guest_responded = 0;
>      dsps->dm_savefile = libxl__device_model_savefile(gc, domid);
>  
> -    if (r_info != NULL) {
> +    if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
>          dss->interval = r_info->interval;
> -        dss->xcflags |= XCFLAGS_CHECKPOINTED;
>          if (libxl_defbool_val(r_info->compression))
>              dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
>      }
> @@ -414,7 +420,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>      }
>  
>      memset(callbacks, 0, sizeof(*callbacks));
> -    if (r_info != NULL) {
> +    if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_REMUS) {
>          callbacks->suspend = libxl__remus_domain_suspend_callback;
>          callbacks->postcopy = libxl__remus_domain_resume_callback;
>          callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback;
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 4fdf9d8..573c844 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -2941,6 +2941,7 @@ struct libxl__domain_save_state {
>      libxl_domain_type type;
>      int live;
>      int debug;
> +    int checkpointed_stream;
>      const libxl_domain_remus_info *remus;
>      /* private */
>      libxl__domain_suspend_state dsps;
> diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
> index a5039e6..12cc57e 100644
> --- a/tools/libxl/libxl_save_callout.c
> +++ b/tools/libxl/libxl_save_callout.c
> @@ -83,7 +83,7 @@ void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
>  
>      const unsigned long argnums[] = {
>          dss->domid, 0, 0, dss->xcflags, dss->hvm,
> -        cbflags,
> +        cbflags, dss->checkpointed_stream,
>      };
>  
>      dss->shs.ao = ao;
> diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
> index b8f0390..9d2cbf6 100644
> --- a/tools/libxl/libxl_save_helper.c
> +++ b/tools/libxl/libxl_save_helper.c
> @@ -248,6 +248,7 @@ int main(int argc, char **argv)
>          uint32_t flags =           strtoul(NEXTARG,0,10);
>          int hvm =                  atoi(NEXTARG);
>          unsigned cbflags =         strtoul(NEXTARG,0,10);
> +        int checkpointed_stream =  strtoul(NEXTARG,0,10);
>          assert(!*++argv);
>  
>          helper_setcallbacks_save(&helper_save_callbacks, cbflags);
> @@ -256,7 +257,7 @@ int main(int argc, char **argv)
>          setup_signals(save_signal_handler);
>  
>          r = xc_domain_save2(xch, io_fd, dom, max_iters, max_factor, flags,
> -                           &helper_save_callbacks, hvm);
> +                           &helper_save_callbacks, hvm, checkpointed_stream);
>          complete(r);
>  
>      } else if (!strcmp(mode,"--restore-domain")) {