Re: [PATCH v5 RFC 13/14] tools/libxc: noarch save code

[Hongyang Yang <yanghy@cn.fujitsu.com>]

On 06/12/2014 02:14 AM, Andrew Cooper wrote:
> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
> Signed-off-by: Frediano Ziglio <frediano.ziglio@citrix.com>
> Signed-off-by: David Vrabel <david.vrabel@citrix.com>
> ---
>   tools/libxc/saverestore/save.c |  545 +++++++++++++++++++++++++++++++++++++++-
>   1 file changed, 544 insertions(+), 1 deletion(-)
>
> diff --git a/tools/libxc/saverestore/save.c b/tools/libxc/saverestore/save.c
> index f6ad734..9ad43a5 100644
> --- a/tools/libxc/saverestore/save.c
> +++ b/tools/libxc/saverestore/save.c
> @@ -1,11 +1,554 @@
> +#include <assert.h>
> +#include <arpa/inet.h>
> +
>   #include "common.h"
>
> +/*
> + * Writes an Image header and Domain header into the stream.
> + */
> +static int write_headers(struct context *ctx, uint16_t guest_type)
> +{
> +    xc_interface *xch = ctx->xch;
...snip...
> +/*
> + * Send all domain memory.  This is the heart of the live migration loop.
> + */
> +static int send_domain_memory(struct context *ctx)
> +{
> +    xc_interface *xch = ctx->xch;
> +    DECLARE_HYPERCALL_BUFFER(unsigned long, to_send);
> +    xc_shadow_op_stats_t stats = { -1, -1 };
> +    unsigned pages_written;
> +    unsigned x, max_iter = 5, dirty_threshold = 50;
> +    xen_pfn_t p;
> +    int rc = -1;
> +
> +    to_send = xc_hypercall_buffer_alloc_pages(
> +        xch, to_send, NRPAGES(bitmap_size(ctx->save.p2m_size)));
> +
> +    ctx->save.batch_pfns = malloc(MAX_BATCH_SIZE * sizeof(*ctx->save.batch_pfns));
> +    ctx->save.deferred_pages = calloc(1, bitmap_size(ctx->save.p2m_size));
> +
> +    if ( !ctx->save.batch_pfns || !to_send || !ctx->save.deferred_pages )
> +    {
> +        ERROR("Unable to allocate memory for to_{send,fix}/batch bitmaps");
> +        goto out;
> +    }
> +
> +    if ( xc_shadow_control(xch, ctx->domid,
> +                           XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY,
> +                           NULL, 0, NULL, 0, NULL) < 0 )
> +    {
> +        PERROR("Failed to enable logdirty");
> +        goto out;
> +    }
> +
> +    for ( x = 0, pages_written = 0; x < max_iter ; ++x )
> +    {
> +        if ( x == 0 )
> +        {
> +            /* First iteration, send all pages. */
> +            memset(to_send, 0xff, bitmap_size(ctx->save.p2m_size));
> +        }
> +        else
> +        {
> +            /* Else consult the dirty bitmap. */
> +            if ( xc_shadow_control(
> +                     xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_CLEAN,
> +                     HYPERCALL_BUFFER(to_send), ctx->save.p2m_size,
> +                     NULL, 0, &stats) != ctx->save.p2m_size )
> +            {
> +                PERROR("Failed to retrieve logdirty bitmap");
> +                rc = -1;
> +                goto out;
> +            }
> +            else
> +                DPRINTF("  Wrote %u pages; stats: faults %"PRIu32", dirty %"PRIu32,
> +                        pages_written, stats.fault_count, stats.dirty_count);
> +            pages_written = 0;
> +
> +            if ( stats.dirty_count < dirty_threshold )
> +                break;
> +        }
> +
> +        DPRINTF("Iteration %u", x);
> +
> +        for ( p = 0 ; p < ctx->save.p2m_size; ++p )
> +        {
> +            if ( test_bit(p, to_send) )
> +            {
> +                rc = add_to_batch(ctx, p);
> +                if ( rc )
> +                    goto out;
> +                ++pages_written;
> +            }
> +        }
> +
> +        rc = flush_batch(ctx);
> +        if ( rc )
> +            goto out;
> +    }
> +
> +    rc = pause_domain(ctx);
> +    if ( rc )
> +        goto out;
> +
> +    if ( xc_shadow_control(
> +             xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_CLEAN,
> +             HYPERCALL_BUFFER(to_send), ctx->save.p2m_size,
> +             NULL, 0, &stats) != ctx->save.p2m_size )
> +    {
> +        PERROR("Failed to retrieve logdirty bitmap");
> +        rc = -1;
> +        goto out;
> +    }
> +
> +    for ( p = 0, pages_written = 0 ; p < ctx->save.p2m_size; ++p )
> +    {
> +        if ( test_bit(p, to_send) || test_bit(p, ctx->save.deferred_pages) )
> +        {
> +            rc = add_to_batch(ctx, p);
> +            if ( rc )
> +                goto out;
> +            ++pages_written;
> +        }
> +    }
> +
> +    rc = flush_batch(ctx);
> +    if ( rc )
> +        goto out;
> +
> +    DPRINTF("  Wrote %u pages", pages_written);
> +    IPRINTF("Sent all pages");
> +
> +  out:
> +    xc_hypercall_buffer_free_pages(xch, to_send,
> +                                   NRPAGES(bitmap_size(ctx->save.p2m_size)));
> +    free(ctx->save.deferred_pages);
> +    free(ctx->save.batch_pfns);
> +    return rc;
> +}
> +
> +/*
> + * Save a domain.
> + */
> +static int save(struct context *ctx, uint16_t guest_type)
> +{
> +    xc_interface *xch = ctx->xch;
> +    int rc, saved_rc = 0, saved_errno = 0;
> +
> +    IPRINTF("Saving domain %d, type %s",
> +            ctx->domid, dhdr_type_to_str(guest_type));
> +
> +    rc = ctx->save.ops.setup(ctx);
> +    if ( rc )
> +        goto err;
> +
> +    rc = write_headers(ctx, guest_type);
> +    if ( rc )
> +        goto err;
> +
> +    rc = ctx->save.ops.start_of_stream(ctx);
> +    if ( rc )
> +        goto err;
> +
> +    rc = send_domain_memory(ctx);
> +    if ( rc )
> +        goto err;
> +
> +    /* Refresh domain information now it has paused. */
> +    if ( (xc_domain_getinfo(xch, ctx->domid, 1, &ctx->dominfo) != 1) ||
> +         (ctx->dominfo.domid != ctx->domid) )
> +    {
> +        PERROR("Unable to refresh domain information");
> +        rc = -1;
> +        goto err;
> +    }
> +    else if ( (!ctx->dominfo.shutdown ||
> +               ctx->dominfo.shutdown_reason != SHUTDOWN_suspend ) &&
> +              !ctx->dominfo.paused )
> +    {
> +        ERROR("Domain has not been suspended");
> +        rc = -1;
> +        goto err;
> +    }
> +
> +    rc = ctx->save.ops.end_of_stream(ctx);
> +    if ( rc )
> +        goto err;
> +
> +    rc = write_end_record(ctx);
> +    if ( rc )
> +        goto err;
> +
> +    xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF,
> +                      NULL, 0, NULL, 0, NULL);

If migration failed because log-dirty already been enabled or there's err after
we enabled log-dirty, we should off shadow op. otherwise we will always fail
when migration, so this op should under error path. The following patch fix it.

diff --git a/tools/libxc/saverestore/save.c b/tools/libxc/saverestore/save.c
index 9ad43a5..6e9d325 100644
--- a/tools/libxc/saverestore/save.c
+++ b/tools/libxc/saverestore/save.c
@@ -474,9 +474,6 @@ static int save(struct context *ctx, uint16_t guest_type)
      if ( rc )
          goto err;

-    xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF,
-                      NULL, 0, NULL, 0, NULL);
-
      IPRINTF("Save successful");
      goto done;

@@ -490,6 +487,9 @@ static int save(struct context *ctx, uint16_t guest_type)
      if ( rc )
          PERROR("Failed to clean up");

+    xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF,
+                      NULL, 0, NULL, 0, NULL);
+
      if ( saved_rc )
      {
          rc = saved_rc;


> +
> +    IPRINTF("Save successful");
> +    goto done;
> +
> + err:
> +    saved_errno = errno;
> +    saved_rc = rc;
> +    PERROR("Save failed");
> +
> + done:
> +    rc = ctx->save.ops.cleanup(ctx);
> +    if ( rc )
> +        PERROR("Failed to clean up");
> +
> +    if ( saved_rc )
> +    {
> +        rc = saved_rc;
> +        errno = saved_errno;
> +    }
> +
> +    return rc;
> +};
> +
>   int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters,
>                       uint32_t max_factor, uint32_t flags,
>                       struct save_callbacks* callbacks, int hvm)
>   {
> +    struct context ctx =
> +        {
> +            .xch = xch,
> +            .fd = io_fd,
> +        };
> +
> +    /* GCC 4.4 (of CentOS 6.x vintage) can' t initialise anonymous unions :( */
> +    ctx.save.callbacks = callbacks;
> +
>       IPRINTF("In experimental %s", __func__);
> -    return -1;
> +
> +    if ( xc_domain_getinfo(xch, dom, 1, &ctx.dominfo) != 1 )
> +    {
> +        PERROR("Failed to get domain info");
> +        return -1;
> +    }
> +
> +    if ( ctx.dominfo.domid != dom )
> +    {
> +        ERROR("Domain %d does not exist", dom);
> +        return -1;
> +    }
> +
> +    ctx.domid = dom;
> +    IPRINTF("Saving domain %d", dom);
> +
> +    ctx.save.p2m_size = xc_domain_maximum_gpfn(xch, dom) + 1;
> +    if ( ctx.save.p2m_size > ~XEN_DOMCTL_PFINFO_LTAB_MASK )
> +    {
> +        errno = E2BIG;
> +        ERROR("Cannot save this big a guest");
> +        return -1;
> +    }
> +
> +    if ( ctx.dominfo.hvm )
> +    {
> +        ctx.ops = common_ops_x86_hvm;
> +        ctx.save.ops = save_ops_x86_hvm;
> +        return save(&ctx, DHDR_TYPE_X86_HVM);
> +    }
> +    else
> +    {
> +        ctx.ops = common_ops_x86_pv;
> +        ctx.save.ops = save_ops_x86_pv;
> +        return save(&ctx, DHDR_TYPE_X86_PV);
> +    }
>   }
>
>   /*
>

-- 
Thanks,
Yang.