From mboxrd@z Thu Jan 1 00:00:00 1970 From: Hongyang Yang Subject: Re: [PATCH v5 RFC 13/14] tools/libxc: noarch save code Date: Wed, 18 Jun 2014 15:08:25 +0800 Message-ID: <53A13AE9.2020701@cn.fujitsu.com> References: <1402510482-21099-1-git-send-email-andrew.cooper3@citrix.com> <1402510482-21099-14-git-send-email-andrew.cooper3@citrix.com> <53A138BA.5000702@cn.fujitsu.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii"; Format="flowed" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <53A138BA.5000702@cn.fujitsu.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Andrew Cooper , Xen-devel Cc: Frediano Ziglio , David Vrabel List-Id: xen-devel@lists.xenproject.org On 06/18/2014 02:59 PM, Hongyang Yang wrote: > On 06/12/2014 02:14 AM, Andrew Cooper wrote: >> Signed-off-by: Andrew Cooper >> Signed-off-by: Frediano Ziglio >> Signed-off-by: David Vrabel >> --- >> tools/libxc/saverestore/save.c | 545 +++++++++++++++++++++++++++++++++++++++- >> 1 file changed, 544 insertions(+), 1 deletion(-) >> >> diff --git a/tools/libxc/saverestore/save.c b/tools/libxc/saverestore/save.c >> index f6ad734..9ad43a5 100644 >> --- a/tools/libxc/saverestore/save.c >> +++ b/tools/libxc/saverestore/save.c >> @@ -1,11 +1,554 @@ >> +#include >> +#include >> + >> #include "common.h" >> >> +/* >> + * Writes an Image header and Domain header into the stream. >> + */ >> +static int write_headers(struct context *ctx, uint16_t guest_type) >> +{ >> + xc_interface *xch = ctx->xch; > ...snip... >> +/* >> + * Send all domain memory. This is the heart of the live migration loop. >> + */ >> +static int send_domain_memory(struct context *ctx) >> +{ >> + xc_interface *xch = ctx->xch; >> + DECLARE_HYPERCALL_BUFFER(unsigned long, to_send); >> + xc_shadow_op_stats_t stats = { -1, -1 }; >> + unsigned pages_written; >> + unsigned x, max_iter = 5, dirty_threshold = 50; >> + xen_pfn_t p; >> + int rc = -1; >> + >> + to_send = xc_hypercall_buffer_alloc_pages( >> + xch, to_send, NRPAGES(bitmap_size(ctx->save.p2m_size))); >> + >> + ctx->save.batch_pfns = malloc(MAX_BATCH_SIZE * >> sizeof(*ctx->save.batch_pfns)); >> + ctx->save.deferred_pages = calloc(1, bitmap_size(ctx->save.p2m_size)); >> + >> + if ( !ctx->save.batch_pfns || !to_send || !ctx->save.deferred_pages ) >> + { >> + ERROR("Unable to allocate memory for to_{send,fix}/batch bitmaps"); >> + goto out; >> + } >> + >> + if ( xc_shadow_control(xch, ctx->domid, >> + XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY, >> + NULL, 0, NULL, 0, NULL) < 0 ) >> + { >> + PERROR("Failed to enable logdirty"); >> + goto out; >> + } >> + >> + for ( x = 0, pages_written = 0; x < max_iter ; ++x ) >> + { >> + if ( x == 0 ) >> + { >> + /* First iteration, send all pages. */ >> + memset(to_send, 0xff, bitmap_size(ctx->save.p2m_size)); >> + } >> + else >> + { >> + /* Else consult the dirty bitmap. */ >> + if ( xc_shadow_control( >> + xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_CLEAN, >> + HYPERCALL_BUFFER(to_send), ctx->save.p2m_size, >> + NULL, 0, &stats) != ctx->save.p2m_size ) >> + { >> + PERROR("Failed to retrieve logdirty bitmap"); >> + rc = -1; >> + goto out; >> + } >> + else >> + DPRINTF(" Wrote %u pages; stats: faults %"PRIu32", dirty >> %"PRIu32, >> + pages_written, stats.fault_count, stats.dirty_count); >> + pages_written = 0; >> + >> + if ( stats.dirty_count < dirty_threshold ) >> + break; >> + } >> + >> + DPRINTF("Iteration %u", x); >> + >> + for ( p = 0 ; p < ctx->save.p2m_size; ++p ) >> + { >> + if ( test_bit(p, to_send) ) >> + { >> + rc = add_to_batch(ctx, p); >> + if ( rc ) >> + goto out; >> + ++pages_written; >> + } >> + } >> + >> + rc = flush_batch(ctx); >> + if ( rc ) >> + goto out; >> + } >> + >> + rc = pause_domain(ctx); >> + if ( rc ) >> + goto out; >> + >> + if ( xc_shadow_control( >> + xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_CLEAN, >> + HYPERCALL_BUFFER(to_send), ctx->save.p2m_size, >> + NULL, 0, &stats) != ctx->save.p2m_size ) >> + { >> + PERROR("Failed to retrieve logdirty bitmap"); >> + rc = -1; >> + goto out; >> + } >> + >> + for ( p = 0, pages_written = 0 ; p < ctx->save.p2m_size; ++p ) >> + { >> + if ( test_bit(p, to_send) || test_bit(p, ctx->save.deferred_pages) ) >> + { >> + rc = add_to_batch(ctx, p); >> + if ( rc ) >> + goto out; >> + ++pages_written; >> + } >> + } >> + >> + rc = flush_batch(ctx); >> + if ( rc ) >> + goto out; >> + >> + DPRINTF(" Wrote %u pages", pages_written); >> + IPRINTF("Sent all pages"); >> + >> + out: >> + xc_hypercall_buffer_free_pages(xch, to_send, >> + NRPAGES(bitmap_size(ctx->save.p2m_size))); >> + free(ctx->save.deferred_pages); >> + free(ctx->save.batch_pfns); >> + return rc; >> +} >> + >> +/* >> + * Save a domain. >> + */ >> +static int save(struct context *ctx, uint16_t guest_type) >> +{ >> + xc_interface *xch = ctx->xch; >> + int rc, saved_rc = 0, saved_errno = 0; >> + >> + IPRINTF("Saving domain %d, type %s", >> + ctx->domid, dhdr_type_to_str(guest_type)); >> + >> + rc = ctx->save.ops.setup(ctx); >> + if ( rc ) >> + goto err; >> + >> + rc = write_headers(ctx, guest_type); >> + if ( rc ) >> + goto err; >> + >> + rc = ctx->save.ops.start_of_stream(ctx); >> + if ( rc ) >> + goto err; >> + >> + rc = send_domain_memory(ctx); >> + if ( rc ) >> + goto err; >> + >> + /* Refresh domain information now it has paused. */ >> + if ( (xc_domain_getinfo(xch, ctx->domid, 1, &ctx->dominfo) != 1) || >> + (ctx->dominfo.domid != ctx->domid) ) >> + { >> + PERROR("Unable to refresh domain information"); >> + rc = -1; >> + goto err; >> + } >> + else if ( (!ctx->dominfo.shutdown || >> + ctx->dominfo.shutdown_reason != SHUTDOWN_suspend ) && >> + !ctx->dominfo.paused ) >> + { >> + ERROR("Domain has not been suspended"); >> + rc = -1; >> + goto err; >> + } >> + >> + rc = ctx->save.ops.end_of_stream(ctx); >> + if ( rc ) >> + goto err; >> + >> + rc = write_end_record(ctx); >> + if ( rc ) >> + goto err; >> + >> + xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF, >> + NULL, 0, NULL, 0, NULL); > > If migration failed because log-dirty already been enabled or there's err after > we enabled log-dirty, we should off shadow op. otherwise we will always fail > when migration, so this op should under error path. The following patch fix it. Or there's another solution, just like the old migration, try to re-enable log dirty if enable log dirty failed. > > diff --git a/tools/libxc/saverestore/save.c b/tools/libxc/saverestore/save.c > index 9ad43a5..6e9d325 100644 > --- a/tools/libxc/saverestore/save.c > +++ b/tools/libxc/saverestore/save.c > @@ -474,9 +474,6 @@ static int save(struct context *ctx, uint16_t guest_type) > if ( rc ) > goto err; > > - xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF, > - NULL, 0, NULL, 0, NULL); > - > IPRINTF("Save successful"); > goto done; > > @@ -490,6 +487,9 @@ static int save(struct context *ctx, uint16_t guest_type) > if ( rc ) > PERROR("Failed to clean up"); > > + xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF, > + NULL, 0, NULL, 0, NULL); > + > if ( saved_rc ) > { > rc = saved_rc; > > >> + >> + IPRINTF("Save successful"); >> + goto done; >> + >> + err: >> + saved_errno = errno; >> + saved_rc = rc; >> + PERROR("Save failed"); >> + >> + done: >> + rc = ctx->save.ops.cleanup(ctx); >> + if ( rc ) >> + PERROR("Failed to clean up"); >> + >> + if ( saved_rc ) >> + { >> + rc = saved_rc; >> + errno = saved_errno; >> + } >> + >> + return rc; >> +}; >> + >> int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t >> max_iters, >> uint32_t max_factor, uint32_t flags, >> struct save_callbacks* callbacks, int hvm) >> { >> + struct context ctx = >> + { >> + .xch = xch, >> + .fd = io_fd, >> + }; >> + >> + /* GCC 4.4 (of CentOS 6.x vintage) can' t initialise anonymous unions :( */ >> + ctx.save.callbacks = callbacks; >> + >> IPRINTF("In experimental %s", __func__); >> - return -1; >> + >> + if ( xc_domain_getinfo(xch, dom, 1, &ctx.dominfo) != 1 ) >> + { >> + PERROR("Failed to get domain info"); >> + return -1; >> + } >> + >> + if ( ctx.dominfo.domid != dom ) >> + { >> + ERROR("Domain %d does not exist", dom); >> + return -1; >> + } >> + >> + ctx.domid = dom; >> + IPRINTF("Saving domain %d", dom); >> + >> + ctx.save.p2m_size = xc_domain_maximum_gpfn(xch, dom) + 1; >> + if ( ctx.save.p2m_size > ~XEN_DOMCTL_PFINFO_LTAB_MASK ) >> + { >> + errno = E2BIG; >> + ERROR("Cannot save this big a guest"); >> + return -1; >> + } >> + >> + if ( ctx.dominfo.hvm ) >> + { >> + ctx.ops = common_ops_x86_hvm; >> + ctx.save.ops = save_ops_x86_hvm; >> + return save(&ctx, DHDR_TYPE_X86_HVM); >> + } >> + else >> + { >> + ctx.ops = common_ops_x86_pv; >> + ctx.save.ops = save_ops_x86_pv; >> + return save(&ctx, DHDR_TYPE_X86_PV); >> + } >> } >> >> /* >> > -- Thanks, Yang.