From mboxrd@z Thu Jan 1 00:00:00 1970 From: Hongyang Yang Subject: Re: [PATCH v5 RFC 13/14] tools/libxc: noarch save code Date: Wed, 18 Jun 2014 14:59:06 +0800 Message-ID: <53A138BA.5000702@cn.fujitsu.com> References: <1402510482-21099-1-git-send-email-andrew.cooper3@citrix.com> <1402510482-21099-14-git-send-email-andrew.cooper3@citrix.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii"; Format="flowed" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1402510482-21099-14-git-send-email-andrew.cooper3@citrix.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Andrew Cooper , Xen-devel Cc: Frediano Ziglio , David Vrabel List-Id: xen-devel@lists.xenproject.org On 06/12/2014 02:14 AM, Andrew Cooper wrote: > Signed-off-by: Andrew Cooper > Signed-off-by: Frediano Ziglio > Signed-off-by: David Vrabel > --- > tools/libxc/saverestore/save.c | 545 +++++++++++++++++++++++++++++++++++++++- > 1 file changed, 544 insertions(+), 1 deletion(-) > > diff --git a/tools/libxc/saverestore/save.c b/tools/libxc/saverestore/save.c > index f6ad734..9ad43a5 100644 > --- a/tools/libxc/saverestore/save.c > +++ b/tools/libxc/saverestore/save.c > @@ -1,11 +1,554 @@ > +#include > +#include > + > #include "common.h" > > +/* > + * Writes an Image header and Domain header into the stream. > + */ > +static int write_headers(struct context *ctx, uint16_t guest_type) > +{ > + xc_interface *xch = ctx->xch; ...snip... > +/* > + * Send all domain memory. This is the heart of the live migration loop. > + */ > +static int send_domain_memory(struct context *ctx) > +{ > + xc_interface *xch = ctx->xch; > + DECLARE_HYPERCALL_BUFFER(unsigned long, to_send); > + xc_shadow_op_stats_t stats = { -1, -1 }; > + unsigned pages_written; > + unsigned x, max_iter = 5, dirty_threshold = 50; > + xen_pfn_t p; > + int rc = -1; > + > + to_send = xc_hypercall_buffer_alloc_pages( > + xch, to_send, NRPAGES(bitmap_size(ctx->save.p2m_size))); > + > + ctx->save.batch_pfns = malloc(MAX_BATCH_SIZE * sizeof(*ctx->save.batch_pfns)); > + ctx->save.deferred_pages = calloc(1, bitmap_size(ctx->save.p2m_size)); > + > + if ( !ctx->save.batch_pfns || !to_send || !ctx->save.deferred_pages ) > + { > + ERROR("Unable to allocate memory for to_{send,fix}/batch bitmaps"); > + goto out; > + } > + > + if ( xc_shadow_control(xch, ctx->domid, > + XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY, > + NULL, 0, NULL, 0, NULL) < 0 ) > + { > + PERROR("Failed to enable logdirty"); > + goto out; > + } > + > + for ( x = 0, pages_written = 0; x < max_iter ; ++x ) > + { > + if ( x == 0 ) > + { > + /* First iteration, send all pages. */ > + memset(to_send, 0xff, bitmap_size(ctx->save.p2m_size)); > + } > + else > + { > + /* Else consult the dirty bitmap. */ > + if ( xc_shadow_control( > + xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_CLEAN, > + HYPERCALL_BUFFER(to_send), ctx->save.p2m_size, > + NULL, 0, &stats) != ctx->save.p2m_size ) > + { > + PERROR("Failed to retrieve logdirty bitmap"); > + rc = -1; > + goto out; > + } > + else > + DPRINTF(" Wrote %u pages; stats: faults %"PRIu32", dirty %"PRIu32, > + pages_written, stats.fault_count, stats.dirty_count); > + pages_written = 0; > + > + if ( stats.dirty_count < dirty_threshold ) > + break; > + } > + > + DPRINTF("Iteration %u", x); > + > + for ( p = 0 ; p < ctx->save.p2m_size; ++p ) > + { > + if ( test_bit(p, to_send) ) > + { > + rc = add_to_batch(ctx, p); > + if ( rc ) > + goto out; > + ++pages_written; > + } > + } > + > + rc = flush_batch(ctx); > + if ( rc ) > + goto out; > + } > + > + rc = pause_domain(ctx); > + if ( rc ) > + goto out; > + > + if ( xc_shadow_control( > + xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_CLEAN, > + HYPERCALL_BUFFER(to_send), ctx->save.p2m_size, > + NULL, 0, &stats) != ctx->save.p2m_size ) > + { > + PERROR("Failed to retrieve logdirty bitmap"); > + rc = -1; > + goto out; > + } > + > + for ( p = 0, pages_written = 0 ; p < ctx->save.p2m_size; ++p ) > + { > + if ( test_bit(p, to_send) || test_bit(p, ctx->save.deferred_pages) ) > + { > + rc = add_to_batch(ctx, p); > + if ( rc ) > + goto out; > + ++pages_written; > + } > + } > + > + rc = flush_batch(ctx); > + if ( rc ) > + goto out; > + > + DPRINTF(" Wrote %u pages", pages_written); > + IPRINTF("Sent all pages"); > + > + out: > + xc_hypercall_buffer_free_pages(xch, to_send, > + NRPAGES(bitmap_size(ctx->save.p2m_size))); > + free(ctx->save.deferred_pages); > + free(ctx->save.batch_pfns); > + return rc; > +} > + > +/* > + * Save a domain. > + */ > +static int save(struct context *ctx, uint16_t guest_type) > +{ > + xc_interface *xch = ctx->xch; > + int rc, saved_rc = 0, saved_errno = 0; > + > + IPRINTF("Saving domain %d, type %s", > + ctx->domid, dhdr_type_to_str(guest_type)); > + > + rc = ctx->save.ops.setup(ctx); > + if ( rc ) > + goto err; > + > + rc = write_headers(ctx, guest_type); > + if ( rc ) > + goto err; > + > + rc = ctx->save.ops.start_of_stream(ctx); > + if ( rc ) > + goto err; > + > + rc = send_domain_memory(ctx); > + if ( rc ) > + goto err; > + > + /* Refresh domain information now it has paused. */ > + if ( (xc_domain_getinfo(xch, ctx->domid, 1, &ctx->dominfo) != 1) || > + (ctx->dominfo.domid != ctx->domid) ) > + { > + PERROR("Unable to refresh domain information"); > + rc = -1; > + goto err; > + } > + else if ( (!ctx->dominfo.shutdown || > + ctx->dominfo.shutdown_reason != SHUTDOWN_suspend ) && > + !ctx->dominfo.paused ) > + { > + ERROR("Domain has not been suspended"); > + rc = -1; > + goto err; > + } > + > + rc = ctx->save.ops.end_of_stream(ctx); > + if ( rc ) > + goto err; > + > + rc = write_end_record(ctx); > + if ( rc ) > + goto err; > + > + xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF, > + NULL, 0, NULL, 0, NULL); If migration failed because log-dirty already been enabled or there's err after we enabled log-dirty, we should off shadow op. otherwise we will always fail when migration, so this op should under error path. The following patch fix it. diff --git a/tools/libxc/saverestore/save.c b/tools/libxc/saverestore/save.c index 9ad43a5..6e9d325 100644 --- a/tools/libxc/saverestore/save.c +++ b/tools/libxc/saverestore/save.c @@ -474,9 +474,6 @@ static int save(struct context *ctx, uint16_t guest_type) if ( rc ) goto err; - xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF, - NULL, 0, NULL, 0, NULL); - IPRINTF("Save successful"); goto done; @@ -490,6 +487,9 @@ static int save(struct context *ctx, uint16_t guest_type) if ( rc ) PERROR("Failed to clean up"); + xc_shadow_control(xch, ctx->domid, XEN_DOMCTL_SHADOW_OP_OFF, + NULL, 0, NULL, 0, NULL); + if ( saved_rc ) { rc = saved_rc; > + > + IPRINTF("Save successful"); > + goto done; > + > + err: > + saved_errno = errno; > + saved_rc = rc; > + PERROR("Save failed"); > + > + done: > + rc = ctx->save.ops.cleanup(ctx); > + if ( rc ) > + PERROR("Failed to clean up"); > + > + if ( saved_rc ) > + { > + rc = saved_rc; > + errno = saved_errno; > + } > + > + return rc; > +}; > + > int xc_domain_save2(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters, > uint32_t max_factor, uint32_t flags, > struct save_callbacks* callbacks, int hvm) > { > + struct context ctx = > + { > + .xch = xch, > + .fd = io_fd, > + }; > + > + /* GCC 4.4 (of CentOS 6.x vintage) can' t initialise anonymous unions :( */ > + ctx.save.callbacks = callbacks; > + > IPRINTF("In experimental %s", __func__); > - return -1; > + > + if ( xc_domain_getinfo(xch, dom, 1, &ctx.dominfo) != 1 ) > + { > + PERROR("Failed to get domain info"); > + return -1; > + } > + > + if ( ctx.dominfo.domid != dom ) > + { > + ERROR("Domain %d does not exist", dom); > + return -1; > + } > + > + ctx.domid = dom; > + IPRINTF("Saving domain %d", dom); > + > + ctx.save.p2m_size = xc_domain_maximum_gpfn(xch, dom) + 1; > + if ( ctx.save.p2m_size > ~XEN_DOMCTL_PFINFO_LTAB_MASK ) > + { > + errno = E2BIG; > + ERROR("Cannot save this big a guest"); > + return -1; > + } > + > + if ( ctx.dominfo.hvm ) > + { > + ctx.ops = common_ops_x86_hvm; > + ctx.save.ops = save_ops_x86_hvm; > + return save(&ctx, DHDR_TYPE_X86_HVM); > + } > + else > + { > + ctx.ops = common_ops_x86_pv; > + ctx.save.ops = save_ops_x86_pv; > + return save(&ctx, DHDR_TYPE_X86_PV); > + } > } > > /* > -- Thanks, Yang.