From mboxrd@z Thu Jan 1 00:00:00 1970 From: Olaf Hering Subject: Re: [PATCH v6 3/3] tools/libxc: use superpages during restore of HVM guest Date: Tue, 29 Aug 2017 11:58:24 +0200 Message-ID: <20170829095823.GA9803@aepfle.de> References: <20170826103332.24570-1-olaf@aepfle.de> <20170826103332.24570-4-olaf@aepfle.de> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="===============8435487581332236571==" Return-path: In-Reply-To: <20170826103332.24570-4-olaf@aepfle.de> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Errors-To: xen-devel-bounces@lists.xen.org Sender: "Xen-devel" To: xen-devel@lists.xen.org, Andrew Cooper , Ian Jackson , Wei Liu List-Id: xen-devel@lists.xenproject.org --===============8435487581332236571== Content-Type: multipart/signed; micalg=pgp-sha1; protocol="application/pgp-signature"; boundary="tThc/1wpZn/ma/RB" Content-Disposition: inline --tThc/1wpZn/ma/RB Content-Type: text/plain; charset=utf-8 Content-Disposition: inline Content-Transfer-Encoding: quoted-printable On Sat, Aug 26, Olaf Hering wrote: > +static int x86_hvm_populate_pfns(struct xc_sr_context *ctx, unsigned cou= nt, > + /* > + * Scan the entire superpage because several batches will fit into > + * a superpage, and it is unknown which pfn triggered the allocation. > + */ > + order =3D SUPERPAGE_1GB_SHIFT; > + pfn =3D min_pfn =3D (min_pfn >> order) << order; Scanning an entire superpage again and again looked expensive, but with the debug change below it turned out that the loop which peeks at each single bit in populated_pfns is likely not a bootleneck. Migrating a domU with a simple workload that touches pages to mark them dirty will set the min_pfn/max_pfn to a large range anyway after the first iteration. This large range may also happen with an idle domU. A small domU takes 78 seconds to migrate, and just the freeing part takes 1.4 seconds. Similar for a large domain, the loop takes 1% of the time. 78 seconds, 1.4 seconds, 2119 calls (8GB, 12*512M memdirty) 695 seconds, 7.6 seconds, 18076 calls (72GB, 12*5G memdirty) Olaf track time spent if decrease_reservation is needed diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h index 0fa0fbea4d..5ec8b6fee6 100644 --- a/tools/libxc/xc_sr_common.h +++ b/tools/libxc/xc_sr_common.h @@ -353,6 +353,9 @@ struct xc_sr_context struct xc_sr_bitmap attempted_1g; struct xc_sr_bitmap attempted_2m; struct xc_sr_bitmap allocated_pfns; + + unsigned long tv_nsec; + unsigned long iterations; } restore; }; } x86_hvm; diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c index 8cd9289d1a..f6aad329e2 100644 --- a/tools/libxc/xc_sr_restore.c +++ b/tools/libxc/xc_sr_restore.c @@ -769,6 +769,7 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uin= t32_t dom, { ctx.restore.ops =3D restore_ops_x86_hvm; if ( restore(&ctx) ) + ; return -1; } else diff --git a/tools/libxc/xc_sr_restore_x86_hvm.c b/tools/libxc/xc_sr_restor= e_x86_hvm.c index 2b0eca0c7c..11758b3f7d 100644 --- a/tools/libxc/xc_sr_restore_x86_hvm.c +++ b/tools/libxc/xc_sr_restore_x86_hvm.c @@ -1,5 +1,6 @@ #include #include +#include =20 #include "xc_sr_common_x86.h" =20 @@ -248,6 +249,12 @@ static int x86_hvm_stream_complete(struct xc_sr_contex= t *ctx) =20 static int x86_hvm_cleanup(struct xc_sr_context *ctx) { + xc_interface *xch =3D ctx->xch; + errno =3D 0; + PERROR("tv_nsec %lu.%lu iterations %lu", + ctx->x86_hvm.restore.tv_nsec / 1000000000UL, + ctx->x86_hvm.restore.tv_nsec % 1000000000UL, + ctx->x86_hvm.restore.iterations); free(ctx->x86_hvm.restore.context); xc_sr_bitmap_free(&ctx->x86_hvm.restore.attempted_1g); xc_sr_bitmap_free(&ctx->x86_hvm.restore.attempted_2m); @@ -440,6 +447,28 @@ static int x86_hvm_allocate_pfn(struct xc_sr_context *= ctx, xen_pfn_t pfn) return rc; } =20 +static void diff_timespec(struct xc_sr_context *ctx, const struct timespec= *old, const struct timespec *new, struct timespec *diff) +{ + xc_interface *xch =3D ctx->xch; + if (new->tv_sec =3D=3D old->tv_sec && new->tv_nsec =3D=3D old->tv_nsec) + PERROR("%s: time did not move: %ld/%ld =3D=3D %ld/%ld", __func__, = old->tv_sec, old->tv_nsec, new->tv_sec, new->tv_nsec); + if ( (new->tv_sec < old->tv_sec) || (new->tv_sec =3D=3D old->tv_sec &&= new->tv_nsec < old->tv_nsec) ) + { + PERROR("%s: time went backwards: %ld/%ld -> %ld/%ld", __func__, ol= d->tv_sec, old->tv_nsec, new->tv_sec, new->tv_nsec); + diff->tv_sec =3D diff->tv_nsec =3D 0; + return; + } + if ((new->tv_nsec - old->tv_nsec) < 0) { + diff->tv_sec =3D new->tv_sec - old->tv_sec - 1; + diff->tv_nsec =3D new->tv_nsec - old->tv_nsec + 1000000000UL; + } else { + diff->tv_sec =3D new->tv_sec - old->tv_sec; + diff->tv_nsec =3D new->tv_nsec - old->tv_nsec; + } + if (diff->tv_sec < 0) + PERROR("%s: time diff broken. old: %ld/%ld new: %ld/%ld diff: %ld/= %ld ", __func__, old->tv_sec, old->tv_nsec, new->tv_sec, new->tv_nsec, diff= ->tv_sec, diff->tv_nsec); +} + static int x86_hvm_populate_pfns(struct xc_sr_context *ctx, unsigned count, const xen_pfn_t *original_pfns, const uint32_t *types) @@ -448,6 +477,7 @@ static int x86_hvm_populate_pfns(struct xc_sr_context *= ctx, unsigned count, xen_pfn_t pfn, min_pfn =3D original_pfns[0], max_pfn =3D original_pfns= [0]; unsigned i, freed =3D 0, order; int rc =3D -1; + struct timespec a, b, d; =20 for ( i =3D 0; i < count; ++i ) { @@ -474,6 +504,8 @@ static int x86_hvm_populate_pfns(struct xc_sr_context *= ctx, unsigned count, } } =20 + if (clock_gettime(CLOCK_MONOTONIC, &a)) + PERROR("clock_gettime start"); /* * Scan the entire superpage because several batches will fit into * a superpage, and it is unknown which pfn triggered the allocation. @@ -504,10 +536,17 @@ static int x86_hvm_populate_pfns(struct xc_sr_context= *ctx, unsigned count, } pfn++; } - if ( freed ) + if ( 0 && freed ) DPRINTF("freed %u between %" PRI_xen_pfn " %" PRI_xen_pfn "\n", freed, min_pfn, max_pfn); =20 + if (clock_gettime(CLOCK_MONOTONIC, &b)) + PERROR("clock_gettime end"); + + diff_timespec(ctx, &a, &b, &d); + ctx->x86_hvm.restore.tv_nsec +=3D d.tv_nsec + (1000000000UL * d.tv_sec= ); + ctx->x86_hvm.restore.iterations++; + rc =3D 0; =20 err: --tThc/1wpZn/ma/RB Content-Type: application/pgp-signature; name="signature.asc" -----BEGIN PGP SIGNATURE----- iF0EARECAB0WIQSkRyP6Rn//f03pRUBdQqD6ppg2fgUCWaU6vQAKCRBdQqD6ppg2 fvrkAKDAmcEH/RyRnnwYll0mLazr3IFofQCg0+Hkn4BRjLRV+Xu5Uk3qv8qSoe0= =/Ac2 -----END PGP SIGNATURE----- --tThc/1wpZn/ma/RB-- --===============8435487581332236571== Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: base64 Content-Disposition: inline X19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18KWGVuLWRldmVs IG1haWxpbmcgbGlzdApYZW4tZGV2ZWxAbGlzdHMueGVuLm9yZwpodHRwczovL2xpc3RzLnhlbi5v cmcveGVuLWRldmVsCg== --===============8435487581332236571==--