* [PATCH 0/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball @ 2015-06-02 9:04 Robert Yang 2015-06-02 9:04 ` [PATCH 1/1] " Robert Yang 0 siblings, 1 reply; 4+ messages in thread From: Robert Yang @ 2015-06-02 9:04 UTC (permalink / raw) To: bitbake-devel The following changes since commit 5cc614ed68d9f763041e4d67134c5d7ada795978: qt4: unconditionally disable gstreamer 0.10 support in qt webkit (2015-05-30 22:26:14 +0100) are available in the git repository at: git://git.pokylinux.org/poky-contrib rbt/hg http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=rbt/hg Robert Yang (1): bitbake: fetch2/hg.py: fix unpack error and mirror tarball bitbake/lib/bb/fetch2/hg.py | 70 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 13 deletions(-) -- 1.7.9.5 ^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH 1/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball 2015-06-02 9:04 [PATCH 0/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball Robert Yang @ 2015-06-02 9:04 ` Robert Yang 2015-06-02 14:34 ` Robert Yang 0 siblings, 1 reply; 4+ messages in thread From: Robert Yang @ 2015-06-02 9:04 UTC (permalink / raw) To: bitbake-devel Fixed: * do_unpack error: abort: repository DL_DIR/hg/vim.googlecode.com/hg/vim not found! * The mirror tarball doesn't work - Add the build_mirror_data to create the tarball - Unpack the mirror tarball when needed * The hg files will put in the dir like git: DL_DIR/hg, it was DL_DIR/hg/path/to/src/uri/path in the past. Signed-off-by: Robert Yang <liezhi.yang@windriver.com> --- bitbake/lib/bb/fetch2/hg.py | 70 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index cdef4aa..2c8c25d 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py @@ -59,10 +59,12 @@ class Hg(FetchMethod): ud.module = ud.parm["module"] - # Create paths to mercurial checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath) - ud.moddir = os.path.join(ud.pkgdir, ud.module) + if 'protocol' in ud.parm: + ud.proto = ud.parm['protocol'] + elif not ud.host: + ud.proto = 'file' + else: + ud.proto = "hg" ud.setup_revisons(d) @@ -71,10 +73,20 @@ class Hg(FetchMethod): elif not ud.revision: ud.revision = self.latest_revision(ud, d) - ud.localfile = ud.moddir + # Create paths to mercurial checkouts + hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ + ud.host, ud.path.replace('/', '.')) + ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/") + ud.pkgdir = os.path.join(hgdir, hgsrcname) + ud.moddir = os.path.join(ud.pkgdir, ud.module) + ud.localfile = ud.moddir ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg" + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True) + def need_update(self, ud, d): revTag = ud.parm.get('rev', 'tip') if revTag == "tip": @@ -142,18 +154,36 @@ class Hg(FetchMethod): def download(self, ud, d): """Fetch url""" + ud.repochanged = not os.path.exists(ud.fullmirror) + logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") + # If the checkout doesn't exist and the mirror tarball does, extract it + if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror): + bb.utils.mkdirhier(ud.pkgdir) + os.chdir(ud.pkgdir) + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) + if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): - updatecmd = self._buildhgcommand(ud, d, "pull") - logger.info("Update " + ud.url) - # update sources there + # Found the source, check whether need pull + updatecmd = self._buildhgcommand(ud, d, "update") os.chdir(ud.moddir) logger.debug(1, "Running %s", updatecmd) - bb.fetch2.check_network_access(d, updatecmd, ud.url) - runfetchcmd(updatecmd, d) - - else: + try: + runfetchcmd(updatecmd, d) + except bb.fetch2.FetchError: + # Runnning pull in the repo + pullcmd = self._buildhgcommand(ud, d, "pull") + logger.info("Pulling " + ud.url) + # update sources there + os.chdir(ud.moddir) + logger.debug(1, "Running %s", pullcmd) + bb.fetch2.check_network_access(d, pullcmd, ud.url) + runfetchcmd(pullcmd, d) + ud.repochanged = True + + # No source found, clone it. + if not os.path.exists(ud.moddir): fetchcmd = self._buildhgcommand(ud, d, "fetch") logger.info("Fetch " + ud.url) # check out sources there @@ -174,6 +204,8 @@ class Hg(FetchMethod): """ Clean the hg dir """ bb.utils.remove(ud.localpath, True) + bb.utils.remove(ud.fullmirror) + bb.utils.remove(ud.fullmirror + ".done") def supports_srcrev(self): return True @@ -195,8 +227,20 @@ class Hg(FetchMethod): """ return "hg:" + ud.moddir + def build_mirror_data(self, ud, d): + # Generate a mirror tarball if needed + if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)): + # it's possible that this symlink points to read-only filesystem with PREMIRROR + if os.path.islink(ud.fullmirror): + os.unlink(ud.fullmirror) + + os.chdir(ud.pkgdir) + logger.info("Creating tarball of hg repository") + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d) + runfetchcmd("touch %s.done" % (ud.fullmirror), d) + def localpath(self, ud, d): - return ud.moddir + return ud.pkgdir def unpack(self, ud, destdir, d): """ -- 1.7.9.5 ^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH 1/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball 2015-06-02 9:04 ` [PATCH 1/1] " Robert Yang @ 2015-06-02 14:34 ` Robert Yang 2015-06-02 17:08 ` Dan McGregor 0 siblings, 1 reply; 4+ messages in thread From: Robert Yang @ 2015-06-02 14:34 UTC (permalink / raw) To: bitbake-devel Sorry, I updated the patch a little in the repo: git://git.pokylinux.org/poky-contrib rbt/hg Add the following code to hg.py: (refer to git.py) + def try_premirror(self, ud, d): + # If we don't do this, updating an existing checkout with only premirrors + # is not possible + if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: + return True + if os.path.exists(ud.moddir): + return False + return True Otherwise we may get the error like the following when there is a local mirror tarball in DL_DIR: ERROR: No checksum specified for /path/to/downloads/hg_vim_vim.googlecode.com_.hg.tar.gz, please add at least one to the recipe: SRC_URI[md5sum] = "18598f5ad0f30083a3715154bd326d26" SRC_URI[sha256sum] = "450c989b89d4738d051ae0628c50fbeaaa7acd15927a1ecc16e0fc0dd38ec25e" ERROR: Function failed: Fetcher failure for URL: 'http://downloads.yoctoproject.org/mirror/sources/hg_vim_vim.googlecode.com_.hg.tar.gz'. Missing SRC_URI checksum // Robert On 06/02/2015 05:04 PM, Robert Yang wrote: > Fixed: > * do_unpack error: > abort: repository DL_DIR/hg/vim.googlecode.com/hg/vim not found! > > * The mirror tarball doesn't work > - Add the build_mirror_data to create the tarball > - Unpack the mirror tarball when needed > > * The hg files will put in the dir like git: DL_DIR/hg, it was > DL_DIR/hg/path/to/src/uri/path in the past. > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > --- > bitbake/lib/bb/fetch2/hg.py | 70 +++++++++++++++++++++++++++++++++++-------- > 1 file changed, 57 insertions(+), 13 deletions(-) > > diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py > index cdef4aa..2c8c25d 100644 > --- a/bitbake/lib/bb/fetch2/hg.py > +++ b/bitbake/lib/bb/fetch2/hg.py > @@ -59,10 +59,12 @@ class Hg(FetchMethod): > > ud.module = ud.parm["module"] > > - # Create paths to mercurial checkouts > - relpath = self._strip_leading_slashes(ud.path) > - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath) > - ud.moddir = os.path.join(ud.pkgdir, ud.module) > + if 'protocol' in ud.parm: > + ud.proto = ud.parm['protocol'] > + elif not ud.host: > + ud.proto = 'file' > + else: > + ud.proto = "hg" > > ud.setup_revisons(d) > > @@ -71,10 +73,20 @@ class Hg(FetchMethod): > elif not ud.revision: > ud.revision = self.latest_revision(ud, d) > > - ud.localfile = ud.moddir > + # Create paths to mercurial checkouts > + hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ > + ud.host, ud.path.replace('/', '.')) > + ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname > + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) > > + hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/") > + ud.pkgdir = os.path.join(hgdir, hgsrcname) > + ud.moddir = os.path.join(ud.pkgdir, ud.module) > + ud.localfile = ud.moddir > ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg" > > + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True) > + > def need_update(self, ud, d): > revTag = ud.parm.get('rev', 'tip') > if revTag == "tip": > @@ -142,18 +154,36 @@ class Hg(FetchMethod): > def download(self, ud, d): > """Fetch url""" > > + ud.repochanged = not os.path.exists(ud.fullmirror) > + > logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") > > + # If the checkout doesn't exist and the mirror tarball does, extract it > + if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror): > + bb.utils.mkdirhier(ud.pkgdir) > + os.chdir(ud.pkgdir) > + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) > + > if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): > - updatecmd = self._buildhgcommand(ud, d, "pull") > - logger.info("Update " + ud.url) > - # update sources there > + # Found the source, check whether need pull > + updatecmd = self._buildhgcommand(ud, d, "update") > os.chdir(ud.moddir) > logger.debug(1, "Running %s", updatecmd) > - bb.fetch2.check_network_access(d, updatecmd, ud.url) > - runfetchcmd(updatecmd, d) > - > - else: > + try: > + runfetchcmd(updatecmd, d) > + except bb.fetch2.FetchError: > + # Runnning pull in the repo > + pullcmd = self._buildhgcommand(ud, d, "pull") > + logger.info("Pulling " + ud.url) > + # update sources there > + os.chdir(ud.moddir) > + logger.debug(1, "Running %s", pullcmd) > + bb.fetch2.check_network_access(d, pullcmd, ud.url) > + runfetchcmd(pullcmd, d) > + ud.repochanged = True > + > + # No source found, clone it. > + if not os.path.exists(ud.moddir): > fetchcmd = self._buildhgcommand(ud, d, "fetch") > logger.info("Fetch " + ud.url) > # check out sources there > @@ -174,6 +204,8 @@ class Hg(FetchMethod): > """ Clean the hg dir """ > > bb.utils.remove(ud.localpath, True) > + bb.utils.remove(ud.fullmirror) > + bb.utils.remove(ud.fullmirror + ".done") > > def supports_srcrev(self): > return True > @@ -195,8 +227,20 @@ class Hg(FetchMethod): > """ > return "hg:" + ud.moddir > > + def build_mirror_data(self, ud, d): > + # Generate a mirror tarball if needed > + if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)): > + # it's possible that this symlink points to read-only filesystem with PREMIRROR > + if os.path.islink(ud.fullmirror): > + os.unlink(ud.fullmirror) > + > + os.chdir(ud.pkgdir) > + logger.info("Creating tarball of hg repository") > + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d) > + runfetchcmd("touch %s.done" % (ud.fullmirror), d) > + > def localpath(self, ud, d): > - return ud.moddir > + return ud.pkgdir > > def unpack(self, ud, destdir, d): > """ > ^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH 1/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball 2015-06-02 14:34 ` Robert Yang @ 2015-06-02 17:08 ` Dan McGregor 0 siblings, 0 replies; 4+ messages in thread From: Dan McGregor @ 2015-06-02 17:08 UTC (permalink / raw) To: Robert Yang; +Cc: bitbake-devel This looks good. It does some things my patch didn't. I still like the idea of sharing the hg checkout though. I'll look at merging them this afternoon or tomorrow. On 2 June 2015 at 08:34, Robert Yang <liezhi.yang@windriver.com> wrote: > Sorry, I updated the patch a little in the repo: > > git://git.pokylinux.org/poky-contrib rbt/hg > > Add the following code to hg.py: (refer to git.py) > > + def try_premirror(self, ud, d): > + # If we don't do this, updating an existing checkout with only > premirrors > + # is not possible > + if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: > + return True > + if os.path.exists(ud.moddir): > + return False > + return True > > Otherwise we may get the error like the following when there is a local > mirror tarball in DL_DIR: > > ERROR: No checksum specified for > /path/to/downloads/hg_vim_vim.googlecode.com_.hg.tar.gz, please add at least > one to the recipe: > SRC_URI[md5sum] = "18598f5ad0f30083a3715154bd326d26" > SRC_URI[sha256sum] = > "450c989b89d4738d051ae0628c50fbeaaa7acd15927a1ecc16e0fc0dd38ec25e" > ERROR: Function failed: Fetcher failure for URL: > 'http://downloads.yoctoproject.org/mirror/sources/hg_vim_vim.googlecode.com_.hg.tar.gz'. > Missing SRC_URI checksum > > // Robert > > > On 06/02/2015 05:04 PM, Robert Yang wrote: >> >> Fixed: >> * do_unpack error: >> abort: repository DL_DIR/hg/vim.googlecode.com/hg/vim not found! >> >> * The mirror tarball doesn't work >> - Add the build_mirror_data to create the tarball >> - Unpack the mirror tarball when needed >> >> * The hg files will put in the dir like git: DL_DIR/hg, it was >> DL_DIR/hg/path/to/src/uri/path in the past. >> >> Signed-off-by: Robert Yang <liezhi.yang@windriver.com> >> --- >> bitbake/lib/bb/fetch2/hg.py | 70 >> +++++++++++++++++++++++++++++++++++-------- >> 1 file changed, 57 insertions(+), 13 deletions(-) >> >> diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py >> index cdef4aa..2c8c25d 100644 >> --- a/bitbake/lib/bb/fetch2/hg.py >> +++ b/bitbake/lib/bb/fetch2/hg.py >> @@ -59,10 +59,12 @@ class Hg(FetchMethod): >> >> ud.module = ud.parm["module"] >> >> - # Create paths to mercurial checkouts >> - relpath = self._strip_leading_slashes(ud.path) >> - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, >> relpath) >> - ud.moddir = os.path.join(ud.pkgdir, ud.module) >> + if 'protocol' in ud.parm: >> + ud.proto = ud.parm['protocol'] >> + elif not ud.host: >> + ud.proto = 'file' >> + else: >> + ud.proto = "hg" >> >> ud.setup_revisons(d) >> >> @@ -71,10 +73,20 @@ class Hg(FetchMethod): >> elif not ud.revision: >> ud.revision = self.latest_revision(ud, d) >> >> - ud.localfile = ud.moddir >> + # Create paths to mercurial checkouts >> + hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ >> + ud.host, ud.path.replace('/', '.')) >> + ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname >> + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), >> ud.mirrortarball) >> >> + hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + >> "/hg/") >> + ud.pkgdir = os.path.join(hgdir, hgsrcname) >> + ud.moddir = os.path.join(ud.pkgdir, ud.module) >> + ud.localfile = ud.moddir >> ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or >> "/usr/bin/env hg" >> >> + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True) >> + >> def need_update(self, ud, d): >> revTag = ud.parm.get('rev', 'tip') >> if revTag == "tip": >> @@ -142,18 +154,36 @@ class Hg(FetchMethod): >> def download(self, ud, d): >> """Fetch url""" >> >> + ud.repochanged = not os.path.exists(ud.fullmirror) >> + >> logger.debug(2, "Fetch: checking for module directory '" + >> ud.moddir + "'") >> >> + # If the checkout doesn't exist and the mirror tarball does, >> extract it >> + if not os.path.exists(ud.pkgdir) and >> os.path.exists(ud.fullmirror): >> + bb.utils.mkdirhier(ud.pkgdir) >> + os.chdir(ud.pkgdir) >> + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) >> + >> if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): >> - updatecmd = self._buildhgcommand(ud, d, "pull") >> - logger.info("Update " + ud.url) >> - # update sources there >> + # Found the source, check whether need pull >> + updatecmd = self._buildhgcommand(ud, d, "update") >> os.chdir(ud.moddir) >> logger.debug(1, "Running %s", updatecmd) >> - bb.fetch2.check_network_access(d, updatecmd, ud.url) >> - runfetchcmd(updatecmd, d) >> - >> - else: >> + try: >> + runfetchcmd(updatecmd, d) >> + except bb.fetch2.FetchError: >> + # Runnning pull in the repo >> + pullcmd = self._buildhgcommand(ud, d, "pull") >> + logger.info("Pulling " + ud.url) >> + # update sources there >> + os.chdir(ud.moddir) >> + logger.debug(1, "Running %s", pullcmd) >> + bb.fetch2.check_network_access(d, pullcmd, ud.url) >> + runfetchcmd(pullcmd, d) >> + ud.repochanged = True >> + >> + # No source found, clone it. >> + if not os.path.exists(ud.moddir): >> fetchcmd = self._buildhgcommand(ud, d, "fetch") >> logger.info("Fetch " + ud.url) >> # check out sources there >> @@ -174,6 +204,8 @@ class Hg(FetchMethod): >> """ Clean the hg dir """ >> >> bb.utils.remove(ud.localpath, True) >> + bb.utils.remove(ud.fullmirror) >> + bb.utils.remove(ud.fullmirror + ".done") >> >> def supports_srcrev(self): >> return True >> @@ -195,8 +227,20 @@ class Hg(FetchMethod): >> """ >> return "hg:" + ud.moddir >> >> + def build_mirror_data(self, ud, d): >> + # Generate a mirror tarball if needed >> + if ud.write_tarballs == "1" and (ud.repochanged or not >> os.path.exists(ud.fullmirror)): >> + # it's possible that this symlink points to read-only >> filesystem with PREMIRROR >> + if os.path.islink(ud.fullmirror): >> + os.unlink(ud.fullmirror) >> + >> + os.chdir(ud.pkgdir) >> + logger.info("Creating tarball of hg repository") >> + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d) >> + runfetchcmd("touch %s.done" % (ud.fullmirror), d) >> + >> def localpath(self, ud, d): >> - return ud.moddir >> + return ud.pkgdir >> >> def unpack(self, ud, destdir, d): >> """ >> > -- > _______________________________________________ > bitbake-devel mailing list > bitbake-devel@lists.openembedded.org > http://lists.openembedded.org/mailman/listinfo/bitbake-devel ^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2015-06-02 17:08 UTC | newest] Thread overview: 4+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2015-06-02 9:04 [PATCH 0/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball Robert Yang 2015-06-02 9:04 ` [PATCH 1/1] " Robert Yang 2015-06-02 14:34 ` Robert Yang 2015-06-02 17:08 ` Dan McGregor
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.