* [PATCH 0/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball
@ 2015-06-02 9:04 Robert Yang
2015-06-02 9:04 ` [PATCH 1/1] " Robert Yang
0 siblings, 1 reply; 4+ messages in thread
From: Robert Yang @ 2015-06-02 9:04 UTC (permalink / raw)
To: bitbake-devel
The following changes since commit 5cc614ed68d9f763041e4d67134c5d7ada795978:
qt4: unconditionally disable gstreamer 0.10 support in qt webkit (2015-05-30 22:26:14 +0100)
are available in the git repository at:
git://git.pokylinux.org/poky-contrib rbt/hg
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=rbt/hg
Robert Yang (1):
bitbake: fetch2/hg.py: fix unpack error and mirror tarball
bitbake/lib/bb/fetch2/hg.py | 70 +++++++++++++++++++++++++++++++++++--------
1 file changed, 57 insertions(+), 13 deletions(-)
--
1.7.9.5
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH 1/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball
2015-06-02 9:04 [PATCH 0/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball Robert Yang
@ 2015-06-02 9:04 ` Robert Yang
2015-06-02 14:34 ` Robert Yang
0 siblings, 1 reply; 4+ messages in thread
From: Robert Yang @ 2015-06-02 9:04 UTC (permalink / raw)
To: bitbake-devel
Fixed:
* do_unpack error:
abort: repository DL_DIR/hg/vim.googlecode.com/hg/vim not found!
* The mirror tarball doesn't work
- Add the build_mirror_data to create the tarball
- Unpack the mirror tarball when needed
* The hg files will put in the dir like git: DL_DIR/hg, it was
DL_DIR/hg/path/to/src/uri/path in the past.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
bitbake/lib/bb/fetch2/hg.py | 70 +++++++++++++++++++++++++++++++++++--------
1 file changed, 57 insertions(+), 13 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index cdef4aa..2c8c25d 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -59,10 +59,12 @@ class Hg(FetchMethod):
ud.module = ud.parm["module"]
- # Create paths to mercurial checkouts
- relpath = self._strip_leading_slashes(ud.path)
- ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
- ud.moddir = os.path.join(ud.pkgdir, ud.module)
+ if 'protocol' in ud.parm:
+ ud.proto = ud.parm['protocol']
+ elif not ud.host:
+ ud.proto = 'file'
+ else:
+ ud.proto = "hg"
ud.setup_revisons(d)
@@ -71,10 +73,20 @@ class Hg(FetchMethod):
elif not ud.revision:
ud.revision = self.latest_revision(ud, d)
- ud.localfile = ud.moddir
+ # Create paths to mercurial checkouts
+ hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
+ ud.host, ud.path.replace('/', '.'))
+ ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
+ ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+ hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/")
+ ud.pkgdir = os.path.join(hgdir, hgsrcname)
+ ud.moddir = os.path.join(ud.pkgdir, ud.module)
+ ud.localfile = ud.moddir
ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg"
+ ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True)
+
def need_update(self, ud, d):
revTag = ud.parm.get('rev', 'tip')
if revTag == "tip":
@@ -142,18 +154,36 @@ class Hg(FetchMethod):
def download(self, ud, d):
"""Fetch url"""
+ ud.repochanged = not os.path.exists(ud.fullmirror)
+
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+ # If the checkout doesn't exist and the mirror tarball does, extract it
+ if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
+ bb.utils.mkdirhier(ud.pkgdir)
+ os.chdir(ud.pkgdir)
+ runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
+
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
- updatecmd = self._buildhgcommand(ud, d, "pull")
- logger.info("Update " + ud.url)
- # update sources there
+ # Found the source, check whether need pull
+ updatecmd = self._buildhgcommand(ud, d, "update")
os.chdir(ud.moddir)
logger.debug(1, "Running %s", updatecmd)
- bb.fetch2.check_network_access(d, updatecmd, ud.url)
- runfetchcmd(updatecmd, d)
-
- else:
+ try:
+ runfetchcmd(updatecmd, d)
+ except bb.fetch2.FetchError:
+ # Runnning pull in the repo
+ pullcmd = self._buildhgcommand(ud, d, "pull")
+ logger.info("Pulling " + ud.url)
+ # update sources there
+ os.chdir(ud.moddir)
+ logger.debug(1, "Running %s", pullcmd)
+ bb.fetch2.check_network_access(d, pullcmd, ud.url)
+ runfetchcmd(pullcmd, d)
+ ud.repochanged = True
+
+ # No source found, clone it.
+ if not os.path.exists(ud.moddir):
fetchcmd = self._buildhgcommand(ud, d, "fetch")
logger.info("Fetch " + ud.url)
# check out sources there
@@ -174,6 +204,8 @@ class Hg(FetchMethod):
""" Clean the hg dir """
bb.utils.remove(ud.localpath, True)
+ bb.utils.remove(ud.fullmirror)
+ bb.utils.remove(ud.fullmirror + ".done")
def supports_srcrev(self):
return True
@@ -195,8 +227,20 @@ class Hg(FetchMethod):
"""
return "hg:" + ud.moddir
+ def build_mirror_data(self, ud, d):
+ # Generate a mirror tarball if needed
+ if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+ # it's possible that this symlink points to read-only filesystem with PREMIRROR
+ if os.path.islink(ud.fullmirror):
+ os.unlink(ud.fullmirror)
+
+ os.chdir(ud.pkgdir)
+ logger.info("Creating tarball of hg repository")
+ runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d)
+ runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+
def localpath(self, ud, d):
- return ud.moddir
+ return ud.pkgdir
def unpack(self, ud, destdir, d):
"""
--
1.7.9.5
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH 1/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball
2015-06-02 9:04 ` [PATCH 1/1] " Robert Yang
@ 2015-06-02 14:34 ` Robert Yang
2015-06-02 17:08 ` Dan McGregor
0 siblings, 1 reply; 4+ messages in thread
From: Robert Yang @ 2015-06-02 14:34 UTC (permalink / raw)
To: bitbake-devel
Sorry, I updated the patch a little in the repo:
git://git.pokylinux.org/poky-contrib rbt/hg
Add the following code to hg.py: (refer to git.py)
+ def try_premirror(self, ud, d):
+ # If we don't do this, updating an existing checkout with only premirrors
+ # is not possible
+ if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+ return True
+ if os.path.exists(ud.moddir):
+ return False
+ return True
Otherwise we may get the error like the following when there is a local
mirror tarball in DL_DIR:
ERROR: No checksum specified for
/path/to/downloads/hg_vim_vim.googlecode.com_.hg.tar.gz, please add at least one
to the recipe:
SRC_URI[md5sum] = "18598f5ad0f30083a3715154bd326d26"
SRC_URI[sha256sum] =
"450c989b89d4738d051ae0628c50fbeaaa7acd15927a1ecc16e0fc0dd38ec25e"
ERROR: Function failed: Fetcher failure for URL:
'http://downloads.yoctoproject.org/mirror/sources/hg_vim_vim.googlecode.com_.hg.tar.gz'.
Missing SRC_URI checksum
// Robert
On 06/02/2015 05:04 PM, Robert Yang wrote:
> Fixed:
> * do_unpack error:
> abort: repository DL_DIR/hg/vim.googlecode.com/hg/vim not found!
>
> * The mirror tarball doesn't work
> - Add the build_mirror_data to create the tarball
> - Unpack the mirror tarball when needed
>
> * The hg files will put in the dir like git: DL_DIR/hg, it was
> DL_DIR/hg/path/to/src/uri/path in the past.
>
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
> bitbake/lib/bb/fetch2/hg.py | 70 +++++++++++++++++++++++++++++++++++--------
> 1 file changed, 57 insertions(+), 13 deletions(-)
>
> diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
> index cdef4aa..2c8c25d 100644
> --- a/bitbake/lib/bb/fetch2/hg.py
> +++ b/bitbake/lib/bb/fetch2/hg.py
> @@ -59,10 +59,12 @@ class Hg(FetchMethod):
>
> ud.module = ud.parm["module"]
>
> - # Create paths to mercurial checkouts
> - relpath = self._strip_leading_slashes(ud.path)
> - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
> - ud.moddir = os.path.join(ud.pkgdir, ud.module)
> + if 'protocol' in ud.parm:
> + ud.proto = ud.parm['protocol']
> + elif not ud.host:
> + ud.proto = 'file'
> + else:
> + ud.proto = "hg"
>
> ud.setup_revisons(d)
>
> @@ -71,10 +73,20 @@ class Hg(FetchMethod):
> elif not ud.revision:
> ud.revision = self.latest_revision(ud, d)
>
> - ud.localfile = ud.moddir
> + # Create paths to mercurial checkouts
> + hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
> + ud.host, ud.path.replace('/', '.'))
> + ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
> + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
>
> + hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/")
> + ud.pkgdir = os.path.join(hgdir, hgsrcname)
> + ud.moddir = os.path.join(ud.pkgdir, ud.module)
> + ud.localfile = ud.moddir
> ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg"
>
> + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True)
> +
> def need_update(self, ud, d):
> revTag = ud.parm.get('rev', 'tip')
> if revTag == "tip":
> @@ -142,18 +154,36 @@ class Hg(FetchMethod):
> def download(self, ud, d):
> """Fetch url"""
>
> + ud.repochanged = not os.path.exists(ud.fullmirror)
> +
> logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
>
> + # If the checkout doesn't exist and the mirror tarball does, extract it
> + if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
> + bb.utils.mkdirhier(ud.pkgdir)
> + os.chdir(ud.pkgdir)
> + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
> +
> if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
> - updatecmd = self._buildhgcommand(ud, d, "pull")
> - logger.info("Update " + ud.url)
> - # update sources there
> + # Found the source, check whether need pull
> + updatecmd = self._buildhgcommand(ud, d, "update")
> os.chdir(ud.moddir)
> logger.debug(1, "Running %s", updatecmd)
> - bb.fetch2.check_network_access(d, updatecmd, ud.url)
> - runfetchcmd(updatecmd, d)
> -
> - else:
> + try:
> + runfetchcmd(updatecmd, d)
> + except bb.fetch2.FetchError:
> + # Runnning pull in the repo
> + pullcmd = self._buildhgcommand(ud, d, "pull")
> + logger.info("Pulling " + ud.url)
> + # update sources there
> + os.chdir(ud.moddir)
> + logger.debug(1, "Running %s", pullcmd)
> + bb.fetch2.check_network_access(d, pullcmd, ud.url)
> + runfetchcmd(pullcmd, d)
> + ud.repochanged = True
> +
> + # No source found, clone it.
> + if not os.path.exists(ud.moddir):
> fetchcmd = self._buildhgcommand(ud, d, "fetch")
> logger.info("Fetch " + ud.url)
> # check out sources there
> @@ -174,6 +204,8 @@ class Hg(FetchMethod):
> """ Clean the hg dir """
>
> bb.utils.remove(ud.localpath, True)
> + bb.utils.remove(ud.fullmirror)
> + bb.utils.remove(ud.fullmirror + ".done")
>
> def supports_srcrev(self):
> return True
> @@ -195,8 +227,20 @@ class Hg(FetchMethod):
> """
> return "hg:" + ud.moddir
>
> + def build_mirror_data(self, ud, d):
> + # Generate a mirror tarball if needed
> + if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)):
> + # it's possible that this symlink points to read-only filesystem with PREMIRROR
> + if os.path.islink(ud.fullmirror):
> + os.unlink(ud.fullmirror)
> +
> + os.chdir(ud.pkgdir)
> + logger.info("Creating tarball of hg repository")
> + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d)
> + runfetchcmd("touch %s.done" % (ud.fullmirror), d)
> +
> def localpath(self, ud, d):
> - return ud.moddir
> + return ud.pkgdir
>
> def unpack(self, ud, destdir, d):
> """
>
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH 1/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball
2015-06-02 14:34 ` Robert Yang
@ 2015-06-02 17:08 ` Dan McGregor
0 siblings, 0 replies; 4+ messages in thread
From: Dan McGregor @ 2015-06-02 17:08 UTC (permalink / raw)
To: Robert Yang; +Cc: bitbake-devel
This looks good. It does some things my patch didn't. I still like the
idea of sharing the hg checkout though. I'll look at merging them this
afternoon or tomorrow.
On 2 June 2015 at 08:34, Robert Yang <liezhi.yang@windriver.com> wrote:
> Sorry, I updated the patch a little in the repo:
>
> git://git.pokylinux.org/poky-contrib rbt/hg
>
> Add the following code to hg.py: (refer to git.py)
>
> + def try_premirror(self, ud, d):
> + # If we don't do this, updating an existing checkout with only
> premirrors
> + # is not possible
> + if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
> + return True
> + if os.path.exists(ud.moddir):
> + return False
> + return True
>
> Otherwise we may get the error like the following when there is a local
> mirror tarball in DL_DIR:
>
> ERROR: No checksum specified for
> /path/to/downloads/hg_vim_vim.googlecode.com_.hg.tar.gz, please add at least
> one to the recipe:
> SRC_URI[md5sum] = "18598f5ad0f30083a3715154bd326d26"
> SRC_URI[sha256sum] =
> "450c989b89d4738d051ae0628c50fbeaaa7acd15927a1ecc16e0fc0dd38ec25e"
> ERROR: Function failed: Fetcher failure for URL:
> 'http://downloads.yoctoproject.org/mirror/sources/hg_vim_vim.googlecode.com_.hg.tar.gz'.
> Missing SRC_URI checksum
>
> // Robert
>
>
> On 06/02/2015 05:04 PM, Robert Yang wrote:
>>
>> Fixed:
>> * do_unpack error:
>> abort: repository DL_DIR/hg/vim.googlecode.com/hg/vim not found!
>>
>> * The mirror tarball doesn't work
>> - Add the build_mirror_data to create the tarball
>> - Unpack the mirror tarball when needed
>>
>> * The hg files will put in the dir like git: DL_DIR/hg, it was
>> DL_DIR/hg/path/to/src/uri/path in the past.
>>
>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> ---
>> bitbake/lib/bb/fetch2/hg.py | 70
>> +++++++++++++++++++++++++++++++++++--------
>> 1 file changed, 57 insertions(+), 13 deletions(-)
>>
>> diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
>> index cdef4aa..2c8c25d 100644
>> --- a/bitbake/lib/bb/fetch2/hg.py
>> +++ b/bitbake/lib/bb/fetch2/hg.py
>> @@ -59,10 +59,12 @@ class Hg(FetchMethod):
>>
>> ud.module = ud.parm["module"]
>>
>> - # Create paths to mercurial checkouts
>> - relpath = self._strip_leading_slashes(ud.path)
>> - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host,
>> relpath)
>> - ud.moddir = os.path.join(ud.pkgdir, ud.module)
>> + if 'protocol' in ud.parm:
>> + ud.proto = ud.parm['protocol']
>> + elif not ud.host:
>> + ud.proto = 'file'
>> + else:
>> + ud.proto = "hg"
>>
>> ud.setup_revisons(d)
>>
>> @@ -71,10 +73,20 @@ class Hg(FetchMethod):
>> elif not ud.revision:
>> ud.revision = self.latest_revision(ud, d)
>>
>> - ud.localfile = ud.moddir
>> + # Create paths to mercurial checkouts
>> + hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
>> + ud.host, ud.path.replace('/', '.'))
>> + ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
>> + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True),
>> ud.mirrortarball)
>>
>> + hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) +
>> "/hg/")
>> + ud.pkgdir = os.path.join(hgdir, hgsrcname)
>> + ud.moddir = os.path.join(ud.pkgdir, ud.module)
>> + ud.localfile = ud.moddir
>> ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or
>> "/usr/bin/env hg"
>>
>> + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True)
>> +
>> def need_update(self, ud, d):
>> revTag = ud.parm.get('rev', 'tip')
>> if revTag == "tip":
>> @@ -142,18 +154,36 @@ class Hg(FetchMethod):
>> def download(self, ud, d):
>> """Fetch url"""
>>
>> + ud.repochanged = not os.path.exists(ud.fullmirror)
>> +
>> logger.debug(2, "Fetch: checking for module directory '" +
>> ud.moddir + "'")
>>
>> + # If the checkout doesn't exist and the mirror tarball does,
>> extract it
>> + if not os.path.exists(ud.pkgdir) and
>> os.path.exists(ud.fullmirror):
>> + bb.utils.mkdirhier(ud.pkgdir)
>> + os.chdir(ud.pkgdir)
>> + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
>> +
>> if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
>> - updatecmd = self._buildhgcommand(ud, d, "pull")
>> - logger.info("Update " + ud.url)
>> - # update sources there
>> + # Found the source, check whether need pull
>> + updatecmd = self._buildhgcommand(ud, d, "update")
>> os.chdir(ud.moddir)
>> logger.debug(1, "Running %s", updatecmd)
>> - bb.fetch2.check_network_access(d, updatecmd, ud.url)
>> - runfetchcmd(updatecmd, d)
>> -
>> - else:
>> + try:
>> + runfetchcmd(updatecmd, d)
>> + except bb.fetch2.FetchError:
>> + # Runnning pull in the repo
>> + pullcmd = self._buildhgcommand(ud, d, "pull")
>> + logger.info("Pulling " + ud.url)
>> + # update sources there
>> + os.chdir(ud.moddir)
>> + logger.debug(1, "Running %s", pullcmd)
>> + bb.fetch2.check_network_access(d, pullcmd, ud.url)
>> + runfetchcmd(pullcmd, d)
>> + ud.repochanged = True
>> +
>> + # No source found, clone it.
>> + if not os.path.exists(ud.moddir):
>> fetchcmd = self._buildhgcommand(ud, d, "fetch")
>> logger.info("Fetch " + ud.url)
>> # check out sources there
>> @@ -174,6 +204,8 @@ class Hg(FetchMethod):
>> """ Clean the hg dir """
>>
>> bb.utils.remove(ud.localpath, True)
>> + bb.utils.remove(ud.fullmirror)
>> + bb.utils.remove(ud.fullmirror + ".done")
>>
>> def supports_srcrev(self):
>> return True
>> @@ -195,8 +227,20 @@ class Hg(FetchMethod):
>> """
>> return "hg:" + ud.moddir
>>
>> + def build_mirror_data(self, ud, d):
>> + # Generate a mirror tarball if needed
>> + if ud.write_tarballs == "1" and (ud.repochanged or not
>> os.path.exists(ud.fullmirror)):
>> + # it's possible that this symlink points to read-only
>> filesystem with PREMIRROR
>> + if os.path.islink(ud.fullmirror):
>> + os.unlink(ud.fullmirror)
>> +
>> + os.chdir(ud.pkgdir)
>> + logger.info("Creating tarball of hg repository")
>> + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d)
>> + runfetchcmd("touch %s.done" % (ud.fullmirror), d)
>> +
>> def localpath(self, ud, d):
>> - return ud.moddir
>> + return ud.pkgdir
>>
>> def unpack(self, ud, destdir, d):
>> """
>>
> --
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2015-06-02 17:08 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2015-06-02 9:04 [PATCH 0/1] bitbake: fetch2/hg.py: fix unpack error and mirror tarball Robert Yang
2015-06-02 9:04 ` [PATCH 1/1] " Robert Yang
2015-06-02 14:34 ` Robert Yang
2015-06-02 17:08 ` Dan McGregor
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.