* [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-30 22:52 ` McClintock Matthew-B29882
2012-07-03 22:32 ` [denzil][PATCH 2/9] bitbake: fetch2/__init__: Make it clearer when uri_replace doesn't return a match Khem Raj
` (9 subsequent siblings)
10 siblings, 1 reply; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
There were some hardcoded behaviours in the system for which backends
support checksums verses which backends recommend them verses which
don't recommend them.
This moves the functionality into specific fetchers and then makes the
general code generic. This cleans up the codebase and fixes some corner
cases such as trying to checksum directories returned by the git fetcher.
(Bitbake rev: ef6d268f7b8527541a7fb044cf95a973be4097f4)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Conflicts:
bitbake/lib/bb/fetch2/__init__.py
Signed-off-by: Khem Raj <kraj@juniper.net>
---
bitbake/lib/bb/fetch2/__init__.py | 56 ++++++++++++++++++++++++-------------
bitbake/lib/bb/fetch2/git.py | 3 ++
bitbake/lib/bb/fetch2/ssh.py | 3 ++
bitbake/lib/bb/fetch2/wget.py | 3 ++
4 files changed, 46 insertions(+), 19 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 414cc2b..ec043a8 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -268,30 +268,31 @@ def verify_checksum(u, ud, d):
matched
"""
- if not ud.type in ["http", "https", "ftp", "ftps"]:
+ if not ud.method.supports_checksum(ud):
return
md5data = bb.utils.md5_file(ud.localpath)
sha256data = bb.utils.sha256_file(ud.localpath)
- # If strict checking enabled and neither sum defined, raise error
- strict = d.getVar("BB_STRICT_CHECKSUM", True) or None
- if (strict and ud.md5_expected == None and ud.sha256_expected == None):
- raise FetchError('No checksum specified for %s, please add at least one to the recipe:\n'
- 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
- (ud.localpath, ud.md5_name, md5data,
- ud.sha256_name, sha256data), u)
-
- # Log missing sums so user can more easily add them
- if ud.md5_expected == None:
- logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
- 'SRC_URI[%s] = "%s"',
- ud.localpath, ud.md5_name, md5data)
-
- if ud.sha256_expected == None:
- logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
- 'SRC_URI[%s] = "%s"',
- ud.localpath, ud.sha256_name, sha256data)
+ if ud.method.recommends_checksum(ud):
+ # If strict checking enabled and neither sum defined, raise error
+ strict = d.getVar("BB_STRICT_CHECKSUM", True) or None
+ if (strict and ud.md5_expected == None and ud.sha256_expected == None):
+ raise FetchError('No checksum specified for %s, please add at least one to the recipe:\n'
+ 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
+ (ud.localpath, ud.md5_name, md5data,
+ ud.sha256_name, sha256data), u)
+
+ # Log missing sums so user can more easily add them
+ if ud.md5_expected == None:
+ logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
+ 'SRC_URI[%s] = "%s"',
+ ud.localpath, ud.md5_name, md5data)
+
+ if ud.sha256_expected == None:
+ logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
+ 'SRC_URI[%s] = "%s"',
+ ud.localpath, ud.sha256_name, sha256data)
md5mismatch = False
sha256mismatch = False
@@ -574,10 +575,14 @@ class FetchData(object):
self.sha256_name = "sha256sum"
if self.md5_name in self.parm:
self.md5_expected = self.parm[self.md5_name]
+ elif self.type not in ["http", "https", "ftp", "ftps"]:
+ self.md5_expected = None
else:
self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
if self.sha256_name in self.parm:
self.sha256_expected = self.parm[self.sha256_name]
+ elif self.type not in ["http", "https", "ftp", "ftps"]:
+ self.sha256_expected = None
else:
self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
@@ -656,6 +661,19 @@ class FetchMethod(object):
"""
return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile)
+ def supports_checksum(self, urldata):
+ """
+ Is localpath something that can be represented by a checksum?
+ """
+ return True
+
+ def recommends_checksum(self, urldata):
+ """
+ Is the backend on where checksumming is recommended (should warnings
+ by displayed if there is no checksum)?
+ """
+ return False
+
def _strip_leading_slashes(self, relpath):
"""
Remove leading slash as os.path.join can't cope
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 61fdc4b..8cea7dd 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -82,6 +82,9 @@ class Git(FetchMethod):
"""
return ud.type in ['git']
+ def supports_checksum(self, urldata):
+ return False
+
def urldata_init(self, ud, d):
"""
init git specific variable within url data
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 91ac15f..8d6434a 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -69,6 +69,9 @@ class SSH(FetchMethod):
def supports(self, url, urldata, d):
return __pattern__.match(url) != None
+ def supports_checksum(self, urldata):
+ return False
+
def localpath(self, url, urldata, d):
m = __pattern__.match(urldata.url)
path = m.group('path')
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index d3be069..30a52fa 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -45,6 +45,9 @@ class Wget(FetchMethod):
"""
return ud.type in ['http', 'https', 'ftp']
+ def recommends_checksum(self, urldata):
+ return True
+
def urldata_init(self, ud, d):
ud.basename = os.path.basename(ud.path)
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* Re: [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums
2012-07-03 22:32 ` [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums Khem Raj
@ 2012-07-30 22:52 ` McClintock Matthew-B29882
2012-07-31 21:39 ` Scott Garman
0 siblings, 1 reply; 20+ messages in thread
From: McClintock Matthew-B29882 @ 2012-07-30 22:52 UTC (permalink / raw)
To: Khem Raj; +Cc: poky@yoctoproject.org
On Tue, Jul 3, 2012 at 5:32 PM, Khem Raj <raj.khem@gmail.com> wrote:
> From: Richard Purdie <richard.purdie@linuxfoundation.org>
>
> There were some hardcoded behaviours in the system for which backends
> support checksums verses which backends recommend them verses which
> don't recommend them.
>
> This moves the functionality into specific fetchers and then makes the
> general code generic. This cleans up the codebase and fixes some corner
> cases such as trying to checksum directories returned by the git fetcher.
>
> (Bitbake rev: ef6d268f7b8527541a7fb044cf95a973be4097f4)
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This patch is causing the following issue in the denzil branch of poky:
NOTE: package libpam-1.1.5-r3: task do_fetch: Started
ERROR: Error executing a python function in
/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb:
IOError: [Errno 2] No such file or directory:
'/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam/./pam.d/*'
ERROR: The stack trace of python calls that resulted in this
exception/failure was:
ERROR: File "base_do_fetch", line 18, in <module>
ERROR:
ERROR: File "base_do_fetch", line 13, in base_do_fetch
ERROR:
ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
line 1037, in download
ERROR: update_stamp(u, ud, self.d)
ERROR:
ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
line 335, in update_stamp
ERROR: verify_checksum(u, ud, d)
ERROR:
ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
line 277, in verify_checksum
ERROR: md5data = bb.utils.md5_file(ud.localpath)
ERROR:
ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/utils.py",
line 482, in md5_file
ERROR: for line in open(filename):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0014: except bb.fetch2.BBFetchException, e:
ERROR: 0015: raise bb.build.FuncFailed(e)
ERROR: 0016:
ERROR: 0017:
ERROR: *** 0018:base_do_fetch(d)
ERROR: 0019:
ERROR: (file: 'base_do_fetch', lineno: 18, function: <module>)
ERROR: 0009: bb.data.update_data(localdata)
ERROR: 0010:
ERROR: 0011: try:
ERROR: 0012: fetcher = bb.fetch2.Fetch(src_uri, localdata)
ERROR: *** 0013: fetcher.download()
ERROR: 0014: except bb.fetch2.BBFetchException, e:
ERROR: 0015: raise bb.build.FuncFailed(e)
ERROR: 0016:
ERROR: 0017:
ERROR: (file: 'base_do_fetch', lineno: 13, function: base_do_fetch)
ERROR: Function failed: base_do_fetch
ERROR: Logfile of failure stored in:
/local/home/mattsm/git/poky-upstream/build-denzil/tmp/work/ppce5500-poky-linux/libpam-1.1.5-r3/temp/log.do_fetch.9368
NOTE: package libpam-1.1.5-r3: task do_fetch: Failed
ERROR: Task 0 (/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb,
do_fetch) failed with exit code '1'
-M
>
> Conflicts:
>
> bitbake/lib/bb/fetch2/__init__.py
>
> Signed-off-by: Khem Raj <kraj@juniper.net>
> ---
> bitbake/lib/bb/fetch2/__init__.py | 56 ++++++++++++++++++++++++-------------
> bitbake/lib/bb/fetch2/git.py | 3 ++
> bitbake/lib/bb/fetch2/ssh.py | 3 ++
> bitbake/lib/bb/fetch2/wget.py | 3 ++
> 4 files changed, 46 insertions(+), 19 deletions(-)
>
> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
> index 414cc2b..ec043a8 100644
> --- a/bitbake/lib/bb/fetch2/__init__.py
> +++ b/bitbake/lib/bb/fetch2/__init__.py
> @@ -268,30 +268,31 @@ def verify_checksum(u, ud, d):
> matched
> """
>
> - if not ud.type in ["http", "https", "ftp", "ftps"]:
> + if not ud.method.supports_checksum(ud):
> return
>
> md5data = bb.utils.md5_file(ud.localpath)
> sha256data = bb.utils.sha256_file(ud.localpath)
>
> - # If strict checking enabled and neither sum defined, raise error
> - strict = d.getVar("BB_STRICT_CHECKSUM", True) or None
> - if (strict and ud.md5_expected == None and ud.sha256_expected == None):
> - raise FetchError('No checksum specified for %s, please add at least one to the recipe:\n'
> - 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
> - (ud.localpath, ud.md5_name, md5data,
> - ud.sha256_name, sha256data), u)
> -
> - # Log missing sums so user can more easily add them
> - if ud.md5_expected == None:
> - logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
> - 'SRC_URI[%s] = "%s"',
> - ud.localpath, ud.md5_name, md5data)
> -
> - if ud.sha256_expected == None:
> - logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
> - 'SRC_URI[%s] = "%s"',
> - ud.localpath, ud.sha256_name, sha256data)
> + if ud.method.recommends_checksum(ud):
> + # If strict checking enabled and neither sum defined, raise error
> + strict = d.getVar("BB_STRICT_CHECKSUM", True) or None
> + if (strict and ud.md5_expected == None and ud.sha256_expected == None):
> + raise FetchError('No checksum specified for %s, please add at least one to the recipe:\n'
> + 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
> + (ud.localpath, ud.md5_name, md5data,
> + ud.sha256_name, sha256data), u)
> +
> + # Log missing sums so user can more easily add them
> + if ud.md5_expected == None:
> + logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
> + 'SRC_URI[%s] = "%s"',
> + ud.localpath, ud.md5_name, md5data)
> +
> + if ud.sha256_expected == None:
> + logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
> + 'SRC_URI[%s] = "%s"',
> + ud.localpath, ud.sha256_name, sha256data)
>
> md5mismatch = False
> sha256mismatch = False
> @@ -574,10 +575,14 @@ class FetchData(object):
> self.sha256_name = "sha256sum"
> if self.md5_name in self.parm:
> self.md5_expected = self.parm[self.md5_name]
> + elif self.type not in ["http", "https", "ftp", "ftps"]:
> + self.md5_expected = None
> else:
> self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
> if self.sha256_name in self.parm:
> self.sha256_expected = self.parm[self.sha256_name]
> + elif self.type not in ["http", "https", "ftp", "ftps"]:
> + self.sha256_expected = None
> else:
> self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
>
> @@ -656,6 +661,19 @@ class FetchMethod(object):
> """
> return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile)
>
> + def supports_checksum(self, urldata):
> + """
> + Is localpath something that can be represented by a checksum?
> + """
> + return True
> +
> + def recommends_checksum(self, urldata):
> + """
> + Is the backend on where checksumming is recommended (should warnings
> + by displayed if there is no checksum)?
> + """
> + return False
> +
> def _strip_leading_slashes(self, relpath):
> """
> Remove leading slash as os.path.join can't cope
> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> index 61fdc4b..8cea7dd 100644
> --- a/bitbake/lib/bb/fetch2/git.py
> +++ b/bitbake/lib/bb/fetch2/git.py
> @@ -82,6 +82,9 @@ class Git(FetchMethod):
> """
> return ud.type in ['git']
>
> + def supports_checksum(self, urldata):
> + return False
> +
> def urldata_init(self, ud, d):
> """
> init git specific variable within url data
> diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
> index 91ac15f..8d6434a 100644
> --- a/bitbake/lib/bb/fetch2/ssh.py
> +++ b/bitbake/lib/bb/fetch2/ssh.py
> @@ -69,6 +69,9 @@ class SSH(FetchMethod):
> def supports(self, url, urldata, d):
> return __pattern__.match(url) != None
>
> + def supports_checksum(self, urldata):
> + return False
> +
> def localpath(self, url, urldata, d):
> m = __pattern__.match(urldata.url)
> path = m.group('path')
> diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
> index d3be069..30a52fa 100644
> --- a/bitbake/lib/bb/fetch2/wget.py
> +++ b/bitbake/lib/bb/fetch2/wget.py
> @@ -45,6 +45,9 @@ class Wget(FetchMethod):
> """
> return ud.type in ['http', 'https', 'ftp']
>
> + def recommends_checksum(self, urldata):
> + return True
> +
> def urldata_init(self, ud, d):
>
> ud.basename = os.path.basename(ud.path)
> --
> 1.7.9.5
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums
2012-07-30 22:52 ` McClintock Matthew-B29882
@ 2012-07-31 21:39 ` Scott Garman
2012-07-31 21:43 ` McClintock Matthew-B29882
0 siblings, 1 reply; 20+ messages in thread
From: Scott Garman @ 2012-07-31 21:39 UTC (permalink / raw)
To: McClintock Matthew-B29882; +Cc: poky@yoctoproject.org
On 07/30/2012 03:52 PM, McClintock Matthew-B29882 wrote:
> On Tue, Jul 3, 2012 at 5:32 PM, Khem Raj <raj.khem@gmail.com> wrote:
>> From: Richard Purdie <richard.purdie@linuxfoundation.org>
>>
>> There were some hardcoded behaviours in the system for which backends
>> support checksums verses which backends recommend them verses which
>> don't recommend them.
>>
>> This moves the functionality into specific fetchers and then makes the
>> general code generic. This cleans up the codebase and fixes some corner
>> cases such as trying to checksum directories returned by the git fetcher.
>>
>> (Bitbake rev: ef6d268f7b8527541a7fb044cf95a973be4097f4)
>>
>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>
> This patch is causing the following issue in the denzil branch of poky:
>
> NOTE: package libpam-1.1.5-r3: task do_fetch: Started
> ERROR: Error executing a python function in
> /local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb:
> IOError: [Errno 2] No such file or directory:
> '/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam/./pam.d/*'
Thanks for testing and reporting this, Matthew. I have rebased my
sgarman/denzil-1.2.2 branch to remove this commit.
I'm going through my denzil queue of patches right now and hope to be on
top of everything by EOD - I will send out an overall status email then
in which I will also announce renaming my branch to simply
sgarman/denzil-next.
Scott
> ERROR: The stack trace of python calls that resulted in this
> exception/failure was:
> ERROR: File "base_do_fetch", line 18, in <module>
> ERROR:
> ERROR: File "base_do_fetch", line 13, in base_do_fetch
> ERROR:
> ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
> line 1037, in download
> ERROR: update_stamp(u, ud, self.d)
> ERROR:
> ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
> line 335, in update_stamp
> ERROR: verify_checksum(u, ud, d)
> ERROR:
> ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
> line 277, in verify_checksum
> ERROR: md5data = bb.utils.md5_file(ud.localpath)
> ERROR:
> ERROR: File "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/utils.py",
> line 482, in md5_file
> ERROR: for line in open(filename):
> ERROR:
> ERROR: The code that was being executed was:
> ERROR: 0014: except bb.fetch2.BBFetchException, e:
> ERROR: 0015: raise bb.build.FuncFailed(e)
> ERROR: 0016:
> ERROR: 0017:
> ERROR: *** 0018:base_do_fetch(d)
> ERROR: 0019:
> ERROR: (file: 'base_do_fetch', lineno: 18, function: <module>)
> ERROR: 0009: bb.data.update_data(localdata)
> ERROR: 0010:
> ERROR: 0011: try:
> ERROR: 0012: fetcher = bb.fetch2.Fetch(src_uri, localdata)
> ERROR: *** 0013: fetcher.download()
> ERROR: 0014: except bb.fetch2.BBFetchException, e:
> ERROR: 0015: raise bb.build.FuncFailed(e)
> ERROR: 0016:
> ERROR: 0017:
> ERROR: (file: 'base_do_fetch', lineno: 13, function: base_do_fetch)
> ERROR: Function failed: base_do_fetch
> ERROR: Logfile of failure stored in:
> /local/home/mattsm/git/poky-upstream/build-denzil/tmp/work/ppce5500-poky-linux/libpam-1.1.5-r3/temp/log.do_fetch.9368
> NOTE: package libpam-1.1.5-r3: task do_fetch: Failed
> ERROR: Task 0 (/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb,
> do_fetch) failed with exit code '1'
>
> -M
--
Scott Garman
Embedded Linux Engineer - Yocto Project
Intel Open Source Technology Center
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums
2012-07-31 21:39 ` Scott Garman
@ 2012-07-31 21:43 ` McClintock Matthew-B29882
2012-07-31 22:03 ` Scott Garman
0 siblings, 1 reply; 20+ messages in thread
From: McClintock Matthew-B29882 @ 2012-07-31 21:43 UTC (permalink / raw)
To: Scott Garman; +Cc: McClintock Matthew-B29882, poky@yoctoproject.org
On Tue, Jul 31, 2012 at 4:39 PM, Scott Garman <scott.a.garman@intel.com> wrote:
> On 07/30/2012 03:52 PM, McClintock Matthew-B29882 wrote:
>>
>> On Tue, Jul 3, 2012 at 5:32 PM, Khem Raj <raj.khem@gmail.com> wrote:
>>>
>>> From: Richard Purdie <richard.purdie@linuxfoundation.org>
>>>
>>> There were some hardcoded behaviours in the system for which backends
>>> support checksums verses which backends recommend them verses which
>>> don't recommend them.
>>>
>>> This moves the functionality into specific fetchers and then makes the
>>> general code generic. This cleans up the codebase and fixes some corner
>>> cases such as trying to checksum directories returned by the git fetcher.
>>>
>>> (Bitbake rev: ef6d268f7b8527541a7fb044cf95a973be4097f4)
>>>
>>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>>
>>
>> This patch is causing the following issue in the denzil branch of poky:
>>
>> NOTE: package libpam-1.1.5-r3: task do_fetch: Started
>> ERROR: Error executing a python function in
>>
>> /local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb:
>> IOError: [Errno 2] No such file or directory:
>>
>> '/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam/./pam.d/*'
>
>
> Thanks for testing and reporting this, Matthew. I have rebased my
> sgarman/denzil-1.2.2 branch to remove this commit.
>
> I'm going through my denzil queue of patches right now and hope to be on top
> of everything by EOD - I will send out an overall status email then in which
> I will also announce renaming my branch to simply sgarman/denzil-next.
Scott,
I have a few more patches on my tree as well:
http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=mattsm/denzil
I will rebase once you are done.
-M
>
> Scott
>
>
>> ERROR: The stack trace of python calls that resulted in this
>> exception/failure was:
>> ERROR: File "base_do_fetch", line 18, in <module>
>> ERROR:
>> ERROR: File "base_do_fetch", line 13, in base_do_fetch
>> ERROR:
>> ERROR: File
>> "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
>> line 1037, in download
>> ERROR: update_stamp(u, ud, self.d)
>> ERROR:
>> ERROR: File
>> "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
>> line 335, in update_stamp
>> ERROR: verify_checksum(u, ud, d)
>> ERROR:
>> ERROR: File
>> "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/fetch2/__init__.py",
>> line 277, in verify_checksum
>> ERROR: md5data = bb.utils.md5_file(ud.localpath)
>> ERROR:
>> ERROR: File
>> "/local/home/mattsm/git/poky-upstream/bitbake/lib/bb/utils.py",
>> line 482, in md5_file
>> ERROR: for line in open(filename):
>> ERROR:
>> ERROR: The code that was being executed was:
>> ERROR: 0014: except bb.fetch2.BBFetchException, e:
>> ERROR: 0015: raise bb.build.FuncFailed(e)
>> ERROR: 0016:
>> ERROR: 0017:
>> ERROR: *** 0018:base_do_fetch(d)
>> ERROR: 0019:
>> ERROR: (file: 'base_do_fetch', lineno: 18, function: <module>)
>> ERROR: 0009: bb.data.update_data(localdata)
>> ERROR: 0010:
>> ERROR: 0011: try:
>> ERROR: 0012: fetcher = bb.fetch2.Fetch(src_uri, localdata)
>> ERROR: *** 0013: fetcher.download()
>> ERROR: 0014: except bb.fetch2.BBFetchException, e:
>> ERROR: 0015: raise bb.build.FuncFailed(e)
>> ERROR: 0016:
>> ERROR: 0017:
>> ERROR: (file: 'base_do_fetch', lineno: 13, function: base_do_fetch)
>> ERROR: Function failed: base_do_fetch
>> ERROR: Logfile of failure stored in:
>>
>> /local/home/mattsm/git/poky-upstream/build-denzil/tmp/work/ppce5500-poky-linux/libpam-1.1.5-r3/temp/log.do_fetch.9368
>> NOTE: package libpam-1.1.5-r3: task do_fetch: Failed
>> ERROR: Task 0
>> (/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb,
>> do_fetch) failed with exit code '1'
>>
>> -M
>
>
>
> --
> Scott Garman
> Embedded Linux Engineer - Yocto Project
> Intel Open Source Technology Center
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums
2012-07-31 21:43 ` McClintock Matthew-B29882
@ 2012-07-31 22:03 ` Scott Garman
2012-07-31 22:06 ` McClintock Matthew-B29882
0 siblings, 1 reply; 20+ messages in thread
From: Scott Garman @ 2012-07-31 22:03 UTC (permalink / raw)
To: McClintock Matthew-B29882; +Cc: poky@yoctoproject.org
On 07/31/2012 02:43 PM, McClintock Matthew-B29882 wrote:
> On Tue, Jul 31, 2012 at 4:39 PM, Scott Garman <scott.a.garman@intel.com> wrote:
>> On 07/30/2012 03:52 PM, McClintock Matthew-B29882 wrote:
>>>
>>> On Tue, Jul 3, 2012 at 5:32 PM, Khem Raj <raj.khem@gmail.com> wrote:
>>>>
>>>> From: Richard Purdie <richard.purdie@linuxfoundation.org>
>>>>
>>>> There were some hardcoded behaviours in the system for which backends
>>>> support checksums verses which backends recommend them verses which
>>>> don't recommend them.
>>>>
>>>> This moves the functionality into specific fetchers and then makes the
>>>> general code generic. This cleans up the codebase and fixes some corner
>>>> cases such as trying to checksum directories returned by the git fetcher.
>>>>
>>>> (Bitbake rev: ef6d268f7b8527541a7fb044cf95a973be4097f4)
>>>>
>>>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>>>
>>>
>>> This patch is causing the following issue in the denzil branch of poky:
>>>
>>> NOTE: package libpam-1.1.5-r3: task do_fetch: Started
>>> ERROR: Error executing a python function in
>>>
>>> /local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb:
>>> IOError: [Errno 2] No such file or directory:
>>>
>>> '/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam/./pam.d/*'
>>
>>
>> Thanks for testing and reporting this, Matthew. I have rebased my
>> sgarman/denzil-1.2.2 branch to remove this commit.
>>
>> I'm going through my denzil queue of patches right now and hope to be on top
>> of everything by EOD - I will send out an overall status email then in which
>> I will also announce renaming my branch to simply sgarman/denzil-next.
>
> Scott,
>
> I have a few more patches on my tree as well:
>
> http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=mattsm/denzil
>
> I will rebase once you are done.
I'm about to merge most of these, but the two recent ones with
"NEEDSTOUPSTREAM" git subjects are not in proper format and will need to
be redone.
Scott
--
Scott Garman
Embedded Linux Engineer - Yocto Project
Intel Open Source Technology Center
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums
2012-07-31 22:03 ` Scott Garman
@ 2012-07-31 22:06 ` McClintock Matthew-B29882
0 siblings, 0 replies; 20+ messages in thread
From: McClintock Matthew-B29882 @ 2012-07-31 22:06 UTC (permalink / raw)
To: Scott Garman; +Cc: McClintock Matthew-B29882, poky@yoctoproject.org
On Tue, Jul 31, 2012 at 5:03 PM, Scott Garman <scott.a.garman@intel.com> wrote:
> On 07/31/2012 02:43 PM, McClintock Matthew-B29882 wrote:
>>
>> On Tue, Jul 31, 2012 at 4:39 PM, Scott Garman <scott.a.garman@intel.com>
>> wrote:
>>>
>>> On 07/30/2012 03:52 PM, McClintock Matthew-B29882 wrote:
>>>>
>>>>
>>>> On Tue, Jul 3, 2012 at 5:32 PM, Khem Raj <raj.khem@gmail.com> wrote:
>>>>>
>>>>>
>>>>> From: Richard Purdie <richard.purdie@linuxfoundation.org>
>>>>>
>>>>> There were some hardcoded behaviours in the system for which backends
>>>>> support checksums verses which backends recommend them verses which
>>>>> don't recommend them.
>>>>>
>>>>> This moves the functionality into specific fetchers and then makes the
>>>>> general code generic. This cleans up the codebase and fixes some corner
>>>>> cases such as trying to checksum directories returned by the git
>>>>> fetcher.
>>>>>
>>>>> (Bitbake rev: ef6d268f7b8527541a7fb044cf95a973be4097f4)
>>>>>
>>>>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>>>>
>>>>
>>>>
>>>> This patch is causing the following issue in the denzil branch of poky:
>>>>
>>>> NOTE: package libpam-1.1.5-r3: task do_fetch: Started
>>>> ERROR: Error executing a python function in
>>>>
>>>>
>>>> /local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam_1.1.5.bb:
>>>> IOError: [Errno 2] No such file or directory:
>>>>
>>>>
>>>> '/local/home/mattsm/git/poky-upstream/meta/recipes-extended/pam/libpam/./pam.d/*'
>>>
>>>
>>>
>>> Thanks for testing and reporting this, Matthew. I have rebased my
>>> sgarman/denzil-1.2.2 branch to remove this commit.
>>>
>>> I'm going through my denzil queue of patches right now and hope to be on
>>> top
>>> of everything by EOD - I will send out an overall status email then in
>>> which
>>> I will also announce renaming my branch to simply sgarman/denzil-next.
>>
>>
>> Scott,
>>
>> I have a few more patches on my tree as well:
>>
>>
>> http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=mattsm/denzil
>>
>> I will rebase once you are done.
>
>
> I'm about to merge most of these, but the two recent ones with
> "NEEDSTOUPSTREAM" git subjects are not in proper format and will need to be
> redone.
Sorry, those are pending upstream patches being accepted. They can be
a) ignored until I can cherry-pick from upstream or b) I can remove
them for you.
-M
>
> Scott
>
>
> --
> Scott Garman
> Embedded Linux Engineer - Yocto Project
> Intel Open Source Technology Center
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
^ permalink raw reply [flat|nested] 20+ messages in thread
* [denzil][PATCH 2/9] bitbake: fetch2/__init__: Make it clearer when uri_replace doesn't return a match
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 3/9] bitbake: fetch2: Fix error handling in uri_replace() Khem Raj
` (8 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
(From Poky rev: dc9976331c5cbb0983adb54f6deb97b9203bacbc)
(Bitbake rev: eb96609864dec95a516e6e687dd6a2f31d523acf)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index ec043a8..b19c84a 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -200,8 +200,10 @@ def uri_replace(ud, uri_find, uri_replace, d):
if basename and not result_decoded[loc].endswith(basename):
result_decoded[loc] = os.path.join(result_decoded[loc], basename)
else:
- return ud.url
+ return None
result = encodeurl(result_decoded)
+ if result == ud.url:
+ return None
logger.debug(2, "For url %s returning %s" % (ud.url, result))
return result
@@ -463,7 +465,7 @@ def try_mirrors(d, origud, mirrors, check = False):
except ValueError:
continue
newuri = uri_replace(origud, find, replace, ld)
- if newuri == origud.url:
+ if not newuri:
continue
try:
ud = FetchData(newuri, ld)
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 3/9] bitbake: fetch2: Fix error handling in uri_replace()
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 1/9] bitbake/fetch: Spell out which fetcher backends support and recommend checksums Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 2/9] bitbake: fetch2/__init__: Make it clearer when uri_replace doesn't return a match Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 4/9] bitbake: fetch2: Only cache data if fn is set, its pointless caching it against a None value Khem Raj
` (7 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
(From Poky rev: 1bfba28a583cb167f60e05ecdf34d0786dc1eec5)
(Bitbake rev: aa7467a764ddcbc7d65af99e88cf093b6ec6d24e)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index b19c84a..3e96282 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -174,7 +174,8 @@ def encodeurl(decoded):
def uri_replace(ud, uri_find, uri_replace, d):
if not ud.url or not uri_find or not uri_replace:
- logger.debug(1, "uri_replace: passed an undefined value, not replacing")
+ logger.error("uri_replace: passed an undefined value, not replacing")
+ return None
uri_decoded = list(decodeurl(ud.url))
uri_find_decoded = list(decodeurl(uri_find))
uri_replace_decoded = list(decodeurl(uri_replace))
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 4/9] bitbake: fetch2: Only cache data if fn is set, its pointless caching it against a None value
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (2 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 3/9] bitbake: fetch2: Fix error handling in uri_replace() Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 5/9] bitbake: fetch2: Ensure when downloading we are consistently in the same directory Khem Raj
` (6 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
(From Poky rev: c2df30bf6d1f8c263a38c45866936c1bf496ece5)
(Bitbake rev: f4b59cc6e1c3ddc168a1678ce39ff402ea1ff4cc)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 3e96282..4b32599 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -922,14 +922,14 @@ class Fetch(object):
self.ud = {}
fn = d.getVar('FILE', True)
- if cache and fn in urldata_cache:
+ if cache and fn and fn in urldata_cache:
self.ud = urldata_cache[fn]
for url in urls:
if url not in self.ud:
self.ud[url] = FetchData(url, d)
- if cache:
+ if fn and cache:
urldata_cache[fn] = self.ud
def localpath(self, url):
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 5/9] bitbake: fetch2: Ensure when downloading we are consistently in the same directory
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (3 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 4/9] bitbake: fetch2: Only cache data if fn is set, its pointless caching it against a None value Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 6/9] bitbake: fetch2: Split try_mirrors into two parts Khem Raj
` (5 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
This assists with build reproducuility. It also avoids errors if cwd
happens not to exist when we call into the fetcher. That situation
would be unusual but I hit it with the unit tests.
(From Poky rev: 86517af9e066c2da1d580fa66b7c7f0340f3403e)
(Bitbake rev: b886c6c15a58643e06ca5ad7a3ff1f7766e4f48c)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 4b32599..33e07bb 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -472,6 +472,8 @@ def try_mirrors(d, origud, mirrors, check = False):
ud = FetchData(newuri, ld)
ud.setup_localpath(ld)
+ os.chdir(ld.getVar("DL_DIR", True))
+
if check:
found = ud.method.checkstatus(newuri, ud, ld)
if found:
@@ -983,6 +985,8 @@ class Fetch(object):
if premirroronly:
self.d.setVar("BB_NO_NETWORK", "1")
+ os.chdir(self.d.getVar("DL_DIR", True))
+
firsterr = None
if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(u, ud, self.d)):
try:
@@ -1042,7 +1046,7 @@ class Fetch(object):
except:
# Finally, try checking uri, u, from MIRRORS
mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
- ret = try_mirrors (self.d, ud, mirrors, True)
+ ret = try_mirrors(self.d, ud, mirrors, True)
if not ret:
raise FetchError("URL %s doesn't work" % u, u)
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 6/9] bitbake: fetch2: Split try_mirrors into two parts
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (4 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 5/9] bitbake: fetch2: Ensure when downloading we are consistently in the same directory Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 7/9] bitbake: fetch2: Explicitly check for mirror tarballs in mirror handling code Khem Raj
` (4 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
There are no functionality changes in this change
(From Poky rev: d222ebb7c75d74fde4fd04ea6feb27e10a862bae)
(Bitbake rev: db62e109cc36380ff8b8918628c9dea14ac9afbc)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Conflicts:
bitbake/lib/bb/fetch2/__init__.py
Signed-off-by: Khem Raj <kraj@juniper.net>
---
bitbake/lib/bb/fetch2/__init__.py | 108 ++++++++++++++++++++-----------------
1 file changed, 59 insertions(+), 49 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 33e07bb..7c05c61 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -450,6 +450,60 @@ def check_network_access(d, info = "", url = None):
else:
logger.debug(1, "Fetcher accessed the network with the command %s" % info)
+def try_mirror_url(newuri, origud, ud, ld, check = False):
+ # Return of None or a value means we're finished
+ # False means try another url
+ try:
+ if check:
+ found = ud.method.checkstatus(newuri, ud, ld)
+ if found:
+ return found
+ return False
+
+ os.chdir(ld.getVar("DL_DIR", True))
+
+ if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld):
+ ud.method.download(newuri, ud, ld)
+ if hasattr(ud.method,"build_mirror_data"):
+ ud.method.build_mirror_data(newuri, ud, ld)
+
+ if not ud.localpath or not os.path.exists(ud.localpath):
+ return False
+
+ if ud.localpath == origud.localpath:
+ return ud.localpath
+
+ # We may be obtaining a mirror tarball which needs further processing by the real fetcher
+ # If that tarball is a local file:// we need to provide a symlink to it
+ dldir = ld.getVar("DL_DIR", True)
+ if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+ open(ud.donestamp, 'w').close()
+ dest = os.path.join(dldir, os.path.basename(ud.localpath))
+ if not os.path.exists(dest):
+ os.symlink(ud.localpath, dest)
+ return None
+ # Otherwise the result is a local file:// and we symlink to it
+ if not os.path.exists(origud.localpath):
+ os.symlink(ud.localpath, origud.localpath)
+ update_stamp(newuri, origud, ld)
+ return ud.localpath
+
+ except bb.fetch2.NetworkAccess:
+ raise
+
+ except bb.fetch2.BBFetchException as e:
+ if isinstance(e, ChecksumError):
+ logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (newuri, origud.url))
+ logger.warn(str(e))
+ else:
+ logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+ logger.debug(1, str(e))
+ try:
+ ud.method.clean(ud, ld)
+ except UnboundLocalError:
+ pass
+ return False
+
def try_mirrors(d, origud, mirrors, check = False):
"""
Try to use a mirrored version of the sources.
@@ -468,56 +522,12 @@ def try_mirrors(d, origud, mirrors, check = False):
newuri = uri_replace(origud, find, replace, ld)
if not newuri:
continue
- try:
- ud = FetchData(newuri, ld)
- ud.setup_localpath(ld)
+ ud = FetchData(newuri, ld)
+ ud.setup_localpath(ld)
- os.chdir(ld.getVar("DL_DIR", True))
-
- if check:
- found = ud.method.checkstatus(newuri, ud, ld)
- if found:
- return found
- continue
-
- if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld):
- ud.method.download(newuri, ud, ld)
- if os.path.exists(ud.localpath):
- open(ud.donestamp, 'w').close()
- if hasattr(ud.method,"build_mirror_data"):
- ud.method.build_mirror_data(newuri, ud, ld)
-
- if not ud.localpath or not os.path.exists(ud.localpath):
- continue
-
- if ud.localpath == origud.localpath:
- return ud.localpath
-
- # We may be obtaining a mirror tarball which needs further processing by the real fetcher
- # If that tarball is a local file:// we need to provide a symlink to it
- dldir = ld.getVar("DL_DIR", True)
- if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
- dest = os.path.join(dldir, os.path.basename(ud.localpath))
- if not os.path.exists(dest):
- os.symlink(ud.localpath, dest)
- return None
- # Otherwise the result is a local file:// and we symlink to it
- if not os.path.exists(origud.localpath):
- os.symlink(ud.localpath, origud.localpath)
- return ud.localpath
-
- except bb.fetch2.NetworkAccess:
- raise
-
- except bb.fetch2.BBFetchException as e:
- logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
- logger.debug(1, str(e))
- try:
- if os.path.isfile(ud.localpath):
- bb.utils.remove(ud.localpath)
- except UnboundLocalError:
- pass
- continue
+ ret = try_mirror_url(newuri, origud, ud, ld, check)
+ if ret != False:
+ return ret
return None
def srcrev_internal_helper(ud, d, name):
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 7/9] bitbake: fetch2: Explicitly check for mirror tarballs in mirror handling code
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (5 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 6/9] bitbake: fetch2: Split try_mirrors into two parts Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 8/9] bitbake: fetch2: Improve mirror looping to consider more cases Khem Raj
` (3 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
With support for things like git:// -> git:// urls, we need to be
more explicity about the mirrortarball check since we need to fall
through to the following code in other cases.
(From Poky rev: 28e858cd6f7509468ef3e527a86820b9e06044db)
(Bitbake rev: a2459f5ca2f517964287f9a7c666a6856434e631)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 7c05c61..b3b3b48 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -476,7 +476,8 @@ def try_mirror_url(newuri, origud, ud, ld, check = False):
# We may be obtaining a mirror tarball which needs further processing by the real fetcher
# If that tarball is a local file:// we need to provide a symlink to it
dldir = ld.getVar("DL_DIR", True)
- if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+ if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
+ and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
open(ud.donestamp, 'w').close()
dest = os.path.join(dldir, os.path.basename(ud.localpath))
if not os.path.exists(dest):
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 8/9] bitbake: fetch2: Improve mirror looping to consider more cases
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (6 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 7/9] bitbake: fetch2: Explicitly check for mirror tarballs in mirror handling code Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-03 22:32 ` [denzil][PATCH 9/9] bitbake: fetch2: Handle errors orruring when building mirror urls Khem Raj
` (2 subsequent siblings)
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Currently we only consider one pass through the mirror list. This doesn't
catch cases where for example you might want to setup a mirror of a mirror
and allow multiple redirection. There is no reason we can't support this
and the patch loops through the list recursively now.
As a safeguard, it will stop if any duplicate urls are found, hence
avoiding circular dependency looping.
(From Poky rev: 0ec0a4412865e54495c07beea1ced8355da58073)
(Bitbake rev: e585730e931e6abdb15ba8a3849c5fd22845b891)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 41 ++++++++++++++++++++++++++-----------
1 file changed, 29 insertions(+), 12 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index b3b3b48..f6ead2f 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -450,6 +450,30 @@ def check_network_access(d, info = "", url = None):
else:
logger.debug(1, "Fetcher accessed the network with the command %s" % info)
+def build_mirroruris(origud, mirrors, ld):
+ uris = []
+ uds = []
+
+ def adduri(uri, ud, uris, uds):
+ for line in mirrors:
+ try:
+ (find, replace) = line
+ except ValueError:
+ continue
+ newuri = uri_replace(ud, find, replace, ld)
+ if not newuri or newuri in uris or newuri == origud.url:
+ continue
+ uris.append(newuri)
+ newud = FetchData(newuri, ld)
+ newud.setup_localpath(ld)
+ uds.append(newud)
+
+ adduri(newuri, newud, uris, uds)
+
+ adduri(None, origud, uris, uds)
+
+ return uris, uds
+
def try_mirror_url(newuri, origud, ud, ld, check = False):
# Return of None or a value means we're finished
# False means try another url
@@ -515,18 +539,11 @@ def try_mirrors(d, origud, mirrors, check = False):
mirrors is the list of mirrors we're going to try
"""
ld = d.createCopy()
- for line in mirrors:
- try:
- (find, replace) = line
- except ValueError:
- continue
- newuri = uri_replace(origud, find, replace, ld)
- if not newuri:
- continue
- ud = FetchData(newuri, ld)
- ud.setup_localpath(ld)
-
- ret = try_mirror_url(newuri, origud, ud, ld, check)
+
+ uris, uds = build_mirroruris(origud, mirrors, ld)
+
+ for index, uri in enumerate(uris):
+ ret = try_mirror_url(uri, origud, uds[index], ld, check)
if ret != False:
return ret
return None
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* [denzil][PATCH 9/9] bitbake: fetch2: Handle errors orruring when building mirror urls
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (7 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 8/9] bitbake: fetch2: Improve mirror looping to consider more cases Khem Raj
@ 2012-07-03 22:32 ` Khem Raj
2012-07-05 15:38 ` [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Scott Garman
2012-07-05 20:06 ` Scott Garman
10 siblings, 0 replies; 20+ messages in thread
From: Khem Raj @ 2012-07-03 22:32 UTC (permalink / raw)
To: poky
From: Richard Purdie <richard.purdie@linuxfoundation.org>
When we build the mirror urls, its possible an error will occur. If it
does, it should just mean we don't attempt this mirror url. The current
code actually aborts *all* the mirrors, not just the failed url.
This patch catches and logs the exception allowing things to continue.
(Bitbake rev: c35cbd1a1403865cf4f59ec88e1881669868103c)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/__init__.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index f6ead2f..9373c44 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -463,9 +463,18 @@ def build_mirroruris(origud, mirrors, ld):
newuri = uri_replace(ud, find, replace, ld)
if not newuri or newuri in uris or newuri == origud.url:
continue
+ try:
+ newud = FetchData(newuri, ld)
+ newud.setup_localpath(ld)
+ except bb.fetch2.BBFetchException as e:
+ logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+ logger.debug(1, str(e))
+ try:
+ ud.method.clean(ud, ld)
+ except UnboundLocalError:
+ pass
+ continue
uris.append(newuri)
- newud = FetchData(newuri, ld)
- newud.setup_localpath(ld)
uds.append(newud)
adduri(newuri, newud, uris, uds)
--
1.7.9.5
^ permalink raw reply related [flat|nested] 20+ messages in thread* Re: [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (8 preceding siblings ...)
2012-07-03 22:32 ` [denzil][PATCH 9/9] bitbake: fetch2: Handle errors orruring when building mirror urls Khem Raj
@ 2012-07-05 15:38 ` Scott Garman
2012-07-05 16:12 ` Richard Purdie
2012-07-05 20:06 ` Scott Garman
10 siblings, 1 reply; 20+ messages in thread
From: Scott Garman @ 2012-07-05 15:38 UTC (permalink / raw)
To: poky; +Cc: Purdie, Richard
On 07/03/2012 03:31 PM, Khem Raj wrote:
> On denzil bitbake was still trying to go out to internet for git repos
> when BB_NO_NETWORK was set. This set of backports fix that issue
> and now BB_NO_NETWORK works as expected
Hi Khem,
Denzil 1.2.1 is already in release candidate phase and going through QA.
I'm not sure if we can take this for 1.2.1, it may have to wait for
1.2.2. Richard should probably weigh in here.
Scott
>
> The following changes since commit 73cdebf60df225ee10f2eb215935be3b61e1b831:
>
> documentation/dev-manual/dev-manual-kernel-appendix.xml: Add note about conflict (2012-06-29 15:54:26 +0100)
>
> are available in the git repository at:
>
> git://git.yoctoproject.org/poky-contrib kraj/denzil-fetcher-update
> http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=kraj/denzil-fetcher-update
>
> Richard Purdie (9):
> bitbake/fetch: Spell out which fetcher backends support and recommend
> checksums
> bitbake: fetch2/__init__: Make it clearer when uri_replace doesn't
> return a match
> bitbake: fetch2: Fix error handling in uri_replace()
> bitbake: fetch2: Only cache data if fn is set, its pointless caching
> it against a None value
> bitbake: fetch2: Ensure when downloading we are consistently in the
> same directory
> bitbake: fetch2: Split try_mirrors into two parts
> bitbake: fetch2: Explicitly check for mirror tarballs in mirror
> handling code
> bitbake: fetch2: Improve mirror looping to consider more cases
> bitbake: fetch2: Handle errors orruring when building mirror urls
>
> bitbake/lib/bb/fetch2/__init__.py | 218 ++++++++++++++++++++++++-------------
> bitbake/lib/bb/fetch2/git.py | 3 +
> bitbake/lib/bb/fetch2/ssh.py | 3 +
> bitbake/lib/bb/fetch2/wget.py | 3 +
> 4 files changed, 149 insertions(+), 78 deletions(-)
>
--
Scott Garman
Embedded Linux Engineer - Yocto Project
Intel Open Source Technology Center
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil
2012-07-05 15:38 ` [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Scott Garman
@ 2012-07-05 16:12 ` Richard Purdie
2012-07-05 17:30 ` Khem Raj
0 siblings, 1 reply; 20+ messages in thread
From: Richard Purdie @ 2012-07-05 16:12 UTC (permalink / raw)
To: Scott Garman; +Cc: poky
On Thu, 2012-07-05 at 08:38 -0700, Scott Garman wrote:
> On 07/03/2012 03:31 PM, Khem Raj wrote:
> > On denzil bitbake was still trying to go out to internet for git repos
> > when BB_NO_NETWORK was set. This set of backports fix that issue
> > and now BB_NO_NETWORK works as expected
>
> Hi Khem,
>
> Denzil 1.2.1 is already in release candidate phase and going through QA.
> I'm not sure if we can take this for 1.2.1, it may have to wait for
> 1.2.2. Richard should probably weigh in here.
Its 1.2.2 material at this point but we should try and get things into
the branch sooner than later after release.
I am worried about this series introducing bugs. We did have some bugs
found in some of the series of fetcher changes I made in master.
Cheers,
Richard
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil
2012-07-05 16:12 ` Richard Purdie
@ 2012-07-05 17:30 ` Khem Raj
2012-07-05 17:43 ` McClintock Matthew-B29882
0 siblings, 1 reply; 20+ messages in thread
From: Khem Raj @ 2012-07-05 17:30 UTC (permalink / raw)
To: Richard Purdie; +Cc: poky@yoctoproject.org
[-- Attachment #1: Type: text/plain, Size: 1142 bytes --]
On Thursday, July 5, 2012, Richard Purdie wrote:
> On Thu, 2012-07-05 at 08:38 -0700, Scott Garman wrote:
> > On 07/03/2012 03:31 PM, Khem Raj wrote:
> > > On denzil bitbake was still trying to go out to internet for git repos
> > > when BB_NO_NETWORK was set. This set of backports fix that issue
> > > and now BB_NO_NETWORK works as expected
> >
> > Hi Khem,
> >
> > Denzil 1.2.1 is already in release candidate phase and going through QA.
> > I'm not sure if we can take this for 1.2.1, it may have to wait for
> > 1.2.2. Richard should probably weigh in here.
>
> Its 1.2.2 material at this point but we should try and get things into
> the branch sooner than later after release.
>
> I am worried about this series introducing bugs. We did have some bugs
> found in some of the series of fetcher changes I made in master.
Sure I am fine for it to get more testing before it goes in, my testing has
been good so far
>
> Cheers,
>
> Richard
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org <javascript:;>
> https://lists.yoctoproject.org/listinfo/poky
>
[-- Attachment #2: Type: text/html, Size: 1666 bytes --]
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil
2012-07-05 17:30 ` Khem Raj
@ 2012-07-05 17:43 ` McClintock Matthew-B29882
0 siblings, 0 replies; 20+ messages in thread
From: McClintock Matthew-B29882 @ 2012-07-05 17:43 UTC (permalink / raw)
To: Khem Raj; +Cc: poky@yoctoproject.org
On Thu, Jul 5, 2012 at 12:30 PM, Khem Raj <raj.khem@gmail.com> wrote:
>
>
> On Thursday, July 5, 2012, Richard Purdie wrote:
>>
>> On Thu, 2012-07-05 at 08:38 -0700, Scott Garman wrote:
>> > On 07/03/2012 03:31 PM, Khem Raj wrote:
>> > > On denzil bitbake was still trying to go out to internet for git repos
>> > > when BB_NO_NETWORK was set. This set of backports fix that issue
>> > > and now BB_NO_NETWORK works as expected
>> >
>> > Hi Khem,
>> >
>> > Denzil 1.2.1 is already in release candidate phase and going through QA.
>> > I'm not sure if we can take this for 1.2.1, it may have to wait for
>> > 1.2.2. Richard should probably weigh in here.
>>
>> Its 1.2.2 material at this point but we should try and get things into
>> the branch sooner than later after release.
>>
>> I am worried about this series introducing bugs. We did have some bugs
>> found in some of the series of fetcher changes I made in master.
>
>
> Sure I am fine for it to get more testing before it goes in, my testing has
> been good so far
I'm all for having these in 1.2.2 - I will be doing lots of testing as well.
-M
>>
>>
>> Cheers,
>>
>> Richard
>>
>> _______________________________________________
>> poky mailing list
>> poky@yoctoproject.org
>> https://lists.yoctoproject.org/listinfo/poky
>
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
>
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil
2012-07-03 22:31 [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Khem Raj
` (9 preceding siblings ...)
2012-07-05 15:38 ` [denzil][PATCH 0/9] bitbake fetch improvements backport to denzil Scott Garman
@ 2012-07-05 20:06 ` Scott Garman
10 siblings, 0 replies; 20+ messages in thread
From: Scott Garman @ 2012-07-05 20:06 UTC (permalink / raw)
To: poky
On 07/03/2012 03:31 PM, Khem Raj wrote:
> On denzil bitbake was still trying to go out to internet for git repos
> when BB_NO_NETWORK was set. This set of backports fix that issue
> and now BB_NO_NETWORK works as expected
Ok, I've now created a contrib branch to start tracking changes for
1.2.2, and have included this series:
http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=sgarman/denzil-next-1.2.2
The usual caveats apply - I can rebase this branch at any time, etc.
Scott
>
> The following changes since commit 73cdebf60df225ee10f2eb215935be3b61e1b831:
>
> documentation/dev-manual/dev-manual-kernel-appendix.xml: Add note about conflict (2012-06-29 15:54:26 +0100)
>
> are available in the git repository at:
>
> git://git.yoctoproject.org/poky-contrib kraj/denzil-fetcher-update
> http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=kraj/denzil-fetcher-update
>
> Richard Purdie (9):
> bitbake/fetch: Spell out which fetcher backends support and recommend
> checksums
> bitbake: fetch2/__init__: Make it clearer when uri_replace doesn't
> return a match
> bitbake: fetch2: Fix error handling in uri_replace()
> bitbake: fetch2: Only cache data if fn is set, its pointless caching
> it against a None value
> bitbake: fetch2: Ensure when downloading we are consistently in the
> same directory
> bitbake: fetch2: Split try_mirrors into two parts
> bitbake: fetch2: Explicitly check for mirror tarballs in mirror
> handling code
> bitbake: fetch2: Improve mirror looping to consider more cases
> bitbake: fetch2: Handle errors orruring when building mirror urls
>
> bitbake/lib/bb/fetch2/__init__.py | 218 ++++++++++++++++++++++++-------------
> bitbake/lib/bb/fetch2/git.py | 3 +
> bitbake/lib/bb/fetch2/ssh.py | 3 +
> bitbake/lib/bb/fetch2/wget.py | 3 +
> 4 files changed, 149 insertions(+), 78 deletions(-)
>
--
Scott Garman
Embedded Linux Engineer - Yocto Project
Intel Open Source Technology Center
^ permalink raw reply [flat|nested] 20+ messages in thread