[PATCH 0/1] SRC_URI checksum support v2

[PATCH 0/1] SRC_URI checksum support v2

[Yu Ke]

This patch add SRC_URI checksum support. With this patch,fetcher
can verify the MD5 and SHA256 checksum of download src with the
value defined in recipes SRC_URI.

This is the V2 patch. The major change compared to V1 is:
moving the logic from poky bbclass to bitbake fetcher, per
Richard's comment

Pull URL: git://git.pokylinux.org/poky-contrib.git
  Branch: kyu3/srcuri-v2
  Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=kyu3/srcuri-v2

Thanks,
    Yu Ke <ke.yu@intel.com>
---


Yu Ke (1):
  FetchData: add SRC_URI checksum

 bitbake/lib/bb/fetch/__init__.py |   36 ++++++++++++++++++++++++++++++++++++
 1 files changed, 36 insertions(+), 0 deletions(-)

[PATCH 1/1] FetchData: add SRC_URI checksum

[Yu Ke]

This patch add the per-recipe SRC_URI checksum verification.

- SRC_URI format
The format of SRC_URI checksum follow OE definition:

1. SRC_URI has single src
SRC_URI = "http://some.domain/file.tar.gz"
SRC_URI[md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"

2. SRC_URI has multiple src, every src need specify name
SRC_URI = "http://some.domain/file1.tar.gz;name=name1 \
           http://some.domain/file2.tar.gz;name=name2 "
SRC_URI[name1.md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[name1.sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"
SRC_URI[name2.md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[name2.sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"

- SRC_URI checking invocation:
the checksum checking is invoked in do_fetch phase,
so it can be invoked manually by

# bitbake -f -c fetch <recipe_name>

if recipes has no SRC_URI checksum item, bitbake will show warning:
"
WARNING: Missing SRC_URI checksum for xxxx.tar.gz, consider to add
SRC_URI[md5sum] = "5c69f16d452b0bb3d44bc3c10556c072"
SRC_URI[sha256sum] = "f4e0ada8d4d516bbb8600a3ee7d9046c9c79e38cd781df9ffc46d8f16acd1768"
"
thus recipe author can add it to recpie file after SRC_URI

Signed-off-by: Yu Ke <ke.yu@intel.com>
---
 bitbake/lib/bb/fetch/__init__.py |   36 ++++++++++++++++++++++++++++++++++++
 1 files changed, 36 insertions(+), 0 deletions(-)

diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index 50955f1..d3360b1 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -231,6 +231,29 @@ def removefile(f):
     except:
         pass
 
+def verify_checksum(d, ud):
+    """
+    verify the MD5 and SHA256 checksum for downloaded src
+    return True if matched, False if not
+    """
+    md5data = bb.utils.md5_file(ud.localpath)
+    sha256data = bb.utils.sha256_file(ud.localpath)
+
+    if (ud.md5_expected == None or ud.sha256_expected == None):
+        bb.warn("Missing SRC_URI checksum for %s, consider to add\n" \
+                "SRC_URI[%s] = \"%s\"\nSRC_URI[%s] = \"%s\"" \
+                % (ud.localpath, ud.md5_name, md5data, ud.sha256_name, sha256data))
+        # TODO: change to "return False" once all recpies has checksum entry
+        return True
+
+    if (ud.md5_expected != md5data or ud.sha256_expected != sha256data):
+        bb.error("The checksums for '%s' did not match." % ud.localpath)
+        bb.error("Expected MD5: '%s' and Got: '%s'" % (ud.md5_expected, md5data))
+        bb.error("Expected SHA256: '%s' and Got: '%s'" % (ud.sha256_expected, sha256data))
+        return False
+
+    return True
+
 def go(d, urls = None):
     """
     Fetch all urls
@@ -283,6 +306,9 @@ def go(d, urls = None):
         else:
             Fetch.write_md5sum(u, ud, d)
 
+        if not verify_checksum(d, ud):
+            raise FetchError("%s checksum mismatch." % u)
+
         bb.utils.unlockfile(lf)
 
 def checkstatus(d, urls = None):
@@ -502,6 +528,16 @@ class FetchData(object):
         if not self.pswd and "pswd" in self.parm:
             self.pswd = self.parm["pswd"]
         self.setup = False
+
+        if "name" in self.parm:
+            self.md5_name = "%s.md5sum" % self.parm["name"]
+            self.sha256_name = "%s.sha256sum" % self.parm["name"]
+        else:
+            self.md5_name = "md5sum"
+            self.sha256_name = "sha256sum"
+        self.md5_expected = bb.data.getVarFlag("SRC_URI", self.md5_name, d)
+        self.sha256_expected = bb.data.getVarFlag("SRC_URI", self.sha256_name, d)
+
         for m in methods:
             if m.supports(url, self, d):
                 self.method = m
-- 
1.7.0.4

Re: [PATCH 1/1] FetchData: add SRC_URI checksum

[Richard Purdie]

Hi Ke,

This patch looks good in general, just a couple of minor things:

On Tue, 2010-12-14 at 15:55 +0800, Yu Ke wrote:
>  
> +def verify_checksum(d, ud):
> +    """
> +    verify the MD5 and SHA256 checksum for downloaded src
> +    return True if matched, False if not
> +    """

This should be more explicit about the missing checksums case.

> +    md5data = bb.utils.md5_file(ud.localpath)
> +    sha256data = bb.utils.sha256_file(ud.localpath)
> +
> +    if (ud.md5_expected == None or ud.sha256_expected == None):
> +        bb.warn("Missing SRC_URI checksum for %s, consider to add\n" \
> +                "SRC_URI[%s] = \"%s\"\nSRC_URI[%s] = \"%s\"" \
> +                % (ud.localpath, ud.md5_name, md5data, ud.sha256_name, sha256data))
> +        # TODO: change to "return False" once all recpies has checksum entry
> +        return True

We can't just redefine "policy" like this in bitbake. I think missing
checksums will always be warnings and we might just make warnings
optionally fatal for bitbake.

What does this do for file:// urls or SCM urls?

Otherwise I'm good with the patch and if you tweak these things I'll
merge it.

Cheers,

Richard

Re: [PATCH 1/1] FetchData: add SRC_URI checksum

[Yu Ke]

On Dec 15, 09:12, Richard Purdie wrote:
> Hi Ke,
> 
> This patch looks good in general, just a couple of minor things:
> 
> On Tue, 2010-12-14 at 15:55 +0800, Yu Ke wrote:
> >  
> > +def verify_checksum(d, ud):
> > +    """
> > +    verify the MD5 and SHA256 checksum for downloaded src
> > +    return True if matched, False if not
> > +    """
> 
> This should be more explicit about the missing checksums case.

thanks for the review. and yes, will do that.

> 
> > +    md5data = bb.utils.md5_file(ud.localpath)
> > +    sha256data = bb.utils.sha256_file(ud.localpath)
> > +
> > +    if (ud.md5_expected == None or ud.sha256_expected == None):
> > +        bb.warn("Missing SRC_URI checksum for %s, consider to add\n" \
> > +                "SRC_URI[%s] = \"%s\"\nSRC_URI[%s] = \"%s\"" \
> > +                % (ud.localpath, ud.md5_name, md5data, ud.sha256_name, sha256data))
> > +        # TODO: change to "return False" once all recpies has checksum entry
> > +        return True
> 
> We can't just redefine "policy" like this in bitbake. I think missing
> checksums will always be warnings and we might just make warnings
> optionally fatal for bitbake.

for the "make warnings optionally fatal in bitbake", do you mean add a variable like "BB_STRICT_CHECKSUM" in config file, and if for example "BB_STRICT_CHECKSUMS" = "1", then fatal, otherwise, just warning?

> 
> What does this do for file:// urls or SCM urls?

Good catch. It should only verify for "http/https/ftp/ftps" protocol, other protocol like local file and SCM url should just return.

Regards
Ke

> 
> Otherwise I'm good with the patch and if you tweak these things I'll
> merge it.
> 
> Cheers,
> 
> Richard
> 
> 
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky

Re: [PATCH 0/1] SRC_URI checksum support v2

[Koen Kooi]

Op 14 dec 2010, om 11:01 heeft Yu Ke het volgende geschreven:

> This patch add SRC_URI checksum support. With this patch,fetcher
> can verify the MD5 and SHA256 checksum of download src with the
> value defined in recipes SRC_URI.

Awesome, I have been waiting for that!

> 
> This is the V2 patch. The major change compared to V1 is:
> moving the logic from poky bbclass to bitbake fetcher, per
> Richard's comment
> 
> Pull URL: git://git.pokylinux.org/poky-contrib.git
>  Branch: kyu3/srcuri-v2
>  Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=kyu3/srcuri-v2
> 
> Thanks,
>    Yu Ke <ke.yu@intel.com>
> ---
> 
> 
> Yu Ke (1):
>  FetchData: add SRC_URI checksum
> 
> bitbake/lib/bb/fetch/__init__.py |   36 ++++++++++++++++++++++++++++++++++++
> 1 files changed, 36 insertions(+), 0 deletions(-)
> 
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky

[PATCH 0/1] SRC_URI checksum support v3

[Yu Ke]

This patch add SRC_URI checksum support. With this patch,fetcher
can verify the MD5 and SHA256 checksum of download src with the
value defined in recipes SRC_URI.

This is the v3 patch with following changes compared with v2:
- add configurable variable BB_STRICT_CHECKSUM to handle checksum missing case
  if checksum is missing and BB_STRICT_CHECKSUM = "1", bitbake will fatal
- add check to only verify checksum for protocol http/https/ftp/ftps, not
  verify checksum for local file and other SCM

Pull URL: git://git.pokylinux.org/poky-contrib.git
  Branch: kyu3/srcuri-v3
  Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=kyu3/srcuri-v3

Thanks,
    Yu Ke <ke.yu@intel.com>
---


Yu Ke (1):
  FetchData: add SRC_URI checksum

 bitbake/lib/bb/fetch/__init__.py |   49 ++++++++++++++++++++++++++++++++++++++
 1 files changed, 49 insertions(+), 0 deletions(-)

[PATCH 1/1] FetchData: add SRC_URI checksum

[Yu Ke]

This patch add the per-recipe SRC_URI checksum verification.

- SRC_URI format
The format of SRC_URI checksum follow OE definition:

1. SRC_URI has single src
SRC_URI = "http://some.domain/file.tar.gz"
SRC_URI[md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"

2. SRC_URI has multiple src, every src need specify name
SRC_URI = "http://some.domain/file1.tar.gz;name=name1 \
           http://some.domain/file2.tar.gz;name=name2 "
SRC_URI[name1.md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[name1.sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"
SRC_URI[name2.md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[name2.sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"

- SRC_URI checking invocation:
the checksum checking is invoked in do_fetch phase,
so it can be invoked manually by

# bitbake -f -c fetch <recipe_name>

if recipes has no SRC_URI checksum item, bitbake will show warning:
"
WARNING: Missing SRC_URI checksum for xxxx.tar.gz, consider to add
SRC_URI[md5sum] = "5c69f16d452b0bb3d44bc3c10556c072"
SRC_URI[sha256sum] = "f4e0ada8d4d516bbb8600a3ee7d9046c9c79e38cd781df9ffc46d8f16acd1768"
"
thus recipe author can add it to recpie file after SRC_URI

- control variable BB_STRICT_CHECKSUM
when SRC_URI checksum is missing, this variable decide pass or not
if BB_STRICT_CHECKSUM = "1", bitbake should fatal in this case, otherwise bitbake just pass

Signed-off-by: Yu Ke <ke.yu@intel.com>
---
 bitbake/lib/bb/fetch/__init__.py |   49 ++++++++++++++++++++++++++++++++++++++
 1 files changed, 49 insertions(+), 0 deletions(-)

diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index 50955f1..387de66 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -231,6 +231,42 @@ def removefile(f):
     except:
         pass
 
+def verify_checksum(d, ud):
+    """
+    verify the MD5 and SHA256 checksum for downloaded src
+
+    return value:
+        - True: checksum matched
+        - False: checksum unmatched
+
+    if checksum is missing in recipes file, "BB_STRICT_CHECKSUM" decide the return value.
+    if BB_STRICT_CHECKSUM = "1" then return false as unmatched, otherwise return true as
+    matched
+    """
+
+    if not ud.type in ["http", "https", "ftp", "ftps"]:
+        return True
+
+    md5data = bb.utils.md5_file(ud.localpath)
+    sha256data = bb.utils.sha256_file(ud.localpath)
+
+    if (ud.md5_expected == None or ud.sha256_expected == None):
+        bb.warn("Missing SRC_URI checksum for %s, consider to add\n" \
+                "SRC_URI[%s] = \"%s\"\nSRC_URI[%s] = \"%s\"" \
+                % (ud.localpath, ud.md5_name, md5data, ud.sha256_name, sha256data))
+        if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1":
+            return False
+        else:
+            return True
+
+    if (ud.md5_expected != md5data or ud.sha256_expected != sha256data):
+        bb.error("The checksums for '%s' did not match." % ud.localpath)
+        bb.error("Expected MD5: '%s' and Got: '%s'" % (ud.md5_expected, md5data))
+        bb.error("Expected SHA256: '%s' and Got: '%s'" % (ud.sha256_expected, sha256data))
+        return False
+
+    return True
+
 def go(d, urls = None):
     """
     Fetch all urls
@@ -283,6 +319,9 @@ def go(d, urls = None):
         else:
             Fetch.write_md5sum(u, ud, d)
 
+        if not verify_checksum(d, ud):
+            raise FetchError("%s checksum mismatch." % u)
+
         bb.utils.unlockfile(lf)
 
 def checkstatus(d, urls = None):
@@ -502,6 +541,16 @@ class FetchData(object):
         if not self.pswd and "pswd" in self.parm:
             self.pswd = self.parm["pswd"]
         self.setup = False
+
+        if "name" in self.parm:
+            self.md5_name = "%s.md5sum" % self.parm["name"]
+            self.sha256_name = "%s.sha256sum" % self.parm["name"]
+        else:
+            self.md5_name = "md5sum"
+            self.sha256_name = "sha256sum"
+        self.md5_expected = bb.data.getVarFlag("SRC_URI", self.md5_name, d)
+        self.sha256_expected = bb.data.getVarFlag("SRC_URI", self.sha256_name, d)
+
         for m in methods:
             if m.supports(url, self, d):
                 self.method = m
-- 
1.7.0.4