[PATCH 0/4] Minor improvements to verify-homepage script

[PATCH 0/4] Minor improvements to verify-homepage script

[Paul Eggleton]

The following changes since commit e44ed8c18e395b9c055aefee113b90708e8a8a2f:

  build-appliance-image: Update to jethro head revision (2015-11-03 14:02:57 +0000)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/verify-homepage
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/verify-homepage

Paul Eggleton (4):
  verify-homepage: use scriptpath to find bitbake path
  verify-homepage: get expanded HOMEPAGE value
  verify-homepage: tidy up output and comments
  verify-homepage: fix recipe file selection

 scripts/contrib/verify-homepage.py | 65 +++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 33 deletions(-)

-- 
2.1.0

[PATCH 1/4] verify-homepage: use scriptpath to find bitbake path

[Paul Eggleton]

We have shared code for this, let's use it.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/contrib/verify-homepage.py | 28 +++++++++-------------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/scripts/contrib/verify-homepage.py b/scripts/contrib/verify-homepage.py
index 86cc82b..9953bab 100755
--- a/scripts/contrib/verify-homepage.py
+++ b/scripts/contrib/verify-homepage.py
@@ -8,26 +8,16 @@ import os
 import subprocess
 import urllib2
 
-def search_bitbakepath():
-    bitbakepath = ""
 
-    # Search path to bitbake lib dir in order to load bb modules
-    if os.path.exists(os.path.join(os.path.dirname(sys.argv[0]), '../../bitbake/lib/bb')):
-        bitbakepath = os.path.join(os.path.dirname(sys.argv[0]), '../../bitbake/lib')
-        bitbakepath = os.path.abspath(bitbakepath)
-    else:
-        # Look for bitbake/bin dir in PATH
-        for pth in os.environ['PATH'].split(':'):
-            if os.path.exists(os.path.join(pth, '../lib/bb')):
-                bitbakepath = os.path.abspath(os.path.join(pth, '../lib'))
-                break
-        if not bitbakepath:
-            sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
-            sys.exit(1)
-    return bitbakepath
-
-# For importing the following modules
-sys.path.insert(0, search_bitbakepath())
+# Allow importing scripts/lib modules
+scripts_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..')
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import scriptpath
+
+# Allow importing bitbake modules
+bitbakepath = scriptpath.add_bitbake_lib_path()
+
 import bb.tinfoil
 
 def wgetHomepage(pn, homepage):
-- 
2.1.0

[PATCH 2/4] verify-homepage: get expanded HOMEPAGE value

[Paul Eggleton]

We tend not to use any variables in HOMEPAGE values, but that doesn't
mean we would never do so.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/contrib/verify-homepage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/contrib/verify-homepage.py b/scripts/contrib/verify-homepage.py
index 9953bab..3e73339 100755
--- a/scripts/contrib/verify-homepage.py
+++ b/scripts/contrib/verify-homepage.py
@@ -35,7 +35,7 @@ def verifyHomepage(bbhandler):
     for pn in pnlist:
         fn = pkg_pn[pn].pop()
         data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data)
-        homepage = data.getVar("HOMEPAGE")
+        homepage = data.getVar("HOMEPAGE", True)
         if homepage:
             try:
                 urllib2.urlopen(homepage, timeout=5)
-- 
2.1.0

[PATCH 3/4] verify-homepage: tidy up output and comments

[Paul Eggleton]

* Set up and use a proper logger
* Tweak output messages and comments

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/contrib/verify-homepage.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/scripts/contrib/verify-homepage.py b/scripts/contrib/verify-homepage.py
index 3e73339..522824b 100755
--- a/scripts/contrib/verify-homepage.py
+++ b/scripts/contrib/verify-homepage.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-# This script is used for verify HOMEPAGE.
+# This script can be used to verify HOMEPAGE values for all recipes in
+# the current configuration.
 # The result is influenced by network environment, since the timeout of connect url is 5 seconds as default.
 
 import sys
@@ -14,16 +15,19 @@ scripts_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/.
 lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
 import scriptpath
+import scriptutils
 
 # Allow importing bitbake modules
 bitbakepath = scriptpath.add_bitbake_lib_path()
 
 import bb.tinfoil
 
+logger = scriptutils.logger_create('verify_homepage')
+
 def wgetHomepage(pn, homepage):
     result = subprocess.call('wget ' + '-q -T 5 -t 1 --spider ' + homepage, shell = True)
     if result:
-        bb.warn("Failed to verify HOMEPAGE (%s) of %s" % (homepage, pn))
+        logger.warn("%s: failed to verify HOMEPAGE: %s " % (pn, homepage))
         return 1
     else:
         return 0
@@ -44,10 +48,9 @@ def verifyHomepage(bbhandler):
     return count
 
 if __name__=='__main__':
-    failcount = 0
     bbhandler = bb.tinfoil.Tinfoil()
     bbhandler.prepare()
-    print "Start to verify HOMEPAGE:"
+    logger.info("Start verifying HOMEPAGE:")
     failcount = verifyHomepage(bbhandler)
-    print "finish to verify HOMEPAGE."
-    print "Summary: %s failed" % failcount
+    logger.info("Finished verifying HOMEPAGE.")
+    logger.info("Summary: %s failed" % failcount)
-- 
2.1.0

[PATCH 4/4] verify-homepage: fix recipe file selection

[Paul Eggleton]

* We need to check all recipe files, not just the preferred ones
  (i.e. we have multiple recipes for different versions of the same
  piece of software). Print the recipe file name (without path) so we
  can tell the difference between them.
* We can skip BBCLASSEXTENDed variants of recipes

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/contrib/verify-homepage.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/scripts/contrib/verify-homepage.py b/scripts/contrib/verify-homepage.py
index 522824b..265ff65 100755
--- a/scripts/contrib/verify-homepage.py
+++ b/scripts/contrib/verify-homepage.py
@@ -36,15 +36,21 @@ def verifyHomepage(bbhandler):
     pkg_pn = bbhandler.cooker.recipecache.pkg_pn
     pnlist = sorted(pkg_pn)
     count = 0
+    checked = []
     for pn in pnlist:
-        fn = pkg_pn[pn].pop()
-        data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data)
-        homepage = data.getVar("HOMEPAGE", True)
-        if homepage:
-            try:
-                urllib2.urlopen(homepage, timeout=5)
-            except Exception:
-                count = count + wgetHomepage(pn, homepage)
+        for fn in pkg_pn[pn]:
+            # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
+            realfn, _ = bb.cache.Cache.virtualfn2realfn(fn)
+            if realfn in checked:
+                continue
+            data = bb.cache.Cache.loadDataFull(realfn, bbhandler.cooker.collection.get_file_appends(realfn), bbhandler.config_data)
+            homepage = data.getVar("HOMEPAGE", True)
+            if homepage:
+                try:
+                    urllib2.urlopen(homepage, timeout=5)
+                except Exception:
+                    count = count + wgetHomepage(os.path.basename(realfn), homepage)
+            checked.append(realfn)
     return count
 
 if __name__=='__main__':
-- 
2.1.0