[PATCH 0/4] Various script improvements

[PATCH 0/4] Various script improvements

[Paul Eggleton]

The following changes since commit f1727bb18f35ff01e53d3d442a6ff3c613639fa6:

  guile: Update to 2.0.11 version (2014-05-21 10:50:37 -0700)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/script-improvements
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/script-improvements

Paul Eggleton (4):
  scripts: consolidate code to find bitbake path
  list-packageconfig-flags: filter out doc and defaultval varflags
  list-packageconfig-flags: improve option parsing
  oe-pkgdata-util: fix help text

 scripts/bitbake-whatchanged                 |  18 ++--
 scripts/buildhistory-diff                   |  28 +++----
 scripts/contrib/list-packageconfig-flags.py | 122 +++++++++++-----------------
 scripts/lib/scriptpath.py                   |  42 ++++++++++
 scripts/oe-pkgdata-util                     |   2 +-
 5 files changed, 111 insertions(+), 101 deletions(-)
 create mode 100644 scripts/lib/scriptpath.py

-- 
1.9.0

[PATCH 1/4] scripts: consolidate code to find bitbake path

[Paul Eggleton]

Several of these scripts were using duplicated code (and slightly
different methods) to find the path to bitbake and add its lib
subdirectory to the Python import path. Add some common code to do this
and change the scripts to use it.

Fixes [YOCTO #5076].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/bitbake-whatchanged                 | 18 ++++++-------
 scripts/buildhistory-diff                   | 28 +++++++++----------
 scripts/contrib/list-packageconfig-flags.py | 29 ++++++++------------
 scripts/lib/scriptpath.py                   | 42 +++++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 43 deletions(-)
 create mode 100644 scripts/lib/scriptpath.py

diff --git a/scripts/bitbake-whatchanged b/scripts/bitbake-whatchanged
index e4497e0..55cfe4b 100755
--- a/scripts/bitbake-whatchanged
+++ b/scripts/bitbake-whatchanged
@@ -27,17 +27,17 @@ import warnings
 import subprocess
 from optparse import OptionParser
 
-# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
-p = subprocess.Popen("bash -c 'echo $(dirname $(which bitbake-diffsigs | grep -v \'^alias\'))/../lib'",
-        shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
 
-err = p.stderr.read()
-if err:
-    print("ERROR: Failed to locate bitbake-diffsigs:", file=sys.stderr)
-    print(err, file=sys.stderr)
-    sys.exit(1)
+import scriptpath
 
-sys.path.insert(0, p.stdout.read().rstrip('\n'))
+# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
+bitbakepath = scriptpath.add_bitbake_lib_path()
+if not bitbakepath:
+    sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
+    sys.exit(1)
 
 import bb.siggen
 import bb.process
diff --git a/scripts/buildhistory-diff b/scripts/buildhistory-diff
index ad50414..dfebcdd 100755
--- a/scripts/buildhistory-diff
+++ b/scripts/buildhistory-diff
@@ -50,24 +50,20 @@ def main():
         parser.print_help()
         sys.exit(1)
 
+    scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
+    lib_path = scripts_path + '/lib'
+    sys.path = sys.path + [lib_path]
+
+    import scriptpath
+
     # Set path to OE lib dir so we can import the buildhistory_analysis module
-    basepath = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])) + '/..')
-    newpath = basepath + '/meta/lib'
+    scriptpath.add_oe_lib_path()
     # Set path to bitbake lib dir so the buildhistory_analysis module can load bb.utils
-    if os.path.exists(basepath + '/bitbake/lib/bb'):
-        bitbakepath = basepath + '/bitbake'
-    else:
-        # look for bitbake/bin dir in PATH
-        bitbakepath = None
-        for pth in os.environ['PATH'].split(':'):
-            if os.path.exists(os.path.join(pth, '../lib/bb')):
-                bitbakepath = os.path.abspath(os.path.join(pth, '..'))
-                break
-        if not bitbakepath:
-            sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
-            sys.exit(1)
-
-    sys.path[0:0] = [newpath, bitbakepath + '/lib']
+    bitbakepath = scriptpath.add_bitbake_lib_path()
+    if not bitbakepath:
+        sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
+        sys.exit(1)
+
     import oe.buildhistory_analysis
 
     fromrev = 'build-minus-1'
diff --git a/scripts/contrib/list-packageconfig-flags.py b/scripts/contrib/list-packageconfig-flags.py
index 371033a..615f91f 100755
--- a/scripts/contrib/list-packageconfig-flags.py
+++ b/scripts/contrib/list-packageconfig-flags.py
@@ -23,26 +23,19 @@ import sys
 import getopt
 import os
 
-def search_bitbakepath():
-    bitbakepath = ""
-
-    # Search path to bitbake lib dir in order to load bb modules
-    if os.path.exists(os.path.join(os.path.dirname(sys.argv[0]), '../../bitbake/lib/bb')):
-        bitbakepath = os.path.join(os.path.dirname(sys.argv[0]), '../../bitbake/lib')
-        bitbakepath = os.path.abspath(bitbakepath)
-    else:
-        # Look for bitbake/bin dir in PATH
-        for pth in os.environ['PATH'].split(':'):
-            if os.path.exists(os.path.join(pth, '../lib/bb')):
-                bitbakepath = os.path.abspath(os.path.join(pth, '../lib'))
-                break
-        if not bitbakepath:
-            sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
-            sys.exit(1)
-    return bitbakepath
+
+scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
+lib_path = os.path.abspath(scripts_path + '/../lib')
+sys.path = sys.path + [lib_path]
+
+import scriptpath
 
 # For importing the following modules
-sys.path.insert(0, search_bitbakepath())
+bitbakepath = scriptpath.add_bitbake_lib_path()
+if not bitbakepath:
+    sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
+    sys.exit(1)
+
 import bb.cache
 import bb.cooker
 import bb.providers
diff --git a/scripts/lib/scriptpath.py b/scripts/lib/scriptpath.py
new file mode 100644
index 0000000..d00317e
--- /dev/null
+++ b/scripts/lib/scriptpath.py
@@ -0,0 +1,42 @@
+# Path utility functions for OE python scripts
+#
+# Copyright (C) 2012-2014 Intel Corporation
+# Copyright (C) 2011 Mentor Graphics Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import os.path
+
+def add_oe_lib_path():
+    basepath = os.path.abspath(os.path.dirname(__file__) + '/../..')
+    newpath = basepath + '/meta/lib'
+    sys.path.insert(0, newpath)
+
+def add_bitbake_lib_path():
+    basepath = os.path.abspath(os.path.dirname(__file__) + '/../..')
+    bitbakepath = None
+    if os.path.exists(basepath + '/bitbake/lib/bb'):
+        bitbakepath = basepath + '/bitbake'
+    else:
+        # look for bitbake/bin dir in PATH
+        for pth in os.environ['PATH'].split(':'):
+            if os.path.exists(os.path.join(pth, '../lib/bb')):
+                bitbakepath = os.path.abspath(os.path.join(pth, '..'))
+                break
+
+    if bitbakepath:
+        sys.path.insert(0, bitbakepath + '/lib')
+    return bitbakepath
-- 
1.9.0

[PATCH 2/4] list-packageconfig-flags: filter out doc and defaultval varflags

[Paul Eggleton]

These are generic flags and shouldn't be listed in the output of this
script.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/contrib/list-packageconfig-flags.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scripts/contrib/list-packageconfig-flags.py b/scripts/contrib/list-packageconfig-flags.py
index 615f91f..3db4298 100755
--- a/scripts/contrib/list-packageconfig-flags.py
+++ b/scripts/contrib/list-packageconfig-flags.py
@@ -83,7 +83,10 @@ def get_recipesdata(bbhandler, preferred):
     data_dict = {}
     for fn in get_fnlist(bbhandler, pkg_pn, preferred):
         data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data)
-        if data.getVarFlags("PACKAGECONFIG"):
+        flags = data.getVarFlags("PACKAGECONFIG")
+        flags.pop('doc', None)
+        flags.pop('defaultval', None)
+        if flags:
             data_dict[fn] = data
 
     return data_dict
@@ -94,6 +97,8 @@ def collect_pkgs(data_dict):
     pkg_dict = {}
     for fn in data_dict:
         pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG")
+        pkgconfigflags.pop('doc', None)
+        pkgconfigflags.pop('defaultval', None)
         pkgname = data_dict[fn].getVar("P", True)
         pkg_dict[pkgname] = sorted(pkgconfigflags.keys())
 
@@ -105,9 +110,6 @@ def collect_flags(pkg_dict):
     flag_dict = {}
     for pkgname, flaglist in pkg_dict.iteritems():
         for flag in flaglist:
-            if flag == "defaultval":
-                continue
-
             if flag in flag_dict:
                 flag_dict[flag].append(pkgname)
             else:
@@ -153,7 +155,7 @@ def display_all(data_dict):
         print('PACKAGECONFIG %s' % packageconfig)
 
         for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").iteritems():
-            if flag == "defaultval":
+            if flag in ["defaultval", "doc"]:
                 continue
             print('PACKAGECONFIG[%s] %s' % (flag, flag_val))
         print ''
-- 
1.9.0

[PATCH 3/4] list-packageconfig-flags: improve option parsing

[Paul Eggleton]

* Use optparse instead of getopt (less code & automatic help)
* Change help text / output to use "recipe" instead of "package"
* Print something to indicate the script is still gathering information

Note that the long options have been renamed as appropriate.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/contrib/list-packageconfig-flags.py | 81 +++++++++++------------------
 1 file changed, 29 insertions(+), 52 deletions(-)

diff --git a/scripts/contrib/list-packageconfig-flags.py b/scripts/contrib/list-packageconfig-flags.py
index 3db4298..598b5c3 100755
--- a/scripts/contrib/list-packageconfig-flags.py
+++ b/scripts/contrib/list-packageconfig-flags.py
@@ -14,13 +14,14 @@
 # along with this program; if not, write to the Free Software Foundation.
 #
 # Copyright (C) 2013 Wind River Systems, Inc.
+# Copyright (C) 2014 Intel Corporation
 #
-# - list available pkgs which have PACKAGECONFIG flags
-# - list available PACKAGECONFIG flags and all affected pkgs
-# - list all pkgs and PACKAGECONFIG information
+# - list available recipes which have PACKAGECONFIG flags
+# - list available PACKAGECONFIG flags and all affected recipes
+# - list all recipes and PACKAGECONFIG information
 
 import sys
-import getopt
+import optparse
 import os
 
 
@@ -41,27 +42,6 @@ import bb.cooker
 import bb.providers
 import bb.tinfoil
 
-usage_body = '''  list available pkgs which have PACKAGECONFIG flags
-
-OPTION:
-  -h, --help    display this help and exit
-  -f, --flag    list available PACKAGECONFIG flags and all affected pkgs
-  -a, --all     list all pkgs and PACKAGECONFIG information
-  -p, --prefer  list pkgs with preferred version
-
-EXAMPLE:
-list-packageconfig-flags.py
-list-packageconfig-flags.py -f
-list-packageconfig-flags.py -a
-list-packageconfig-flags.py -p
-list-packageconfig-flags.py -f -p
-list-packageconfig-flags.py -a -p
-'''
-
-def usage():
-    print 'Usage: %s [-f|-a] [-p]' % os.path.basename(sys.argv[0])
-    print usage_body
-
 def get_fnlist(bbhandler, pkg_pn, preferred):
     ''' Get all recipe file names '''
     if preferred:
@@ -119,13 +99,13 @@ def collect_flags(pkg_dict):
 
 def display_pkgs(pkg_dict):
     ''' Display available pkgs which have PACKAGECONFIG flags '''
-    pkgname_len = len("PACKAGE NAME") + 1
+    pkgname_len = len("RECIPE NAME") + 1
     for pkgname in pkg_dict:
         if pkgname_len < len(pkgname):
             pkgname_len = len(pkgname)
     pkgname_len += 1
 
-    header = '%-*s%s' % (pkgname_len, str("PACKAGE NAME"), str("PACKAGECONFIG FLAGS"))
+    header = '%-*s%s' % (pkgname_len, str("RECIPE NAME"), str("PACKAGECONFIG FLAGS"))
     print header
     print str("").ljust(len(header), '=')
     for pkgname in sorted(pkg_dict):
@@ -136,7 +116,7 @@ def display_flags(flag_dict):
     ''' Display available PACKAGECONFIG flags and all affected pkgs '''
     flag_len = len("PACKAGECONFIG FLAG") + 5
 
-    header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("PACKAGE NAMES"))
+    header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("RECIPE NAMES"))
     print header
     print str("").ljust(len(header), '=')
 
@@ -161,43 +141,40 @@ def display_all(data_dict):
         print ''
 
 def main():
-    listtype = 'pkgs'
-    preferred = False
     pkg_dict = {}
     flag_dict = {}
 
     # Collect and validate input
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "hfap", ["help", "flag", "all", "prefer"])
-    except getopt.GetoptError, err:
-        print >> sys.stderr,'%s' % str(err)
-        usage()
-        sys.exit(2)
-    for opt, value in opts:
-        if opt in ('-h', '--help'):
-            usage()
-            sys.exit(0)
-        elif opt in ('-f', '--flag'):
-            listtype = 'flags'
-        elif opt in ('-a', '--all'):
-            listtype = 'all'
-        elif opt in ('-p', '--prefer'):
-            preferred = True
-        else:
-            assert False, "unhandled option"
+    parser = optparse.OptionParser(
+        description = "Lists recipes and PACKAGECONFIG flags. Without -a or -f, recipes and their available PACKAGECONFIG flags are listed.",
+        usage = """
+    %prog [options]""")
+
+    parser.add_option("-f", "--flags",
+            help = "list available PACKAGECONFIG flags and affected recipes",
+            action="store_const", dest="listtype", const="flags", default="recipes")
+    parser.add_option("-a", "--all",
+            help = "list all recipes and PACKAGECONFIG information",
+            action="store_const", dest="listtype", const="all")
+    parser.add_option("-p", "--preferred-only",
+            help = "where multiple recipe versions are available, list only the preferred version",
+            action="store_true", dest="preferred", default=False)
+
+    options, args = parser.parse_args(sys.argv)
 
     bbhandler = bb.tinfoil.Tinfoil()
     bbhandler.prepare()
-    data_dict = get_recipesdata(bbhandler, preferred)
+    print("Gathering recipe data...")
+    data_dict = get_recipesdata(bbhandler, options.preferred)
 
-    if listtype == 'flags':
+    if options.listtype == 'flags':
         pkg_dict = collect_pkgs(data_dict)
         flag_dict = collect_flags(pkg_dict)
         display_flags(flag_dict)
-    elif listtype == 'pkgs':
+    elif options.listtype == 'recipes':
         pkg_dict = collect_pkgs(data_dict)
         display_pkgs(pkg_dict)
-    elif listtype == 'all':
+    elif options.listtype == 'all':
         display_all(data_dict)
 
 if __name__ == "__main__":
-- 
1.9.0

[PATCH 4/4] oe-pkgdata-util: fix help text

[Paul Eggleton]

This was copy/pasted from another script and not corrected.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index a373116..bf87547 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -303,7 +303,7 @@ Available commands:
         packages''')
 
     parser.add_option("-d", "--debug",
-            help = "Report all SRCREV values, not just ones where AUTOREV has been used",
+            help = "Enable debug output",
             action="store_true", dest="debug", default=False)
 
     options, args = parser.parse_args(sys.argv)
-- 
1.9.0