[PATCH 0/6] oe-pkgdata-util improvements

[PATCH 0/6] oe-pkgdata-util improvements

[Paul Eggleton]

Some improvements for the oe-pkgdata-util script making it a little more
useful outside of its usage within do_rootfs.


The following changes since commit 8081dcb03f54efd551d1c8fe8a0484f8270053e0:

  pigz: bump to 2.3.1 (2013-10-26 16:08:30 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/oe-pkgdata-util
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/oe-pkgdata-util

Paul Eggleton (6):
  scripts/oe-pkgdata-util: remove remnants of former pkgdata structure
  scripts/oe-pkgdata-util: improve help text and command line parsing
  scripts/oe-pkgdata-util: check path arguments to ensure they exist
  scripts/oe-pkgdata-util: add ability to search for a target path
  scripts/oe-pkgdata-util: add ability to look up runtime package names
  scripts/oe-pkgdata-util: add ability to find a recipe from a target
    package

 scripts/oe-pkgdata-util | 209 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 165 insertions(+), 44 deletions(-)

-- 
1.8.1.2

[PATCH 1/6] scripts/oe-pkgdata-util: remove remnants of former pkgdata structure

[Paul Eggleton]

OE-Core commit 1b8e4abd2d9c0901d38d89d0f944fe1ffd019379 removed the
vendor-os argument from the command line, and the code using the package
architectures, so clean these items up.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index dcdfe25..c0fd50d 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -35,8 +35,8 @@ import fnmatch
 import re
 
 def usage():
-    print("syntax: oe-pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"\n \
-                    read-value [-d] <pkgdatadir> <vendor-os> <value-name> \"<package-name>_<package_architecture>\"");
+    print("syntax: oe-pkgdata-util glob [-d] <pkgdatadir> <pkglist> \"<globs>\"\n \
+                    read-value [-d] <pkgdatadir> <value-name> \"<pkgs>\"");
 
 
 
@@ -55,10 +55,11 @@ def glob(args):
     with open(pkglist_file, 'r') as f:
         for line in f:
             fields = line.rstrip().split()
-            if len(fields) < 2:
+            if not fields:
                 continue
             pkg = fields[0]
-            arch = fields[1]
+            # We don't care about other args (used to need the package architecture but the
+            # new pkgdata structure avoids the need for that)
 
             # Skip packages for which there is no point applying globs
             if skipregex.search(pkg):
@@ -172,9 +173,8 @@ def read_value(args):
     for package in packages:
         pkg_split = package.split('_')
         pkg_name = pkg_split[0]
-        pkg_arch = '_'.join(pkg_split[1:])
         if debug:
-            print "package: name: '%s', arch: '%s'" % (pkg_name, pkg_arch)
+            print "package: '%s'" % pkg_name
         revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg_name)
         if debug:
             print(revlink)
-- 
1.8.1.2

[PATCH 2/6] scripts/oe-pkgdata-util: improve help text and command line parsing

[Paul Eggleton]

* Use optparse to parse command line
* Make help text actually helpful by describing what each command does
* Drop comment at the top listing the commands which is now superfluous

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 78 ++++++++++++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 40 deletions(-)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index c0fd50d..e34fcbe 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -4,7 +4,7 @@
 #
 # Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
 #
-# Copyright 2012 Intel Corporation
+# Copyright 2012-2013 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -19,28 +19,16 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
-#
-# Currently only has two functions:
-# 1) glob - mapping of packages to their dev/dbg/doc/locale etc. counterparts.
-# 2) read-value - mapping of packagenames to their location in
-#    pkgdata and then returns value of selected variable (e.g. PKGSIZE)
-# Could be extended in future to perform other useful querying functions on the
-# pkgdata though.
-#
 
 import sys
 import os
 import os.path
 import fnmatch
 import re
-
-def usage():
-    print("syntax: oe-pkgdata-util glob [-d] <pkgdatadir> <pkglist> \"<globs>\"\n \
-                    read-value [-d] <pkgdatadir> <value-name> \"<pkgs>\"");
-
+import optparse
 
 
-def glob(args):
+def glob(args, usage):
     if len(args) < 3:
         usage()
         sys.exit(1)
@@ -151,7 +139,7 @@ def glob(args):
 
     print("\n".join(mappedpkgs))
 
-def read_value(args):
+def read_value(args, usage):
     if len(args) < 3:
         usage()
         sys.exit(1)
@@ -186,28 +174,38 @@ def read_value(args):
                 qvar = "%s_%s" % (var, mappedpkg)
             print(readvar(revlink, qvar))
 
-# Too lazy to use getopt
-debug = False
-noopt = False
-args = []
-for arg in sys.argv[1:]:
-    if arg == "--":
-        noopt = True
+
+def main():
+    parser = optparse.OptionParser(
+        usage = '''%prog [options] <command> <arguments>
+
+Available commands:
+    glob <pkgdatadir> <pkglistfile> "<globs>"
+        expand one or more glob expressions over the packages listed in
+        pkglistfile (one package per line)
+    read-value <pkgdatadir> <value-name> "<pkgs>"
+        read the named value from the pkgdata files for the specified
+        packages''')
+
+    parser.add_option("-d", "--debug",
+            help = "Report all SRCREV values, not just ones where AUTOREV has been used",
+            action="store_true", dest="debug")
+
+    options, args = parser.parse_args(sys.argv)
+    args = args[1:]
+
+    if len(args) < 1:
+        parser.print_help()
+        sys.exit(1)
+
+    if args[0] == "glob":
+        glob(args[1:], parser.print_help)
+    elif args[0] == "read-value":
+        read_value(args[1:], parser.print_help)
     else:
-        if not noopt:
-            if arg == "-d":
-                debug = True
-                continue
-        args.append(arg)
-
-if len(args) < 1:
-    usage()
-    sys.exit(1)
-
-if args[0] == "glob":
-    glob(args[1:])
-elif args[0] == "read-value":
-    read_value(args[1:])
-else:
-    usage()
-    sys.exit(1)
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
-- 
1.8.1.2

[PATCH 3/6] scripts/oe-pkgdata-util: check path arguments to ensure they exist

[Paul Eggleton]

Show an error if the specified paths don't exist.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index e34fcbe..01fccd2 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -37,6 +37,14 @@ def glob(args, usage):
     pkglist_file = args[1]
     globs = args[2].split()
 
+    if not os.path.exists(pkgdata_dir):
+        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
+        sys.exit(1)
+
+    if not os.path.exists(pkglist_file):
+        print('ERROR: Unable to find package list file %s' % pkglist_file)
+        sys.exit(1)
+
     skipregex = re.compile("-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-")
 
     mappedpkgs = set()
@@ -148,6 +156,10 @@ def read_value(args, usage):
     var = args[1]
     packages = args[2].split()
 
+    if not os.path.exists(pkgdata_dir):
+        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
+        sys.exit(1)
+
     def readvar(pkgdata_file, var):
         val = ""
         with open(pkgdata_file, 'r') as f:
-- 
1.8.1.2

[PATCH 4/6] scripts/oe-pkgdata-util: add ability to search for a target path

[Paul Eggleton]

Add ability to search for a target path in produced packages, in order
to find which package provides a specific file.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index 01fccd2..2d896d0 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -186,6 +186,38 @@ def read_value(args, usage):
                 qvar = "%s_%s" % (var, mappedpkg)
             print(readvar(revlink, qvar))
 
+def find_path(args, usage):
+    if len(args) < 2:
+        usage()
+        sys.exit(1)
+
+    pkgdata_dir = args[0]
+    targetpath = args[1]
+
+    if not os.path.exists(pkgdata_dir):
+        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
+        sys.exit(1)
+
+    import ast
+    import fnmatch
+
+    for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
+        for fn in files:
+            pkgsplitname = '/packages-split/%s' % fn
+            with open(os.path.join(root,fn)) as f:
+                for line in f:
+                    if line.startswith('FILES_INFO:'):
+                        val = line.split(':', 1)[1].strip().replace('\\\'', '\'')
+                        dictval = ast.literal_eval(val)
+                        for parent, dirlist in dictval.items():
+                            idx = parent.find(pkgsplitname)
+                            if idx > -1:
+                                parent = parent[idx+len(pkgsplitname):]
+                            for basename in dirlist:
+                                fullpth = os.path.join(parent, basename)
+                                if fnmatch.fnmatchcase(fullpth, targetpath):
+                                    print("%s: %s" % (fn, fullpth))
+
 
 def main():
     parser = optparse.OptionParser(
@@ -195,6 +227,8 @@ Available commands:
     glob <pkgdatadir> <pkglistfile> "<globs>"
         expand one or more glob expressions over the packages listed in
         pkglistfile (one package per line)
+    find-path <pkgdatadir> <path>
+        find the package providing the specified path (wildcards * ? allowed)
     read-value <pkgdatadir> <value-name> "<pkgs>"
         read the named value from the pkgdata files for the specified
         packages''')
@@ -212,6 +246,8 @@ Available commands:
 
     if args[0] == "glob":
         glob(args[1:], parser.print_help)
+    elif args[0] == "find-path":
+        find_path(args[1:], parser.print_help)
     elif args[0] == "read-value":
         read_value(args[1:], parser.print_help)
     else:
-- 
1.8.1.2

[PATCH 5/6] scripts/oe-pkgdata-util: add ability to look up runtime package names

[Paul Eggleton]

Add a "lookup-pkg" command to oe-pkgdata-util that can be used to find
the runtime name of a package (after e.g. Debian library package
renaming).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index 2d896d0..08773e9 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -26,6 +26,7 @@ import os.path
 import fnmatch
 import re
 import optparse
+from collections import defaultdict
 
 
 def glob(args, usage):
@@ -186,6 +187,38 @@ def read_value(args, usage):
                 qvar = "%s_%s" % (var, mappedpkg)
             print(readvar(revlink, qvar))
 
+def lookup_pkg(args, usage):
+    if len(args) < 2:
+        usage()
+        sys.exit(1)
+
+    pkgdata_dir = args[0]
+    pkgs = args[1].split()
+
+    if not os.path.exists(pkgdata_dir):
+        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
+        sys.exit(1)
+
+    mappings = defaultdict(list)
+    for pkg in pkgs:
+        pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg)
+        if os.path.exists(pkgfile):
+            with open(pkgfile, 'r') as f:
+                for line in f:
+                    fields = line.rstrip().split(': ')
+                    if fields[0] == 'PKG_%s' % pkg:
+                        mappings[pkg].append(fields[1])
+                        break
+    if len(mappings) < len(pkgs):
+        missing = list(set(pkgs) - set(mappings.keys()))
+        sys.stderr.write("ERROR: the following packages could not be found: %s\n" % ', '.join(missing))
+        sys.exit(1)
+
+    items = []
+    for pkg in pkgs:
+        items.extend(mappings.get(pkg, []))
+    print '\n'.join(items)
+
 def find_path(args, usage):
     if len(args) < 2:
         usage()
@@ -227,6 +260,9 @@ Available commands:
     glob <pkgdatadir> <pkglistfile> "<globs>"
         expand one or more glob expressions over the packages listed in
         pkglistfile (one package per line)
+    lookup-pkg <pkgdatadir> "<recipe-pkgs>"
+        look up the specified recipe-space package name(s) to see what the
+        final runtime package name is (e.g. eglibc becomes libc6)
     find-path <pkgdatadir> <path>
         find the package providing the specified path (wildcards * ? allowed)
     read-value <pkgdatadir> <value-name> "<pkgs>"
@@ -246,6 +282,8 @@ Available commands:
 
     if args[0] == "glob":
         glob(args[1:], parser.print_help)
+    elif args[0] == "lookup-pkg":
+        lookup_pkg(args[1:], parser.print_help)
     elif args[0] == "find-path":
         find_path(args[1:], parser.print_help)
     elif args[0] == "read-value":
-- 
1.8.1.2

[PATCH 6/6] scripts/oe-pkgdata-util: add ability to find a recipe from a target package

[Paul Eggleton]

Add a "lookup-recipe" command to show which recipe produced a particular
package.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/oe-pkgdata-util | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index 08773e9..80cacc5 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -219,6 +219,38 @@ def lookup_pkg(args, usage):
         items.extend(mappings.get(pkg, []))
     print '\n'.join(items)
 
+def lookup_recipe(args, usage):
+    if len(args) < 2:
+        usage()
+        sys.exit(1)
+
+    pkgdata_dir = args[0]
+    pkgs = args[1].split()
+
+    if not os.path.exists(pkgdata_dir):
+        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
+        sys.exit(1)
+
+    mappings = defaultdict(list)
+    for pkg in pkgs:
+        pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
+        if os.path.exists(pkgfile):
+            with open(pkgfile, 'r') as f:
+                for line in f:
+                    fields = line.rstrip().split(': ')
+                    if fields[0] == 'PN':
+                        mappings[pkg].append(fields[1])
+                        break
+    if len(mappings) < len(pkgs):
+        missing = list(set(pkgs) - set(mappings.keys()))
+        sys.stderr.write("ERROR: the following packages could not be found: %s\n" % ', '.join(missing))
+        sys.exit(1)
+
+    items = []
+    for pkg in pkgs:
+        items.extend(mappings.get(pkg, []))
+    print '\n'.join(items)
+
 def find_path(args, usage):
     if len(args) < 2:
         usage()
@@ -263,6 +295,9 @@ Available commands:
     lookup-pkg <pkgdatadir> "<recipe-pkgs>"
         look up the specified recipe-space package name(s) to see what the
         final runtime package name is (e.g. eglibc becomes libc6)
+    lookup-recipe <pkgdatadir> "<pkgs>"
+        look up the specified package(s) to see which recipe they were
+        produced by
     find-path <pkgdatadir> <path>
         find the package providing the specified path (wildcards * ? allowed)
     read-value <pkgdatadir> <value-name> "<pkgs>"
@@ -284,6 +319,8 @@ Available commands:
         glob(args[1:], parser.print_help)
     elif args[0] == "lookup-pkg":
         lookup_pkg(args[1:], parser.print_help)
+    elif args[0] == "lookup-recipe":
+        lookup_recipe(args[1:], parser.print_help)
     elif args[0] == "find-path":
         find_path(args[1:], parser.print_help)
     elif args[0] == "read-value":
-- 
1.8.1.2