[PATCH 0/7] recipetool improvements

[PATCH 0/7] recipetool improvements

[Paul Eggleton]

Make the following improvements to recipetool create (and by extension,
devtool add):
* Improve creation of recipes for binary deb/ipk/rpm package files with -b
* General supporting improvements to patch_recipe_* functions
* Set a fixed SRCREV when pointing to a git repo by default rather than
  using ${AUTOREV}


The following changes since commit bb4ead9b7b1400c37a72d148d9775bdf4210ec37:

  linux-yocto/4.4: integrate v4.4.11 (2016-05-25 07:49:55 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/recipetool-fixes2-oe
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool-fixes2-oe

Paul Eggleton (7):
  lib/oe/recipeutils: split out patch_recipe_lines()
  lib/oe/recipeutils: patch_recipe_lines: allow omitting trailing newlines
  lib/oe/recipeutils: fix insertion of variable values
  recipetool: create: support extracting SUMMARY and HOMEPAGE
  recipetool: create: extract variable values from .deb/.ipk/.rpm
  recipetool: create: use ${BP} for subdir for binary packages
  recipetool / devtool: set a fixed SRCREV by default when fetching from git

 meta/lib/oe/recipeutils.py                |  40 ++++++--
 meta/lib/oeqa/selftest/devtool.py         |   6 +-
 scripts/lib/devtool/standard.py           |   3 +
 scripts/lib/recipetool/create.py          | 165 +++++++++++++++++++++++-------
 scripts/lib/recipetool/create_buildsys.py |  37 ++++---
 scripts/lib/recipetool/create_npm.py      |   4 +-
 6 files changed, 193 insertions(+), 62 deletions(-)

-- 
2.5.5

[PATCH 1/7] lib/oe/recipeutils: split out patch_recipe_lines()

[Paul Eggleton]

Split out a function from patch_recipe_file() that takes just the lines
as input so we can edit recipe lines in memory. This will be used within
recipetool to ensure we insert new values in the right place.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/lib/oe/recipeutils.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 6c7adb5..ef82755 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -158,9 +158,8 @@ def split_var_value(value, assignment=True):
     return outlist
 
 
-def patch_recipe_file(fn, values, patch=False, relpath=''):
-    """Update or insert variable values into a recipe file (assuming you
-       have already identified the exact file you want to update.)
+def patch_recipe_lines(fromlines, values):
+    """Update or insert variable values into lines from a recipe.
        Note that some manual inspection/intervention may be required
        since this cannot handle all situations.
     """
@@ -247,8 +246,7 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
 
     # First run - establish which values we want to set are already in the file
     varlist = [re.escape(item) for item in values.keys()]
-    with open(fn, 'r') as f:
-        changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc)
+    bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
     # Second run - actually set everything
     modifying = True
     varlist.extend(recipe_progression_restrs)
@@ -260,6 +258,21 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
         for k in remainingnames.keys():
             outputvalue(k, tolines)
 
+    return changed, tolines
+
+
+def patch_recipe_file(fn, values, patch=False, relpath=''):
+    """Update or insert variable values into a recipe file (assuming you
+       have already identified the exact file you want to update.)
+       Note that some manual inspection/intervention may be required
+       since this cannot handle all situations.
+    """
+
+    with open(fn, 'r') as f:
+        fromlines = f.readlines()
+
+    _, tolines = patch_recipe_lines(fromlines, values)
+
     if patch:
         relfn = os.path.relpath(fn, relpath)
         diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
-- 
2.5.5

[PATCH 2/7] lib/oe/recipeutils: patch_recipe_lines: allow omitting trailing newlines

[Paul Eggleton]

This function was assuming that what you wanted was that output lines
had trailing newline characters. If you're just outputting each line
verbatim to a text file then that's fine, but sometimes you start with
the assumption that the lines don't have trailing newlines; thus we
shouldn't allow for the possibility that the caller doesn't want them
and add a parameter to control it.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/lib/oe/recipeutils.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index ef82755..b9f6ada 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -158,7 +158,7 @@ def split_var_value(value, assignment=True):
     return outlist
 
 
-def patch_recipe_lines(fromlines, values):
+def patch_recipe_lines(fromlines, values, trailing_newline=True):
     """Update or insert variable values into lines from a recipe.
        Note that some manual inspection/intervention may be required
        since this cannot handle all situations.
@@ -166,6 +166,11 @@ def patch_recipe_lines(fromlines, values):
 
     import bb.utils
 
+    if trailing_newline:
+        newline = '\n'
+    else:
+        newline = ''
+
     recipe_progression_res = []
     recipe_progression_restrs = []
     for item in recipe_progression:
@@ -196,7 +201,7 @@ def patch_recipe_lines(fromlines, values):
     def outputvalue(name, lines, rewindcomments=False):
         if values[name] is None:
             return
-        rawtext = '%s = "%s"\n' % (name, values[name])
+        rawtext = '%s = "%s"%s' % (name, values[name], newline)
         addlines = []
         if name in nowrap_vars:
             addlines.append(rawtext)
@@ -204,19 +209,19 @@ def patch_recipe_lines(fromlines, values):
             splitvalue = split_var_value(values[name], assignment=False)
             if len(splitvalue) > 1:
                 linesplit = ' \\\n' + (' ' * (len(name) + 4))
-                addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit))
+                addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline))
             else:
                 addlines.append(rawtext)
         else:
             wrapped = textwrap.wrap(rawtext)
             for wrapline in wrapped[:-1]:
-                addlines.append('%s \\\n' % wrapline)
-            addlines.append('%s\n' % wrapped[-1])
+                addlines.append('%s \\%s' % (wrapline, newline))
+            addlines.append('%s%s' % (wrapped[-1], newline))
         if rewindcomments:
             # Ensure we insert the lines before any leading comments
             # (that we'd want to ensure remain leading the next value)
             for i, ln in reversed(list(enumerate(lines))):
-                if ln[0] != '#':
+                if not ln.startswith('#'):
                     lines[i+1:i+1] = addlines
                     break
             else:
-- 
2.5.5

[PATCH 3/7] lib/oe/recipeutils: fix insertion of variable values

[Paul Eggleton]

Add some more variables in appropriate places in recipe_progression such
that the patch_recipe_* functions are able to insert variables in the
right place within a recipe.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/lib/oe/recipeutils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index b9f6ada..3e17873 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -19,7 +19,7 @@ from collections import OrderedDict, defaultdict
 
 
 # Help us to find places to insert values
-recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()']
+recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()']
 # Variables that sometimes are a bit long but shouldn't be wrapped
 nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]']
 list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
-- 
2.5.5

[PATCH 4/7] recipetool: create: support extracting SUMMARY and HOMEPAGE

[Paul Eggleton]

Allow plugins to set any variable value through the extravalues dict,
and use this to support extracting SUMMARY and HOMEPAGE values from spec
files included with the source; additionally translate "License:" to a
comment next to the LICENSE field (we have our own logic for setting
LICENSE, but it will often be useful to see what the spec file says if
one is present).

Also use the same mechanism for setting the same variables for node.js
modules; this was already supported but wasn't inserting the settings in
the appropriate place in the file which this will now do.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py          | 49 +++++++++++++++++--------------
 scripts/lib/recipetool/create_buildsys.py | 37 +++++++++++++++--------
 scripts/lib/recipetool/create_npm.py      |  4 +--
 3 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index aade40b..66c881a 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -331,6 +331,7 @@ def create_recipe(args):
     import bb.process
     import tempfile
     import shutil
+    import oe.recipeutils
 
     pkgarch = ""
     if args.machine:
@@ -429,7 +430,8 @@ def create_recipe(args):
     lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
     lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
     lines_before.append('# (Feel free to remove these comments when editing.)')
-    lines_before.append('#')
+    # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
+    lines_before.append('')
 
     licvalues = guess_license(srctree_use)
     lic_files_chksum = []
@@ -561,28 +563,28 @@ def create_recipe(args):
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
     extrafiles = extravalues.pop('extrafiles', {})
+    extra_pn = extravalues.pop('PN', None)
+    extra_pv = extravalues.pop('PV', None)
 
-    if not realpv:
-        realpv = extravalues.get('PV', None)
-        if realpv:
-            if not validate_pv(realpv):
-                realpv = None
-            else:
-                realpv = realpv.lower().split()[0]
-                if '_' in realpv:
-                    realpv = realpv.replace('_', '-')
-    if not pn:
-        pn = extravalues.get('PN', None)
-        if pn:
-            if pn.startswith('GNU '):
-                pn = pn[4:]
-            if ' ' in pn:
-                # Probably a descriptive identifier rather than a proper name
-                pn = None
-            else:
-                pn = pn.lower()
-                if '_' in pn:
-                    pn = pn.replace('_', '-')
+    if extra_pv and not realpv:
+        realpv = extra_pv
+        if not validate_pv(realpv):
+            realpv = None
+        else:
+            realpv = realpv.lower().split()[0]
+            if '_' in realpv:
+                realpv = realpv.replace('_', '-')
+    if extra_pn and not pn:
+        pn = extra_pn
+        if pn.startswith('GNU '):
+            pn = pn[4:]
+        if ' ' in pn:
+            # Probably a descriptive identifier rather than a proper name
+            pn = None
+        else:
+            pn = pn.lower()
+            if '_' in pn:
+                pn = pn.replace('_', '-')
 
     if not outfile:
         if not pn:
@@ -662,6 +664,9 @@ def create_recipe(args):
         outlines.append('')
     outlines.extend(lines_after)
 
+    if extravalues:
+        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
+
     if args.extract_to:
         scriptutils.git_convert_standalone_clone(srctree)
         if os.path.isdir(args.extract_to):
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index f84ec3d..37d161e 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -830,22 +830,35 @@ class SpecFileRecipeHandler(RecipeHandler):
         if 'PV' in extravalues and 'PN' in extravalues:
             return
         filelist = RecipeHandler.checkfiles(srctree, ['*.spec'], recursive=True)
-        pn = None
-        pv = None
+        valuemap = {'Name': 'PN',
+                    'Version': 'PV',
+                    'Summary': 'SUMMARY',
+                    'Url': 'HOMEPAGE',
+                    'License': 'LICENSE'}
+        foundvalues = {}
         for fileitem in filelist:
             linecount = 0
             with open(fileitem, 'r') as f:
                 for line in f:
-                    if line.startswith('Name:') and not pn:
-                        pn = line.split(':')[1].strip()
-                    if line.startswith('Version:') and not pv:
-                        pv = line.split(':')[1].strip()
-            if pv or pn:
-                if pv and not 'PV' in extravalues and validate_pv(pv):
-                    extravalues['PV'] = pv
-                if pn and not 'PN' in extravalues:
-                    extravalues['PN'] = pn
-                break
+                    for value, varname in valuemap.iteritems():
+                        if line.startswith(value + ':') and not varname in foundvalues:
+                            foundvalues[varname] = line.split(':', 1)[1].strip()
+                            break
+                    if len(foundvalues) == len(valuemap):
+                        break
+        if 'PV' in foundvalues:
+            if not validate_pv(foundvalues['PV']):
+                del foundvalues['PV']
+        license = foundvalues.pop('LICENSE', None)
+        if license:
+            liccomment = '# NOTE: spec file indicates the license may be "%s"' % license
+            for i, line in enumerate(lines_before):
+                if line.startswith('LICENSE ='):
+                    lines_before.insert(i, liccomment)
+                    break
+            else:
+                lines_before.append(liccomment)
+        extravalues.update(foundvalues)
 
 def register_recipe_handlers(handlers):
     # Set priorities with some gaps so that other plugins can insert
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index cc4fb42..ffbcb49 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -104,9 +104,9 @@ class NpmRecipeHandler(RecipeHandler):
                 classes.append('npm')
                 handled.append('buildsystem')
                 if 'description' in data:
-                    lines_before.append('SUMMARY = "%s"' % data['description'])
+                    extravalues['SUMMARY'] = data['description']
                 if 'homepage' in data:
-                    lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
+                    extravalues['HOMEPAGE'] = data['homepage']
 
                 # Shrinkwrap
                 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
-- 
2.5.5

[PATCH 5/7] recipetool: create: extract variable values from .deb/.ipk/.rpm

[Paul Eggleton]

Extract the metadata from package files and use it to set variable
values in the recipe (including recipe name and version, LICENSE,
SUMMARY, DESCRIPTION, SECTION and HOMEPAGE). For LICENSE we take care
not to step on any value determined by our license file scan; if there
is one we simply add a comment above the LICENSE setting so the user can
resolve it.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py | 105 +++++++++++++++++++++++++++++++++++----
 1 file changed, 94 insertions(+), 11 deletions(-)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 66c881a..ad26189 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -261,7 +261,11 @@ def determine_from_filename(srcfile):
         namepart = srcfile.split('.tar.')[0].lower()
     else:
         namepart = os.path.splitext(srcfile)[0].lower()
-    splitval = namepart.rsplit('_', 1)
+    if is_package(srcfile):
+        # Force getting the value from the package metadata
+        return None, None
+    else:
+        splitval = namepart.rsplit('_', 1)
     if len(splitval) == 1:
         splitval = namepart.rsplit('-', 1)
     pn = splitval[0].replace('_', '-')
@@ -327,6 +331,13 @@ def reformat_git_uri(uri):
             return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '')
     return uri
 
+def is_package(url):
+    '''Check if a URL points to a package'''
+    checkurl = url.split(';', 1)[0]
+    if checkurl.endswith(('.deb', '.ipk', '.rpm', '.srpm')):
+        return True
+    return False
+
 def create_recipe(args):
     import bb.process
     import tempfile
@@ -337,6 +348,7 @@ def create_recipe(args):
     if args.machine:
         pkgarch = "${MACHINE_ARCH}"
 
+    extravalues = {}
     checksums = (None, None)
     tempsrc = ''
     srcsubdir = ''
@@ -382,6 +394,33 @@ def create_recipe(args):
                     if '<html' in f.read(100).lower():
                         logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
                         sys.exit(1)
+
+        if is_package(fetchuri):
+            tmpfdir = tempfile.mkdtemp(prefix='recipetool-')
+            try:
+                pkgfile = None
+                try:
+                    fileuri = fetchuri + ';unpack=0'
+                    scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev)
+                    for root, _, files in os.walk(tmpfdir):
+                        for f in files:
+                            pkgfile = os.path.join(root, f)
+                            break
+                except bb.fetch2.BBFetchException as e:
+                    logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip())
+
+                if pkgfile:
+                    if pkgfile.endswith(('.deb', '.ipk')):
+                        stdout, _ = bb.process.run('ar x %s control.tar.gz' % pkgfile, cwd=tmpfdir)
+                        stdout, _ = bb.process.run('tar xf control.tar.gz ./control', cwd=tmpfdir)
+                        values = convert_debian(tmpfdir)
+                        extravalues.update(values)
+                    elif pkgfile.endswith(('.rpm', '.srpm')):
+                        stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir)
+                        values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml'))
+                        extravalues.update(values)
+            finally:
+                shutil.rmtree(tmpfdir)
     else:
         # Assume we're pointing to an existing source tree
         if args.extract_to:
@@ -458,6 +497,13 @@ def create_recipe(args):
         lines_before.append('# will not be in most cases) you must specify the correct value before using this')
         lines_before.append('# recipe for anything other than initial testing/development!')
         licenses = ['CLOSED']
+    pkg_license = extravalues.pop('LICENSE', None)
+    if pkg_license:
+        if licenses == ['Unknown']:
+            lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
+            licenses = [pkg_license]
+        else:
+            lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
     lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
     lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
     lines_before.append('')
@@ -558,7 +604,6 @@ def create_recipe(args):
         classes.append('bin_package')
         handled.append('buildsystem')
 
-    extravalues = {}
     for handler in handlers:
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
@@ -665,6 +710,9 @@ def create_recipe(args):
     outlines.extend(lines_after)
 
     if extravalues:
+        if 'LICENSE' in extravalues and not licvalues:
+            # Don't blow away 'CLOSED' value that comments say we set
+            del extravalues['LICENSE']
         _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
 
     if args.extract_to:
@@ -913,6 +961,12 @@ def convert_pkginfo(pkginfofile):
     return values
 
 def convert_debian(debpath):
+    value_map = {'Package': 'PN',
+                 'Version': 'PV',
+                 'Section': 'SECTION',
+                 'License': 'LICENSE',
+                 'Homepage': 'HOMEPAGE'}
+
     # FIXME extend this mapping - perhaps use distro_alias.inc?
     depmap = {'libz-dev': 'zlib'}
 
@@ -922,34 +976,63 @@ def convert_debian(debpath):
         indesc = False
         for line in f:
             if indesc:
-                if line.strip():
+                if line.startswith(' '):
                     if line.startswith(' This package contains'):
                         indesc = False
                     else:
-                        values['DESCRIPTION'] += ' ' + line.strip()
+                        if 'DESCRIPTION' in values:
+                            values['DESCRIPTION'] += ' ' + line.strip()
+                        else:
+                            values['DESCRIPTION'] = line.strip()
                 else:
                     indesc = False
-            else:
+            if not indesc:
                 splitline = line.split(':', 1)
-                key = line[0]
-                value = line[1]
+                if len(splitline) < 2:
+                    continue
+                key = splitline[0]
+                value = splitline[1].strip()
                 if key == 'Build-Depends':
                     for dep in value.split(','):
                         dep = dep.split()[0]
                         mapped = depmap.get(dep, '')
                         if mapped:
                             depends.append(mapped)
-                elif key == 'Section':
-                    values['SECTION'] = value
                 elif key == 'Description':
                     values['SUMMARY'] = value
                     indesc = True
+                else:
+                    varname = value_map.get(key, None)
+                    if varname:
+                        values[varname] = value
 
-    if depends:
-        values['DEPENDS'] = ' '.join(depends)
+    #if depends:
+    #    values['DEPENDS'] = ' '.join(depends)
 
     return values
 
+def convert_rpm_xml(xmlfile):
+    '''Converts the output from rpm -qp --xml to a set of variable values'''
+    import xml.etree.ElementTree as ElementTree
+    rpmtag_map = {'Name': 'PN',
+                  'Version': 'PV',
+                  'Summary': 'SUMMARY',
+                  'Description': 'DESCRIPTION',
+                  'License': 'LICENSE',
+                  'Url': 'HOMEPAGE'}
+
+    values = {}
+    tree = ElementTree.parse(xmlfile)
+    root = tree.getroot()
+    for child in root:
+        if child.tag == 'rpmTag':
+            name = child.attrib.get('name', None)
+            if name:
+                varname = rpmtag_map.get(name, None)
+                if varname:
+                    values[varname] = child[0].text
+    return values
+
 
 def register_commands(subparsers):
     parser_create = subparsers.add_parser('create',
-- 
2.5.5

[PATCH 6/7] recipetool: create: use ${BP} for subdir for binary packages

[Paul Eggleton]

If we use ${BP} for the subdirectory, the default value of S will work
rather than having to have an ugly value derived from the package
file name in both places. This does mean that we have to assume the
default though (we can't just let the normal logic work because the
value of BP is the default until later on, so the replacement doesn't
work).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index ad26189..1899a0d 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -359,7 +359,7 @@ def create_recipe(args):
         if args.binary:
             # Assume the archive contains the directory structure verbatim
             # so we need to extract to a subdirectory
-            fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0]
+            fetchuri += ';subdir=${BP}'
         srcuri = fetchuri
         rev_re = re.compile(';rev=([^;]+)')
         res = rev_re.search(srcuri)
@@ -566,7 +566,9 @@ def create_recipe(args):
         lines_before.append('SRCREV = "%s"' % srcrev)
     lines_before.append('')
 
-    if srcsubdir:
+    if srcsubdir and not args.binary:
+        # (for binary packages we explicitly specify subdir= when fetching to
+        # match the default value of S, so we don't need to set it in that case)
         lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
         lines_before.append('')
 
-- 
2.5.5

[PATCH 7/7] recipetool / devtool: set a fixed SRCREV by default when fetching from git

[Paul Eggleton]

If fetching source from a git repository, typically within OpenEmbedded
we encourage setting SRCREV to a fixed revision, so change to do that by
default and add a  -a/--autorev option to use "${AUTOREV}" instead.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/lib/oeqa/selftest/devtool.py | 6 ++++--
 scripts/lib/devtool/standard.py   | 3 +++
 scripts/lib/recipetool/create.py  | 5 +++++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index 132a73d..b64f9b3 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -207,12 +207,14 @@ class DevtoolTests(DevtoolBase):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         self.track_for_cleanup(tempdir)
         pn = 'dbus-wait'
+        srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
         # We choose an https:// git URL here to check rewriting the URL works
         url = 'https://git.yoctoproject.org/git/dbus-wait'
         # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf
         # instead of the directory name
         result = runCmd('git clone %s noname' % url, cwd=tempdir)
         srcdir = os.path.join(tempdir, 'noname')
+        result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir)
         self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory')
         # Test devtool add
         self.track_for_cleanup(self.workspacedir)
@@ -235,7 +237,7 @@ class DevtoolTests(DevtoolBase):
         checkvars['S'] = '${WORKDIR}/git'
         checkvars['PV'] = '0.1+git${SRCPV}'
         checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https'
-        checkvars['SRCREV'] = '${AUTOREV}'
+        checkvars['SRCREV'] = srcrev
         checkvars['DEPENDS'] = set(['dbus'])
         self._test_recipe_contents(recipefile, checkvars, [])
 
@@ -345,7 +347,7 @@ class DevtoolTests(DevtoolBase):
         self.track_for_cleanup(self.workspacedir)
         self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
         self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
-        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
+        result = runCmd('devtool add %s %s -a -f %s' % (testrecipe, srcdir, url))
         self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output)
         self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')
         # Test devtool status
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 77a82d5..3be3214 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -144,6 +144,8 @@ def add(args, config, basepath, workspace):
         extracmdopts += ' --also-native'
     if args.src_subdir:
         extracmdopts += ' --src-subdir "%s"' % args.src_subdir
+    if args.autorev:
+        extracmdopts += ' -a'
 
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
@@ -1390,6 +1392,7 @@ def register_commands(subparsers, context):
     parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
     parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
     parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
+    parser_add.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
     parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
     parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
     parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 1899a0d..4a59363 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -563,6 +563,10 @@ def create_recipe(args):
         lines_before.append('')
         lines_before.append('# Modify these as desired')
         lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
+        if not args.autorev and srcrev == '${AUTOREV}':
+            if os.path.exists(os.path.join(srctree, '.git')):
+                (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+            srcrev = stdout.rstrip()
         lines_before.append('SRCREV = "%s"' % srcrev)
     lines_before.append('')
 
@@ -1049,5 +1053,6 @@ def register_commands(subparsers):
     parser_create.add_argument('-b', '--binary', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true')
     parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
     parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
+    parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
     parser_create.set_defaults(func=create_recipe)
 
-- 
2.5.5