[PATCH 0/4] devtool related changes, part 2

[PATCH 0/4] devtool related changes, part 2

[Paul Eggleton]

The following changes since commit 18cd0ce6a55c9065c3f1bf223b47d817b5efcd8f:

  data: escape '$' in shell variable assignment (2014-12-18 10:24:45 +0000)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib paule/devtool-bb2
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/devtool-bb2

Paul Eggleton (4):
  siggen: ensure nostamp tasks force dependent tasks to re-execute
  utils: add basic metadata manipulation functions
  bitbake-layers: add commands for adding and removing layers
  cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN

 bin/bitbake-layers |  54 +++++++++++++++++++++++
 lib/bb/cooker.py   |   7 +--
 lib/bb/siggen.py   |   6 +++
 lib/bb/utils.py    | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 191 insertions(+), 3 deletions(-)

-- 
1.9.3

[PATCH 1/4] siggen: ensure nostamp tasks force dependent tasks to re-execute

[Paul Eggleton]

If a nostamp task is depended on by a non-nostamp task, then we want the
signature of that task to change such that it re-executes afterwards.
This is an unusual situation, but we want this to work in OE in
externalsrc.bbclass so that compilation happens every time it is
requested.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/siggen.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 5103073..0c77d72 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -187,6 +187,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 self.file_checksum_values[k][f] = cs
                 data = data + cs
 
+        taskdep = dataCache.task_deps[fn]
+        if 'nostamp' in taskdep and task in taskdep['nostamp']:
+            # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
+            import uuid
+            data = data + str(uuid.uuid4())
+
         taint = self.read_taint(fn, task, dataCache.stamp[fn])
         if taint:
             data = data + taint
-- 
1.9.3

[PATCH 2/4] utils: add basic metadata manipulation functions

[Paul Eggleton]

* Add a generic edit_metadata_file() function to modify variable
  assignments in any metadata file (conf, bb, bbappend) using a callback
  for flexibility
* Add a specific edit_bblayers_conf() function to modify
  conf/bblayers.conf and add and/or remove layers from the BBLAYERS
  value within it.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/utils.py | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 127 insertions(+)

diff --git a/lib/bb/utils.py b/lib/bb/utils.py
index d7c5067..f26349f 100644
--- a/lib/bb/utils.py
+++ b/lib/bb/utils.py
@@ -915,3 +915,130 @@ def exec_flat_python_func(func, *args, **kwargs):
     comp = bb.utils.better_compile(code, '<string>', '<string>')
     bb.utils.better_exec(comp, context, code, '<string>')
     return context['retval']
+
+def edit_metadata_file(meta_file, variables, func):
+    """Edit a recipe or config file and modify one or more specified
+    variable values set in the file using a specified callback function.
+    The file is only written to if the value(s) actually change.
+    """
+    var_res = {}
+    for var in variables:
+        var_res[var] = re.compile(r'^%s[ \t]*[?=]+' % var)
+
+    updated = False
+    varset_start = ''
+    newlines = []
+    in_var = None
+    full_value = ''
+
+    def handle_var_end():
+        (newvalue, indent, minbreak) = func(in_var, full_value)
+        if newvalue != full_value:
+            if isinstance(newvalue, list):
+                intentspc = ' ' * indent
+                if minbreak:
+                    # First item on first line
+                    if len(newvalue) == 1:
+                        newlines.append('%s "%s"\n' % (varset_start, newvalue[0]))
+                    else:
+                        newlines.append('%s "%s\\\n' % (varset_start, newvalue[0]))
+                        for item in newvalue[1:]:
+                            newlines.append('%s%s \\\n' % (intentspc, item))
+                        newlines.append('%s"\n' % indentspc)
+                else:
+                    # No item on first line
+                    newlines.append('%s " \\\n' % varset_start)
+                    for item in newvalue:
+                        newlines.append('%s%s \\\n' % (intentspc, item))
+                    newlines.append('%s"\n' % intentspc)
+            else:
+                newlines.append('%s "%s"\n' % (varset_start, newvalue))
+            return True
+        return False
+
+    with open(meta_file, 'r') as f:
+        for line in f:
+            if in_var:
+                value = line.rstrip()
+                full_value += value[:-1]
+                if value.endswith('"') or value.endswith("'"):
+                    if handle_var_end():
+                        updated = True
+                    in_var = None
+            else:
+                matched = False
+                for (varname, var_re) in var_res.iteritems():
+                    if var_re.match(line):
+                        splitvalue = line.split('"', 1)
+                        varset_start = splitvalue[0].rstrip()
+                        value = splitvalue[1].rstrip()
+                        if value.endswith('\\'):
+                            value = value[:-1]
+                        full_value = value
+                        if value.endswith('"') or value.endswith("'"):
+                            if handle_var_end():
+                                updated = True
+                        else:
+                            in_var = varname
+                        matched = True
+                        break
+                if not matched:
+                    newlines.append(line)
+    if updated:
+        with open(meta_file, 'w') as f:
+            f.writelines(newlines)
+
+def edit_bblayers_conf(bblayers_conf, add, remove):
+    """Edit bblayers.conf, adding and/or removing layers"""
+
+    import fnmatch
+
+    def remove_trailing_sep(pth):
+        if pth and pth[-1] == os.sep:
+            pth = pth[:-1]
+        return pth
+
+    def layerlist_param(value):
+        if not value:
+            return []
+        elif isinstance(value, list):
+            return [remove_trailing_sep(x) for x in value]
+        else:
+            return [remove_trailing_sep(value)]
+
+    notadded = []
+    notremoved = []
+
+    addlayers = layerlist_param(add)
+    removelayers = layerlist_param(remove)
+
+    def handle_bblayers(varname, origvalue):
+        updated = False
+        bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
+        if removelayers:
+            for removelayer in removelayers:
+                matched = False
+                for layer in bblayers:
+                    if fnmatch.fnmatch(layer, removelayer):
+                        updated = True
+                        matched = True
+                        bblayers.remove(layer)
+                        break
+                if not matched:
+                    notremoved.append(removelayer)
+        if addlayers:
+            for addlayer in addlayers:
+                if addlayer not in bblayers:
+                    updated = True
+                    bblayers.append(addlayer)
+                else:
+                    notadded.append(addlayer)
+
+        if updated:
+            return (bblayers, 2, False)
+        else:
+            return (origvalue, 2, False)
+
+    edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers)
+    return (notadded, notremoved)
+
-- 
1.9.3

[PATCH 3/4] bitbake-layers: add commands for adding and removing layers

[Paul Eggleton]

Add add-layer and remove-layer commands for easily adding and removing
layers to/from bblayers.conf.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 bin/bitbake-layers | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/bin/bitbake-layers b/bin/bitbake-layers
index 9964040..edb4d2f 100755
--- a/bin/bitbake-layers
+++ b/bin/bitbake-layers
@@ -103,6 +103,60 @@ class Commands(cmd.Cmd):
             logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), layerpri))
 
 
+    def do_add_layer(self, dirname):
+        """Add a layer to bblayers.conf
+
+usage: add-layer <layerdir>
+"""
+        if not dirname:
+            sys.stderr.write("Please specify the layer directory to add\n")
+            return
+
+        layerdir = os.path.abspath(dirname)
+        if not os.path.exists(layerdir):
+            sys.stderr.write("Specified layer directory doesn't exist\n")
+            return
+
+        layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
+        if not os.path.exists(layer_conf):
+            sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n")
+            return
+
+        bblayers_conf = os.path.join('conf', 'bblayers.conf')
+        if not os.path.exists(bblayers_conf):
+            sys.stderr.write("Unable to find bblayers.conf\n")
+            return
+
+        (notadded, _) = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None)
+        if notadded:
+            for item in notadded:
+                sys.stderr.write("Specified layer %s not in BBLAYERS\n" % item)
+
+
+    def do_remove_layer(self, dirname):
+        """Remove a layer from bblayers.conf
+
+usage: remove-layer <layerdir>
+"""
+        if not dirname:
+            sys.stderr.write("Please specify the layer directory to remove\n")
+            return
+
+        bblayers_conf = os.path.join('conf', 'bblayers.conf')
+        if not os.path.exists(bblayers_conf):
+            sys.stderr.write("Unable to find bblayers.conf\n")
+            return
+
+        if dirname.startswith('*'):
+            layerdir = dirname
+        else:
+            layerdir = os.path.abspath(dirname)
+        (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir)
+        if notremoved:
+            for item in notremoved:
+                sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
+
+
     def version_str(self, pe, pv, pr = None):
         verstr = "%s" % pv
         if pr:
-- 
1.9.3

[PATCH 4/4] cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN

[Paul Eggleton]

Add a BBFILE_PATTERN_IGNORE_EMPTY variable to allow ignoring the fact
that a regex specified in BBFILE_PATTERN for a particular collection
doesn't match any recipes. This will be used in OpenEmbedded in the
workspace layers created by "devtool" which may not always contain any
recipes (which is not cause for warning the user).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/cooker.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 16fd4ad..0d9b85e 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1385,7 +1385,7 @@ class BBCooker:
                 raise bb.BBHandledException()
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
-            self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn)
+            self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data)
             self.state = state.running
             return None
 
@@ -1610,7 +1610,7 @@ class CookerCollectFiles(object):
                     filelist.append(filename)
         return filelist
 
-    def collection_priorities(self, pkgfns):
+    def collection_priorities(self, pkgfns, d):
 
         priorities = {}
 
@@ -1639,7 +1639,8 @@ class CookerCollectFiles(object):
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
             if regex in unmatched:
-                collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+                if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1':
+                    collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
 
         return priorities
 
-- 
1.9.3