* [PATCH 1/4] siggen: ensure nostamp tasks force dependent tasks to re-execute
2014-12-19 10:20 [PATCH 0/4] devtool related changes, part 2 Paul Eggleton
@ 2014-12-19 10:20 ` Paul Eggleton
2014-12-19 10:20 ` [PATCH 2/4] utils: add basic metadata manipulation functions Paul Eggleton
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2014-12-19 10:20 UTC (permalink / raw)
To: bitbake-devel
If a nostamp task is depended on by a non-nostamp task, then we want the
signature of that task to change such that it re-executes afterwards.
This is an unusual situation, but we want this to work in OE in
externalsrc.bbclass so that compilation happens every time it is
requested.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/siggen.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 5103073..0c77d72 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -187,6 +187,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
self.file_checksum_values[k][f] = cs
data = data + cs
+ taskdep = dataCache.task_deps[fn]
+ if 'nostamp' in taskdep and task in taskdep['nostamp']:
+ # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
+ import uuid
+ data = data + str(uuid.uuid4())
+
taint = self.read_taint(fn, task, dataCache.stamp[fn])
if taint:
data = data + taint
--
1.9.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/4] utils: add basic metadata manipulation functions
2014-12-19 10:20 [PATCH 0/4] devtool related changes, part 2 Paul Eggleton
2014-12-19 10:20 ` [PATCH 1/4] siggen: ensure nostamp tasks force dependent tasks to re-execute Paul Eggleton
@ 2014-12-19 10:20 ` Paul Eggleton
2014-12-19 10:20 ` [PATCH 3/4] bitbake-layers: add commands for adding and removing layers Paul Eggleton
2014-12-19 10:20 ` [PATCH 4/4] cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN Paul Eggleton
3 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2014-12-19 10:20 UTC (permalink / raw)
To: bitbake-devel
* Add a generic edit_metadata_file() function to modify variable
assignments in any metadata file (conf, bb, bbappend) using a callback
for flexibility
* Add a specific edit_bblayers_conf() function to modify
conf/bblayers.conf and add and/or remove layers from the BBLAYERS
value within it.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/utils.py | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 127 insertions(+)
diff --git a/lib/bb/utils.py b/lib/bb/utils.py
index d7c5067..f26349f 100644
--- a/lib/bb/utils.py
+++ b/lib/bb/utils.py
@@ -915,3 +915,130 @@ def exec_flat_python_func(func, *args, **kwargs):
comp = bb.utils.better_compile(code, '<string>', '<string>')
bb.utils.better_exec(comp, context, code, '<string>')
return context['retval']
+
+def edit_metadata_file(meta_file, variables, func):
+ """Edit a recipe or config file and modify one or more specified
+ variable values set in the file using a specified callback function.
+ The file is only written to if the value(s) actually change.
+ """
+ var_res = {}
+ for var in variables:
+ var_res[var] = re.compile(r'^%s[ \t]*[?=]+' % var)
+
+ updated = False
+ varset_start = ''
+ newlines = []
+ in_var = None
+ full_value = ''
+
+ def handle_var_end():
+ (newvalue, indent, minbreak) = func(in_var, full_value)
+ if newvalue != full_value:
+ if isinstance(newvalue, list):
+ intentspc = ' ' * indent
+ if minbreak:
+ # First item on first line
+ if len(newvalue) == 1:
+ newlines.append('%s "%s"\n' % (varset_start, newvalue[0]))
+ else:
+ newlines.append('%s "%s\\\n' % (varset_start, newvalue[0]))
+ for item in newvalue[1:]:
+ newlines.append('%s%s \\\n' % (intentspc, item))
+ newlines.append('%s"\n' % indentspc)
+ else:
+ # No item on first line
+ newlines.append('%s " \\\n' % varset_start)
+ for item in newvalue:
+ newlines.append('%s%s \\\n' % (intentspc, item))
+ newlines.append('%s"\n' % intentspc)
+ else:
+ newlines.append('%s "%s"\n' % (varset_start, newvalue))
+ return True
+ return False
+
+ with open(meta_file, 'r') as f:
+ for line in f:
+ if in_var:
+ value = line.rstrip()
+ full_value += value[:-1]
+ if value.endswith('"') or value.endswith("'"):
+ if handle_var_end():
+ updated = True
+ in_var = None
+ else:
+ matched = False
+ for (varname, var_re) in var_res.iteritems():
+ if var_re.match(line):
+ splitvalue = line.split('"', 1)
+ varset_start = splitvalue[0].rstrip()
+ value = splitvalue[1].rstrip()
+ if value.endswith('\\'):
+ value = value[:-1]
+ full_value = value
+ if value.endswith('"') or value.endswith("'"):
+ if handle_var_end():
+ updated = True
+ else:
+ in_var = varname
+ matched = True
+ break
+ if not matched:
+ newlines.append(line)
+ if updated:
+ with open(meta_file, 'w') as f:
+ f.writelines(newlines)
+
+def edit_bblayers_conf(bblayers_conf, add, remove):
+ """Edit bblayers.conf, adding and/or removing layers"""
+
+ import fnmatch
+
+ def remove_trailing_sep(pth):
+ if pth and pth[-1] == os.sep:
+ pth = pth[:-1]
+ return pth
+
+ def layerlist_param(value):
+ if not value:
+ return []
+ elif isinstance(value, list):
+ return [remove_trailing_sep(x) for x in value]
+ else:
+ return [remove_trailing_sep(value)]
+
+ notadded = []
+ notremoved = []
+
+ addlayers = layerlist_param(add)
+ removelayers = layerlist_param(remove)
+
+ def handle_bblayers(varname, origvalue):
+ updated = False
+ bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
+ if removelayers:
+ for removelayer in removelayers:
+ matched = False
+ for layer in bblayers:
+ if fnmatch.fnmatch(layer, removelayer):
+ updated = True
+ matched = True
+ bblayers.remove(layer)
+ break
+ if not matched:
+ notremoved.append(removelayer)
+ if addlayers:
+ for addlayer in addlayers:
+ if addlayer not in bblayers:
+ updated = True
+ bblayers.append(addlayer)
+ else:
+ notadded.append(addlayer)
+
+ if updated:
+ return (bblayers, 2, False)
+ else:
+ return (origvalue, 2, False)
+
+ edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers)
+ return (notadded, notremoved)
+
--
1.9.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/4] bitbake-layers: add commands for adding and removing layers
2014-12-19 10:20 [PATCH 0/4] devtool related changes, part 2 Paul Eggleton
2014-12-19 10:20 ` [PATCH 1/4] siggen: ensure nostamp tasks force dependent tasks to re-execute Paul Eggleton
2014-12-19 10:20 ` [PATCH 2/4] utils: add basic metadata manipulation functions Paul Eggleton
@ 2014-12-19 10:20 ` Paul Eggleton
2014-12-19 10:20 ` [PATCH 4/4] cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN Paul Eggleton
3 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2014-12-19 10:20 UTC (permalink / raw)
To: bitbake-devel
Add add-layer and remove-layer commands for easily adding and removing
layers to/from bblayers.conf.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bin/bitbake-layers | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)
diff --git a/bin/bitbake-layers b/bin/bitbake-layers
index 9964040..edb4d2f 100755
--- a/bin/bitbake-layers
+++ b/bin/bitbake-layers
@@ -103,6 +103,60 @@ class Commands(cmd.Cmd):
logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), layerpri))
+ def do_add_layer(self, dirname):
+ """Add a layer to bblayers.conf
+
+usage: add-layer <layerdir>
+"""
+ if not dirname:
+ sys.stderr.write("Please specify the layer directory to add\n")
+ return
+
+ layerdir = os.path.abspath(dirname)
+ if not os.path.exists(layerdir):
+ sys.stderr.write("Specified layer directory doesn't exist\n")
+ return
+
+ layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
+ if not os.path.exists(layer_conf):
+ sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n")
+ return
+
+ bblayers_conf = os.path.join('conf', 'bblayers.conf')
+ if not os.path.exists(bblayers_conf):
+ sys.stderr.write("Unable to find bblayers.conf\n")
+ return
+
+ (notadded, _) = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None)
+ if notadded:
+ for item in notadded:
+ sys.stderr.write("Specified layer %s not in BBLAYERS\n" % item)
+
+
+ def do_remove_layer(self, dirname):
+ """Remove a layer from bblayers.conf
+
+usage: remove-layer <layerdir>
+"""
+ if not dirname:
+ sys.stderr.write("Please specify the layer directory to remove\n")
+ return
+
+ bblayers_conf = os.path.join('conf', 'bblayers.conf')
+ if not os.path.exists(bblayers_conf):
+ sys.stderr.write("Unable to find bblayers.conf\n")
+ return
+
+ if dirname.startswith('*'):
+ layerdir = dirname
+ else:
+ layerdir = os.path.abspath(dirname)
+ (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir)
+ if notremoved:
+ for item in notremoved:
+ sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
+
+
def version_str(self, pe, pv, pr = None):
verstr = "%s" % pv
if pr:
--
1.9.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 4/4] cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN
2014-12-19 10:20 [PATCH 0/4] devtool related changes, part 2 Paul Eggleton
` (2 preceding siblings ...)
2014-12-19 10:20 ` [PATCH 3/4] bitbake-layers: add commands for adding and removing layers Paul Eggleton
@ 2014-12-19 10:20 ` Paul Eggleton
3 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2014-12-19 10:20 UTC (permalink / raw)
To: bitbake-devel
Add a BBFILE_PATTERN_IGNORE_EMPTY variable to allow ignoring the fact
that a regex specified in BBFILE_PATTERN for a particular collection
doesn't match any recipes. This will be used in OpenEmbedded in the
workspace layers created by "devtool" which may not always contain any
recipes (which is not cause for warning the user).
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/cooker.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 16fd4ad..0d9b85e 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1385,7 +1385,7 @@ class BBCooker:
raise bb.BBHandledException()
self.show_appends_with_no_recipes()
self.handlePrefProviders()
- self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn)
+ self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data)
self.state = state.running
return None
@@ -1610,7 +1610,7 @@ class CookerCollectFiles(object):
filelist.append(filename)
return filelist
- def collection_priorities(self, pkgfns):
+ def collection_priorities(self, pkgfns, d):
priorities = {}
@@ -1639,7 +1639,8 @@ class CookerCollectFiles(object):
for collection, pattern, regex, _ in self.bbfile_config_priorities:
if regex in unmatched:
- collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+ if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1':
+ collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
return priorities
--
1.9.3
^ permalink raw reply related [flat|nested] 5+ messages in thread