* [PATCH 1/5] tinfoil: create simple interface for bitbake-based utilities
2012-08-27 20:44 [PATCH 0/5] Refactoring + improve bitbake-diffsigs Paul Eggleton
@ 2012-08-27 20:44 ` Paul Eggleton
2012-08-27 20:44 ` [PATCH 2/5] lib/bb/siggen: replace tabs with spaces Paul Eggleton
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Paul Eggleton @ 2012-08-27 20:44 UTC (permalink / raw)
To: bitbake-devel
The code to initialise BitBake within bitbake-layers should be useful
for other utilities that need to query configuration or recipe
information, so refactor it out into its own class, "Tinfoil" (to
continue with our cooking metaphor).
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bitbake/bin/bitbake-layers | 146 ++++++++++++++------------------------------
bitbake/lib/bb/tinfoil.py | 98 +++++++++++++++++++++++++++++
2 files changed, 145 insertions(+), 99 deletions(-)
create mode 100644 bitbake/lib/bb/tinfoil.py
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
index f473711..fa4e767 100755
--- a/bitbake/bin/bitbake-layers
+++ b/bitbake/bin/bitbake-layers
@@ -6,10 +6,22 @@
# Copyright (C) 2011 Mentor Graphics Corporation
# Copyright (C) 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import cmd
import logging
-import warnings
import os
import sys
import fnmatch
@@ -23,26 +35,14 @@ import bb.cache
import bb.cooker
import bb.providers
import bb.utils
-from bb.cooker import state
-import bb.fetch2
+import bb.tinfoil
logger = logging.getLogger('BitBake')
-warnings.filterwarnings("ignore", category=DeprecationWarning)
def main(args):
- # Set up logging
- console = logging.StreamHandler(sys.stdout)
- format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
- bb.msg.addDefaultlogFilter(console)
- console.setFormatter(format)
- logger.addHandler(console)
-
- initialenv = os.environ.copy()
- bb.utils.clean_environment()
-
- cmds = Commands(initialenv)
+ cmds = Commands()
if args:
# Allow user to specify e.g. show-layers instead of show_layers
args = [args[0].replace('-', '_')] + args[1:]
@@ -53,46 +53,11 @@ def main(args):
class Commands(cmd.Cmd):
- def __init__(self, initialenv):
+ def __init__(self):
cmd.Cmd.__init__(self)
+ self.bbhandler = bb.tinfoil.Tinfoil()
self.returncode = 0
- self.config = Config(parse_only=True)
- self.cooker = bb.cooker.BBCooker(self.config,
- self.register_idle_function,
- initialenv)
- self.config_data = self.cooker.configuration.data
- bb.providers.logger.setLevel(logging.ERROR)
- self.cooker_data = None
- self.bblayers = (self.config_data.getVar('BBLAYERS', True) or "").split()
-
- def register_idle_function(self, function, data):
- pass
-
- def prepare_cooker(self):
- sys.stderr.write("Parsing recipes..")
- logger.setLevel(logging.WARNING)
-
- try:
- while self.cooker.state in (state.initial, state.parsing):
- self.cooker.updateCache()
- except KeyboardInterrupt:
- self.cooker.shutdown()
- self.cooker.updateCache()
- sys.exit(2)
-
- logger.setLevel(logging.INFO)
- sys.stderr.write("done.\n")
-
- self.cooker_data = self.cooker.status
- self.cooker_data.appends = self.cooker.appendlist
-
- def check_prepare_cooker(self, config_only = False):
- if not self.cooker_data:
- if config_only:
- self.cooker.parseConfiguration()
- self.cooker_data = self.cooker.status
- else:
- self.prepare_cooker()
+ self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split()
def default(self, line):
"""Handle unrecognised commands"""
@@ -117,13 +82,13 @@ class Commands(cmd.Cmd):
def do_show_layers(self, args):
"""show current configured layers"""
- self.check_prepare_cooker(config_only = True)
+ self.bbhandler.prepare(config_only = True)
logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
logger.plain('=' * 74)
for layerdir in self.bblayers:
layername = self.get_layer_name(layerdir)
layerpri = 0
- for layer, _, regex, pri in self.cooker.status.bbfile_config_priorities:
+ for layer, _, regex, pri in self.bbhandler.cooker.status.bbfile_config_priorities:
if regex.match(os.path.join(layerdir, 'test')):
layerpri = pri
break
@@ -154,7 +119,7 @@ Options:
recipes with the ones they overlay indented underneath
-s only list overlayed recipes where the version is the same
"""
- self.check_prepare_cooker()
+ self.bbhandler.prepare()
show_filenames = False
show_same_ver_only = False
@@ -186,7 +151,7 @@ Options:
# factor - however, each layer.conf is free to either prepend or append to
# BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
# not be exactly the order present in bblayers.conf either.
- bbpath = str(self.config_data.getVar('BBPATH', True))
+ bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
overlayed_class_found = False
for (classfile, classdirs) in classes.items():
if len(classdirs) > 1:
@@ -237,7 +202,7 @@ Options:
-m only list where multiple recipes (in the same layer or different
layers) exist for the same recipe name
"""
- self.check_prepare_cooker()
+ self.bbhandler.prepare()
show_filenames = False
show_multi_provider_only = False
@@ -259,15 +224,15 @@ Options:
def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only):
- pkg_pn = self.cooker.status.pkg_pn
- (latest_versions, preferred_versions) = bb.providers.findProviders(self.cooker.configuration.data, self.cooker.status, pkg_pn)
- allproviders = bb.providers.allProviders(self.cooker.status)
+ pkg_pn = self.bbhandler.cooker.status.pkg_pn
+ (latest_versions, preferred_versions) = bb.providers.findProviders(self.bbhandler.cooker.configuration.data, self.bbhandler.cooker.status, pkg_pn)
+ allproviders = bb.providers.allProviders(self.bbhandler.cooker.status)
# Ensure we list skipped recipes
# We are largely guessing about PN, PV and the preferred version here,
# but we have no choice since skipped recipes are not fully parsed
- skiplist = self.cooker.skiplist.keys()
- skiplist.sort( key=lambda fileitem: self.cooker.calc_bbfile_priority(fileitem) )
+ skiplist = self.bbhandler.cooker.skiplist.keys()
+ skiplist.sort( key=lambda fileitem: self.bbhandler.cooker.calc_bbfile_priority(fileitem) )
skiplist.reverse()
for fn in skiplist:
recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
@@ -375,7 +340,7 @@ build results (as the layer priority order has effectively changed).
logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
return
- self.check_prepare_cooker()
+ self.bbhandler.prepare()
layers = self.bblayers
if len(arglist) > 2:
layernames = arglist[:-1]
@@ -405,8 +370,8 @@ build results (as the layer priority order has effectively changed).
appended_recipes = []
for layer in layers:
overlayed = []
- for f in self.cooker.overlayed.iterkeys():
- for of in self.cooker.overlayed[f]:
+ for f in self.bbhandler.cooker.overlayed.iterkeys():
+ for of in self.bbhandler.cooker.overlayed[f]:
if of.startswith(layer):
overlayed.append(of)
@@ -430,8 +395,8 @@ build results (as the layer priority order has effectively changed).
logger.warn('Overwriting file %s', fdest)
bb.utils.copyfile(f1full, fdest)
if ext == '.bb':
- if f1 in self.cooker_data.appends:
- appends = self.cooker_data.appends[f1]
+ if f1 in self.bbhandler.cooker.appendlist:
+ appends = self.bbhandler.cooker.appendlist[f1]
if appends:
logger.plain(' Applying appends to %s' % fdest )
for appendname in appends:
@@ -440,9 +405,9 @@ build results (as the layer priority order has effectively changed).
appended_recipes.append(f1)
# Take care of when some layers are excluded and yet we have included bbappends for those recipes
- for recipename in self.cooker_data.appends.iterkeys():
+ for recipename in self.bbhandler.cooker.appendlist.iterkeys():
if recipename not in appended_recipes:
- appends = self.cooker_data.appends[recipename]
+ appends = self.bbhandler.cooker.appendlist[recipename]
first_append = None
for appendname in appends:
layer = layer_path_match(appendname)
@@ -460,14 +425,14 @@ build results (as the layer priority order has effectively changed).
# have come from)
first_regex = None
layerdir = layers[0]
- for layername, pattern, regex, _ in self.cooker.status.bbfile_config_priorities:
+ for layername, pattern, regex, _ in self.bbhandler.cooker.status.bbfile_config_priorities:
if regex.match(os.path.join(layerdir, 'test')):
first_regex = regex
break
if first_regex:
# Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
- bbfiles = str(self.config_data.getVar('BBFILES', True)).split()
+ bbfiles = str(self.bbhandler.config_data.getVar('BBFILES', True)).split()
bbfiles_layer = []
for item in bbfiles:
if first_regex.match(item):
@@ -490,7 +455,7 @@ build results (as the layer priority order has effectively changed).
logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
def get_file_layer(self, filename):
- for layer, _, regex, _ in self.cooker.status.bbfile_config_priorities:
+ for layer, _, regex, _ in self.bbhandler.cooker.status.bbfile_config_priorities:
if regex.match(filename):
for layerdir in self.bblayers:
if regex.match(os.path.join(layerdir, 'test')):
@@ -516,14 +481,14 @@ usage: show-appends
Recipes are listed with the bbappends that apply to them as subitems.
"""
- self.check_prepare_cooker()
- if not self.cooker_data.appends:
+ self.bbhandler.prepare()
+ if not self.bbhandler.cooker.appendlist:
logger.plain('No append files found')
return
logger.plain('=== Appended recipes ===')
- pnlist = list(self.cooker_data.pkg_pn.keys())
+ pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys())
pnlist.sort()
for pn in pnlist:
self.show_appends_for_pn(pn)
@@ -531,19 +496,19 @@ Recipes are listed with the bbappends that apply to them as subitems.
self.show_appends_for_skipped()
def show_appends_for_pn(self, pn):
- filenames = self.cooker_data.pkg_pn[pn]
+ filenames = self.bbhandler.cooker_data.pkg_pn[pn]
best = bb.providers.findBestProvider(pn,
- self.cooker.configuration.data,
- self.cooker_data,
- self.cooker_data.pkg_pn)
+ self.bbhandler.cooker.configuration.data,
+ self.bbhandler.cooker_data,
+ self.bbhandler.cooker_data.pkg_pn)
best_filename = os.path.basename(best[3])
self.show_appends_output(filenames, best_filename)
def show_appends_for_skipped(self):
filenames = [os.path.basename(f)
- for f in self.cooker.skiplist.iterkeys()]
+ for f in self.bbhandler.cooker.skiplist.iterkeys()]
self.show_appends_output(filenames, None, " (skipped)")
def show_appends_output(self, filenames, best_filename, name_suffix = ''):
@@ -569,7 +534,7 @@ Recipes are listed with the bbappends that apply to them as subitems.
continue
basename = os.path.basename(filename)
- appends = self.cooker_data.appends.get(basename)
+ appends = self.bbhandler.cooker.appendlist.get(basename)
if appends:
appended.append((basename, list(appends)))
else:
@@ -577,22 +542,5 @@ Recipes are listed with the bbappends that apply to them as subitems.
return appended, notappended
-class Config(object):
- def __init__(self, **options):
- self.pkgs_to_build = []
- self.debug_domains = []
- self.extra_assume_provided = []
- self.prefile = []
- self.postfile = []
- self.debug = 0
- self.__dict__.update(options)
-
- def __getattr__(self, attribute):
- try:
- return super(Config, self).__getattribute__(attribute)
- except AttributeError:
- return None
-
-
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]) or 0)
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
new file mode 100644
index 0000000..73d8fe9
--- /dev/null
+++ b/bitbake/lib/bb/tinfoil.py
@@ -0,0 +1,98 @@
+# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
+#
+# Copyright (C) 2012 Intel Corporation
+# Copyright (C) 2011 Mentor Graphics Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import logging
+import warnings
+import os
+import sys
+
+import bb.cache
+import bb.cooker
+import bb.providers
+import bb.utils
+from bb.cooker import state
+import bb.fetch2
+
+class Tinfoil:
+ def __init__(self):
+ # Needed to avoid deprecation warnings with python 2.6
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+ # Set up logging
+ self.logger = logging.getLogger('BitBake')
+ console = logging.StreamHandler(sys.stdout)
+ format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+ bb.msg.addDefaultlogFilter(console)
+ console.setFormatter(format)
+ self.logger.addHandler(console)
+
+ initialenv = os.environ.copy()
+ bb.utils.clean_environment()
+ self.config = TinfoilConfig(parse_only=True)
+ self.cooker = bb.cooker.BBCooker(self.config,
+ self.register_idle_function,
+ initialenv)
+ self.config_data = self.cooker.configuration.data
+ bb.providers.logger.setLevel(logging.ERROR)
+ self.cooker_data = None
+
+ def register_idle_function(self, function, data):
+ pass
+
+ def parseRecipes(self):
+ sys.stderr.write("Parsing recipes..")
+ self.logger.setLevel(logging.WARNING)
+
+ try:
+ while self.cooker.state in (state.initial, state.parsing):
+ self.cooker.updateCache()
+ except KeyboardInterrupt:
+ self.cooker.shutdown()
+ self.cooker.updateCache()
+ sys.exit(2)
+
+ self.logger.setLevel(logging.INFO)
+ sys.stderr.write("done.\n")
+
+ self.cooker_data = self.cooker.status
+
+ def prepare(self, config_only = False):
+ if not self.cooker_data:
+ if config_only:
+ self.cooker.parseConfiguration()
+ self.cooker_data = self.cooker.status
+ else:
+ self.parseRecipes()
+
+
+class TinfoilConfig(object):
+ def __init__(self, **options):
+ self.pkgs_to_build = []
+ self.debug_domains = []
+ self.extra_assume_provided = []
+ self.prefile = []
+ self.postfile = []
+ self.debug = 0
+ self.__dict__.update(options)
+
+ def __getattr__(self, attribute):
+ try:
+ return super(TinfoilConfig, self).__getattribute__(attribute)
+ except AttributeError:
+ return None
+
--
1.7.9.5
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 2/5] lib/bb/siggen: replace tabs with spaces
2012-08-27 20:44 [PATCH 0/5] Refactoring + improve bitbake-diffsigs Paul Eggleton
2012-08-27 20:44 ` [PATCH 1/5] tinfoil: create simple interface for bitbake-based utilities Paul Eggleton
@ 2012-08-27 20:44 ` Paul Eggleton
2012-08-27 20:44 ` [PATCH 3/5] lib/bb/siggen.py: insert a colon between class and recipe name Paul Eggleton
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Paul Eggleton @ 2012-08-27 20:44 UTC (permalink / raw)
To: bitbake-devel
We had one section of the code mixing tabs with spaces, which is
particularly undesirable with python code.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bitbake/lib/bb/siggen.py | 34 +++++++++++++++++-----------------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 02a4268..b04a8bc 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -185,8 +185,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
if task in dataCache.file_checksums[fn]:
checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
for (f,cs) in checksums:
- self.file_checksum_values[k][f] = cs
- data = data + cs
+ self.file_checksum_values[k][f] = cs
+ data = data + cs
taint = self.read_taint(fn, task, dataCache.stamp[fn])
if taint:
@@ -372,24 +372,24 @@ def compare_sigfiles(a, b):
changed, added, removed = dict_diff(a, b)
if added:
for dep in added:
- bdep_found = False
- if removed:
- for bdep in removed:
- if a[dep] == b[bdep]:
- #print "Dependency on task %s was replaced by %s with same hash" % (dep, bdep)
- bdep_found = True
- if not bdep_found:
+ bdep_found = False
+ if removed:
+ for bdep in removed:
+ if a[dep] == b[bdep]:
+ #print "Dependency on task %s was replaced by %s with same hash" % (dep, bdep)
+ bdep_found = True
+ if not bdep_found:
print "Dependency on task %s was added with hash %s" % (dep, a[dep])
if removed:
for dep in removed:
- adep_found = False
- if added:
- for adep in added:
- if a[adep] == b[dep]:
- #print "Dependency on task %s was replaced by %s with same hash" % (adep, dep)
- adep_found = True
- if not adep_found:
- print "Dependency on task %s was removed with hash %s" % (dep, b[dep])
+ adep_found = False
+ if added:
+ for adep in added:
+ if a[adep] == b[dep]:
+ #print "Dependency on task %s was replaced by %s with same hash" % (adep, dep)
+ adep_found = True
+ if not adep_found:
+ print "Dependency on task %s was removed with hash %s" % (dep, b[dep])
if changed:
for dep in changed:
print "Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep])
--
1.7.9.5
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 3/5] lib/bb/siggen.py: insert a colon between class and recipe name
2012-08-27 20:44 [PATCH 0/5] Refactoring + improve bitbake-diffsigs Paul Eggleton
2012-08-27 20:44 ` [PATCH 1/5] tinfoil: create simple interface for bitbake-based utilities Paul Eggleton
2012-08-27 20:44 ` [PATCH 2/5] lib/bb/siggen: replace tabs with spaces Paul Eggleton
@ 2012-08-27 20:44 ` Paul Eggleton
2012-08-27 20:44 ` [PATCH 4/5] lib/bb/siggen.py: make signature dump/compare functions return a list Paul Eggleton
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Paul Eggleton @ 2012-08-27 20:44 UTC (permalink / raw)
To: bitbake-devel
before:
virtual:nativeautomake_1.12.1.bb.do_compile
after:
virtual:native:automake_1.12.1.bb.do_compile
This separation ensures that the key is readable, and if necessary,
parsable.
Unfortunately this invalidates previous native sstate signatures with
OE-Core - not much that can be done about that; however that occurs
frequently during the development cycle so it's par for the course.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bitbake/lib/bb/siggen.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index b04a8bc..d0a4d18 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -290,7 +290,7 @@ def dump_this_task(outfile, d):
def clean_basepath(a):
if a.startswith("virtual:"):
- b = a.rsplit(":", 1)[0] + a.rsplit("/", 1)[1]
+ b = a.rsplit(":", 1)[0] + ":" + a.rsplit("/", 1)[1]
else:
b = a.rsplit("/", 1)[1]
return b
--
1.7.9.5
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 4/5] lib/bb/siggen.py: make signature dump/compare functions return a list
2012-08-27 20:44 [PATCH 0/5] Refactoring + improve bitbake-diffsigs Paul Eggleton
` (2 preceding siblings ...)
2012-08-27 20:44 ` [PATCH 3/5] lib/bb/siggen.py: insert a colon between class and recipe name Paul Eggleton
@ 2012-08-27 20:44 ` Paul Eggleton
2012-08-27 20:44 ` [PATCH 5/5] bitbake-diffsigs: allow specifying task & follow deps recursively Paul Eggleton
2012-09-07 11:09 ` [PATCH 0/5] Refactoring + improve bitbake-diffsigs Richard Purdie
5 siblings, 0 replies; 7+ messages in thread
From: Paul Eggleton @ 2012-08-27 20:44 UTC (permalink / raw)
To: bitbake-devel
These functions become a little bit more reusable if they return a list
containing the output rather than just printing it.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bitbake/bin/bitbake-diffsigs | 7 +++--
bitbake/bin/bitbake-dumpsig | 4 ++-
bitbake/lib/bb/siggen.py | 68 +++++++++++++++++++++++-------------------
3 files changed, 46 insertions(+), 33 deletions(-)
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
index 5eb77ce..146cab8 100755
--- a/bitbake/bin/bitbake-diffsigs
+++ b/bitbake/bin/bitbake-diffsigs
@@ -7,6 +7,9 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), '
import bb.siggen
if len(sys.argv) > 2:
- bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
+ output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
else:
- bb.siggen.dump_sigfile(sys.argv[1])
+ output = bb.siggen.dump_sigfile(sys.argv[1])
+
+if output:
+ print '\n'.join(output)
diff --git a/bitbake/bin/bitbake-dumpsig b/bitbake/bin/bitbake-dumpsig
index 59740c8..ccbc412 100755
--- a/bitbake/bin/bitbake-dumpsig
+++ b/bitbake/bin/bitbake-dumpsig
@@ -6,4 +6,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), '
import bb.siggen
-bb.siggen.dump_sigfile(sys.argv[1])
+output = bb.siggen.dump_sigfile(sys.argv[1])
+if output:
+ print '\n'.join(output)
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index d0a4d18..8d1501b 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -302,6 +302,8 @@ def clean_basepaths(a):
return b
def compare_sigfiles(a, b):
+ output = []
+
p1 = pickle.Unpickler(open(a, "rb"))
a_data = p1.load()
p2 = pickle.Unpickler(open(b, "rb"))
@@ -320,50 +322,50 @@ def compare_sigfiles(a, b):
return changed, added, removed
if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
- print "basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist'])
+ output.append("basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist']))
if a_data['basewhitelist'] and b_data['basewhitelist']:
- print "changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])
+ output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
- print "taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist'])
+ output.append("taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
if a_data['taskwhitelist'] and b_data['taskwhitelist']:
- print "changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])
+ output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
if a_data['taskdeps'] != b_data['taskdeps']:
- print "Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))
+ output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
if a_data['basehash'] != b_data['basehash']:
- print "basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash'])
+ output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash']))
changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
if changed:
for dep in changed:
- print "List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])
+ output.append("List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
- print "changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])
+ output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
if added:
for dep in added:
- print "Dependency on variable %s was added" % (dep)
+ output.append("Dependency on variable %s was added" % (dep))
if removed:
for dep in removed:
- print "Dependency on Variable %s was removed" % (dep)
+ output.append("Dependency on Variable %s was removed" % (dep))
changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
if changed:
for dep in changed:
- print "Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep])
+ output.append("Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
if changed:
for f in changed:
- print "Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f])
+ output.append("Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f]))
if added:
for f in added:
- print "Dependency on checksum of file %s was added" % (f)
+ output.append("Dependency on checksum of file %s was added" % (f))
if removed:
for f in removed:
- print "Dependency on checksum of file %s was removed" % (f)
+ output.append("Dependency on checksum of file %s was removed" % (f))
if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
@@ -376,58 +378,64 @@ def compare_sigfiles(a, b):
if removed:
for bdep in removed:
if a[dep] == b[bdep]:
- #print "Dependency on task %s was replaced by %s with same hash" % (dep, bdep)
+ #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
bdep_found = True
if not bdep_found:
- print "Dependency on task %s was added with hash %s" % (dep, a[dep])
+ output.append("Dependency on task %s was added with hash %s" % (dep, a[dep]))
if removed:
for dep in removed:
adep_found = False
if added:
for adep in added:
if a[adep] == b[dep]:
- #print "Dependency on task %s was replaced by %s with same hash" % (adep, dep)
+ #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
adep_found = True
if not adep_found:
- print "Dependency on task %s was removed with hash %s" % (dep, b[dep])
+ output.append("Dependency on task %s was removed with hash %s" % (dep, b[dep]))
if changed:
for dep in changed:
- print "Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep])
+ output.append("Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep]))
a_taint = a_data.get('taint', None)
b_taint = b_data.get('taint', None)
if a_taint != b_taint:
- print "Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint)
+ output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint))
+
+ return output
def dump_sigfile(a):
+ output = []
+
p1 = pickle.Unpickler(open(a, "rb"))
a_data = p1.load()
- print "basewhitelist: %s" % (a_data['basewhitelist'])
+ output.append("basewhitelist: %s" % (a_data['basewhitelist']))
- print "taskwhitelist: %s" % (a_data['taskwhitelist'])
+ output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
- print "Task dependencies: %s" % (sorted(a_data['taskdeps']))
+ output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
- print "basehash: %s" % (a_data['basehash'])
+ output.append("basehash: %s" % (a_data['basehash']))
for dep in a_data['gendeps']:
- print "List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep])
+ output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
for dep in a_data['varvals']:
- print "Variable %s value is %s" % (dep, a_data['varvals'][dep])
+ output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
if 'runtaskdeps' in a_data:
- print "Tasks this task depends on: %s" % (a_data['runtaskdeps'])
+ output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
if 'file_checksum_values' in a_data:
- print "This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])
+ output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
if 'runtaskhashes' in a_data:
for dep in a_data['runtaskhashes']:
- print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])
+ output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
if 'taint' in a_data:
- print "Tainted (by forced/invalidated task): %s" % a_data['taint']
+ output.append("Tainted (by forced/invalidated task): %s" % a_data['taint'])
+
+ return output
--
1.7.9.5
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 5/5] bitbake-diffsigs: allow specifying task & follow deps recursively
2012-08-27 20:44 [PATCH 0/5] Refactoring + improve bitbake-diffsigs Paul Eggleton
` (3 preceding siblings ...)
2012-08-27 20:44 ` [PATCH 4/5] lib/bb/siggen.py: make signature dump/compare functions return a list Paul Eggleton
@ 2012-08-27 20:44 ` Paul Eggleton
2012-09-07 11:09 ` [PATCH 0/5] Refactoring + improve bitbake-diffsigs Richard Purdie
5 siblings, 0 replies; 7+ messages in thread
From: Paul Eggleton @ 2012-08-27 20:44 UTC (permalink / raw)
To: bitbake-devel
Add the ability to compare the two most recent runs of a specified task,
and follow dependent hash changes recursively. This enables you to trace
back and find exactly why a task was re-run after the fact.
Note that this relies on the metadata providing a function, hooked in
as bb.siggen.find_siginfo, which allows searching in the appropriate
places to find signature data files.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bitbake/bin/bitbake-diffsigs | 97 +++++++++++++++++++++++++++++++++++++++---
bitbake/lib/bb/siggen.py | 17 +++++---
2 files changed, 102 insertions(+), 12 deletions(-)
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
index 146cab8..62f6827 100755
--- a/bitbake/bin/bitbake-diffsigs
+++ b/bitbake/bin/bitbake-diffsigs
@@ -1,15 +1,102 @@
#!/usr/bin/env python
+
+# bitbake-diffsigs
+# BitBake task signature data comparison utility
+#
+# Copyright (C) 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
import os
import sys
import warnings
+import fnmatch
+import optparse
+import logging
+
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+import bb.tinfoil
import bb.siggen
-if len(sys.argv) > 2:
- output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
+logger = logging.getLogger('BitBake')
+
+def find_compare_task(bbhandler, pn, taskname):
+ """ Find the most recent signature files for the specified PN/task and compare them """
+
+ if not hasattr(bb.siggen, 'find_siginfo'):
+ logger.error('Metadata does not support finding signature data files')
+ sys.exit(1)
+
+ filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+ latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
+ if not latestfiles:
+ logger.error('No sigdata files found matching %s %s' % (pn, taskname))
+ sys.exit(1)
+ elif len(latestfiles) < 2:
+ logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
+ sys.exit(1)
+ else:
+ # Define recursion callback
+ def recursecb(key, hash1, hash2):
+ hashes = [hash1, hash2]
+ hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data)
+
+ recout = []
+ if len(hashfiles) == 2:
+ out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
+ recout.extend(list(' ' + l for l in out2))
+ else:
+ recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+
+ return recout
+
+ # Recurse into signature comparison
+ output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb)
+ if output:
+ print '\n'.join(output)
+ sys.exit(0)
+
+
+
+parser = optparse.OptionParser(
+ usage = """
+ %prog -t recipename taskname
+ %prog sigdatafile1 sigdatafile2
+ %prog sigdatafile1""")
+
+parser.add_option("-t", "--task",
+ help = "find the signature data files for last two runs of the specified task and compare them",
+ action="store_true", dest="taskmode")
+
+options, args = parser.parse_args(sys.argv)
+
+if len(args) == 1:
+ parser.print_help()
else:
- output = bb.siggen.dump_sigfile(sys.argv[1])
+ tinfoil = bb.tinfoil.Tinfoil()
+ if options.taskmode:
+ if len(args) < 3:
+ logger.error("Please specify a recipe and task name")
+ sys.exit(1)
+ tinfoil.prepare(config_only = True)
+ find_compare_task(tinfoil, args[1], args[2])
+ else:
+ if len(args) == 2:
+ output = bb.siggen.dump_sigfile(sys.argv[1])
+ else:
+ output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
-if output:
- print '\n'.join(output)
+ if output:
+ print '\n'.join(output)
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 8d1501b..8fe59b9 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -301,7 +301,7 @@ def clean_basepaths(a):
b[clean_basepath(x)] = a[x]
return b
-def compare_sigfiles(a, b):
+def compare_sigfiles(a, b, recursecb = None):
output = []
p1 = pickle.Unpickler(open(a, "rb"))
@@ -369,8 +369,8 @@ def compare_sigfiles(a, b):
if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
- a = clean_basepaths(a_data['runtaskhashes'])
- b = clean_basepaths(b_data['runtaskhashes'])
+ a = a_data['runtaskhashes']
+ b = b_data['runtaskhashes']
changed, added, removed = dict_diff(a, b)
if added:
for dep in added:
@@ -381,7 +381,7 @@ def compare_sigfiles(a, b):
#output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
bdep_found = True
if not bdep_found:
- output.append("Dependency on task %s was added with hash %s" % (dep, a[dep]))
+ output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), a[dep]))
if removed:
for dep in removed:
adep_found = False
@@ -391,11 +391,14 @@ def compare_sigfiles(a, b):
#output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
adep_found = True
if not adep_found:
- output.append("Dependency on task %s was removed with hash %s" % (dep, b[dep]))
+ output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), b[dep]))
if changed:
for dep in changed:
- output.append("Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep]))
-
+ output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep]))
+ if callable(recursecb):
+ recout = recursecb(dep, a[dep], b[dep])
+ if recout:
+ output.extend(recout)
a_taint = a_data.get('taint', None)
b_taint = b_data.get('taint', None)
--
1.7.9.5
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH 0/5] Refactoring + improve bitbake-diffsigs
2012-08-27 20:44 [PATCH 0/5] Refactoring + improve bitbake-diffsigs Paul Eggleton
` (4 preceding siblings ...)
2012-08-27 20:44 ` [PATCH 5/5] bitbake-diffsigs: allow specifying task & follow deps recursively Paul Eggleton
@ 2012-09-07 11:09 ` Richard Purdie
5 siblings, 0 replies; 7+ messages in thread
From: Richard Purdie @ 2012-09-07 11:09 UTC (permalink / raw)
To: Paul Eggleton; +Cc: bitbake-devel
On Mon, 2012-08-27 at 21:44 +0100, Paul Eggleton wrote:
> Refactor out a class which can be used as the basis for bitbake-based
> utilities, and use it to improve bitbake-diffsigs to enable easily
> specifying a recipe and task and determine what the difference was in
> the signature data between the last two executions.
>
> This change requires some support code within the metadata (since
> BitBake itself knows nothing about the implementation of shared state);
> a patch to implement this in OE-Core has been sent to the OE-Core
> mailing list.
>
> Note that this is similar in spirit but does not directly overlap with
> Robert's bitbake-whatchanged script - that allows you to determine what
> will be done before it happens, whereas this tool is useful for
> determining why a rebuild occurred afterwards.
>
>
> The following changes (against poky, but apply cleanly with -p2 against
> bitbake master) are available in the git repository at:
>
> git://git.yoctoproject.org/poky-contrib paule/bbdiffsigs
> http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bbdiffsigs
>
> Paul Eggleton (5):
> tinfoil: create simple interface for bitbake-based utilities
> lib/bb/siggen: replace tabs with spaces
> lib/bb/siggen.py: insert a colon between class and recipe name
> lib/bb/siggen.py: make signature dump/compare functions return a list
> bitbake-diffsigs: allow specifying task & follow deps recursively
Merged to master, thanks.
Nice to have this working better :)
Cheers,
Richard
^ permalink raw reply [flat|nested] 7+ messages in thread