* [OE-core][PATCH v3 1/8] bitbake: cooker: Split file collections per multiconfig
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
@ 2020-06-03 2:53 ` Joshua Watt
2020-06-03 2:53 ` [OE-core][PATCH v3 2/8] bitbake: cache: Use multiconfig aware caches Joshua Watt
` (6 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Splits the cooker to track a collection per multiconfig instead of a
single collection for all multiconfigs. Practically speaking, this
allows each multiconfigs to each have different BBMASKs that apply to it
instead of each one using the mask specified in the base configuration.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/cache.py | 2 +-
bitbake/lib/bb/command.py | 31 ++++++--
bitbake/lib/bb/cooker.py | 139 ++++++++++++++++++++--------------
bitbake/lib/bb/runqueue.py | 11 +--
bitbake/lib/bb/tinfoil.py | 26 ++++---
bitbake/lib/bblayers/query.py | 4 +-
6 files changed, 129 insertions(+), 84 deletions(-)
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index d1be83617b..aa5ec5b591 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -623,7 +623,7 @@ class Cache(NoCache):
self.remove(fn)
return False
- if appends != info_array[0].appends:
+ if tuple(appends) != tuple(info_array[0].appends):
logger.debug(2, "Cache: appends for %s changed", fn)
logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
self.remove(fn)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index 6abf38668b..f45b6e5320 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -232,7 +232,11 @@ class CommandsSync:
def matchFile(self, command, params):
fMatch = params[0]
- return command.cooker.matchFile(fMatch)
+ try:
+ mc = params[0]
+ except IndexError:
+ mc = ''
+ return command.cooker.matchFile(fMatch, mc)
matchFile.needconfig = False
def getUIHandlerNum(self, command, params):
@@ -396,21 +400,33 @@ class CommandsSync:
# Return list sorted by reverse priority order
import bb.cache
skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
- key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
+ key=lambda x: (-command.cooker.collections[mc].calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
return list(skipdict.items())
getSkippedRecipes.readonly = True
def getOverlayedRecipes(self, command, params):
- return list(command.cooker.collection.overlayed.items())
+ try:
+ mc = params[0]
+ except IndexError:
+ mc = ''
+ return list(command.cooker.collections[mc].overlayed.items())
getOverlayedRecipes.readonly = True
def getFileAppends(self, command, params):
fn = params[0]
- return command.cooker.collection.get_file_appends(fn)
+ try:
+ mc = params[1]
+ except IndexError:
+ mc = ''
+ return command.cooker.collections[mc].get_file_appends(fn)
getFileAppends.readonly = True
def getAllAppends(self, command, params):
- return command.cooker.collection.bbappends
+ try:
+ mc = params[0]
+ except IndexError:
+ mc = ''
+ return command.cooker.collections[mc].bbappends
getAllAppends.readonly = True
def findProviders(self, command, params):
@@ -496,6 +512,7 @@ class CommandsSync:
for the recipe.
"""
fn = params[0]
+ mc = bb.runqueue.mc_from_tid(fn)
appends = params[1]
appendlist = params[2]
if len(params) > 3:
@@ -507,7 +524,7 @@ class CommandsSync:
if appendlist is not None:
appendfiles = appendlist
else:
- appendfiles = command.cooker.collection.get_file_appends(fn)
+ appendfiles = command.cooker.collections[mc].get_file_appends(fn)
else:
appendfiles = []
# We are calling bb.cache locally here rather than on the server,
@@ -517,7 +534,7 @@ class CommandsSync:
if config_data:
# We have to use a different function here if we're passing in a datastore
# NOTE: we took a copy above, so we don't do it here again
- envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)['']
+ envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)['']
else:
# Use the standard path
parser = bb.cache.NoCache(command.cooker.databuilder)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index e527e23114..8f45233c8d 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -525,7 +525,7 @@ class BBCooker:
self.parseConfiguration()
fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
- fn = self.matchFile(fn)
+ fn = self.matchFile(fn, mc)
fn = bb.cache.realfn2virtual(fn, cls, mc)
elif len(pkgs_to_build) == 1:
mc = mc_base(pkgs_to_build[0])
@@ -542,7 +542,7 @@ class BBCooker:
if fn:
try:
bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
- envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
+ envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn))
except Exception as e:
parselog.exception("Unable to read %s", fn)
raise
@@ -929,26 +929,33 @@ class BBCooker:
logger.info("Task dependencies saved to 'task-depends.dot'")
def show_appends_with_no_recipes(self):
+ appends_without_recipes = {}
# Determine which bbappends haven't been applied
-
- # First get list of recipes, including skipped
- recipefns = list(self.recipecaches[''].pkg_fn.keys())
- recipefns.extend(self.skiplist.keys())
-
- # Work out list of bbappends that have been applied
- applied_appends = []
- for fn in recipefns:
- applied_appends.extend(self.collection.get_file_appends(fn))
-
- appends_without_recipes = []
- for _, appendfn in self.collection.bbappends:
- if not appendfn in applied_appends:
- appends_without_recipes.append(appendfn)
-
- if appends_without_recipes:
- msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes)
- warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
- False) or "no"
+ for mc in self.multiconfigs:
+ # First get list of recipes, including skipped
+ recipefns = list(self.recipecaches[mc].pkg_fn.keys())
+ recipefns.extend(self.skiplist.keys())
+
+ # Work out list of bbappends that have been applied
+ applied_appends = []
+ for fn in recipefns:
+ applied_appends.extend(self.collections[mc].get_file_appends(fn))
+
+ appends_without_recipes[mc] = []
+ for _, appendfn in self.collections[mc].bbappends:
+ if not appendfn in applied_appends:
+ appends_without_recipes[mc].append(appendfn)
+
+ msgs = []
+ for mc in sorted(appends_without_recipes.keys()):
+ if appends_without_recipes[mc]:
+ msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default',
+ '\n '.join(appends_without_recipes[mc])))
+
+ if msgs:
+ msg = "\n".join(msgs)
+ warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
+ False) or "no"
if warn_only.lower() in ("1", "yes", "true"):
bb.warn(msg)
else:
@@ -1249,15 +1256,15 @@ class BBCooker:
if siggen_cache:
bb.parse.siggen.checksum_cache.mtime_cache.clear()
- def matchFiles(self, bf):
+ def matchFiles(self, bf, mc=''):
"""
Find the .bb files which match the expression in 'buildfile'.
"""
if bf.startswith("/") or bf.startswith("../"):
bf = os.path.abspath(bf)
- self.collection = CookerCollectFiles(self.bbfile_config_priorities)
- filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
+ self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+ filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
try:
os.stat(bf)
bf = os.path.abspath(bf)
@@ -1270,12 +1277,12 @@ class BBCooker:
matches.append(f)
return matches
- def matchFile(self, buildfile):
+ def matchFile(self, buildfile, mc=''):
"""
Find the .bb file which matches the expression in 'buildfile'.
Raise an error if multiple files
"""
- matches = self.matchFiles(buildfile)
+ matches = self.matchFiles(buildfile, mc)
if len(matches) != 1:
if matches:
msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
@@ -1316,14 +1323,14 @@ class BBCooker:
task = "do_%s" % task
fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
- fn = self.matchFile(fn)
+ fn = self.matchFile(fn, mc)
self.buildSetVars()
self.reset_mtime_caches()
bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
- infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
+ infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn))
infos = dict(infos)
fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1552,14 +1559,24 @@ class BBCooker:
for dep in self.configuration.extra_assume_provided:
self.recipecaches[mc].ignored_dependencies.add(dep)
- self.collection = CookerCollectFiles(self.bbfile_config_priorities)
- (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+ self.collections = {}
+
+ mcfilelist = {}
+ total_masked = 0
+ searchdirs = set()
+ for mc in self.multiconfigs:
+ self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+ (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+
+ mcfilelist[mc] = filelist
+ total_masked += masked
+ searchdirs |= set(search)
# Add inotify watches for directories searched for bb/bbappend files
for dirent in searchdirs:
self.add_filewatch([[dirent]], dirs=True)
- self.parser = CookerParser(self, filelist, masked)
+ self.parser = CookerParser(self, mcfilelist, total_masked)
self.parsecache_valid = True
self.state = state.parsing
@@ -1571,7 +1588,7 @@ class BBCooker:
self.show_appends_with_no_recipes()
self.handlePrefProviders()
for mc in self.multiconfigs:
- self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
+ self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
self.state = state.running
# Send an event listing all stamps reachable after parsing
@@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event):
class CookerCollectFiles(object):
- def __init__(self, priorities):
+ def __init__(self, priorities, mc=''):
+ self.mc = mc
self.bbappends = []
# Priorities is a list of tupples, with the second element as the pattern.
# We need to sort the list with the longest pattern first, and so on to
@@ -1846,7 +1864,7 @@ class CookerCollectFiles(object):
(bbappend, filename) = b
if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
filelist.append(filename)
- return filelist
+ return tuple(filelist)
def collection_priorities(self, pkgfns, d):
@@ -1882,7 +1900,8 @@ class CookerCollectFiles(object):
for collection, pattern, regex, _ in self.bbfile_config_priorities:
if regex in unmatched:
if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
- collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+ collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
+ collection, pattern))
return priorities
@@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process):
bb.event.LogHandler.filter = origfilter
class CookerParser(object):
- def __init__(self, cooker, filelist, masked):
- self.filelist = filelist
+ def __init__(self, cooker, mcfilelist, masked):
+ self.mcfilelist = mcfilelist
self.cooker = cooker
self.cfgdata = cooker.data
self.cfghash = cooker.data_hash
@@ -1993,25 +2012,27 @@ class CookerParser(object):
self.skipped = 0
self.virtuals = 0
- self.total = len(filelist)
self.current = 0
self.process_names = []
self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
- self.fromcache = []
- self.willparse = []
- for filename in self.filelist:
- appends = self.cooker.collection.get_file_appends(filename)
- if not self.bb_cache.cacheValid(filename, appends):
- self.willparse.append((filename, appends))
- else:
- self.fromcache.append((filename, appends))
- self.toparse = self.total - len(self.fromcache)
+ self.fromcache = set()
+ self.willparse = set()
+ for mc in self.cooker.multiconfigs:
+ for filename in self.mcfilelist[mc]:
+ appends = self.cooker.collections[mc].get_file_appends(filename)
+ if not self.bb_cache.cacheValid(filename, appends):
+ self.willparse.add((filename, appends))
+ else:
+ self.fromcache.add((filename, appends))
+
+ self.total = len(self.fromcache) + len(self.willparse)
+ self.toparse = len(self.willparse)
self.progress_chunk = int(max(self.toparse / 100, 1))
self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
- multiprocessing.cpu_count()), len(self.willparse))
+ multiprocessing.cpu_count()), self.toparse)
self.start()
self.haveshutdown = False
@@ -2032,7 +2053,7 @@ class CookerParser(object):
def chunkify(lst,n):
return [lst[i::n] for i in range(n)]
- self.jobs = chunkify(self.willparse, self.num_processes)
+ self.jobs = chunkify(list(self.willparse), self.num_processes)
for i in range(0, self.num_processes):
parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
@@ -2095,9 +2116,9 @@ class CookerParser(object):
print("Processed parsing statistics saved to %s" % (pout))
def load_cached(self):
- for filename, appends in self.fromcache:
- cached, infos = self.bb_cache.load(filename, appends)
- yield not cached, infos
+ for mc, filename, appends in self.fromcache:
+ cached, infos = self.bb_cache.load(mc, filename, appends)
+ yield not cached, mc, infos
def parse_generator(self):
while True:
@@ -2119,7 +2140,7 @@ class CookerParser(object):
result = []
parsed = None
try:
- parsed, result = next(self.results)
+ parsed, mc, result = next(self.results)
except StopIteration:
self.shutdown()
return False
@@ -2181,7 +2202,11 @@ class CookerParser(object):
return True
def reparse(self, filename):
- infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
- for vfn, info_array in infos:
- (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
- self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
+ to_reparse = set()
+ for mc in self.cooker.multiconfigs:
+ to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+
+ for mc, filename, appends in to_reparse:
+ infos = self.bb_cache.parse(filename, appends)
+ for vfn, info_array in infos:
+ self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 16f076f3b1..3d54c2b88a 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1557,7 +1557,8 @@ class RunQueue:
def rq_dump_sigfn(self, fn, options):
bb_cache = bb.cache.NoCache(self.cooker.databuilder)
- the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
+ mc = bb.runqueue.mc_from_tid(fn)
+ the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn))
siggen = bb.parse.siggen
dataCaches = self.rqdata.dataCaches
siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute:
if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
if not mc in self.rq.fakeworker:
self.rq.start_fakeworker(self, mc)
- self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+ self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
self.rq.fakeworker[mc].process.stdin.flush()
else:
- self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+ self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
self.rq.worker[mc].process.stdin.flush()
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute:
self.rq.state = runQueueFailed
self.stats.taskFailed()
return True
- self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+ self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
self.rq.fakeworker[mc].process.stdin.flush()
else:
- self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+ self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
self.rq.worker[mc].process.stdin.flush()
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index 8c9b6b8ca5..7e32dca092 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter:
class TinfoilCookerCollectionAdapter:
""" cooker.collection adapter """
- def __init__(self, tinfoil):
+ def __init__(self, tinfoil, mc=''):
self.tinfoil = tinfoil
+ self.mc = mc
def get_file_appends(self, fn):
- return self.tinfoil.get_file_appends(fn)
+ return self.tinfoil.get_file_appends(fn, self.mc)
def __getattr__(self, name):
if name == 'overlayed':
- return self.tinfoil.get_overlayed_recipes()
+ return self.tinfoil.get_overlayed_recipes(self.mc)
elif name == 'bbappends':
- return self.tinfoil.run_command('getAllAppends')
+ return self.tinfoil.run_command('getAllAppends', self.mc)
else:
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter:
def __init__(self, tinfoil):
self.tinfoil = tinfoil
- self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
+ self.collections = {}
self.recipecaches = {}
self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split():
+ self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil)
self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
self._cache = {}
def __getattr__(self, name):
@@ -196,7 +198,7 @@ class TinfoilCookerAdapter:
if name in self._cache:
return self._cache[name]
elif name == 'skiplist':
- attrvalue = self.tinfoil.get_skipped_recipes()
+ attrvalue = self.tinfoil.get_skipped_recipes(self.mc)
elif name == 'bbfile_config_priorities':
ret = self.tinfoil.run_command('getLayerPriorities')
bbfile_config_priorities = []
@@ -492,18 +494,18 @@ class Tinfoil:
raise Exception('Not connected to server (did you call .prepare()?)')
return self.server_connection.events.waitEvent(timeout)
- def get_overlayed_recipes(self):
+ def get_overlayed_recipes(self, mc=''):
"""
Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
"""
- return defaultdict(list, self.run_command('getOverlayedRecipes'))
+ return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
- def get_skipped_recipes(self):
+ def get_skipped_recipes(self, mc=''):
"""
Find recipes which were skipped (i.e. SkipRecipe was raised
during parsing).
"""
- return OrderedDict(self.run_command('getSkippedRecipes'))
+ return OrderedDict(self.run_command('getSkippedRecipes', mc))
def get_all_providers(self, mc=''):
return defaultdict(list, self.run_command('allProviders', mc))
@@ -534,11 +536,11 @@ class Tinfoil:
raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
return best[3]
- def get_file_appends(self, fn):
+ def get_file_appends(self, fn, mc=''):
"""
Find the bbappends for a recipe file
"""
- return self.run_command('getFileAppends', fn)
+ return self.run_command('getFileAppends', fn, mc)
def all_recipes(self, mc='', sort=True):
"""
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py
index e2cc310532..ee2db0efed 100644
--- a/bitbake/lib/bblayers/query.py
+++ b/bitbake/lib/bblayers/query.py
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems.
def get_appends_for_files(self, filenames):
appended, notappended = [], []
for filename in filenames:
- _, cls, _ = bb.cache.virtualfn2realfn(filename)
+ _, cls, mc = bb.cache.virtualfn2realfn(filename)
if cls:
continue
basename = os.path.basename(filename)
- appends = self.tinfoil.cooker.collection.get_file_appends(basename)
+ appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename)
if appends:
appended.append((basename, list(appends)))
else:
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [OE-core][PATCH v3 2/8] bitbake: cache: Use multiconfig aware caches
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
2020-06-03 2:53 ` [OE-core][PATCH v3 1/8] bitbake: cooker: Split file collections per multiconfig Joshua Watt
@ 2020-06-03 2:53 ` Joshua Watt
2020-06-03 2:54 ` [OE-core][PATCH v3 3/8] bitbake: lib: Add support for Logging Adapters Joshua Watt
` (5 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:53 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Splits the parsing cache to maintain one cache per multiconfig instead
of one global cache. This is necessary now that the files and appends
can vary for each multiconfig. A bb.cache.MulticonfigCache
dictionary-like proxy object is created instead of a single
bb.cache.Cache object. This object will create and properly initialize
bb.cache.Cache object for each multiconfig, and each of these caches has
a dedicated cache file with a name based on the multiconfig.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/cache.py | 143 +++++++++++++++++++++++++++++----------
bitbake/lib/bb/cooker.py | 42 +++++++-----
2 files changed, 133 insertions(+), 52 deletions(-)
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index aa5ec5b591..954418384b 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -19,7 +19,7 @@
import os
import logging
import pickle
-from collections import defaultdict
+from collections import defaultdict, Mapping
import bb.utils
import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache")
__cache_version__ = "152"
-def getCacheFile(path, filename, data_hash):
- return os.path.join(path, filename + "." + data_hash)
+def getCacheFile(path, filename, mc, data_hash):
+ mcspec = ''
+ if mc:
+ mcspec = ".%s" % mc
+ return os.path.join(path, filename + mcspec + "." + data_hash)
# RecipeInfoCommon defines common data retrieving methods
# from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache):
"""
BitBake Cache implementation
"""
-
- def __init__(self, databuilder, data_hash, caches_array):
+ def __init__(self, databuilder, mc, data_hash, caches_array):
super().__init__(databuilder)
data = databuilder.data
# Pass caches_array information into Cache Constructor
# It will be used later for deciding whether we
# need extra cache file dump/load support
+ self.mc = mc
self.caches_array = caches_array
self.cachedir = data.getVar("CACHE")
self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache):
return
self.has_cache = True
- self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
+
+ def getCacheFile(self, cachefile):
+ return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
+
+ def prepare_cache(self, progress):
+ if not self.has_cache:
+ return 0
+
+ loaded = 0
+
+ self.cachefile = self.getCacheFile("bb_cache.dat")
logger.debug(1, "Cache dir: %s", self.cachedir)
bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache):
cache_ok = True
if self.caches_array:
for cache_class in self.caches_array:
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+ cachefile = self.getCacheFile(cache_class.cachefile)
cache_ok = cache_ok and os.path.exists(cachefile)
cache_class.init_cacheData(self)
if cache_ok:
- self.load_cachefile()
+ loaded = self.load_cachefile(progress)
elif os.path.isfile(self.cachefile):
logger.info("Out of date cache found, rebuilding...")
else:
logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
# We don't use the symlink, its just for debugging convinience
- symlink = os.path.join(self.cachedir, "bb_cache.dat")
+ if self.mc:
+ symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc)
+ else:
+ symlink = os.path.join(self.cachedir, "bb_cache.dat")
+
if os.path.exists(symlink):
bb.utils.remove(symlink)
try:
@@ -406,21 +423,30 @@ class Cache(NoCache):
except OSError:
pass
- def load_cachefile(self):
- cachesize = 0
- previous_progress = 0
- previous_percent = 0
+ return loaded
+
+ def cachesize(self):
+ if not self.has_cache:
+ return 0
- # Calculate the correct cachesize of all those cache files
+ cachesize = 0
for cache_class in self.caches_array:
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
- with open(cachefile, "rb") as cachefile:
- cachesize += os.fstat(cachefile.fileno()).st_size
+ cachefile = self.getCacheFile(cache_class.cachefile)
+ try:
+ with open(cachefile, "rb") as cachefile:
+ cachesize += os.fstat(cachefile.fileno()).st_size
+ except FileNotFoundError:
+ pass
- bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
+ return cachesize
+
+ def load_cachefile(self, progress):
+ cachesize = self.cachesize()
+ previous_progress = 0
+ previous_percent = 0
for cache_class in self.caches_array:
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+ cachefile = self.getCacheFile(cache_class.cachefile)
logger.debug(1, 'Loading cache file: %s' % cachefile)
with open(cachefile, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache):
self.depends_cache[key] = [value]
# only fire events on even percentage boundaries
current_progress = cachefile.tell() + previous_progress
- if current_progress > cachesize:
- # we might have calculated incorrect total size because a file
- # might've been written out just after we checked its size
- cachesize = current_progress
- current_percent = 100 * current_progress / cachesize
- if current_percent > previous_percent:
- previous_percent = current_percent
- bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
- self.data)
+ progress(cachefile.tell() + previous_progress)
previous_progress += current_progress
- # Note: depends cache number is corresponding to the parsing file numbers.
- # The same file has several caches, still regarded as one item in the cache
- bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
- len(self.depends_cache)),
- self.data)
+ return len(self.depends_cache)
def parse(self, filename, appends):
"""Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache):
for cache_class in self.caches_array:
cache_class_name = cache_class.__name__
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+ cachefile = self.getCacheFile(cache_class.cachefile)
with open(cachefile, "wb") as f:
p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache):
return bb.parse.cached_mtime_noerror(cachefile)
def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
- if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
+ if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
cacheData.add_from_recipeinfo(filename, info_array)
if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache):
info_array.append(cache_class(realfn, data))
self.add_info(file_name, info_array, cacheData, parsed)
+class MulticonfigCache(Mapping):
+ def __init__(self, databuilder, data_hash, caches_array):
+ def progress(p):
+ nonlocal current_progress
+ nonlocal previous_progress
+ nonlocal previous_percent
+ nonlocal cachesize
+
+ current_progress = previous_progress + p
+
+ if current_progress > cachesize:
+ # we might have calculated incorrect total size because a file
+ # might've been written out just after we checked its size
+ cachesize = current_progress
+ current_percent = 100 * current_progress / cachesize
+ if current_percent > previous_percent:
+ previous_percent = current_percent
+ bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
+ databuilder.data)
+
+
+ cachesize = 0
+ current_progress = 0
+ previous_progress = 0
+ previous_percent = 0
+ self.__caches = {}
+
+ for mc, mcdata in databuilder.mcdata.items():
+ self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array)
+
+ cachesize += self.__caches[mc].cachesize()
+
+ bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data)
+ loaded = 0
+
+ for c in self.__caches.values():
+ loaded += c.prepare_cache(progress)
+ previous_progress = current_progress
+
+ # Note: depends cache number is corresponding to the parsing file numbers.
+ # The same file has several caches, still regarded as one item in the cache
+ bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data)
+
+ def __len__(self):
+ return len(self.__caches)
+
+ def __getitem__(self, key):
+ return self.__caches[key]
+
+ def __contains__(self, key):
+ return key in self.__caches
+
+ def __iter__(self):
+ for k in self.__caches:
+ yield k
+
+ def keys(self):
+ return self.__caches[key]
+
def init(cooker):
"""
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 8f45233c8d..50526d52b2 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -541,8 +541,8 @@ class BBCooker:
if fn:
try:
- bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
- envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn))
+ bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
+ envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
except Exception as e:
parselog.exception("Unable to read %s", fn)
raise
@@ -1328,9 +1328,9 @@ class BBCooker:
self.buildSetVars()
self.reset_mtime_caches()
- bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
+ bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
- infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn))
+ infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
infos = dict(infos)
fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1968,7 +1968,7 @@ class Parser(multiprocessing.Process):
except queue.Full:
pending.append(result)
- def parse(self, filename, appends):
+ def parse(self, mc, cache, filename, appends):
try:
origfilter = bb.event.LogHandler.filter
# Record the filename we're parsing into any events generated
@@ -1982,7 +1982,7 @@ class Parser(multiprocessing.Process):
bb.event.set_class_handlers(self.handlers.copy())
bb.event.LogHandler.filter = parse_filter
- return True, self.bb_cache.parse(filename, appends)
+ return True, mc, cache.parse(filename, appends)
except Exception as exc:
tb = sys.exc_info()[2]
exc.recipe = filename
@@ -2016,16 +2016,16 @@ class CookerParser(object):
self.current = 0
self.process_names = []
- self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
+ self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
self.fromcache = set()
self.willparse = set()
for mc in self.cooker.multiconfigs:
for filename in self.mcfilelist[mc]:
appends = self.cooker.collections[mc].get_file_appends(filename)
- if not self.bb_cache.cacheValid(filename, appends):
- self.willparse.add((filename, appends))
+ if not self.bb_caches[mc].cacheValid(filename, appends):
+ self.willparse.add((mc, self.bb_caches[mc], filename, appends))
else:
- self.fromcache.add((filename, appends))
+ self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
self.total = len(self.fromcache) + len(self.willparse)
self.toparse = len(self.willparse)
@@ -2043,7 +2043,6 @@ class CookerParser(object):
if self.toparse:
bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
def init():
- Parser.bb_cache = self.bb_cache
bb.utils.set_process_name(multiprocessing.current_process().name)
multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
@@ -2099,7 +2098,11 @@ class CookerParser(object):
else:
process.join()
- sync = threading.Thread(target=self.bb_cache.sync)
+ def sync_caches():
+ for c in self.bb_caches.values():
+ c.sync()
+
+ sync = threading.Thread(target=sync_caches)
sync.start()
multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
bb.codeparser.parser_cache_savemerge()
@@ -2116,8 +2119,8 @@ class CookerParser(object):
print("Processed parsing statistics saved to %s" % (pout))
def load_cached(self):
- for mc, filename, appends in self.fromcache:
- cached, infos = self.bb_cache.load(mc, filename, appends)
+ for mc, cache, filename, appends in self.fromcache:
+ cached, infos = cache.load(filename, appends)
yield not cached, mc, infos
def parse_generator(self):
@@ -2196,8 +2199,13 @@ class CookerParser(object):
if info_array[0].skipped:
self.skipped += 1
self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
- (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
- self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
+ (fn, cls, fnmc) = bb.cache.virtualfn2realfn(virtualfn)
+
+ if fnmc == mc:
+ cache = self.cooker.recipecaches[mc]
+ else:
+ cache = None
+ self.bb_caches[mc].add_info(virtualfn, info_array, cache,
parsed=parsed, watcher = self.cooker.add_filewatch)
return True
@@ -2207,6 +2215,6 @@ class CookerParser(object):
to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
for mc, filename, appends in to_reparse:
- infos = self.bb_cache.parse(filename, appends)
+ infos = self.bb_caches[mc].parse(filename, appends)
for vfn, info_array in infos:
self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [OE-core][PATCH v3 3/8] bitbake: lib: Add support for Logging Adapters
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
2020-06-03 2:53 ` [OE-core][PATCH v3 1/8] bitbake: cooker: Split file collections per multiconfig Joshua Watt
2020-06-03 2:53 ` [OE-core][PATCH v3 2/8] bitbake: cache: Use multiconfig aware caches Joshua Watt
@ 2020-06-03 2:54 ` Joshua Watt
2020-06-03 2:54 ` [OE-core][PATCH v3 4/8] bitbake: lib: Add PrefixLoggerAdapter helper Joshua Watt
` (4 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:54 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Creates a BBLoggingAdapter class that is monkey patched in place of the
logginer.LoggingAdapter. The new adapter is compatible with the BBLogger
class API, allowing adapters to be created for bitbake loggers. A new
BBLoggerMixin class is used to reduce code duplication between the BBLogger
and BBLoggerAdapter classes.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/__init__.py | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index b96466e654..2e2966c3b9 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -35,12 +35,14 @@ class NullHandler(logging.Handler):
def emit(self, record):
pass
-Logger = logging.getLoggerClass()
-class BBLogger(Logger):
- def __init__(self, name):
+class BBLoggerMixin(object):
+ def __init__(self, *args, **kwargs):
+ # Does nothing to allow calling super() from derived classes
+ pass
+
+ def setup_bblogger(self, name):
if name.split(".")[0] == "BitBake":
self.debug = self.bbdebug
- Logger.__init__(self, name)
def bbdebug(self, level, msg, *args, **kwargs):
loglevel = logging.DEBUG - level + 1
@@ -60,10 +62,22 @@ class BBLogger(Logger):
def verbnote(self, msg, *args, **kwargs):
return self.log(logging.INFO + 2, msg, *args, **kwargs)
+Logger = logging.getLoggerClass()
+class BBLogger(Logger, BBLoggerMixin):
+ def __init__(self, name, *args, **kwargs):
+ self.setup_bblogger(name)
+ super().__init__(name, *args, **kwargs)
logging.raiseExceptions = False
logging.setLoggerClass(BBLogger)
+class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin):
+ def __init__(self, logger, *args, **kwargs):
+ self.setup_bblogger(logger.name)
+ super().__init__(logger, *args, **kwargs)
+
+logging.LoggerAdapter = BBLoggerAdapter
+
logger = logging.getLogger("BitBake")
logger.addHandler(NullHandler())
logger.setLevel(logging.DEBUG - 2)
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [OE-core][PATCH v3 4/8] bitbake: lib: Add PrefixLoggerAdapter helper
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
` (2 preceding siblings ...)
2020-06-03 2:54 ` [OE-core][PATCH v3 3/8] bitbake: lib: Add support for Logging Adapters Joshua Watt
@ 2020-06-03 2:54 ` Joshua Watt
2020-06-04 21:00 ` Richard Purdie
2020-06-03 2:54 ` [OE-core][PATCH v3 5/8] bitbake: cache: Improve logging Joshua Watt
` (3 subsequent siblings)
7 siblings, 1 reply; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:54 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Adds a helper logger adapter to add a prefix to all log messages. This
is useful to distinguish log messages between multiple instances of a
object.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/__init__.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index 2e2966c3b9..1de32041ed 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -84,6 +84,14 @@ logger.setLevel(logging.DEBUG - 2)
mainlogger = logging.getLogger("BitBake.Main")
+class PrefixLoggerAdapter(logging.LoggerAdapter):
+ def __init__(self, prefix, logger):
+ super().__init__(logger, {})
+ self.__msg_prefix = prefix
+
+ def process(self, msg, kwargs):
+ return "%s%s" %(self.__msg_prefix, msg), kwargs
+
# This has to be imported after the setLoggerClass, as the import of bb.msg
# can result in construction of the various loggers.
import bb.msg
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [OE-core][PATCH v3 4/8] bitbake: lib: Add PrefixLoggerAdapter helper
2020-06-03 2:54 ` [OE-core][PATCH v3 4/8] bitbake: lib: Add PrefixLoggerAdapter helper Joshua Watt
@ 2020-06-04 21:00 ` Richard Purdie
0 siblings, 0 replies; 11+ messages in thread
From: Richard Purdie @ 2020-06-04 21:00 UTC (permalink / raw)
To: Joshua Watt, openembedded-core; +Cc: bitbake-devel
On Tue, 2020-06-02 at 21:54 -0500, Joshua Watt wrote:
> Adds a helper logger adapter to add a prefix to all log messages.
> This
> is useful to distinguish log messages between multiple instances of a
> object.
>
> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
> ---
> bitbake/lib/bb/__init__.py | 8 ++++++++
> 1 file changed, 8 insertions(+)
>
> diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
> index 2e2966c3b9..1de32041ed 100644
> --- a/bitbake/lib/bb/__init__.py
> +++ b/bitbake/lib/bb/__init__.py
> @@ -84,6 +84,14 @@ logger.setLevel(logging.DEBUG - 2)
>
> mainlogger = logging.getLogger("BitBake.Main")
>
> +class PrefixLoggerAdapter(logging.LoggerAdapter):
> + def __init__(self, prefix, logger):
> + super().__init__(logger, {})
> + self.__msg_prefix = prefix
> +
> + def process(self, msg, kwargs):
> + return "%s%s" %(self.__msg_prefix, msg), kwargs
> +
Ignoring the mailing list issue, I think there is something odd going
on with different python versions somewhere in here.
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/1004
https://autobuilder.yoctoproject.org/typhoon/#/builders/107/builds/585
https://autobuilder.yoctoproject.org/typhoon/#/builders/15/builds/2255
(all debian9)
and then selftest issues:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/1020
Cheers,
Richard
^ permalink raw reply [flat|nested] 11+ messages in thread
* [OE-core][PATCH v3 5/8] bitbake: cache: Improve logging
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
` (3 preceding siblings ...)
2020-06-03 2:54 ` [OE-core][PATCH v3 4/8] bitbake: lib: Add PrefixLoggerAdapter helper Joshua Watt
@ 2020-06-03 2:54 ` Joshua Watt
2020-06-03 2:54 ` [OE-core][PATCH v3 6/8] bitbake: cache: Cache size optimization Joshua Watt
` (2 subsequent siblings)
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:54 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Improves the logging of Cache objects by prefixing the log messages with
the multiconfig name of the cache, so as to distinguish between multiple
instances of the class. Also adds a more log messages.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/cache.py | 65 ++++++++++++++++++++++-------------------
1 file changed, 35 insertions(+), 30 deletions(-)
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 954418384b..b34bfa9b5a 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -21,6 +21,7 @@ import logging
import pickle
from collections import defaultdict, Mapping
import bb.utils
+from bb import PrefixLoggerAdapter
import re
logger = logging.getLogger("BitBake.Cache")
@@ -365,6 +366,7 @@ class Cache(NoCache):
# It will be used later for deciding whether we
# need extra cache file dump/load support
self.mc = mc
+ self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger)
self.caches_array = caches_array
self.cachedir = data.getVar("CACHE")
self.clean = set()
@@ -377,8 +379,8 @@ class Cache(NoCache):
if self.cachedir in [None, '']:
self.has_cache = False
- logger.info("Not using a cache. "
- "Set CACHE = <directory> to enable.")
+ self.logger.info("Not using a cache. "
+ "Set CACHE = <directory> to enable.")
return
self.has_cache = True
@@ -394,21 +396,23 @@ class Cache(NoCache):
self.cachefile = self.getCacheFile("bb_cache.dat")
- logger.debug(1, "Cache dir: %s", self.cachedir)
+ self.logger.debug(1, "Cache dir: %s", self.cachedir)
bb.utils.mkdirhier(self.cachedir)
cache_ok = True
if self.caches_array:
for cache_class in self.caches_array:
cachefile = self.getCacheFile(cache_class.cachefile)
- cache_ok = cache_ok and os.path.exists(cachefile)
+ cache_exists = os.path.exists(cachefile)
+ self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists)
+ cache_ok = cache_ok and cache_exists
cache_class.init_cacheData(self)
if cache_ok:
loaded = self.load_cachefile(progress)
elif os.path.isfile(self.cachefile):
- logger.info("Out of date cache found, rebuilding...")
+ self.logger.info("Out of date cache found, rebuilding...")
else:
- logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
+ self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
# We don't use the symlink, its just for debugging convinience
if self.mc:
@@ -447,7 +451,7 @@ class Cache(NoCache):
for cache_class in self.caches_array:
cachefile = self.getCacheFile(cache_class.cachefile)
- logger.debug(1, 'Loading cache file: %s' % cachefile)
+ self.logger.debug(1, 'Loading cache file: %s' % cachefile)
with open(cachefile, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
# Check cache version information
@@ -455,14 +459,14 @@ class Cache(NoCache):
cache_ver = pickled.load()
bitbake_ver = pickled.load()
except Exception:
- logger.info('Invalid cache, rebuilding...')
+ self.logger.info('Invalid cache, rebuilding...')
return
if cache_ver != __cache_version__:
- logger.info('Cache version mismatch, rebuilding...')
+ self.logger.info('Cache version mismatch, rebuilding...')
return
elif bitbake_ver != bb.__version__:
- logger.info('Bitbake version mismatch, rebuilding...')
+ self.logger.info('Bitbake version mismatch, rebuilding...')
return
# Load the rest of the cache file
@@ -494,7 +498,7 @@ class Cache(NoCache):
def parse(self, filename, appends):
"""Parse the specified filename, returning the recipe information"""
- logger.debug(1, "Parsing %s", filename)
+ self.logger.debug(1, "Parsing %s", filename)
infos = []
datastores = self.load_bbfile(filename, appends)
depends = []
@@ -548,7 +552,7 @@ class Cache(NoCache):
cached, infos = self.load(fn, appends)
for virtualfn, info_array in infos:
if info_array[0].skipped:
- logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
+ self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
skipped += 1
else:
self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -584,21 +588,21 @@ class Cache(NoCache):
# File isn't in depends_cache
if not fn in self.depends_cache:
- logger.debug(2, "Cache: %s is not cached", fn)
+ self.logger.debug(2, "%s is not cached", fn)
return False
mtime = bb.parse.cached_mtime_noerror(fn)
# Check file still exists
if mtime == 0:
- logger.debug(2, "Cache: %s no longer exists", fn)
+ self.logger.debug(2, "%s no longer exists", fn)
self.remove(fn)
return False
info_array = self.depends_cache[fn]
# Check the file's timestamp
if mtime != info_array[0].timestamp:
- logger.debug(2, "Cache: %s changed", fn)
+ self.logger.debug(2, "%s changed", fn)
self.remove(fn)
return False
@@ -609,14 +613,14 @@ class Cache(NoCache):
fmtime = bb.parse.cached_mtime_noerror(f)
# Check if file still exists
if old_mtime != 0 and fmtime == 0:
- logger.debug(2, "Cache: %s's dependency %s was removed",
- fn, f)
+ self.logger.debug(2, "%s's dependency %s was removed",
+ fn, f)
self.remove(fn)
return False
if (fmtime != old_mtime):
- logger.debug(2, "Cache: %s's dependency %s changed",
- fn, f)
+ self.logger.debug(2, "%s's dependency %s changed",
+ fn, f)
self.remove(fn)
return False
@@ -632,14 +636,14 @@ class Cache(NoCache):
continue
f, exist = f.split(":")
if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
- logger.debug(2, "Cache: %s's file checksum list file %s changed",
- fn, f)
+ self.logger.debug(2, "%s's file checksum list file %s changed",
+ fn, f)
self.remove(fn)
return False
if tuple(appends) != tuple(info_array[0].appends):
- logger.debug(2, "Cache: appends for %s changed", fn)
- logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
+ self.logger.debug(2, "appends for %s changed", fn)
+ self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
self.remove(fn)
return False
@@ -648,10 +652,10 @@ class Cache(NoCache):
virtualfn = variant2virtual(fn, cls)
self.clean.add(virtualfn)
if virtualfn not in self.depends_cache:
- logger.debug(2, "Cache: %s is not cached", virtualfn)
+ self.logger.debug(2, "%s is not cached", virtualfn)
invalid = True
elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
- logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn)
+ self.logger.debug(2, "Extra caches missing for %s?" % virtualfn)
invalid = True
# If any one of the variants is not present, mark as invalid for all
@@ -659,10 +663,10 @@ class Cache(NoCache):
for cls in info_array[0].variants:
virtualfn = variant2virtual(fn, cls)
if virtualfn in self.clean:
- logger.debug(2, "Cache: Removing %s from cache", virtualfn)
+ self.logger.debug(2, "Removing %s from cache", virtualfn)
self.clean.remove(virtualfn)
if fn in self.clean:
- logger.debug(2, "Cache: Marking %s as not clean", fn)
+ self.logger.debug(2, "Marking %s as not clean", fn)
self.clean.remove(fn)
return False
@@ -675,10 +679,10 @@ class Cache(NoCache):
Called from the parser in error cases
"""
if fn in self.depends_cache:
- logger.debug(1, "Removing %s from cache", fn)
+ self.logger.debug(1, "Removing %s from cache", fn)
del self.depends_cache[fn]
if fn in self.clean:
- logger.debug(1, "Marking %s as unclean", fn)
+ self.logger.debug(1, "Marking %s as unclean", fn)
self.clean.remove(fn)
def sync(self):
@@ -691,12 +695,13 @@ class Cache(NoCache):
return
if self.cacheclean:
- logger.debug(2, "Cache is clean, not saving.")
+ self.logger.debug(2, "Cache is clean, not saving.")
return
for cache_class in self.caches_array:
cache_class_name = cache_class.__name__
cachefile = self.getCacheFile(cache_class.cachefile)
+ self.logger.debug(2, "Writing %s", cachefile)
with open(cachefile, "wb") as f:
p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
p.dump(__cache_version__)
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [OE-core][PATCH v3 6/8] bitbake: cache: Cache size optimization
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
` (4 preceding siblings ...)
2020-06-03 2:54 ` [OE-core][PATCH v3 5/8] bitbake: cache: Improve logging Joshua Watt
@ 2020-06-03 2:54 ` Joshua Watt
2020-06-03 2:54 ` [OE-core][PATCH v3 7/8] bitbake: tests: Add tests for BBMASK in multiconfig Joshua Watt
2020-06-03 2:54 ` [OE-core][PATCH v3 8/8] bitbake: command: Move split_mc_pn to runqueue Joshua Watt
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:54 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Now that there is a cache object per multiconfig, it is not necessary
for each cache object to parse all other multiconfigs. Instead, each
cache now only parses the files for it's multiconfig.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/cache.py | 22 ++++++++++++++++++----
bitbake/lib/bb/cooker.py | 8 +-------
2 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index b34bfa9b5a..df78d5b701 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -328,7 +328,7 @@ class NoCache(object):
bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
return bb_data[virtual]
- def load_bbfile(self, bbfile, appends, virtonly = False):
+ def load_bbfile(self, bbfile, appends, virtonly = False, mc=None):
"""
Load and parse one .bb build file
Return the data and whether parsing resulted in the file being skipped
@@ -341,6 +341,10 @@ class NoCache(object):
datastores = parse_recipe(bb_data, bbfile, appends, mc)
return datastores
+ if mc is not None:
+ bb_data = self.databuilder.mcdata[mc].createCopy()
+ return parse_recipe(bb_data, bbfile, appends, mc)
+
bb_data = self.data.createCopy()
datastores = parse_recipe(bb_data, bbfile, appends)
@@ -500,7 +504,7 @@ class Cache(NoCache):
"""Parse the specified filename, returning the recipe information"""
self.logger.debug(1, "Parsing %s", filename)
infos = []
- datastores = self.load_bbfile(filename, appends)
+ datastores = self.load_bbfile(filename, appends, mc=self.mc)
depends = []
variants = []
# Process the "real" fn last so we can store variants list
@@ -720,8 +724,18 @@ class Cache(NoCache):
return bb.parse.cached_mtime_noerror(cachefile)
def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
- if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
- cacheData.add_from_recipeinfo(filename, info_array)
+ if self.mc is not None:
+ (fn, cls, mc) = virtualfn2realfn(filename)
+ if mc:
+ self.logger.error("Unexpected multiconfig %s", virtualfn)
+ return
+
+ vfn = realfn2virtual(fn, cls, self.mc)
+ else:
+ vfn = filename
+
+ if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
+ cacheData.add_from_recipeinfo(vfn, info_array)
if watcher:
watcher(info_array[0].file_depends)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 50526d52b2..effd02442c 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -2199,13 +2199,7 @@ class CookerParser(object):
if info_array[0].skipped:
self.skipped += 1
self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
- (fn, cls, fnmc) = bb.cache.virtualfn2realfn(virtualfn)
-
- if fnmc == mc:
- cache = self.cooker.recipecaches[mc]
- else:
- cache = None
- self.bb_caches[mc].add_info(virtualfn, info_array, cache,
+ self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
parsed=parsed, watcher = self.cooker.add_filewatch)
return True
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [OE-core][PATCH v3 7/8] bitbake: tests: Add tests for BBMASK in multiconfig
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
` (5 preceding siblings ...)
2020-06-03 2:54 ` [OE-core][PATCH v3 6/8] bitbake: cache: Cache size optimization Joshua Watt
@ 2020-06-03 2:54 ` Joshua Watt
2020-06-03 2:54 ` [OE-core][PATCH v3 8/8] bitbake: command: Move split_mc_pn to runqueue Joshua Watt
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:54 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
Adds a test to validate that multiconfigs can independently mask off
recipes by setting BBMASK. See the test description for further
information about how the test works.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
.../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 3 ++-
.../runqueue-tests/conf/multiconfig/mc1.conf | 1 +
.../runqueue-tests/conf/multiconfig/mc2.conf | 1 +
.../runqueue-tests/recipes/fails-mc/fails-mc1.bb | 5 +++++
.../runqueue-tests/recipes/fails-mc/fails-mc2.bb | 4 ++++
bitbake/lib/bb/tests/runqueue.py | 15 +++++++++++++++
6 files changed, 28 insertions(+), 1 deletion(-)
create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb
create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 5e451fc2c0..efebf001a9 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -1,7 +1,8 @@
CACHE = "${TOPDIR}/cache"
THISDIR = "${@os.path.dirname(d.getVar('FILE'))}"
COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}"
-BBFILES = "${COREBASE}/recipes/*.bb"
+EXTRA_BBFILES ?= ""
+BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}"
PROVIDES = "${PN}"
PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
PF = "${BB_CURRENT_MC}:${PN}"
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
index ecf23e1c73..f34b8dcccf 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
@@ -1 +1,2 @@
TMPDIR = "${TOPDIR}/mc1/"
+BBMASK += "recipes/fails-mc/fails-mc1.bb"
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
index eef338e4cc..c3360fc5c8 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
@@ -1 +1,2 @@
TMPDIR = "${TOPDIR}/mc2/"
+BBMASK += "recipes/fails-mc/fails-mc2.bb"
diff --git a/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb b/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb
new file mode 100644
index 0000000000..17a181fffb
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb
@@ -0,0 +1,5 @@
+python () {
+ if d.getVar("BB_CURRENT_MC") == "mc1":
+ bb.fatal("Multiconfig is mc1")
+}
+
diff --git a/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb b/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
new file mode 100644
index 0000000000..cc69e7b82d
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
@@ -0,0 +1,4 @@
+python () {
+ if d.getVar("BB_CURRENT_MC") == "mc2":
+ bb.fatal("Multiconfig is mc2")
+}
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index 4ba12a0772..091b5e41e0 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -232,6 +232,21 @@ class RunQueueTests(unittest.TestCase):
expected.remove(x)
self.assertEqual(set(tasks), set(expected))
+ def test_multiconfig_bbmask(self):
+ # This test validates that multiconfigs can independently mask off
+ # recipes they do not want with BBMASK. It works by having recipes
+ # that will fail to parse for mc1 and mc2, then making each multiconfig
+ # build the one that does parse. This ensures that the recipes are in
+ # each multiconfigs BBFILES, but each is masking only the one that
+ # doesn't parse
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BBMULTICONFIG" : "mc1 mc2",
+ "BB_SIGNATURE_HANDLER" : "basic",
+ "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
+ }
+ cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"]
+ self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
@unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
def test_hashserv_single(self):
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [OE-core][PATCH v3 8/8] bitbake: command: Move split_mc_pn to runqueue
2020-06-03 2:53 ` [OE-core][PATCH v3 0/8] Add support for per-multiconfig BBMASK Joshua Watt
` (6 preceding siblings ...)
2020-06-03 2:54 ` [OE-core][PATCH v3 7/8] bitbake: tests: Add tests for BBMASK in multiconfig Joshua Watt
@ 2020-06-03 2:54 ` Joshua Watt
7 siblings, 0 replies; 11+ messages in thread
From: Joshua Watt @ 2020-06-03 2:54 UTC (permalink / raw)
To: openembedded-core; +Cc: Joshua Watt
All of the other multiconfig splitting functions are located in runqueue
so move the function to split a pn/fn there also so that its easier to
see them all together. Fixes a case where the findBestProvider() command
wasn't working for multiconfig because it was looking for a prefix of
"multiconfig:" instead of the newer "mc:"
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/command.py | 8 +-------
bitbake/lib/bb/runqueue.py | 6 ++++++
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index f45b6e5320..8ed3d8f679 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -138,12 +138,6 @@ class Command:
def reset(self):
self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
-def split_mc_pn(pn):
- if pn.startswith("multiconfig:"):
- _, mc, pn = pn.split(":", 2)
- return (mc, pn)
- return ('', pn)
-
class CommandsSync:
"""
A class of synchronous commands
@@ -438,7 +432,7 @@ class CommandsSync:
findProviders.readonly = True
def findBestProvider(self, command, params):
- (mc, pn) = split_mc_pn(params[0])
+ (mc, pn) = bb.runqueue.split_pn(params[0])
return command.cooker.findBestProvider(pn, mc)
findBestProvider.readonly = True
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 3d54c2b88a..cb5aa80f2f 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -46,6 +46,12 @@ def split_tid(tid):
(mc, fn, taskname, _) = split_tid_mcfn(tid)
return (mc, fn, taskname)
+def split_pn(pn):
+ if pn.startswith("mc:"):
+ _, mc, pn = pn.split(":", 2)
+ return (mc, pn)
+ return ('', pn)
+
def split_tid_mcfn(tid):
if tid.startswith('mc:'):
elems = tid.split(':')
--
2.26.2
^ permalink raw reply related [flat|nested] 11+ messages in thread