* [PATCH 0/2] Implement file checksums
@ 2012-05-22 23:23 Paul Eggleton
2012-05-22 23:23 ` [PATCH 1/2] bitbake: refactor out codeparser cache into a separate class Paul Eggleton
` (2 more replies)
0 siblings, 3 replies; 8+ messages in thread
From: Paul Eggleton @ 2012-05-22 23:23 UTC (permalink / raw)
To: bitbake-devel
This is 2/3 of the changes required to implement local file checksums;
the other third is a patch to base.bbclass which will be sent to the
OE-Core list.
The following changes (against Poky, but apply cleanly against BitBake
master with -p2) are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/file-checksum-bb
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/file-checksum-bb
Paul Eggleton (2):
bitbake: refactor out codeparser cache into a separate class
bitbake: implement checksums for local files in SRC_URI
bitbake/lib/bb/cache.py | 129 ++++++++++++++++++++++++-
bitbake/lib/bb/checksum.py | 90 +++++++++++++++++
bitbake/lib/bb/codeparser.py | 191 +++++++++++--------------------------
bitbake/lib/bb/cooker.py | 2 +
bitbake/lib/bb/fetch2/__init__.py | 85 +++++++++++++++++
bitbake/lib/bb/siggen.py | 24 +++++
6 files changed, 382 insertions(+), 139 deletions(-)
create mode 100644 bitbake/lib/bb/checksum.py
--
1.7.9.5
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 1/2] bitbake: refactor out codeparser cache into a separate class 2012-05-22 23:23 [PATCH 0/2] Implement file checksums Paul Eggleton @ 2012-05-22 23:23 ` Paul Eggleton 2012-05-22 23:23 ` [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI Paul Eggleton 2012-05-23 10:25 ` [PATCH 0/2] Implement file checksums Richard Purdie 2 siblings, 0 replies; 8+ messages in thread From: Paul Eggleton @ 2012-05-22 23:23 UTC (permalink / raw) To: bitbake-devel We want to be able to reuse most this functionality for the file checksum cache. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> --- bitbake/lib/bb/cache.py | 116 ++++++++++++++++++++++++- bitbake/lib/bb/codeparser.py | 191 +++++++++++++----------------------------- 2 files changed, 171 insertions(+), 136 deletions(-) diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 47e814b..36e6356 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -1,11 +1,12 @@ # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # -# BitBake 'Event' implementation +# BitBake Cache implementation # # Caching of bitbake variables before task execution # Copyright (C) 2006 Richard Purdie +# Copyright (C) 2012 Intel Corporation # but small sections based on code from bin/bitbake: # Copyright (C) 2003, 2004 Chris Larson @@ -703,4 +704,115 @@ class CacheData(object): for info in info_array: info.add_cacheData(self, fn) - + +class MultiProcessCache(object): + """ + BitBake multi-process cache implementation + + Used by the codeparser & file checksum caches + """ + + def __init__(self): + self.cachefile = None + self.cachedata = self.create_cachedata() + self.cachedata_extras = self.create_cachedata() + + def init_cache(self, d): + cachedir = (d.getVar("PERSISTENT_DIR", True) or + d.getVar("CACHE", True)) + if cachedir in [None, '']: + return + bb.utils.mkdirhier(cachedir) + self.cachefile = os.path.join(cachedir, self.__class__.cache_file_name) + logger.debug(1, "Using cache in '%s'", self.cachefile) + + try: + p = pickle.Unpickler(file(self.cachefile, "rb")) + data, version = p.load() + except: + return + + if version != self.__class__.CACHE_VERSION: + return + + self.cachedata = data + + def internSet(self, items): + new = set() + for i in items: + new.add(intern(i)) + return new + + def compress_keys(self, data): + # Override in subclasses if desired + return + + def create_cachedata(self): + data = [{}] + return data + + def save_extras(self, d): + if not self.cachefile: + return + + glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True) + + i = os.getpid() + lf = None + while not lf: + lf = bb.utils.lockfile(self.cachefile + ".lock." + str(i), retry=False) + if not lf or os.path.exists(self.cachefile + "-" + str(i)): + if lf: + bb.utils.unlockfile(lf) + lf = None + i = i + 1 + continue + + p = pickle.Pickler(file(self.cachefile + "-" + str(i), "wb"), -1) + p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION]) + + bb.utils.unlockfile(lf) + bb.utils.unlockfile(glf) + + def merge_data(self, source, dest): + for j in range(0,len(dest)): + for h in source[j]: + if h not in dest[j]: + dest[j][h] = source[j][h] + + def save_merge(self, d): + if not self.cachefile: + return + + glf = bb.utils.lockfile(self.cachefile + ".lock") + + try: + p = pickle.Unpickler(file(self.cachefile, "rb")) + data, version = p.load() + except (IOError, EOFError): + data, version = None, None + + if version != self.__class__.CACHE_VERSION: + data = self.create_cachedata() + + for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: + f = os.path.join(os.path.dirname(self.cachefile), f) + try: + p = pickle.Unpickler(file(f, "rb")) + extradata, version = p.load() + except (IOError, EOFError): + extradata, version = self.create_cachedata(), None + + if version != self.__class__.CACHE_VERSION: + continue + + self.merge_data(extradata, data) + os.unlink(f) + + self.compress_keys(data) + + p = pickle.Pickler(file(self.cachefile, "wb"), -1) + p.dump([data, self.__class__.CACHE_VERSION]) + + bb.utils.unlockfile(glf) + diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py index af2e194..d7d3f51 100644 --- a/bitbake/lib/bb/codeparser.py +++ b/bitbake/lib/bb/codeparser.py @@ -5,10 +5,10 @@ import os.path import bb.utils, bb.data from itertools import chain from pysh import pyshyacc, pyshlex, sherrors +from bb.cache import MultiProcessCache logger = logging.getLogger('BitBake.CodeParser') -PARSERCACHE_VERSION = 2 try: import cPickle as pickle @@ -32,133 +32,56 @@ def check_indent(codestr): return codestr -pythonparsecache = {} -shellparsecache = {} -pythonparsecacheextras = {} -shellparsecacheextras = {} - -def parser_cachefile(d): - cachedir = (d.getVar("PERSISTENT_DIR", True) or - d.getVar("CACHE", True)) - if cachedir in [None, '']: - return None - bb.utils.mkdirhier(cachedir) - cachefile = os.path.join(cachedir, "bb_codeparser.dat") - logger.debug(1, "Using cache in '%s' for codeparser cache", cachefile) - return cachefile - -def parser_cache_init(d): - global pythonparsecache - global shellparsecache - - cachefile = parser_cachefile(d) - if not cachefile: +class CodeParserCache(MultiProcessCache): + cache_file_name = "bb_codeparser.dat" + CACHE_VERSION = 2 + + def __init__(self): + MultiProcessCache.__init__(self) + self.pythoncache = self.cachedata[0] + self.shellcache = self.cachedata[1] + self.pythoncacheextras = self.cachedata_extras[0] + self.shellcacheextras = self.cachedata_extras[1] + + def init_cache(self, d): + MultiProcessCache.init_cache(self, d) + + # cachedata gets re-assigned in the parent + self.pythoncache = self.cachedata[0] + self.shellcache = self.cachedata[1] + + def compress_keys(self, data): + # When the dicts are originally created, python calls intern() on the set keys + # which significantly improves memory usage. Sadly the pickle/unpickle process + # doesn't call intern() on the keys and results in the same strings being duplicated + # in memory. This also means pickle will save the same string multiple times in + # the cache file. By interning the data here, the cache file shrinks dramatically + # meaning faster load times and the reloaded cache files also consume much less + # memory. This is worth any performance hit from this loops and the use of the + # intern() data storage. + # Python 3.x may behave better in this area + for h in data[0]: + data[0][h]["refs"] = self.internSet(data[0][h]["refs"]) + data[0][h]["execs"] = self.internSet(data[0][h]["execs"]) + for h in data[1]: + data[1][h]["execs"] = self.internSet(data[1][h]["execs"]) return - try: - p = pickle.Unpickler(file(cachefile, "rb")) - data, version = p.load() - except: - return + def create_cachedata(self): + data = [{}, {}] + return data - if version != PARSERCACHE_VERSION: - return +codeparsercache = CodeParserCache() - pythonparsecache = data[0] - shellparsecache = data[1] +def parser_cache_init(d): + codeparsercache.init_cache(d) def parser_cache_save(d): - cachefile = parser_cachefile(d) - if not cachefile: - return - - glf = bb.utils.lockfile(cachefile + ".lock", shared=True) - - i = os.getpid() - lf = None - while not lf: - shellcache = {} - pythoncache = {} - - lf = bb.utils.lockfile(cachefile + ".lock." + str(i), retry=False) - if not lf or os.path.exists(cachefile + "-" + str(i)): - if lf: - bb.utils.unlockfile(lf) - lf = None - i = i + 1 - continue - - shellcache = shellparsecacheextras - pythoncache = pythonparsecacheextras - - p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1) - p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION]) - - bb.utils.unlockfile(lf) - bb.utils.unlockfile(glf) - -def internSet(items): - new = set() - for i in items: - new.add(intern(i)) - return new + codeparsercache.save_extras(d) def parser_cache_savemerge(d): - cachefile = parser_cachefile(d) - if not cachefile: - return - - glf = bb.utils.lockfile(cachefile + ".lock") - - try: - p = pickle.Unpickler(file(cachefile, "rb")) - data, version = p.load() - except (IOError, EOFError): - data, version = None, None - - if version != PARSERCACHE_VERSION: - data = [{}, {}] - - for f in [y for y in os.listdir(os.path.dirname(cachefile)) if y.startswith(os.path.basename(cachefile) + '-')]: - f = os.path.join(os.path.dirname(cachefile), f) - try: - p = pickle.Unpickler(file(f, "rb")) - extradata, version = p.load() - except (IOError, EOFError): - extradata, version = [{}, {}], None - - if version != PARSERCACHE_VERSION: - continue - - for h in extradata[0]: - if h not in data[0]: - data[0][h] = extradata[0][h] - for h in extradata[1]: - if h not in data[1]: - data[1][h] = extradata[1][h] - os.unlink(f) - - # When the dicts are originally created, python calls intern() on the set keys - # which significantly improves memory usage. Sadly the pickle/unpickle process - # doesn't call intern() on the keys and results in the same strings being duplicated - # in memory. This also means pickle will save the same string multiple times in - # the cache file. By interning the data here, the cache file shrinks dramatically - # meaning faster load times and the reloaded cache files also consume much less - # memory. This is worth any performance hit from this loops and the use of the - # intern() data storage. - # Python 3.x may behave better in this area - for h in data[0]: - data[0][h]["refs"] = internSet(data[0][h]["refs"]) - data[0][h]["execs"] = internSet(data[0][h]["execs"]) - for h in data[1]: - data[1][h]["execs"] = internSet(data[1][h]["execs"]) - - p = pickle.Pickler(file(cachefile, "wb"), -1) - p.dump([data, PARSERCACHE_VERSION]) - - bb.utils.unlockfile(glf) - + codeparsercache.save_merge(d) Logger = logging.getLoggerClass() class BufferedLogger(Logger): @@ -235,14 +158,14 @@ class PythonParser(): def parse_python(self, node): h = hash(str(node)) - if h in pythonparsecache: - self.references = pythonparsecache[h]["refs"] - self.execs = pythonparsecache[h]["execs"] + if h in codeparsercache.pythoncache: + self.references = codeparsercache.pythoncache[h]["refs"] + self.execs = codeparsercache.pythoncache[h]["execs"] return - if h in pythonparsecacheextras: - self.references = pythonparsecacheextras[h]["refs"] - self.execs = pythonparsecacheextras[h]["execs"] + if h in codeparsercache.pythoncacheextras: + self.references = codeparsercache.pythoncacheextras[h]["refs"] + self.execs = codeparsercache.pythoncacheextras[h]["execs"] return @@ -256,9 +179,9 @@ class PythonParser(): self.references.update(self.var_references) self.references.update(self.var_execs) - pythonparsecacheextras[h] = {} - pythonparsecacheextras[h]["refs"] = self.references - pythonparsecacheextras[h]["execs"] = self.execs + codeparsercache.pythoncacheextras[h] = {} + codeparsercache.pythoncacheextras[h]["refs"] = self.references + codeparsercache.pythoncacheextras[h]["execs"] = self.execs class ShellParser(): def __init__(self, name, log): @@ -276,12 +199,12 @@ class ShellParser(): h = hash(str(value)) - if h in shellparsecache: - self.execs = shellparsecache[h]["execs"] + if h in codeparsercache.shellcache: + self.execs = codeparsercache.shellcache[h]["execs"] return self.execs - if h in shellparsecacheextras: - self.execs = shellparsecacheextras[h]["execs"] + if h in codeparsercache.shellcacheextras: + self.execs = codeparsercache.shellcacheextras[h]["execs"] return self.execs try: @@ -293,8 +216,8 @@ class ShellParser(): self.process_tokens(token) self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs) - shellparsecacheextras[h] = {} - shellparsecacheextras[h]["execs"] = self.execs + codeparsercache.shellcacheextras[h] = {} + codeparsercache.shellcacheextras[h]["execs"] = self.execs return self.execs -- 1.7.9.5 ^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI 2012-05-22 23:23 [PATCH 0/2] Implement file checksums Paul Eggleton 2012-05-22 23:23 ` [PATCH 1/2] bitbake: refactor out codeparser cache into a separate class Paul Eggleton @ 2012-05-22 23:23 ` Paul Eggleton 2012-05-22 23:45 ` Mark Hatle 2012-05-23 10:25 ` [PATCH 0/2] Implement file checksums Richard Purdie 2 siblings, 1 reply; 8+ messages in thread From: Paul Eggleton @ 2012-05-22 23:23 UTC (permalink / raw) To: bitbake-devel Gathers a list of paths to have checksums calculated at parse time, and processes these when calculating task hashes. Checksums are cached with the file's current mtime. Thus, changing any local file in SRC_URI will now cause the do_fetch taskhash to change, thus forcing a rebuild. This change adds very roughly about an 8% increase in parse time (a few seconds) and maybe a few seconds during runqueue generation, so a fairly moderate performance hit. Note that since paths are resolved at parse time, this will not force a rebuild when files are introduced which would cause that resolved path to be different - for example, where a machine-specific version of a file was added without otherwise changing the recipe. This will need to be handled in a future update. Code to hook this into the signature generator was courtesy of Richard Purdie <richard.purdie@linuxfoundation.org>. Implements [YOCTO #2044]. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> --- bitbake/lib/bb/cache.py | 13 ++++-- bitbake/lib/bb/checksum.py | 90 +++++++++++++++++++++++++++++++++++++ bitbake/lib/bb/cooker.py | 2 + bitbake/lib/bb/fetch2/__init__.py | 85 +++++++++++++++++++++++++++++++++++ bitbake/lib/bb/siggen.py | 24 ++++++++++ 5 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 bitbake/lib/bb/checksum.py diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 36e6356..dea2a80 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -43,7 +43,7 @@ except ImportError: logger.info("Importing cPickle failed. " "Falling back to a very slow implementation.") -__cache_version__ = "143" +__cache_version__ = "144" def getCacheFile(path, filename, data_hash): return os.path.join(path, filename + "." + data_hash) @@ -76,9 +76,13 @@ class RecipeInfoCommon(object): for task in tasks) @classmethod - def flaglist(cls, flag, varlist, metadata): - return dict((var, metadata.getVarFlag(var, flag, True)) + def flaglist(cls, flag, varlist, metadata, squash=False): + out_dict = dict((var, metadata.getVarFlag(var, flag, True)) for var in varlist) + if squash: + return dict((k,v) for (k,v) in out_dict.iteritems() if v) + else: + return out_dict @classmethod def getvar(cls, var, metadata): @@ -128,6 +132,7 @@ class CoreRecipeInfo(RecipeInfoCommon): self.stamp = self.getvar('STAMP', metadata) self.stamp_base = self.flaglist('stamp-base', self.tasks, metadata) self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) + self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True) self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) self.depends = self.depvar('DEPENDS', metadata) self.provides = self.depvar('PROVIDES', metadata) @@ -154,6 +159,7 @@ class CoreRecipeInfo(RecipeInfoCommon): cachedata.stamp = {} cachedata.stamp_base = {} cachedata.stamp_extrainfo = {} + cachedata.file_checksums = {} cachedata.fn_provides = {} cachedata.pn_provides = defaultdict(list) cachedata.all_depends = [] @@ -185,6 +191,7 @@ class CoreRecipeInfo(RecipeInfoCommon): cachedata.stamp[fn] = self.stamp cachedata.stamp_base[fn] = self.stamp_base cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo + cachedata.file_checksums[fn] = self.file_checksums provides = [self.pn] for provide in self.provides: diff --git a/bitbake/lib/bb/checksum.py b/bitbake/lib/bb/checksum.py new file mode 100644 index 0000000..514ff0b --- /dev/null +++ b/bitbake/lib/bb/checksum.py @@ -0,0 +1,90 @@ +# Local file checksum cache implementation +# +# Copyright (C) 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import stat +import bb.utils +import logging +from bb.cache import MultiProcessCache + +logger = logging.getLogger("BitBake.Cache") + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info("Importing cPickle failed. " + "Falling back to a very slow implementation.") + + +# mtime cache (non-persistent) +# based upon the assumption that files do not change during bitbake run +class FileMtimeCache(object): + cache = {} + + def cached_mtime(self, f): + if f not in self.cache: + self.cache[f] = os.stat(f)[stat.ST_MTIME] + return self.cache[f] + + def cached_mtime_noerror(self, f): + if f not in self.cache: + try: + self.cache[f] = os.stat(f)[stat.ST_MTIME] + except OSError: + return 0 + return self.cache[f] + + def update_mtime(self, f): + self.cache[f] = os.stat(f)[stat.ST_MTIME] + return self.cache[f] + + def clear(self): + self.cache.clear() + +# Checksum + mtime cache (persistent) +class FileChecksumCache(MultiProcessCache): + cache_file_name = "local_file_checksum_cache.dat" + CACHE_VERSION = 1 + + def __init__(self): + self.mtime_cache = FileMtimeCache() + MultiProcessCache.__init__(self) + + def get_checksum(self, f): + entry = self.cachedata[0].get(f) + cmtime = self.mtime_cache.cached_mtime(f) + if entry: + (mtime, hashval) = entry + if cmtime == mtime: + return hashval + else: + bb.debug(2, "file %s changed mtime, recompute checksum" % f) + + hashval = bb.utils.md5_file(f) + self.cachedata_extras[0][f] = (cmtime, hashval) + return hashval + + def merge_data(self, source, dest): + for h in source[0]: + if h in dest: + (smtime, _) = source[0][h] + (dmtime, _) = dest[0][h] + if smtime > dmtime: + dest[0][h] = source[0][h] + else: + dest[0][h] = source[0][h] diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index dea0aad..8ad4922 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -1570,6 +1570,7 @@ class CookerParser(object): def init(): Parser.cfg = self.cfgdata multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cfgdata,), exitpriority=1) + multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, args=(self.cfgdata,), exitpriority=1) self.feeder_quit = multiprocessing.Queue(maxsize=1) self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) @@ -1618,6 +1619,7 @@ class CookerParser(object): sync.start() multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) bb.codeparser.parser_cache_savemerge(self.cooker.configuration.data) + bb.fetch.fetcher_parse_done(self.cooker.configuration.data) def load_cached(self): for filename, appends in self.fromcache: diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 0b976c4..d4b6c3e 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -8,6 +8,7 @@ BitBake build tools. """ # Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2012 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,9 +31,11 @@ import os, re import logging import urllib import bb.persist_data, bb.utils +import bb.checksum from bb import data __version__ = "2" +_checksum_cache = bb.checksum.FileChecksumCache() logger = logging.getLogger("BitBake.Fetcher") @@ -233,10 +236,18 @@ def fetcher_init(d): else: raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) + _checksum_cache.init_cache(d) + for m in methods: if hasattr(m, "init"): m.init(d) +def fetcher_parse_save(d): + _checksum_cache.save_extras(d) + +def fetcher_parse_done(d): + _checksum_cache.save_merge(d) + def fetcher_compare_revisions(d): """ Compare the revisions in the persistant cache with current values and @@ -553,6 +564,80 @@ def srcrev_internal_helper(ud, d, name): return rev + +def get_checksum_file_list(d): + """ Get a list of files checksum in SRC_URI + + Returns the all resolved local path of all local file entries in + SRC_URI as a space-separated string + """ + fetch = Fetch([], d) + + dl_dir = d.getVar('DL_DIR', True) + filelist = [] + for u in fetch.urls: + ud = fetch.ud[u] + + if isinstance(ud.method, local.Local): + ud.setup_localpath(d) + f = ud.localpath + if f.startswith(dl_dir): + # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else + if os.path.exists(f): + bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f))) + else: + bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f))) + continue + filelist.append(f) + + return " ".join(filelist) + + +def get_file_checksums(filelist, pn): + """Get a list of the checksums for a list of local files + + Returns the checksums for a list of local files, caching the results as + it proceeds + + """ + + def checksum_file(f): + try: + checksum = _checksum_cache.get_checksum(f) + except OSError as e: + import traceback + bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e)) + return None + return checksum + + checksums = [] + for pth in filelist.split(): + checksum = None + if '*' in pth: + # Handle globs + import glob + for f in glob.glob(pth): + checksum = checksum_file(f) + if checksum: + checksums.append((f, checksum)) + elif os.path.isdir(pth): + # Handle directories + for root, dirs, files in os.walk(pth): + for name in files: + fullpth = os.path.join(root, name) + checksum = checksum_file(fullpth) + if checksum: + checksums.append((fullpth, checksum)) + else: + checksum = checksum_file(pth) + + if checksum: + checksums.append((pth, checksum)) + + checksums.sort() + return checksums + + class FetchData(object): """ A class which represents the fetcher state for a given URI. diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 5a0b80e..daf5677 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -60,6 +60,7 @@ class SignatureGeneratorBasic(SignatureGenerator): self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} + self.file_checksum_values = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") @@ -152,6 +153,7 @@ class SignatureGeneratorBasic(SignatureGenerator): k = fn + "." + task data = dataCache.basetaskhash[k] self.runtaskdeps[k] = [] + self.file_checksum_values[k] = {} recipename = dataCache.pkg_fn[fn] for dep in sorted(deps, key=clean_basepath): depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] @@ -161,6 +163,12 @@ class SignatureGeneratorBasic(SignatureGenerator): bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) data = data + self.taskhash[dep] self.runtaskdeps[k].append(dep) + + if task in dataCache.file_checksums[fn]: + checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) + for (f,cs) in checksums: + self.file_checksum_values[k][f] = cs + data = data + cs h = hashlib.md5(data).hexdigest() self.taskhash[k] = h #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) @@ -197,6 +205,7 @@ class SignatureGeneratorBasic(SignatureGenerator): if runtime and k in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[k] + data['file_checksum_values'] = self.file_checksum_values[k] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.taskhash[dep] @@ -304,6 +313,18 @@ def compare_sigfiles(a, b): for dep in changed: print "Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]) + changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) + if changed: + for f in changed: + print "Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f]) + if added: + for f in added: + print "Dependency on checksum of file %s was added" % (f) + if removed: + for f in removed: + print "Dependency on checksum of file %s was removed" % (f) + + if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: a = clean_basepaths(a_data['runtaskhashes']) b = clean_basepaths(b_data['runtaskhashes']) @@ -353,6 +374,9 @@ def dump_sigfile(a): if 'runtaskdeps' in a_data: print "Tasks this task depends on: %s" % (a_data['runtaskdeps']) + if 'file_checksum_values' in a_data: + print "This task depends on the checksums of files: %s" % (a_data['file_checksum_values']) + if 'runtaskhashes' in a_data: for dep in a_data['runtaskhashes']: print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]) -- 1.7.9.5 ^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI 2012-05-22 23:23 ` [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI Paul Eggleton @ 2012-05-22 23:45 ` Mark Hatle 2012-05-22 23:50 ` Paul Eggleton 0 siblings, 1 reply; 8+ messages in thread From: Mark Hatle @ 2012-05-22 23:45 UTC (permalink / raw) To: Paul Eggleton; +Cc: bitbake-devel On 5/22/12 6:23 PM, Paul Eggleton wrote: > Gathers a list of paths to have checksums calculated at parse time, and > processes these when calculating task hashes. Checksums are cached with > the file's current mtime. Thus, changing any local file in SRC_URI will > now cause the do_fetch taskhash to change, thus forcing a rebuild. Does the mtime change invalidate the checksum, or just cause the checksum to be re-interpreted? The issue I see is that you share a ccache file with someone else, their files may simply have a different mtime on them. From reading the code below, I think the comment is just confusing me. The checksum is computed and stored bases on a hash + mtime. If the mtime changes, that will cause the system to recalculate the checksum, which may end up being the same.. (and if it is, no rebuild) right? --Mark > This change adds very roughly about an 8% increase in parse time (a few > seconds) and maybe a few seconds during runqueue generation, so a fairly > moderate performance hit. > > Note that since paths are resolved at parse time, this will not force > a rebuild when files are introduced which would cause that resolved path > to be different - for example, where a machine-specific version of a file > was added without otherwise changing the recipe. This will need to be > handled in a future update. > > Code to hook this into the signature generator was courtesy of > Richard Purdie<richard.purdie@linuxfoundation.org>. > > Implements [YOCTO #2044]. > > Signed-off-by: Paul Eggleton<paul.eggleton@linux.intel.com> > --- > bitbake/lib/bb/cache.py | 13 ++++-- > bitbake/lib/bb/checksum.py | 90 +++++++++++++++++++++++++++++++++++++ > bitbake/lib/bb/cooker.py | 2 + > bitbake/lib/bb/fetch2/__init__.py | 85 +++++++++++++++++++++++++++++++++++ > bitbake/lib/bb/siggen.py | 24 ++++++++++ > 5 files changed, 211 insertions(+), 3 deletions(-) > create mode 100644 bitbake/lib/bb/checksum.py > > diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py > index 36e6356..dea2a80 100644 > --- a/bitbake/lib/bb/cache.py > +++ b/bitbake/lib/bb/cache.py > @@ -43,7 +43,7 @@ except ImportError: > logger.info("Importing cPickle failed. " > "Falling back to a very slow implementation.") > > -__cache_version__ = "143" > +__cache_version__ = "144" > > def getCacheFile(path, filename, data_hash): > return os.path.join(path, filename + "." + data_hash) > @@ -76,9 +76,13 @@ class RecipeInfoCommon(object): > for task in tasks) > > @classmethod > - def flaglist(cls, flag, varlist, metadata): > - return dict((var, metadata.getVarFlag(var, flag, True)) > + def flaglist(cls, flag, varlist, metadata, squash=False): > + out_dict = dict((var, metadata.getVarFlag(var, flag, True)) > for var in varlist) > + if squash: > + return dict((k,v) for (k,v) in out_dict.iteritems() if v) > + else: > + return out_dict > > @classmethod > def getvar(cls, var, metadata): > @@ -128,6 +132,7 @@ class CoreRecipeInfo(RecipeInfoCommon): > self.stamp = self.getvar('STAMP', metadata) > self.stamp_base = self.flaglist('stamp-base', self.tasks, metadata) > self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) > + self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True) > self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) > self.depends = self.depvar('DEPENDS', metadata) > self.provides = self.depvar('PROVIDES', metadata) > @@ -154,6 +159,7 @@ class CoreRecipeInfo(RecipeInfoCommon): > cachedata.stamp = {} > cachedata.stamp_base = {} > cachedata.stamp_extrainfo = {} > + cachedata.file_checksums = {} > cachedata.fn_provides = {} > cachedata.pn_provides = defaultdict(list) > cachedata.all_depends = [] > @@ -185,6 +191,7 @@ class CoreRecipeInfo(RecipeInfoCommon): > cachedata.stamp[fn] = self.stamp > cachedata.stamp_base[fn] = self.stamp_base > cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo > + cachedata.file_checksums[fn] = self.file_checksums > > provides = [self.pn] > for provide in self.provides: > diff --git a/bitbake/lib/bb/checksum.py b/bitbake/lib/bb/checksum.py > new file mode 100644 > index 0000000..514ff0b > --- /dev/null > +++ b/bitbake/lib/bb/checksum.py > @@ -0,0 +1,90 @@ > +# Local file checksum cache implementation > +# > +# Copyright (C) 2012 Intel Corporation > +# > +# This program is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License version 2 as > +# published by the Free Software Foundation. > +# > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License along > +# with this program; if not, write to the Free Software Foundation, Inc., > +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. > + > +import os > +import stat > +import bb.utils > +import logging > +from bb.cache import MultiProcessCache > + > +logger = logging.getLogger("BitBake.Cache") > + > +try: > + import cPickle as pickle > +except ImportError: > + import pickle > + logger.info("Importing cPickle failed. " > + "Falling back to a very slow implementation.") > + > + > +# mtime cache (non-persistent) > +# based upon the assumption that files do not change during bitbake run > +class FileMtimeCache(object): > + cache = {} > + > + def cached_mtime(self, f): > + if f not in self.cache: > + self.cache[f] = os.stat(f)[stat.ST_MTIME] > + return self.cache[f] > + > + def cached_mtime_noerror(self, f): > + if f not in self.cache: > + try: > + self.cache[f] = os.stat(f)[stat.ST_MTIME] > + except OSError: > + return 0 > + return self.cache[f] > + > + def update_mtime(self, f): > + self.cache[f] = os.stat(f)[stat.ST_MTIME] > + return self.cache[f] > + > + def clear(self): > + self.cache.clear() > + > +# Checksum + mtime cache (persistent) > +class FileChecksumCache(MultiProcessCache): > + cache_file_name = "local_file_checksum_cache.dat" > + CACHE_VERSION = 1 > + > + def __init__(self): > + self.mtime_cache = FileMtimeCache() > + MultiProcessCache.__init__(self) > + > + def get_checksum(self, f): > + entry = self.cachedata[0].get(f) > + cmtime = self.mtime_cache.cached_mtime(f) > + if entry: > + (mtime, hashval) = entry > + if cmtime == mtime: > + return hashval > + else: > + bb.debug(2, "file %s changed mtime, recompute checksum" % f) > + > + hashval = bb.utils.md5_file(f) > + self.cachedata_extras[0][f] = (cmtime, hashval) > + return hashval > + > + def merge_data(self, source, dest): > + for h in source[0]: > + if h in dest: > + (smtime, _) = source[0][h] > + (dmtime, _) = dest[0][h] > + if smtime> dmtime: > + dest[0][h] = source[0][h] > + else: > + dest[0][h] = source[0][h] > diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py > index dea0aad..8ad4922 100644 > --- a/bitbake/lib/bb/cooker.py > +++ b/bitbake/lib/bb/cooker.py > @@ -1570,6 +1570,7 @@ class CookerParser(object): > def init(): > Parser.cfg = self.cfgdata > multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cfgdata,), exitpriority=1) > + multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, args=(self.cfgdata,), exitpriority=1) > > self.feeder_quit = multiprocessing.Queue(maxsize=1) > self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) > @@ -1618,6 +1619,7 @@ class CookerParser(object): > sync.start() > multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) > bb.codeparser.parser_cache_savemerge(self.cooker.configuration.data) > + bb.fetch.fetcher_parse_done(self.cooker.configuration.data) > > def load_cached(self): > for filename, appends in self.fromcache: > diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py > index 0b976c4..d4b6c3e 100644 > --- a/bitbake/lib/bb/fetch2/__init__.py > +++ b/bitbake/lib/bb/fetch2/__init__.py > @@ -8,6 +8,7 @@ BitBake build tools. > """ > > # Copyright (C) 2003, 2004 Chris Larson > +# Copyright (C) 2012 Intel Corporation > # > # This program is free software; you can redistribute it and/or modify > # it under the terms of the GNU General Public License version 2 as > @@ -30,9 +31,11 @@ import os, re > import logging > import urllib > import bb.persist_data, bb.utils > +import bb.checksum > from bb import data > > __version__ = "2" > +_checksum_cache = bb.checksum.FileChecksumCache() > > logger = logging.getLogger("BitBake.Fetcher") > > @@ -233,10 +236,18 @@ def fetcher_init(d): > else: > raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) > > + _checksum_cache.init_cache(d) > + > for m in methods: > if hasattr(m, "init"): > m.init(d) > > +def fetcher_parse_save(d): > + _checksum_cache.save_extras(d) > + > +def fetcher_parse_done(d): > + _checksum_cache.save_merge(d) > + > def fetcher_compare_revisions(d): > """ > Compare the revisions in the persistant cache with current values and > @@ -553,6 +564,80 @@ def srcrev_internal_helper(ud, d, name): > > return rev > > + > +def get_checksum_file_list(d): > + """ Get a list of files checksum in SRC_URI > + > + Returns the all resolved local path of all local file entries in > + SRC_URI as a space-separated string > + """ > + fetch = Fetch([], d) > + > + dl_dir = d.getVar('DL_DIR', True) > + filelist = [] > + for u in fetch.urls: > + ud = fetch.ud[u] > + > + if isinstance(ud.method, local.Local): > + ud.setup_localpath(d) > + f = ud.localpath > + if f.startswith(dl_dir): > + # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else > + if os.path.exists(f): > + bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f))) > + else: > + bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f))) > + continue > + filelist.append(f) > + > + return " ".join(filelist) > + > + > +def get_file_checksums(filelist, pn): > + """Get a list of the checksums for a list of local files > + > + Returns the checksums for a list of local files, caching the results as > + it proceeds > + > + """ > + > + def checksum_file(f): > + try: > + checksum = _checksum_cache.get_checksum(f) > + except OSError as e: > + import traceback > + bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e)) > + return None > + return checksum > + > + checksums = [] > + for pth in filelist.split(): > + checksum = None > + if '*' in pth: > + # Handle globs > + import glob > + for f in glob.glob(pth): > + checksum = checksum_file(f) > + if checksum: > + checksums.append((f, checksum)) > + elif os.path.isdir(pth): > + # Handle directories > + for root, dirs, files in os.walk(pth): > + for name in files: > + fullpth = os.path.join(root, name) > + checksum = checksum_file(fullpth) > + if checksum: > + checksums.append((fullpth, checksum)) > + else: > + checksum = checksum_file(pth) > + > + if checksum: > + checksums.append((pth, checksum)) > + > + checksums.sort() > + return checksums > + > + > class FetchData(object): > """ > A class which represents the fetcher state for a given URI. > diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py > index 5a0b80e..daf5677 100644 > --- a/bitbake/lib/bb/siggen.py > +++ b/bitbake/lib/bb/siggen.py > @@ -60,6 +60,7 @@ class SignatureGeneratorBasic(SignatureGenerator): > self.taskhash = {} > self.taskdeps = {} > self.runtaskdeps = {} > + self.file_checksum_values = {} > self.gendeps = {} > self.lookupcache = {} > self.pkgnameextract = re.compile("(?P<fn>.*)\..*") > @@ -152,6 +153,7 @@ class SignatureGeneratorBasic(SignatureGenerator): > k = fn + "." + task > data = dataCache.basetaskhash[k] > self.runtaskdeps[k] = [] > + self.file_checksum_values[k] = {} > recipename = dataCache.pkg_fn[fn] > for dep in sorted(deps, key=clean_basepath): > depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] > @@ -161,6 +163,12 @@ class SignatureGeneratorBasic(SignatureGenerator): > bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) > data = data + self.taskhash[dep] > self.runtaskdeps[k].append(dep) > + > + if task in dataCache.file_checksums[fn]: > + checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) > + for (f,cs) in checksums: > + self.file_checksum_values[k][f] = cs > + data = data + cs > h = hashlib.md5(data).hexdigest() > self.taskhash[k] = h > #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) > @@ -197,6 +205,7 @@ class SignatureGeneratorBasic(SignatureGenerator): > > if runtime and k in self.taskhash: > data['runtaskdeps'] = self.runtaskdeps[k] > + data['file_checksum_values'] = self.file_checksum_values[k] > data['runtaskhashes'] = {} > for dep in data['runtaskdeps']: > data['runtaskhashes'][dep] = self.taskhash[dep] > @@ -304,6 +313,18 @@ def compare_sigfiles(a, b): > for dep in changed: > print "Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]) > > + changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) > + if changed: > + for f in changed: > + print "Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f]) > + if added: > + for f in added: > + print "Dependency on checksum of file %s was added" % (f) > + if removed: > + for f in removed: > + print "Dependency on checksum of file %s was removed" % (f) > + > + > if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: > a = clean_basepaths(a_data['runtaskhashes']) > b = clean_basepaths(b_data['runtaskhashes']) > @@ -353,6 +374,9 @@ def dump_sigfile(a): > if 'runtaskdeps' in a_data: > print "Tasks this task depends on: %s" % (a_data['runtaskdeps']) > > + if 'file_checksum_values' in a_data: > + print "This task depends on the checksums of files: %s" % (a_data['file_checksum_values']) > + > if 'runtaskhashes' in a_data: > for dep in a_data['runtaskhashes']: > print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]) ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI 2012-05-22 23:45 ` Mark Hatle @ 2012-05-22 23:50 ` Paul Eggleton 2012-05-22 23:55 ` Paul Eggleton 0 siblings, 1 reply; 8+ messages in thread From: Paul Eggleton @ 2012-05-22 23:50 UTC (permalink / raw) To: Mark Hatle; +Cc: bitbake-devel On Tuesday 22 May 2012 18:45:23 you wrote: > On 5/22/12 6:23 PM, Paul Eggleton wrote: > > Gathers a list of paths to have checksums calculated at parse time, and > > processes these when calculating task hashes. Checksums are cached with > > the file's current mtime. Thus, changing any local file in SRC_URI will > > now cause the do_fetch taskhash to change, thus forcing a rebuild. > > Does the mtime change invalidate the checksum, or just cause the checksum to > be re-interpreted? The latter. > The issue I see is that you share a ccache file with someone else, their > files may simply have a different mtime on them. I'm guessing s/ccache/sstate cache/ ? This will be fine - only the checksum of the file contents goes into the sstate signature. > From reading the code below, I think the comment is just confusing me. The > checksum is computed and stored bases on a hash + mtime. If the mtime > changes, that will cause the system to recalculate the checksum, which may > end up being the same.. (and if it is, no rebuild) right? Correct. Cheers, Paul -- Paul Eggleton Intel Open Source Technology Centre ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI 2012-05-22 23:50 ` Paul Eggleton @ 2012-05-22 23:55 ` Paul Eggleton 2012-05-23 9:42 ` Richard Purdie 0 siblings, 1 reply; 8+ messages in thread From: Paul Eggleton @ 2012-05-22 23:55 UTC (permalink / raw) To: Mark Hatle; +Cc: bitbake-devel On Wednesday 23 May 2012 00:50:48 Paul Eggleton wrote: > On Tuesday 22 May 2012 18:45:23 you wrote: > > On 5/22/12 6:23 PM, Paul Eggleton wrote: > > > Gathers a list of paths to have checksums calculated at parse time, and > > > processes these when calculating task hashes. Checksums are cached with > > > the file's current mtime. Thus, changing any local file in SRC_URI will > > > now cause the do_fetch taskhash to change, thus forcing a rebuild. > > > > Does the mtime change invalidate the checksum, or just cause the checksum > > to be re-interpreted? > > The latter. Er, I think I may have misread your question. To be totally clear - the mtime is not a component of the checksum; we merely store it next to the checksum in a cache so that we don't have to re-compute the checksum if the file hasn't been modified. If mtime changes but the file content does not, the checksum will be re-computed but will not change. Cheers, Paul -- Paul Eggleton Intel Open Source Technology Centre ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI 2012-05-22 23:55 ` Paul Eggleton @ 2012-05-23 9:42 ` Richard Purdie 0 siblings, 0 replies; 8+ messages in thread From: Richard Purdie @ 2012-05-23 9:42 UTC (permalink / raw) To: Paul Eggleton; +Cc: bitbake-devel On Wed, 2012-05-23 at 00:55 +0100, Paul Eggleton wrote: > On Wednesday 23 May 2012 00:50:48 Paul Eggleton wrote: > > On Tuesday 22 May 2012 18:45:23 you wrote: > > > On 5/22/12 6:23 PM, Paul Eggleton wrote: > > > > Gathers a list of paths to have checksums calculated at parse time, and > > > > processes these when calculating task hashes. Checksums are cached with > > > > the file's current mtime. Thus, changing any local file in SRC_URI will > > > > now cause the do_fetch taskhash to change, thus forcing a rebuild. > > > > > > Does the mtime change invalidate the checksum, or just cause the checksum > > > to be re-interpreted? > > > > The latter. > > Er, I think I may have misread your question. To be totally clear - the mtime > is not a component of the checksum; we merely store it next to the checksum in > a cache so that we don't have to re-compute the checksum if the file hasn't > been modified. If mtime changes but the file content does not, the checksum will > be re-computed but will not change. Just to be completely clear, this is purely a performance issue, we don't want to recompute the checksums for all the files at each bitbake invocation as his would be slow. We therefore just recompute the checksum when mtime changes. You can therefore happily touch a file and it won't trigger a rebuild. Any change to the contents will rebuild the recipe though through the changed sstate checksum. Cheers, Richard ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 0/2] Implement file checksums 2012-05-22 23:23 [PATCH 0/2] Implement file checksums Paul Eggleton 2012-05-22 23:23 ` [PATCH 1/2] bitbake: refactor out codeparser cache into a separate class Paul Eggleton 2012-05-22 23:23 ` [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI Paul Eggleton @ 2012-05-23 10:25 ` Richard Purdie 2 siblings, 0 replies; 8+ messages in thread From: Richard Purdie @ 2012-05-23 10:25 UTC (permalink / raw) To: Paul Eggleton; +Cc: bitbake-devel On Wed, 2012-05-23 at 00:23 +0100, Paul Eggleton wrote: > This is 2/3 of the changes required to implement local file checksums; > the other third is a patch to base.bbclass which will be sent to the > OE-Core list. > > > The following changes (against Poky, but apply cleanly against BitBake > master with -p2) are available in the git repository at: > > git://git.yoctoproject.org/poky-contrib paule/file-checksum-bb > http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/file-checksum-bb > > Paul Eggleton (2): > bitbake: refactor out codeparser cache into a separate class > bitbake: implement checksums for local files in SRC_URI Merged to master, thanks. Richard ^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2012-05-23 10:36 UTC | newest] Thread overview: 8+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2012-05-22 23:23 [PATCH 0/2] Implement file checksums Paul Eggleton 2012-05-22 23:23 ` [PATCH 1/2] bitbake: refactor out codeparser cache into a separate class Paul Eggleton 2012-05-22 23:23 ` [PATCH 2/2] bitbake: implement checksums for local files in SRC_URI Paul Eggleton 2012-05-22 23:45 ` Mark Hatle 2012-05-22 23:50 ` Paul Eggleton 2012-05-22 23:55 ` Paul Eggleton 2012-05-23 9:42 ` Richard Purdie 2012-05-23 10:25 ` [PATCH 0/2] Implement file checksums Richard Purdie
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.