* [PATCH 1/5] Richard Purdie : sstatesig/sstate: Add support for locked down sstate cache usage
2014-09-11 15:04 [PATCH 0/5] Add support for locked down sstate Hongxu Jia
@ 2014-09-11 15:04 ` Hongxu Jia
2014-09-11 15:04 ` [PATCH 2/5] Richard Purdie : sstatesig: Improve to handle locking of multiple machines Hongxu Jia
` (3 subsequent siblings)
4 siblings, 0 replies; 10+ messages in thread
From: Hongxu Jia @ 2014-09-11 15:04 UTC (permalink / raw)
To: openembedded-core, mark.hatle, richard.purdie
From: "git@opal.openembedded.org" <git@opal.openembedded.org>
Module: openembedded-core.git
Branch: master-next
Commit: a12e33a584a77df4bdd9ad6a5d1a58f4dde10317
URL: http://git.openembedded.org/?p=openembedded-core.git&a=commit;h=a12e33a584a77df4bdd9ad6a5d1a58f4dde10317
Author: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Fri Sep 5 10:40:02 2014 +0100
sstatesig/sstate: Add support for locked down sstate cache usage
I've been giving things some thought, specifically why sstate doesn't
get used more and why we have people requesting external toolchains. I'm
guessing the issue is that people don't like how often sstate can change
and the lack of an easy way to lock it down.
Locking it down is actually quite easy so patch implements some basics
of how you can do this (for example to a specific toolchain). With an
addition like this to local.conf (or wherever):
SIGGEN_LOCKEDSIGS = "\
gcc-cross:do_populate_sysroot:a8d91b35b98e1494957a2ddaf4598956 \
eglibc:do_populate_sysroot:13e8c68553dc61f9d67564f13b9b2d67 \
eglibc:do_packagedata:bfca0db1782c719d373f8636282596ee \
gcc-cross:do_packagedata:4b601ff4f67601395ee49c46701122f6 \
"
the code at the end of the email will force the hashes to those values
for the recipes mentioned. The system would then find and use those
specific objects from the sstate cache instead of trying to build
anything.
Obviously this is a little simplistic, you might need to put an override
against this to only apply those revisions for a specific architecture
for example. You'd also probably want to put code in the sstate hash
validation code to ensure it really did install these from sstate since
if it didn't you'd want to abort the build.
This patch also implements support to add to bitbake -S which dumps the
locked sstate checksums for each task into a ready prepared include file
locked-sigs.inc (currently placed into cwd). There is a function,
bb.parse.siggen.dump_lockedsigs() which can be called to trigger the
same functionality from task space.
A warning is added to sstate.bbclass through a call back into the siggen
class to warn if objects are not used from the locked cache. The
SIGGEN_ENFORCE_LOCKEDSIGS variable controls whether this is just a warning
or a fatal error.
A script is provided to generate sstate directory from a locked-sigs file.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/sstate.bbclass | 3 ++
meta/lib/oe/sstatesig.py | 74 +++++++++++++++++++++++++++++++++++++++++++++
scripts/gen-lockedsig-cache | 40 ++++++++++++++++++++++++
3 files changed, 117 insertions(+)
create mode 100755 scripts/gen-lockedsig-cache
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 67e0c1d..0cb5235 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -712,6 +712,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
evdata['found'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) )
bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
+ if hasattr(bb.parse.siggen, "checkhashes"):
+ bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
+
return ret
BB_SETSCENE_DEPVALID = "setscene_depvalid"
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 4188873..7b860c5 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -61,6 +61,16 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
# Default to keep dependencies
return True
+def sstate_lockedsigs(d):
+ sigs = {}
+ lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS", True) or "").split()
+ for ls in lockedsigs:
+ pn, task, h = ls.split(":", 2)
+ if pn not in sigs:
+ sigs[pn] = {}
+ sigs[pn][task] = h
+ return sigs
+
class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
name = "OEBasic"
def init_rundepcheck(self, data):
@@ -75,10 +85,74 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
def init_rundepcheck(self, data):
self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split()
self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split()
+ self.lockedsigs = sstate_lockedsigs(data)
+ self.lockedhashes = {}
+ self.lockedpnmap = {}
pass
def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
+ def get_taskdata(self):
+ data = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskdata()
+ return (data, self.lockedpnmap)
+
+ def set_taskdata(self, data):
+ coredata, self.lockedpnmap = data
+ super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata)
+
+ def dump_sigs(self, dataCache, options):
+ self.dump_lockedsigs()
+ return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
+
+ def get_taskhash(self, fn, task, deps, dataCache):
+ recipename = dataCache.pkg_fn[fn]
+ self.lockedpnmap[fn] = recipename
+ if recipename in self.lockedsigs:
+ if task in self.lockedsigs[recipename]:
+ k = fn + "." + task
+ h = self.lockedsigs[recipename][task]
+ self.lockedhashes[k] = h
+ self.taskhash[k] = h
+ #bb.warn("Using %s %s %s" % (recipename, task, h))
+ return h
+ h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
+ #bb.warn("%s %s %s" % (recipename, task, h))
+ return h
+
+ def dump_sigtask(self, fn, task, stampbase, runtime):
+ k = fn + "." + task
+ if k in self.lockedhashes:
+ return
+ super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
+
+ def dump_lockedsigs(self):
+ bb.plain("Writing locked sigs to " + os.getcwd() + "/locked-sigs.inc")
+ with open("locked-sigs.inc", "w") as f:
+ f.write('SIGGEN_LOCKEDSIGS = "\\\n')
+ #for fn in self.taskdeps:
+ for k in self.runtaskdeps:
+ #k = fn + "." + task
+ fn = k.rsplit(".",1)[0]
+ task = k.rsplit(".",1)[1]
+ if k not in self.taskhash:
+ continue
+ f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
+ f.write(' "\n')
+
+ def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
+ enforce = (d.getVar("SIGGEN_ENFORCE_LOCKEDSIGS", True) or "1") == "1"
+ msgs = []
+ for task in range(len(sq_fn)):
+ if task not in ret:
+ for pn in self.lockedsigs:
+ if sq_hash[task] in self.lockedsigs[pn].itervalues():
+ msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task]))
+ if msgs and enforce:
+ bb.fatal("\n".join(msgs))
+ elif msgs:
+ bb.warn("\n".join(msgs))
+
+
# Insert these classes into siggen's namespace so it can see and select them
bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic
bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash
diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache
new file mode 100755
index 0000000..dfb282e
--- /dev/null
+++ b/scripts/gen-lockedsig-cache
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+#
+# gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir>
+#
+
+import os
+import sys
+import glob
+import shutil
+import errno
+
+def mkdir(d):
+ try:
+ os.makedirs(d)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise e
+
+if len(sys.argv) < 3:
+ print("Incorrect number of arguments specified")
+ sys.exit(1)
+
+sigs = []
+with open(sys.argv[1]) as f:
+ for l in f.readlines():
+ if ":" in l:
+ sigs.append(l.split(":")[2].split()[0])
+
+files = set()
+for s in sigs:
+ p = sys.argv[2] + "/" + s[:2] + "/*" + s + "*"
+ files |= set(glob.glob(p))
+ p = sys.argv[2] + "/*/" + s[:2] + "/*" + s + "*"
+ files |= set(glob.glob(p))
+
+for f in files:
+ dst = f.replace(sys.argv[2], sys.argv[3])
+ mkdir(os.path.dirname(dst))
+ os.link(f, dst)
+
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 2/5] Richard Purdie : sstatesig: Improve to handle locking of multiple machines
2014-09-11 15:04 [PATCH 0/5] Add support for locked down sstate Hongxu Jia
2014-09-11 15:04 ` [PATCH 1/5] Richard Purdie : sstatesig/sstate: Add support for locked down sstate cache usage Hongxu Jia
@ 2014-09-11 15:04 ` Hongxu Jia
2014-09-11 15:04 ` [PATCH 3/5] Richard Purdie : siggen/runqueue/bitbake-worker: Improve siggen data transfer interface Hongxu Jia
` (2 subsequent siblings)
4 siblings, 0 replies; 10+ messages in thread
From: Hongxu Jia @ 2014-09-11 15:04 UTC (permalink / raw)
To: openembedded-core, mark.hatle, richard.purdie
From: "git@opal.openembedded.org" <git@opal.openembedded.org>
Module: openembedded-core.git
Branch: master-next
Commit: a702eb50674ba7017c63829272082d4005237f13
URL: http://git.openembedded.org/?p=openembedded-core.git&a=commit;h=a702eb50674ba7017c63829272082d4005237f13
Author: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Fri Sep 5 11:55:18 2014 +0100
sstatesig: Improve to handle locking of multiple machines
Instead of a single monolithic SIGGEN_LOCKEDSIGS, split this into
separate variables, one per sstate package architecture. Add in
a new SIGGEN_LOCKEDSIGS_TYPES variable which lists the package
architectures to load in.
SIGGEN_LOCKEDSIGS_TYPES is made machine specific using overrides.
Also sort the hashes in the lists by PN to make diffing them easier.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/lib/oe/sstatesig.py | 41 ++++++++++++++++++++++++++++-------------
1 file changed, 28 insertions(+), 13 deletions(-)
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 7b860c5..add2619 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -63,12 +63,14 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
def sstate_lockedsigs(d):
sigs = {}
- lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS", True) or "").split()
- for ls in lockedsigs:
- pn, task, h = ls.split(":", 2)
- if pn not in sigs:
- sigs[pn] = {}
- sigs[pn][task] = h
+ types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split()
+ for t in types:
+ lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
+ for ls in lockedsigs:
+ pn, task, h = ls.split(":", 2)
+ if pn not in sigs:
+ sigs[pn] = {}
+ sigs[pn][task] = h
return sigs
class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
@@ -88,16 +90,18 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
self.lockedsigs = sstate_lockedsigs(data)
self.lockedhashes = {}
self.lockedpnmap = {}
+ self.lockedhashfn = {}
+ self.machine = data.getVar("MACHINE", True)
pass
def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
def get_taskdata(self):
data = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskdata()
- return (data, self.lockedpnmap)
+ return (data, self.lockedpnmap, self.lockedhashfn)
def set_taskdata(self, data):
- coredata, self.lockedpnmap = data
+ coredata, self.lockedpnmap, self.lockedhashfn = data
super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata)
def dump_sigs(self, dataCache, options):
@@ -107,6 +111,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
def get_taskhash(self, fn, task, deps, dataCache):
recipename = dataCache.pkg_fn[fn]
self.lockedpnmap[fn] = recipename
+ self.lockedhashfn[fn] = dataCache.hashfn[fn]
if recipename in self.lockedsigs:
if task in self.lockedsigs[recipename]:
k = fn + "." + task
@@ -127,17 +132,27 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
def dump_lockedsigs(self):
bb.plain("Writing locked sigs to " + os.getcwd() + "/locked-sigs.inc")
+ types = {}
+ for k in self.runtaskdeps:
+ fn = k.rsplit(".",1)[0]
+ t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
+ if t not in types:
+ types[t] = []
+ types[t].append(k)
+
with open("locked-sigs.inc", "w") as f:
- f.write('SIGGEN_LOCKEDSIGS = "\\\n')
- #for fn in self.taskdeps:
- for k in self.runtaskdeps:
- #k = fn + "." + task
+ for t in types:
+ f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
+ types[t].sort()
+ sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
+ for k in sortedk:
fn = k.rsplit(".",1)[0]
task = k.rsplit(".",1)[1]
if k not in self.taskhash:
continue
f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
- f.write(' "\n')
+ f.write(' "\n')
+ f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
enforce = (d.getVar("SIGGEN_ENFORCE_LOCKEDSIGS", True) or "1") == "1"
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 3/5] Richard Purdie : siggen/runqueue/bitbake-worker: Improve siggen data transfer interface
2014-09-11 15:04 [PATCH 0/5] Add support for locked down sstate Hongxu Jia
2014-09-11 15:04 ` [PATCH 1/5] Richard Purdie : sstatesig/sstate: Add support for locked down sstate cache usage Hongxu Jia
2014-09-11 15:04 ` [PATCH 2/5] Richard Purdie : sstatesig: Improve to handle locking of multiple machines Hongxu Jia
@ 2014-09-11 15:04 ` Hongxu Jia
2014-09-11 15:04 ` [PATCH 4/5] sstate/sstatesig: optimize the support for locked down sstate cache usage Hongxu Jia
2014-09-11 15:04 ` [PATCH 5/5] sstatesig: incremental dump lockedsigs Hongxu Jia
4 siblings, 0 replies; 10+ messages in thread
From: Hongxu Jia @ 2014-09-11 15:04 UTC (permalink / raw)
To: openembedded-core, mark.hatle, richard.purdie
From: Richard Purdie <richard.purdie@linuxfoundation.org>
We need to transfer some of the siggen data from the core/cooker into
the worker instances. There was a partial API created for this but
its ugly and its not possible to extend it from the siggen class.
This patch completes the interface/abstraction for the data and
means the class can extend/customise it in any siggen class.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/bin/bitbake-worker | 2 +-
bitbake/lib/bb/runqueue.py | 4 +---
bitbake/lib/bb/siggen.py | 9 +++++----
3 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index c7992f7..b2935f6 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -168,7 +168,7 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat
data.setVar("BUILDNAME", workerdata["buildname"])
data.setVar("DATE", workerdata["date"])
data.setVar("TIME", workerdata["time"])
- bb.parse.siggen.set_taskdata(workerdata["hashes"], workerdata["hash_deps"], workerdata["sigchecksums"])
+ bb.parse.siggen.set_taskdata(workerdata["sigdata"])
ret = 0
try:
the_data = bb.cache.Cache.loadDataFull(fn, appends, data)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 39df794..6d9cf3f 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -879,9 +879,7 @@ class RunQueue:
"fakerootenv" : self.rqdata.dataCache.fakerootenv,
"fakerootdirs" : self.rqdata.dataCache.fakerootdirs,
"fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv,
- "hashes" : bb.parse.siggen.taskhash,
- "hash_deps" : bb.parse.siggen.runtaskdeps,
- "sigchecksums" : bb.parse.siggen.file_checksum_values,
+ "sigdata" : bb.parse.siggen.get_taskdata(),
"runq_hash" : self.rqdata.runq_hash,
"logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
"logdefaultverbose" : bb.msg.loggerDefaultVerbose,
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 548f50d..86d9ca0 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -197,10 +197,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
#d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
return h
- def set_taskdata(self, hashes, deps, checksums):
- self.runtaskdeps = deps
- self.taskhash = hashes
- self.file_checksum_values = checksums
+ def get_taskdata(self):
+ return (self.runtaskdeps, self.taskhash, self.file_checksum_values)
+
+ def set_taskdata(self, data):
+ self.runtaskdeps, self.taskhash, self.file_checksum_values = data
def dump_sigtask(self, fn, task, stampbase, runtime):
k = fn + "." + task
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 4/5] sstate/sstatesig: optimize the support for locked down sstate cache usage
2014-09-11 15:04 [PATCH 0/5] Add support for locked down sstate Hongxu Jia
` (2 preceding siblings ...)
2014-09-11 15:04 ` [PATCH 3/5] Richard Purdie : siggen/runqueue/bitbake-worker: Improve siggen data transfer interface Hongxu Jia
@ 2014-09-11 15:04 ` Hongxu Jia
2014-09-16 17:00 ` Richard Purdie
2014-09-11 15:04 ` [PATCH 5/5] sstatesig: incremental dump lockedsigs Hongxu Jia
4 siblings, 1 reply; 10+ messages in thread
From: Hongxu Jia @ 2014-09-11 15:04 UTC (permalink / raw)
To: openembedded-core, mark.hatle, richard.purdie
This fix is based on Richard Purdie's 'sstatesig: Improve to handle
locking of multiple machines' which located in master-next.
Add code in the sstate hash validation code to ensure it really did
install these from sstate since if it didn't should to warn/abort
the build. The judgment condition is:
1) If a build is replaced by locked sstate-cache, it will triger a
warn/error;
2) If objects are not used from the locked cache, it will triger a
warn/error;
3) Use SIGGEN_LOCKEDSIGS_CHECK_LEVEL variable controls whether this
is just a warning or a fatal error or nothing to report.
Use SIGGEN_DUMP_LOCKEDSIGS variable controls whether to dump
lockedsigs. Add a event handler at 'bb.event.BuildCompleted', so
while the locked sstate-cache was created, it will dump the lockedsigs
files rather than manually invoking 'bitbake -S **' again.
Use SIGGEN_LOCKEDSIGS_CONFIG variable controls where to dump the
lockedsigs file, the default is placed into ${SSTATE_DIR}/locked-sigs.inc.
[YOCTO #6639]
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/classes/sstate.bbclass | 14 ++++++++++++++
meta/lib/oe/sstatesig.py | 43 +++++++++++++++++++++++++++----------------
2 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 0cb5235..7a6a107 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -802,3 +802,17 @@ python sstate_eventhandler() {
bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
}
+SIGGEN_LOCKEDSIGS_CONFIG ?= "${SSTATE_DIR}/locked-sigs.inc"
+addhandler sstate_dump_lockedsig
+sstate_dump_lockedsig[eventmask] = "bb.event.BuildCompleted"
+python sstate_dump_lockedsig() {
+ d = e.data
+ if d.getVar('SIGGEN_DUMP_LOCKEDSIGS', True) == '1':
+ if e.getFailures():
+ return
+
+ if hasattr(bb.parse.siggen, "dump_lockedsigs"):
+ lockedsigs = d.getVar('SIGGEN_LOCKEDSIGS_CONFIG', True)
+ bb.parse.siggen.dump_lockedsigs(lockedsigs)
+}
+
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index add2619..c9edd80 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -92,6 +92,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
self.lockedpnmap = {}
self.lockedhashfn = {}
self.machine = data.getVar("MACHINE", True)
+ self.checkmsgs = []
pass
def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
@@ -109,18 +110,24 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
def get_taskhash(self, fn, task, deps, dataCache):
+ h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
+
recipename = dataCache.pkg_fn[fn]
self.lockedpnmap[fn] = recipename
self.lockedhashfn[fn] = dataCache.hashfn[fn]
if recipename in self.lockedsigs:
if task in self.lockedsigs[recipename]:
k = fn + "." + task
- h = self.lockedsigs[recipename][task]
- self.lockedhashes[k] = h
- self.taskhash[k] = h
+ h_locked = self.lockedsigs[recipename][task]
+ self.lockedhashes[k] = h_locked
+ self.taskhash[k] = h_locked
#bb.warn("Using %s %s %s" % (recipename, task, h))
- return h
- h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
+
+ if h != h_locked:
+ self.checkmsgs.append('The %s:%s sig (%s) changed, use locked sig %s to instead'
+ % (recipename, task, h, h_locked))
+
+ return h_locked
#bb.warn("%s %s %s" % (recipename, task, h))
return h
@@ -130,8 +137,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
return
super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
- def dump_lockedsigs(self):
- bb.plain("Writing locked sigs to " + os.getcwd() + "/locked-sigs.inc")
+ def dump_lockedsigs(self, where_to_dump=None):
+ if not where_to_dump:
+ where_to_dump = os.getcwd() + "/locked-sigs.inc"
+
+ bb.plain("Writing locked sigs to %s" % where_to_dump)
types = {}
for k in self.runtaskdeps:
fn = k.rsplit(".",1)[0]
@@ -140,11 +150,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
types[t] = []
types[t].append(k)
- with open("locked-sigs.inc", "w") as f:
+ with open(where_to_dump, "w") as f:
for t in types:
f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
types[t].sort()
- sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
+ sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
for k in sortedk:
fn = k.rsplit(".",1)[0]
task = k.rsplit(".",1)[1]
@@ -155,17 +165,18 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
- enforce = (d.getVar("SIGGEN_ENFORCE_LOCKEDSIGS", True) or "1") == "1"
- msgs = []
+ checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
for task in range(len(sq_fn)):
if task not in ret:
for pn in self.lockedsigs:
if sq_hash[task] in self.lockedsigs[pn].itervalues():
- msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task]))
- if msgs and enforce:
- bb.fatal("\n".join(msgs))
- elif msgs:
- bb.warn("\n".join(msgs))
+ self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
+ % (pn, sq_task[task], sq_hash[task]))
+
+ if self.checkmsgs and checklevel == 'warn':
+ bb.warn("\n".join(self.checkmsgs))
+ elif self.checkmsgs and checklevel == 'error':
+ bb.fatal("\n".join(self.checkmsgs))
# Insert these classes into siggen's namespace so it can see and select them
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 4/5] sstate/sstatesig: optimize the support for locked down sstate cache usage
2014-09-11 15:04 ` [PATCH 4/5] sstate/sstatesig: optimize the support for locked down sstate cache usage Hongxu Jia
@ 2014-09-16 17:00 ` Richard Purdie
2014-09-17 2:46 ` Hongxu Jia
0 siblings, 1 reply; 10+ messages in thread
From: Richard Purdie @ 2014-09-16 17:00 UTC (permalink / raw)
To: Hongxu Jia; +Cc: openembedded-core
On Thu, 2014-09-11 at 23:04 +0800, Hongxu Jia wrote:
> This fix is based on Richard Purdie's 'sstatesig: Improve to handle
> locking of multiple machines' which located in master-next.
>
> Add code in the sstate hash validation code to ensure it really did
> install these from sstate since if it didn't should to warn/abort
> the build. The judgment condition is:
> 1) If a build is replaced by locked sstate-cache, it will triger a
> warn/error;
> 2) If objects are not used from the locked cache, it will triger a
> warn/error;
> 3) Use SIGGEN_LOCKEDSIGS_CHECK_LEVEL variable controls whether this
> is just a warning or a fatal error or nothing to report.
>
> Use SIGGEN_DUMP_LOCKEDSIGS variable controls whether to dump
> lockedsigs. Add a event handler at 'bb.event.BuildCompleted', so
> while the locked sstate-cache was created, it will dump the lockedsigs
> files rather than manually invoking 'bitbake -S **' again.
>
> Use SIGGEN_LOCKEDSIGS_CONFIG variable controls where to dump the
> lockedsigs file, the default is placed into ${SSTATE_DIR}/locked-sigs.inc.
>
> [YOCTO #6639]
>
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
> meta/classes/sstate.bbclass | 14 ++++++++++++++
> meta/lib/oe/sstatesig.py | 43 +++++++++++++++++++++++++++----------------
> 2 files changed, 41 insertions(+), 16 deletions(-)
>
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 0cb5235..7a6a107 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -802,3 +802,17 @@ python sstate_eventhandler() {
> bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
> }
>
> +SIGGEN_LOCKEDSIGS_CONFIG ?= "${SSTATE_DIR}/locked-sigs.inc"
> +addhandler sstate_dump_lockedsig
> +sstate_dump_lockedsig[eventmask] = "bb.event.BuildCompleted"
> +python sstate_dump_lockedsig() {
> + d = e.data
> + if d.getVar('SIGGEN_DUMP_LOCKEDSIGS', True) == '1':
> + if e.getFailures():
> + return
> +
> + if hasattr(bb.parse.siggen, "dump_lockedsigs"):
> + lockedsigs = d.getVar('SIGGEN_LOCKEDSIGS_CONFIG', True)
> + bb.parse.siggen.dump_lockedsigs(lockedsigs)
> +}
Firstly, I think the change to sstate.bbclass should be as a separate
patch and I'm not sure I like that patch as it stands. Should this
perhaps be as a separate class?
> diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
> index add2619..c9edd80 100644
> --- a/meta/lib/oe/sstatesig.py
> +++ b/meta/lib/oe/sstatesig.py
> @@ -92,6 +92,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> self.lockedpnmap = {}
> self.lockedhashfn = {}
> self.machine = data.getVar("MACHINE", True)
> + self.checkmsgs = []
I think this should be called "mismatch_msgs" instead.
> pass
> def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
> return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
> @@ -109,18 +110,24 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
>
> def get_taskhash(self, fn, task, deps, dataCache):
> + h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
> +
> recipename = dataCache.pkg_fn[fn]
> self.lockedpnmap[fn] = recipename
> self.lockedhashfn[fn] = dataCache.hashfn[fn]
> if recipename in self.lockedsigs:
> if task in self.lockedsigs[recipename]:
> k = fn + "." + task
> - h = self.lockedsigs[recipename][task]
> - self.lockedhashes[k] = h
> - self.taskhash[k] = h
> + h_locked = self.lockedsigs[recipename][task]
> + self.lockedhashes[k] = h_locked
> + self.taskhash[k] = h_locked
> #bb.warn("Using %s %s %s" % (recipename, task, h))
> - return h
> - h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
> +
> + if h != h_locked:
> + self.checkmsgs.append('The %s:%s sig (%s) changed, use locked sig %s to instead'
> + % (recipename, task, h, h_locked))
> +
> + return h_locked
> #bb.warn("%s %s %s" % (recipename, task, h))
> return h
>
> @@ -130,8 +137,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> return
> super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
>
> - def dump_lockedsigs(self):
> - bb.plain("Writing locked sigs to " + os.getcwd() + "/locked-sigs.inc")
> + def dump_lockedsigs(self, where_to_dump=None):
where_to_dump -> sigfile
> + if not where_to_dump:
> + where_to_dump = os.getcwd() + "/locked-sigs.inc"
> +
> + bb.plain("Writing locked sigs to %s" % where_to_dump)
> types = {}
> for k in self.runtaskdeps:
> fn = k.rsplit(".",1)[0]
> @@ -140,11 +150,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> types[t] = []
> types[t].append(k)
>
> - with open("locked-sigs.inc", "w") as f:
> + with open(where_to_dump, "w") as f:
> for t in types:
> f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
> types[t].sort()
> - sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
> + sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
> for k in sortedk:
> fn = k.rsplit(".",1)[0]
> task = k.rsplit(".",1)[1]
> @@ -155,17 +165,18 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
>
> def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
> - enforce = (d.getVar("SIGGEN_ENFORCE_LOCKEDSIGS", True) or "1") == "1"
> - msgs = []
> + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
Perhaps this should default to error?
> for task in range(len(sq_fn)):
> if task not in ret:
> for pn in self.lockedsigs:
> if sq_hash[task] in self.lockedsigs[pn].itervalues():
> - msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task]))
> - if msgs and enforce:
> - bb.fatal("\n".join(msgs))
> - elif msgs:
> - bb.warn("\n".join(msgs))
> + self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
> + % (pn, sq_task[task], sq_hash[task]))
> +
> + if self.checkmsgs and checklevel == 'warn':
> + bb.warn("\n".join(self.checkmsgs))
> + elif self.checkmsgs and checklevel == 'error':
> + bb.fatal("\n".join(self.checkmsgs))
>
>
> # Insert these classes into siggen's namespace so it can see and select them
I have an updated version of this patch at
http://git.yoctoproject.org/cgit.cgi/poky-contrib/commit/?h=rpurdie/t222&id=17c255cd72810616139e36ae8ad30cc4daefe9d1
if you're ok with the changes although I do need to update the commit
message.
Cheers,
Richard
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 4/5] sstate/sstatesig: optimize the support for locked down sstate cache usage
2014-09-16 17:00 ` Richard Purdie
@ 2014-09-17 2:46 ` Hongxu Jia
0 siblings, 0 replies; 10+ messages in thread
From: Hongxu Jia @ 2014-09-17 2:46 UTC (permalink / raw)
To: Richard Purdie; +Cc: openembedded-core
On 09/17/2014 01:00 AM, Richard Purdie wrote:
> On Thu, 2014-09-11 at 23:04 +0800, Hongxu Jia wrote:
>> This fix is based on Richard Purdie's 'sstatesig: Improve to handle
>> locking of multiple machines' which located in master-next.
>>
>> Add code in the sstate hash validation code to ensure it really did
>> install these from sstate since if it didn't should to warn/abort
>> the build. The judgment condition is:
>> 1) If a build is replaced by locked sstate-cache, it will triger a
>> warn/error;
>> 2) If objects are not used from the locked cache, it will triger a
>> warn/error;
>> 3) Use SIGGEN_LOCKEDSIGS_CHECK_LEVEL variable controls whether this
>> is just a warning or a fatal error or nothing to report.
>>
>> Use SIGGEN_DUMP_LOCKEDSIGS variable controls whether to dump
>> lockedsigs. Add a event handler at 'bb.event.BuildCompleted', so
>> while the locked sstate-cache was created, it will dump the lockedsigs
>> files rather than manually invoking 'bitbake -S **' again.
>>
>> Use SIGGEN_LOCKEDSIGS_CONFIG variable controls where to dump the
>> lockedsigs file, the default is placed into ${SSTATE_DIR}/locked-sigs.inc.
>>
>> [YOCTO #6639]
>>
>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
>> ---
>> meta/classes/sstate.bbclass | 14 ++++++++++++++
>> meta/lib/oe/sstatesig.py | 43 +++++++++++++++++++++++++++----------------
>> 2 files changed, 41 insertions(+), 16 deletions(-)
>>
>> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
>> index 0cb5235..7a6a107 100644
>> --- a/meta/classes/sstate.bbclass
>> +++ b/meta/classes/sstate.bbclass
>> @@ -802,3 +802,17 @@ python sstate_eventhandler() {
>> bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
>> }
>>
>> +SIGGEN_LOCKEDSIGS_CONFIG ?= "${SSTATE_DIR}/locked-sigs.inc"
>> +addhandler sstate_dump_lockedsig
>> +sstate_dump_lockedsig[eventmask] = "bb.event.BuildCompleted"
>> +python sstate_dump_lockedsig() {
>> + d = e.data
>> + if d.getVar('SIGGEN_DUMP_LOCKEDSIGS', True) == '1':
>> + if e.getFailures():
>> + return
>> +
>> + if hasattr(bb.parse.siggen, "dump_lockedsigs"):
>> + lockedsigs = d.getVar('SIGGEN_LOCKEDSIGS_CONFIG', True)
>> + bb.parse.siggen.dump_lockedsigs(lockedsigs)
>> +}
>
> Firstly, I think the change to sstate.bbclass should be as a separate
> patch and I'm not sure I like that patch as it stands. Should this
> perhaps be as a separate class?
It provides a method to dump lockedsigs at recipe building time,
so no need to invoke 'bitbake -S **' again.
And I will name a new class 'sstate_lockedsig' and move the
event handler to it.
>> diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
>> index add2619..c9edd80 100644
>> --- a/meta/lib/oe/sstatesig.py
>> +++ b/meta/lib/oe/sstatesig.py
>> @@ -92,6 +92,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> self.lockedpnmap = {}
>> self.lockedhashfn = {}
>> self.machine = data.getVar("MACHINE", True)
>> + self.checkmsgs = []
> I think this should be called "mismatch_msgs" instead.
Agree
>> pass
>> def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
>> return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
>> @@ -109,18 +110,24 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
>>
>> def get_taskhash(self, fn, task, deps, dataCache):
>> + h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
>> +
>> recipename = dataCache.pkg_fn[fn]
>> self.lockedpnmap[fn] = recipename
>> self.lockedhashfn[fn] = dataCache.hashfn[fn]
>> if recipename in self.lockedsigs:
>> if task in self.lockedsigs[recipename]:
>> k = fn + "." + task
>> - h = self.lockedsigs[recipename][task]
>> - self.lockedhashes[k] = h
>> - self.taskhash[k] = h
>> + h_locked = self.lockedsigs[recipename][task]
>> + self.lockedhashes[k] = h_locked
>> + self.taskhash[k] = h_locked
>> #bb.warn("Using %s %s %s" % (recipename, task, h))
>> - return h
>> - h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
>> +
>> + if h != h_locked:
>> + self.checkmsgs.append('The %s:%s sig (%s) changed, use locked sig %s to instead'
>> + % (recipename, task, h, h_locked))
>> +
>> + return h_locked
>> #bb.warn("%s %s %s" % (recipename, task, h))
>> return h
>>
>> @@ -130,8 +137,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> return
>> super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
>>
>> - def dump_lockedsigs(self):
>> - bb.plain("Writing locked sigs to " + os.getcwd() + "/locked-sigs.inc")
>> + def dump_lockedsigs(self, where_to_dump=None):
> where_to_dump -> sigfile
Agree
>> + if not where_to_dump:
>> + where_to_dump = os.getcwd() + "/locked-sigs.inc"
>> +
>> + bb.plain("Writing locked sigs to %s" % where_to_dump)
>> types = {}
>> for k in self.runtaskdeps:
>> fn = k.rsplit(".",1)[0]
>> @@ -140,11 +150,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> types[t] = []
>> types[t].append(k)
>>
>> - with open("locked-sigs.inc", "w") as f:
>> + with open(where_to_dump, "w") as f:
>> for t in types:
>> f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
>> types[t].sort()
>> - sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
>> + sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
>> for k in sortedk:
>> fn = k.rsplit(".",1)[0]
>> task = k.rsplit(".",1)[1]
>> @@ -155,17 +165,18 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
>>
>> def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
>> - enforce = (d.getVar("SIGGEN_ENFORCE_LOCKEDSIGS", True) or "1") == "1"
>> - msgs = []
>> + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
> Perhaps this should default to error?
Agree, I will initial it with 'error' in class 'sstate'
>> for task in range(len(sq_fn)):
>> if task not in ret:
>> for pn in self.lockedsigs:
>> if sq_hash[task] in self.lockedsigs[pn].itervalues():
>> - msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task]))
>> - if msgs and enforce:
>> - bb.fatal("\n".join(msgs))
>> - elif msgs:
>> - bb.warn("\n".join(msgs))
>> + self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
>> + % (pn, sq_task[task], sq_hash[task]))
>> +
>> + if self.checkmsgs and checklevel == 'warn':
>> + bb.warn("\n".join(self.checkmsgs))
>> + elif self.checkmsgs and checklevel == 'error':
>> + bb.fatal("\n".join(self.checkmsgs))
>>
>>
>> # Insert these classes into siggen's namespace so it can see and select them
>
> I have an updated version of this patch at
> http://git.yoctoproject.org/cgit.cgi/poky-contrib/commit/?h=rpurdie/t222&id=17c255cd72810616139e36ae8ad30cc4daefe9d1
> if you're ok with the changes although I do need to update the commit
> message.
Got it, I will rebase it according the latest changes
//Hongxu
>
> Cheers,
>
> Richard
>
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 5/5] sstatesig: incremental dump lockedsigs
2014-09-11 15:04 [PATCH 0/5] Add support for locked down sstate Hongxu Jia
` (3 preceding siblings ...)
2014-09-11 15:04 ` [PATCH 4/5] sstate/sstatesig: optimize the support for locked down sstate cache usage Hongxu Jia
@ 2014-09-11 15:04 ` Hongxu Jia
2014-09-16 16:57 ` Richard Purdie
4 siblings, 1 reply; 10+ messages in thread
From: Hongxu Jia @ 2014-09-11 15:04 UTC (permalink / raw)
To: openembedded-core, mark.hatle, richard.purdie
While including an existed lockedsigs file to build from
locked sstate-cache, and we want to incremental build the
locked sstate-cache. Config with:
...
SIGGEN_DUMP_LOCKEDSIGS = '1'
SIGGEN_LOCKEDSIGS_CONFIG = "${TOPDIR}/locked-sigs.inc"
require ${SIGGEN_LOCKEDSIGS_CONFIG}
...
So we support to dump lockedsigs file incrementally
Since we improve to handle locking of multiple machines and
add a new SIGGEN_LOCKEDSIGS_TYPES variable which lists the
package architectures to load in. We should add package
architectures as type to self.lockedsigs to reflect that.
Such as:
{recipename:{task:{hash1}}} --> {type:{recipename:{task:{hash1}}}}
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/lib/oe/sstatesig.py | 67 ++++++++++++++++++++++++++++++++----------------
1 file changed, 45 insertions(+), 22 deletions(-)
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index c9edd80..8b491a6 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -65,12 +65,15 @@ def sstate_lockedsigs(d):
sigs = {}
types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split()
for t in types:
+ if t not in sigs:
+ sigs[t] = {}
+
lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
for ls in lockedsigs:
pn, task, h = ls.split(":", 2)
- if pn not in sigs:
- sigs[pn] = {}
- sigs[pn][task] = h
+ if pn not in sigs[t]:
+ sigs[t][pn] = {}
+ sigs[t][pn][task] = h
return sigs
class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
@@ -115,10 +118,14 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
recipename = dataCache.pkg_fn[fn]
self.lockedpnmap[fn] = recipename
self.lockedhashfn[fn] = dataCache.hashfn[fn]
- if recipename in self.lockedsigs:
- if task in self.lockedsigs[recipename]:
+
+ t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
+ if t not in self.lockedsigs:
+ return h
+ if recipename in self.lockedsigs[t]:
+ if task in self.lockedsigs[t][recipename]:
k = fn + "." + task
- h_locked = self.lockedsigs[recipename][task]
+ h_locked = self.lockedsigs[t][recipename][task]
self.lockedhashes[k] = h_locked
self.taskhash[k] = h_locked
#bb.warn("Using %s %s %s" % (recipename, task, h))
@@ -137,6 +144,19 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
return
super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
+ def update_lockedsigs(self, type, recipename, task, hash):
+ if type not in self.lockedsigs:
+ self.lockedsigs[type] = {}
+
+ if recipename not in self.lockedsigs[type]:
+ self.lockedsigs[type][recipename] = {}
+
+ if task not in self.lockedsigs[type][recipename]:
+ self.lockedsigs[type][recipename][task] = hash
+ elif hash != self.lockedsigs[type][recipename][task]:
+ bb.warn('Conflict %s %s %s (new %s, old %s)' %
+ (type, recipename, task, hash, self.lockedsigs[type][recipename][task]))
+
def dump_lockedsigs(self, where_to_dump=None):
if not where_to_dump:
where_to_dump = os.getcwd() + "/locked-sigs.inc"
@@ -146,30 +166,33 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
for k in self.runtaskdeps:
fn = k.rsplit(".",1)[0]
t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
- if t not in types:
- types[t] = []
- types[t].append(k)
+ recipename = self.lockedpnmap[fn]
+ task = k.rsplit(".",1)[1]
+ hash = self.taskhash[k]
+ self.update_lockedsigs(t, recipename, task, hash)
with open(where_to_dump, "w") as f:
- for t in types:
- f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
- types[t].sort()
- sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
- for k in sortedk:
- fn = k.rsplit(".",1)[0]
- task = k.rsplit(".",1)[1]
- if k not in self.taskhash:
- continue
- f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
+ for type in sorted(self.lockedsigs):
+ f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % type)
+
+ for recipename in sorted(self.lockedsigs[type]):
+ for task in sorted(self.lockedsigs[type][recipename]):
+ hash = self.lockedsigs[type][recipename][task]
+ f.write(" " + recipename + ":" + task + ":" + hash + " \\\n")
f.write(' "\n')
- f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
+
+ f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(self.lockedsigs.keys())))
def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
for task in range(len(sq_fn)):
if task not in ret:
- for pn in self.lockedsigs:
- if sq_hash[task] in self.lockedsigs[pn].itervalues():
+ t = sq_hashfn[task].split(" ")[1].split(":")[5]
+ if t not in self.lockedsigs:
+ continue
+
+ for pn in self.lockedsigs[t]:
+ if sq_hash[task] in self.lockedsigs[t][pn].itervalues():
self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
% (pn, sq_task[task], sq_hash[task]))
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 5/5] sstatesig: incremental dump lockedsigs
2014-09-11 15:04 ` [PATCH 5/5] sstatesig: incremental dump lockedsigs Hongxu Jia
@ 2014-09-16 16:57 ` Richard Purdie
2014-09-17 6:26 ` Hongxu Jia
0 siblings, 1 reply; 10+ messages in thread
From: Richard Purdie @ 2014-09-16 16:57 UTC (permalink / raw)
To: Hongxu Jia; +Cc: openembedded-core
On Thu, 2014-09-11 at 23:04 +0800, Hongxu Jia wrote:
> While including an existed lockedsigs file to build from
> locked sstate-cache, and we want to incremental build the
> locked sstate-cache. Config with:
> ...
> SIGGEN_DUMP_LOCKEDSIGS = '1'
> SIGGEN_LOCKEDSIGS_CONFIG = "${TOPDIR}/locked-sigs.inc"
> require ${SIGGEN_LOCKEDSIGS_CONFIG}
> ...
> So we support to dump lockedsigs file incrementally
>
> Since we improve to handle locking of multiple machines and
> add a new SIGGEN_LOCKEDSIGS_TYPES variable which lists the
> package architectures to load in. We should add package
> architectures as type to self.lockedsigs to reflect that.
> Such as:
> {recipename:{task:{hash1}}} --> {type:{recipename:{task:{hash1}}}}
>
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
> meta/lib/oe/sstatesig.py | 67 ++++++++++++++++++++++++++++++++----------------
> 1 file changed, 45 insertions(+), 22 deletions(-)
I've stared long and hard at this patch and I simply don't understand
why we need to complicate the structure of the lockedsigs variable.
I understand wanting to generate an incremental sig file, that much is
fine, I just don't like the other seemingly unrelated change.
Regardless, they should be split into separate patches.
Can you explain more about what you're trying to do?
Cheers,
Richard
> diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
> index c9edd80..8b491a6 100644
> --- a/meta/lib/oe/sstatesig.py
> +++ b/meta/lib/oe/sstatesig.py
> @@ -65,12 +65,15 @@ def sstate_lockedsigs(d):
> sigs = {}
> types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split()
> for t in types:
> + if t not in sigs:
> + sigs[t] = {}
> +
> lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
> for ls in lockedsigs:
> pn, task, h = ls.split(":", 2)
> - if pn not in sigs:
> - sigs[pn] = {}
> - sigs[pn][task] = h
> + if pn not in sigs[t]:
> + sigs[t][pn] = {}
> + sigs[t][pn][task] = h
> return sigs
>
> class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
> @@ -115,10 +118,14 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> recipename = dataCache.pkg_fn[fn]
> self.lockedpnmap[fn] = recipename
> self.lockedhashfn[fn] = dataCache.hashfn[fn]
> - if recipename in self.lockedsigs:
> - if task in self.lockedsigs[recipename]:
> +
> + t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
> + if t not in self.lockedsigs:
> + return h
> + if recipename in self.lockedsigs[t]:
> + if task in self.lockedsigs[t][recipename]:
> k = fn + "." + task
> - h_locked = self.lockedsigs[recipename][task]
> + h_locked = self.lockedsigs[t][recipename][task]
> self.lockedhashes[k] = h_locked
> self.taskhash[k] = h_locked
> #bb.warn("Using %s %s %s" % (recipename, task, h))
> @@ -137,6 +144,19 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> return
> super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
>
> + def update_lockedsigs(self, type, recipename, task, hash):
> + if type not in self.lockedsigs:
> + self.lockedsigs[type] = {}
> +
> + if recipename not in self.lockedsigs[type]:
> + self.lockedsigs[type][recipename] = {}
> +
> + if task not in self.lockedsigs[type][recipename]:
> + self.lockedsigs[type][recipename][task] = hash
> + elif hash != self.lockedsigs[type][recipename][task]:
> + bb.warn('Conflict %s %s %s (new %s, old %s)' %
> + (type, recipename, task, hash, self.lockedsigs[type][recipename][task]))
> +
> def dump_lockedsigs(self, where_to_dump=None):
> if not where_to_dump:
> where_to_dump = os.getcwd() + "/locked-sigs.inc"
> @@ -146,30 +166,33 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
> for k in self.runtaskdeps:
> fn = k.rsplit(".",1)[0]
> t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
> - if t not in types:
> - types[t] = []
> - types[t].append(k)
> + recipename = self.lockedpnmap[fn]
> + task = k.rsplit(".",1)[1]
> + hash = self.taskhash[k]
> + self.update_lockedsigs(t, recipename, task, hash)
>
> with open(where_to_dump, "w") as f:
> - for t in types:
> - f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
> - types[t].sort()
> - sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
> - for k in sortedk:
> - fn = k.rsplit(".",1)[0]
> - task = k.rsplit(".",1)[1]
> - if k not in self.taskhash:
> - continue
> - f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
> + for type in sorted(self.lockedsigs):
> + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % type)
> +
> + for recipename in sorted(self.lockedsigs[type]):
> + for task in sorted(self.lockedsigs[type][recipename]):
> + hash = self.lockedsigs[type][recipename][task]
> + f.write(" " + recipename + ":" + task + ":" + hash + " \\\n")
> f.write(' "\n')
> - f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
> +
> + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(self.lockedsigs.keys())))
>
> def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
> checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
> for task in range(len(sq_fn)):
> if task not in ret:
> - for pn in self.lockedsigs:
> - if sq_hash[task] in self.lockedsigs[pn].itervalues():
> + t = sq_hashfn[task].split(" ")[1].split(":")[5]
> + if t not in self.lockedsigs:
> + continue
> +
> + for pn in self.lockedsigs[t]:
> + if sq_hash[task] in self.lockedsigs[t][pn].itervalues():
> self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
> % (pn, sq_task[task], sq_hash[task]))
>
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 5/5] sstatesig: incremental dump lockedsigs
2014-09-16 16:57 ` Richard Purdie
@ 2014-09-17 6:26 ` Hongxu Jia
0 siblings, 0 replies; 10+ messages in thread
From: Hongxu Jia @ 2014-09-17 6:26 UTC (permalink / raw)
To: Richard Purdie; +Cc: openembedded-core
On 09/17/2014 12:57 AM, Richard Purdie wrote:
> On Thu, 2014-09-11 at 23:04 +0800, Hongxu Jia wrote:
>> While including an existed lockedsigs file to build from
>> locked sstate-cache, and we want to incremental build the
>> locked sstate-cache. Config with:
>> ...
>> SIGGEN_DUMP_LOCKEDSIGS = '1'
>> SIGGEN_LOCKEDSIGS_CONFIG = "${TOPDIR}/locked-sigs.inc"
>> require ${SIGGEN_LOCKEDSIGS_CONFIG}
>> ...
>> So we support to dump lockedsigs file incrementally
>>
>> Since we improve to handle locking of multiple machines and
>> add a new SIGGEN_LOCKEDSIGS_TYPES variable which lists the
>> package architectures to load in. We should add package
>> architectures as type to self.lockedsigs to reflect that.
>> Such as:
>> {recipename:{task:{hash1}}} --> {type:{recipename:{task:{hash1}}}}
>>
>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
>> ---
>> meta/lib/oe/sstatesig.py | 67 ++++++++++++++++++++++++++++++++----------------
>> 1 file changed, 45 insertions(+), 22 deletions(-)
> I've stared long and hard at this patch and I simply don't understand
> why we need to complicate the structure of the lockedsigs variable.
Ok, I will try to simplify and not complicate the structure of the
lockedsigs variable.
> I understand wanting to generate an incremental sig file, that much is
> fine, I just don't like the other seemingly unrelated change.
> Regardless, they should be split into separate patches.
>
> Can you explain more about what you're trying to do?
I want to save self.lockedsigs to the new sig file.
The idea of incremental sig is:
New sig file = Old sig file (if available) + New sig items in current build.
The self.lockedsigs contains the old locked-sigs.inc but missing 'type'
(machine),
that's why I complicate it.
I will limit the modification within the dump_lockedsigs in V2,
and I have to add two variables 'self.lockedsigs_types' and
'self.lockedsigs_raw' keep old sig file.
...
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -93,6 +93,10 @@ class
SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
self.lockedhashfn = {}
self.machine = data.getVar("MACHINE", True)
self.mismatch_msgs = []
+ self.lockedsigs_types = (data.getVar("SIGGEN_LOCKEDSIGS_TYPES",
True) or "").split()
+ self.lockedsigs_raw = {}
+ for t in self.lockedsigs_types:
+ self.lockedsigs_raw[t] =
(data.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
...
V2 incoming
//Hongxu
> Cheers,
>
> Richard
>
>
>> diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
>> index c9edd80..8b491a6 100644
>> --- a/meta/lib/oe/sstatesig.py
>> +++ b/meta/lib/oe/sstatesig.py
>> @@ -65,12 +65,15 @@ def sstate_lockedsigs(d):
>> sigs = {}
>> types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split()
>> for t in types:
>> + if t not in sigs:
>> + sigs[t] = {}
>> +
>> lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
>> for ls in lockedsigs:
>> pn, task, h = ls.split(":", 2)
>> - if pn not in sigs:
>> - sigs[pn] = {}
>> - sigs[pn][task] = h
>> + if pn not in sigs[t]:
>> + sigs[t][pn] = {}
>> + sigs[t][pn][task] = h
>> return sigs
>>
>> class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
>> @@ -115,10 +118,14 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> recipename = dataCache.pkg_fn[fn]
>> self.lockedpnmap[fn] = recipename
>> self.lockedhashfn[fn] = dataCache.hashfn[fn]
>> - if recipename in self.lockedsigs:
>> - if task in self.lockedsigs[recipename]:
>> +
>> + t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
>> + if t not in self.lockedsigs:
>> + return h
>> + if recipename in self.lockedsigs[t]:
>> + if task in self.lockedsigs[t][recipename]:
>> k = fn + "." + task
>> - h_locked = self.lockedsigs[recipename][task]
>> + h_locked = self.lockedsigs[t][recipename][task]
>> self.lockedhashes[k] = h_locked
>> self.taskhash[k] = h_locked
>> #bb.warn("Using %s %s %s" % (recipename, task, h))
>> @@ -137,6 +144,19 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> return
>> super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
>>
>> + def update_lockedsigs(self, type, recipename, task, hash):
>> + if type not in self.lockedsigs:
>> + self.lockedsigs[type] = {}
>> +
>> + if recipename not in self.lockedsigs[type]:
>> + self.lockedsigs[type][recipename] = {}
>> +
>> + if task not in self.lockedsigs[type][recipename]:
>> + self.lockedsigs[type][recipename][task] = hash
>> + elif hash != self.lockedsigs[type][recipename][task]:
>> + bb.warn('Conflict %s %s %s (new %s, old %s)' %
>> + (type, recipename, task, hash, self.lockedsigs[type][recipename][task]))
>> +
>> def dump_lockedsigs(self, where_to_dump=None):
>> if not where_to_dump:
>> where_to_dump = os.getcwd() + "/locked-sigs.inc"
>> @@ -146,30 +166,33 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
>> for k in self.runtaskdeps:
>> fn = k.rsplit(".",1)[0]
>> t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
>> - if t not in types:
>> - types[t] = []
>> - types[t].append(k)
>> + recipename = self.lockedpnmap[fn]
>> + task = k.rsplit(".",1)[1]
>> + hash = self.taskhash[k]
>> + self.update_lockedsigs(t, recipename, task, hash)
>>
>> with open(where_to_dump, "w") as f:
>> - for t in types:
>> - f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
>> - types[t].sort()
>> - sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]])
>> - for k in sortedk:
>> - fn = k.rsplit(".",1)[0]
>> - task = k.rsplit(".",1)[1]
>> - if k not in self.taskhash:
>> - continue
>> - f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
>> + for type in sorted(self.lockedsigs):
>> + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % type)
>> +
>> + for recipename in sorted(self.lockedsigs[type]):
>> + for task in sorted(self.lockedsigs[type][recipename]):
>> + hash = self.lockedsigs[type][recipename][task]
>> + f.write(" " + recipename + ":" + task + ":" + hash + " \\\n")
>> f.write(' "\n')
>> - f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
>> +
>> + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(self.lockedsigs.keys())))
>>
>> def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
>> checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
>> for task in range(len(sq_fn)):
>> if task not in ret:
>> - for pn in self.lockedsigs:
>> - if sq_hash[task] in self.lockedsigs[pn].itervalues():
>> + t = sq_hashfn[task].split(" ")[1].split(":")[5]
>> + if t not in self.lockedsigs:
>> + continue
>> +
>> + for pn in self.lockedsigs[t]:
>> + if sq_hash[task] in self.lockedsigs[t][pn].itervalues():
>> self.checkmsgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
>> % (pn, sq_task[task], sq_hash[task]))
>>
>
^ permalink raw reply [flat|nested] 10+ messages in thread