[PATCH 0/5] extensible sdk changes

[PATCH 0/5] extensible sdk changes

[Randy Witt]

This set of changes is to make sure the locked signatures are enforced as
part of the extensible sdk usage.

Therefore it also allows for the unlocking of recipes that are part of the
devtool workspace.

The following changes since commit 14e2b90893f57570c2002317b5afb0e61d15e9ca:

  toasterconf.json: Add DL_DIR and SSTATE_DIR to poky toasterconf (2016-04-07 14:58:10 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib sigs
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=sigs

Randy Witt (5):
  sstatesig.py: Split single locked sigs check into multiple checks
  sstatesig.py: Improve the SIGGEN_LOCKEDSIGS_TASKSIG_CHECK message
  populate_sdk_ext.bbclass: Enable locked sigs errors
  sstatesig.py: Add a method to "unlock" recipes
  devtool: Create unlocked-sigs.inc containing items in the workspace

 meta/classes/populate_sdk_ext.bbclass | 16 +++++++--
 meta/classes/sstate.bbclass           |  8 ++++-
 meta/lib/oe/sstatesig.py              | 63 ++++++++++++++++++++++++++++-------
 meta/lib/oeqa/selftest/signing.py     |  4 +--
 scripts/devtool                       | 20 +++++++++++
 5 files changed, 94 insertions(+), 17 deletions(-)

-- 
2.5.5

[PATCH 1/5] sstatesig.py: Split single locked sigs check into multiple checks

[Randy Witt]

Add the SIGGEN_LOCKEDSIGS_TASKSIG_CHECK and
SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK variables to replace
SIGGEN_LOCKEDSIGS_CHECK_LEVEL.

SIGGEN_LOCKEDSIGS_TASKSIG_CHECK will no control whether there is a
warning or error if a task's hash in the locked signature file doesn't match
the computed hash from the current metadata.

SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK will control whther there is a
warning or error if a task that supports sstate is in the locked
signature file, but no sstate exists for the task.

Previously you could only have warning/errors for both controlled by
SIGGEN_LOCKEDSIGS_CHECK_LEVEL. This was an issue in the extensible sdk,
because we know sstate won't exist for certain items in the reverse
dependencies list for tasks. However, we still want to error if task
signatures don't match.

[YOCTO #9195]

Signed-off-by: Randy Witt <randy.e.witt@linux.intel.com>
---
 meta/classes/sstate.bbclass       |  8 +++++++-
 meta/lib/oe/sstatesig.py          | 27 +++++++++++++++++++++------
 meta/lib/oeqa/selftest/signing.py |  4 ++--
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 3234e79..8c62327 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -53,7 +53,13 @@ SSTATEPOSTINSTFUNCS = ""
 EXTRA_STAGING_FIXMES ?= ""
 SSTATECLEANFUNCS = ""
 
-SIGGEN_LOCKEDSIGS_CHECK_LEVEL ?= 'error'
+# Check whether sstate exists for tasks that support sstate and are in the
+# locked signatures file.
+SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
+
+# Check whether the task's computed hash matches the task's hash in the
+# locked signatures file.
+SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
 
 # The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
 # not sign)
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 5828a9d..b2319ff 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -189,20 +189,35 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
             f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys())))
 
     def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
-        checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True)
+        warn_msgs = []
+        error_msgs = []
+        sstate_missing_msgs = []
+
         for task in range(len(sq_fn)):
             if task not in ret:
                 for pn in self.lockedsigs:
                     if sq_hash[task] in self.lockedsigs[pn].itervalues():
                         if sq_task[task] == 'do_shared_workdir':
                             continue
-                        self.mismatch_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
+                        sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
                                                % (pn, sq_task[task], sq_hash[task]))
 
-        if self.mismatch_msgs and checklevel == 'warn':
-            bb.warn("\n".join(self.mismatch_msgs))
-        elif self.mismatch_msgs and checklevel == 'error':
-            bb.fatal("\n".join(self.mismatch_msgs))
+        checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK", True)
+        if checklevel == 'warn':
+            warn_msgs += self.mismatch_msgs
+        elif checklevel == 'error':
+            error_msgs += self.mismatch_msgs
+
+        checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK", True)
+        if checklevel == 'warn':
+            warn_msgs += sstate_missing_msgs
+        elif checklevel == 'error':
+            error_msgs += sstate_missing_msgs
+
+        if warn_msgs:
+            bb.warn("\n".join(warn_msgs))
+        if error_msgs:
+            bb.fatal("\n".join(error_msgs))
 
 
 # Insert these classes into siggen's namespace so it can see and select them
diff --git a/meta/lib/oeqa/selftest/signing.py b/meta/lib/oeqa/selftest/signing.py
index d2b3f00..1babca0 100644
--- a/meta/lib/oeqa/selftest/signing.py
+++ b/meta/lib/oeqa/selftest/signing.py
@@ -160,7 +160,7 @@ class LockedSignatures(oeSelfTest):
         bitbake('-S none %s' % test_recipe)
 
         feature = 'require %s\n' % locked_sigs_file
-        feature += 'SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "warn"\n'
+        feature += 'SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n'
         self.write_config(feature)
 
         # Build a locked recipe
@@ -180,7 +180,7 @@ class LockedSignatures(oeSelfTest):
         ret = bitbake(test_recipe)
 
         # Verify you get the warning and that the real task *isn't* run (i.e. the locked signature has worked)
-        patt = r'WARNING: The %s:do_package sig \S+ changed, use locked sig \S+ to instead' % test_recipe
+        patt = r'WARNING: The %s:do_package sig is computed to be \S+, but the sig is locked to \S+ in SIGGEN_LOCKEDSIGS\S+' % test_recipe
         found_warn = re.search(patt, ret.output)
 
         self.assertIsNotNone(found_warn, "Didn't find the expected warning message. Output: %s" % ret.output)
-- 
2.5.5

[PATCH 2/5] sstatesig.py: Improve the SIGGEN_LOCKEDSIGS_TASKSIG_CHECK message

[Randy Witt]

The previous message when signatures didn't match between the metadata
and the locked signatures file, the message output was a bit confusing.

Now the message should be of the form:

The zlib-native:do_install sig is computed to be
53531910a2a7848432da89def942a91a, but the sig is locked to
d25ba9035f7ccb308e51bbe1066e8d27 in SIGGEN_LOCKEDSIGS_t-x86-64

which will hopefully be more useful in understanding the problem.

[YOCTO #9195]

Signed-off-by: Randy Witt <randy.e.witt@linux.intel.com>
---
 meta/lib/oe/sstatesig.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index b2319ff..009adea 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -65,12 +65,13 @@ def sstate_lockedsigs(d):
     sigs = {}
     types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split()
     for t in types:
-        lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split()
+        siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t
+        lockedsigs = (d.getVar(siggen_lockedsigs_var, True) or "").split()
         for ls in lockedsigs:
             pn, task, h = ls.split(":", 2)
             if pn not in sigs:
                 sigs[pn] = {}
-            sigs[pn][task] = h
+            sigs[pn][task] = [h, siggen_lockedsigs_var]
     return sigs
 
 class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
@@ -138,14 +139,15 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
         if recipename in self.lockedsigs:
             if task in self.lockedsigs[recipename]:
                 k = fn + "." + task
-                h_locked = self.lockedsigs[recipename][task]
+                h_locked = self.lockedsigs[recipename][task][0]
+                var = self.lockedsigs[recipename][task][1]
                 self.lockedhashes[k] = h_locked
                 self.taskhash[k] = h_locked
                 #bb.warn("Using %s %s %s" % (recipename, task, h))
 
                 if h != h_locked:
-                    self.mismatch_msgs.append('The %s:%s sig (%s) changed, use locked sig %s to instead'
-                                          % (recipename, task, h, h_locked))
+                    self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s'
+                                          % (recipename, task, h, h_locked, var))
 
                 return h_locked
         #bb.warn("%s %s %s" % (recipename, task, h))
-- 
2.5.5

[PATCH 3/5] populate_sdk_ext.bbclass: Enable locked sigs errors

[Randy Witt]

With the extensible sdk we want there to be an error if a task tries to
run without signatures that match locked-sigs.inc. This patch enables
that error.

[YOCTO #9195]

Signed-off-by: Randy Witt <randy.e.witt@linux.intel.com>
---
 meta/classes/populate_sdk_ext.bbclass | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 4ad190b..d1977a7 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -209,8 +209,15 @@ python copy_buildsystem () {
             # Bypass the default connectivity check if any
             f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
 
-            # Ensure locked sstate cache objects are re-used without error
-            f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "none"\n\n')
+            # This warning will come out if reverse dependencies for a task
+            # don't have sstate as well as the task itself. We already know
+            # this will be the case for the extensible sdk, so turn off the
+            # warning.
+            f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')
+
+            # Error if the sigs in the locked-signature file don't match
+            # the sig computed from the metadata.
+            f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "error"\n\n')
 
             # Hide the config information from bitbake output (since it's fixed within the SDK)
             f.write('BUILDCFG_HEADER = ""\n')
-- 
2.5.5

[PATCH 4/5] sstatesig.py: Add a method to "unlock" recipes

[Randy Witt]

In order to support workflows using devtool where a user might want to
modify tasks that exist in locked-sigs.inc, there must be a way to unlock
recipes.

This patch adds that support by allowing the user to add recipes to
SIGGEN_UNLOCKED_RECIPES. Recipes that exist in that variable will have
all their tasks unlocked, as well as any tasks that depend on that
recipe.

For example if foo->bar->baz, if you unlock baz, it will also unlock bar
so that foo can be rebuilt without explicitly specifying bar as being
unlocked.

[YOCTO #9195]

Signed-off-by: Randy Witt <randy.e.witt@linux.intel.com>
---
 meta/lib/oe/sstatesig.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 009adea..01dce66 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -94,6 +94,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
         self.lockedhashfn = {}
         self.machine = data.getVar("MACHINE", True)
         self.mismatch_msgs = []
+        self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES", True) or
+                                "").split()
+        self.unlockedrecipes = { k: "" for k in self.unlockedrecipes }
         pass
 
     def tasks_resolved(self, virtmap, virtpnmap, dataCache):
@@ -136,7 +139,26 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
         recipename = dataCache.pkg_fn[fn]
         self.lockedpnmap[fn] = recipename
         self.lockedhashfn[fn] = dataCache.hashfn[fn]
-        if recipename in self.lockedsigs:
+
+        unlocked = False
+        if recipename in self.unlockedrecipes:
+            unlocked = True
+        else:
+            def recipename_from_dep(dep):
+                # The dep entry will look something like
+                # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task,
+                # ...
+                fn = dep.rsplit('.', 1)[0]
+                return dataCache.pkg_fn[fn]
+
+            # If any unlocked recipe is in the direct dependencies then the
+            # current recipe should be unlocked as well.
+            depnames = [ recipename_from_dep(x) for x in deps ]
+            if any(x in y for y in depnames for x in self.unlockedrecipes):
+                self.unlockedrecipes[recipename] = ''
+                unlocked = True
+
+        if not unlocked and recipename in self.lockedsigs:
             if task in self.lockedsigs[recipename]:
                 k = fn + "." + task
                 h_locked = self.lockedsigs[recipename][task][0]
-- 
2.5.5

[PATCH 5/5] devtool: Create unlocked-sigs.inc containing items in the workspace

[Randy Witt]

When a recipe is added to the workspace, the signatures for the tasks
will change. This means that bitbake must be told to allow the
signatures to be different if they are in locked-sigs.inc.

This is done by creating an unlocked-sigs.inc file which contains all
the recipes in the workspace each time devtool reads the workspace.

So not only will necessary things get added, previously added items will
be removed by virtue of them no longer being in the workspace.

This also makes sure that the extensible sdk picks up unlocked-sigs.inc
as part of the configuration.

[YOCTO #9195]

Signed-off-by: Randy Witt <randy.e.witt@linux.intel.com>
---
 meta/classes/populate_sdk_ext.bbclass |  5 +++++
 scripts/devtool                       | 20 ++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index d1977a7..87518d1 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -140,6 +140,10 @@ python copy_buildsystem () {
     with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
         config.write(f)
 
+    unlockedsigs =  os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc')
+    with open(unlockedsigs, 'w') as f:
+        pass
+
     # Create a layer for new recipes / appends
     bbpath = d.getVar('BBPATH', True)
     bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
@@ -238,6 +242,7 @@ python copy_buildsystem () {
                     f.write(line.strip() + '\n')
 
             f.write('require conf/locked-sigs.inc\n')
+            f.write('require conf/unlocked-sigs.inc\n')
 
     if os.path.exists(builddir + '/conf/auto.conf'):
         if derivative:
diff --git a/scripts/devtool b/scripts/devtool
index e1198b1..4780390 100755
--- a/scripts/devtool
+++ b/scripts/devtool
@@ -125,6 +125,25 @@ def read_workspace():
                                      'recipefile': recipefile}
                     logger.debug('Found recipe %s' % workspace[pn])
 
+def create_unlockedsigs():
+    """ This function will make unlocked-sigs.inc match the recipes in the
+    workspace. This runs on every run of devtool, but it lets us ensure
+    the unlocked items are in sync with the workspace. """
+
+    confdir = os.path.join(basepath, 'conf')
+    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
+    bb.utils.mkdirhier(confdir)
+    with open(os.path.join(confdir, 'unlocked-sigs.inc'), 'w') as f:
+        f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
+                "# This layer was created by the OpenEmbedded devtool" +
+                " utility in order to\n" +
+                "# contain recipes that are unlocked.\n")
+
+        f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
+        for pn in workspace:
+            f.write('    ' + pn)
+        f.write('"')
+
 def create_workspace(args, config, basepath, workspace):
     if args.layerpath:
         workspacedir = os.path.abspath(args.layerpath)
@@ -299,6 +318,7 @@ def main():
 
     if not getattr(args, 'no_workspace', False):
         read_workspace()
+        create_unlockedsigs()
 
     try:
         ret = args.func(args, config, basepath, workspace)
-- 
2.5.5