[PATCH v2] sstate.bbclass: Only sign packages at the time of their creation

[Tobias Hagelborn <tobiasha@axis.com>]

The purpose of the change is to never sign a package not created by
the build itself.

sstate_create_package is refactored into Python and re-designed
to handle signing inside the function. Thus, the signing should never apply
to existing sstate packages. The function is therefore renamed into
sstate_create_and_sign_package.
The creation of the archive remains in a separate shellscript function.
---
Changes since v1 based on Rickard's review comments:

- No temporary sub-directory creation unless signing is enabled
- No datastore copy
- Incorporated sstate_create_pkgdirs into sstate_create_and_sign_package
  sstate_create_and_sign_package was the only user and now that it is all
  Python function, it can also host the sstate_create_pkgdirs functionality
- Generally, we have tried to stay close to the original file behavior in
  terms of rename and hardlink into place. (The update_file() helper function)

  Validation:
  - Run in our environment since a few days back with no known issues
    so far. ~ 5000 image builds / day, both signed and unsigned.

 meta/classes-global/sstate.bbclass | 137 ++++++++++++++++++++---------
 1 file changed, 96 insertions(+), 41 deletions(-)

diff --git a/meta/classes-global/sstate.bbclass b/meta/classes-global/sstate.bbclass
index efe7f69775..13dd3fdaf4 100644
--- a/meta/classes-global/sstate.bbclass
+++ b/meta/classes-global/sstate.bbclass
@@ -703,9 +703,7 @@ def sstate_package(ss, d):
     if d.getVar('SSTATE_SKIP_CREATION') == '1':
         return
 
-    sstate_create_package = ['sstate_report_unihash', 'sstate_create_pkgdirs', 'sstate_create_package']
-    if d.getVar('SSTATE_SIG_KEY'):
-        sstate_create_package.append('sstate_sign_package')
+    sstate_create_package = ['sstate_report_unihash', 'sstate_create_and_sign_package']
 
     for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
              sstate_create_package + \
@@ -817,19 +815,100 @@ python sstate_create_pkgdirs () {
         bb.utils.mkdirhier(os.path.dirname(d.getVar('SSTATE_PKG')))
 }
 
-#
-# Shell function to generate a sstate package from a directory
-# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
-#
-sstate_create_package () {
-	# Exit early if it already exists
-	if [ -e ${SSTATE_PKG} ]; then
-		touch ${SSTATE_PKG} 2>/dev/null || true
-		return
-	fi
+# Create a sstate package
+# If enabled, sign the package.
+# Package and signature are created in a sub-directory
+# and renamed in place once created.
+python sstate_create_and_sign_package () {
+    from pathlib import Path
+
+    # Best effort touch
+    def touch(file):
+        try:
+            file.touch()
+        except:
+            pass
+
+    def update_file(src, dst, force=False):
+        if dst.is_symlink() and not dst.exists():
+            force=True
+        try:
+            # This relies on that src is a temporary file that can be renamed
+            # or left as is.
+            if force:
+                src.rename(dst)
+            else:
+                os.link(src, dst)
+            return True
+        except:
+            pass
+
+        if dst.exists():
+            touch(dst)
+
+        return False
 
-	TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
+    sign_pkg = (
+        bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG")) and
+        bool(d.getVar("SSTATE_SIG_KEY"))
+    )
+
+    sstate_pkg = Path(d.getVar("SSTATE_PKG"))
+    sstate_pkg_sig = Path(str(sstate_pkg) + ".sig")
+    if sign_pkg:
+        if sstate_pkg.exists() and sstate_pkg_sig.exists():
+            touch(sstate_pkg)
+            touch(sstate_pkg_sig)
+            return
+    else:
+        if sstate_pkg.exists():
+            touch(sstate_pkg)
+            return
+
+    # Create the required sstate directory if it is not present.
+    if not sstate_pkg.parent.is_dir():
+        with bb.utils.umask(0o002):
+            bb.utils.mkdirhier(str(sstate_pkg.parent))
+
+    if sign_pkg:
+        from tempfile import TemporaryDirectory
+        with TemporaryDirectory(dir=sstate_pkg.parent) as tmp_dir:
+            tmp_pkg = Path(tmp_dir) / sstate_pkg.name
+            d.setVar("TMP_SSTATE_PKG", str(tmp_pkg))
+            bb.build.exec_func('sstate_archive_package', d)
+
+            from oe.gpg_sign import get_signer
+            signer = get_signer(d, 'local')
+            signer.detach_sign(str(tmp_pkg), d.getVar('SSTATE_SIG_KEY'), None,
+                                d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
+
+            tmp_pkg_sig = Path(tmp_dir) / sstate_pkg_sig.name
+            if not update_file(tmp_pkg_sig, sstate_pkg_sig):
+                # If the created signature file could not be copied into place,
+                # then we should not use the sstate package either.
+                return
+
+            # If the .sig file was updated, then the sstate package must also
+            # be updated.
+            update_file(tmp_pkg, sstate_pkg, force=True)
+    else:
+        from tempfile import NamedTemporaryFile
+        with NamedTemporaryFile(prefix=sstate_pkg.name, dir=sstate_pkg.parent) as tmp_pkg_fd:
+            tmp_pkg = tmp_pkg_fd.name
+            d.setVar("TMP_SSTATE_PKG", str(tmp_pkg))
+            bb.build.exec_func('sstate_archive_package',d)
+            update_file(tmp_pkg, sstate_pkg)
+            # update_file() may have renamed tmp_pkg, which must exist when the
+            # NamedTemporaryFile() context handler ends.
+            touch(Path(tmp_pkg))
+
+}
 
+# Shell function to generate a sstate package from a directory
+# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
+# The calling function handles moving the sstate package into the final
+# destination.
+sstate_archive_package () {
 	OPT="-cS"
 	ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
 	# Use pzstd if available
@@ -840,42 +919,18 @@ sstate_create_package () {
 	# Need to handle empty directories
 	if [ "$(ls -A)" ]; then
 		set +e
-		tar -I "$ZSTD" $OPT -f $TFILE *
+		tar -I "$ZSTD" $OPT -f ${TMP_SSTATE_PKG} *
 		ret=$?
 		if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
 			exit 1
 		fi
 		set -e
 	else
-		tar -I "$ZSTD" $OPT --file=$TFILE --files-from=/dev/null
-	fi
-	chmod 0664 $TFILE
-	# Skip if it was already created by some other process
-	if [ -h ${SSTATE_PKG} ] && [ ! -e ${SSTATE_PKG} ]; then
-		# There is a symbolic link, but it links to nothing.
-		# Forcefully replace it with the new file.
-		ln -f $TFILE ${SSTATE_PKG} || true
-	elif [ ! -e ${SSTATE_PKG} ]; then
-		# Move into place using ln to attempt an atomic op.
-		# Abort if it already exists
-		ln $TFILE ${SSTATE_PKG} || true
-	else
-		touch ${SSTATE_PKG} 2>/dev/null || true
+		tar -I "$ZSTD" $OPT --file=${TMP_SSTATE_PKG} --files-from=/dev/null
 	fi
-	rm $TFILE
+	chmod 0664 ${TMP_SSTATE_PKG}
 }
 
-python sstate_sign_package () {
-    from oe.gpg_sign import get_signer
-
-
-    signer = get_signer(d, 'local')
-    sstate_pkg = d.getVar('SSTATE_PKG')
-    if os.path.exists(sstate_pkg + '.sig'):
-        os.unlink(sstate_pkg + '.sig')
-    signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
-                       d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
-}
 
 python sstate_report_unihash() {
     report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
-- 
2.30.2