[PATCH 1/6] package_manager: Remove rpm specific pieces of create_packages_dir()

[PATCH 1/6] package_manager: Remove rpm specific pieces of create_packages_dir()

[Richard Purdie]

This function is generic, tweak the variable names and move out the rpm specific
directory name to make it truly generic and reusable for deb/ipk.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oe/package_manager.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 6011e873503..f1dbe48a720 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -621,7 +621,7 @@ class PackageManager(object, metaclass=ABCMeta):
             return res
         return _append(uris, base_paths)
 
-def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependencies):
+def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencies):
     """
     Go through our do_package_write_X dependencies and hardlink the packages we depend
     upon into the repo directory. This prevents us seeing other packages that may
@@ -636,15 +636,13 @@ def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependenci
     seendirs = set()
     multilibs = {}
    
-    rpm_subrepo_dir = oe.path.join(rpm_repo_dir, "rpm")
-
-    bb.utils.remove(rpm_subrepo_dir, recurse=True)
-    bb.utils.mkdirhier(rpm_subrepo_dir)
+    bb.utils.remove(subrepo_dir, recurse=True)
+    bb.utils.mkdirhier(subrepo_dir)
 
     # Detect bitbake -b usage
     nodeps = d.getVar("BB_LIMITEDDEPS") or False
     if nodeps or not filterbydependencies:
-        oe.path.symlink(deploydir, rpm_subrepo_dir, True)
+        oe.path.symlink(deploydir, subrepo_dir, True)
         return
 
     start = None
@@ -655,24 +653,24 @@ def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependenci
             break
     if start is None:
         bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
-    rpmdeps = set()
+    pkgdeps = set()
     start = [start]
     seen = set(start)
-    # Support direct dependencies (do_rootfs -> rpms)
-    # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> rpms)
+    # Support direct dependencies (do_rootfs -> do_package_write_X)
+    # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> do_package_write_X)
     while start:
         next = []
         for dep2 in start:
             for dep in taskdepdata[dep2][3]:
                 if taskdepdata[dep][0] != pn:
                     if "do_" + taskname in dep:
-                        rpmdeps.add(dep)
+                        pkgdeps.add(dep)
                 elif dep not in seen:
                     next.append(dep)
                     seen.add(dep)
         start = next
 
-    for dep in rpmdeps:
+    for dep in pkgdeps:
         c = taskdepdata[dep][0]
         manifest, d2 = oe.sstatesig.find_sstate_manifest(c, taskdepdata[dep][2], taskname, d, multilibs)
         if not manifest:
@@ -683,7 +681,7 @@ def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependenci
             for l in f:
                 l = l.strip()
                 dest = l.replace(deploydir, "")
-                dest = rpm_subrepo_dir + dest
+                dest = subrepo_dir + dest
                 if l.endswith("/"):
                     if dest not in seendirs:
                         bb.utils.mkdirhier(dest)
@@ -725,7 +723,7 @@ class RpmPM(PackageManager):
             self.primary_arch = self.d.getVar('MACHINE_ARCH')
 
         self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir)
-        create_packages_dir(self.d, self.rpm_repo_dir, d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies)
+        create_packages_dir(self.d, oe.path.join(self.rpm_repo_dir, "rpm"), d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies)
 
         self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name)
         if not os.path.exists(self.d.expand('${T}/saved_packaging_data')):
-- 
2.17.1

[PATCH 2/6] package_manager/sdk: Use filtered copies of the deploy ipk/deb directories

[Richard Purdie]

Similar to rpm, use copies of the ipk/deb directories for rootfs construction.
This means the image creation code can no longer "see" recipes wich aren't in its
dependency chain which is good for a variety of reasons including determinism,
incompatible recipe (e.g. systemd/sysvinit) package conflicts and locking
performance.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oe/package_manager.py         | 13 +++++++++----
 meta/lib/oe/sdk.py                     | 20 ++++++++++++++++----
 meta/lib/oeqa/utils/package_manager.py |  6 ++++--
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index f1dbe48a720..0f1c5899fd5 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -1124,19 +1124,21 @@ class OpkgDpkgPM(PackageManager):
         self.mark_packages("unpacked", registered_pkgs.split())
 
 class OpkgPM(OpkgDpkgPM):
-    def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
+    def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True):
         super(OpkgPM, self).__init__(d, target_rootfs)
 
         self.config_file = config_file
         self.pkg_archs = archs
         self.task_name = task_name
 
-        self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK")
+        self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
         self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS")
 
+        create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
+
         opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
         if opkg_lib_dir[0] == "/":
             opkg_lib_dir = opkg_lib_dir[1:]
@@ -1501,9 +1503,12 @@ class OpkgPM(OpkgDpkgPM):
         return tmp_dir
 
 class DpkgPM(OpkgDpkgPM):
-    def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
+    def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True):
         super(DpkgPM, self).__init__(d, target_rootfs)
-        self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB')
+        self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), deb_repo_workdir)
+
+        create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_DEB"), "package_write_deb", filterbydependencies)
+
         if apt_conf_dir is None:
             self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
         else:
diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
index f20441ccf66..153b07d76b4 100644
--- a/meta/lib/oe/sdk.py
+++ b/meta/lib/oe/sdk.py
@@ -227,11 +227,17 @@ class OpkgSdk(Sdk):
         self.host_manifest = OpkgManifest(d, self.manifest_dir,
                                           Manifest.MANIFEST_TYPE_SDK_HOST)
 
+        ipk_repo_workdir = "oe-sdk-repo"
+        if "sdk_ext" in d.getVar("BB_RUNTASK"):
+            ipk_repo_workdir = "oe-sdk-ext-repo"
+
         self.target_pm = OpkgPM(d, self.sdk_target_sysroot, self.target_conf,
-                                self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"))
+                                self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"), 
+                                ipk_repo_workdir=ipk_repo_workdir)
 
         self.host_pm = OpkgPM(d, self.sdk_host_sysroot, self.host_conf,
-                              self.d.getVar("SDK_PACKAGE_ARCHS"))
+                              self.d.getVar("SDK_PACKAGE_ARCHS"),
+                                ipk_repo_workdir=ipk_repo_workdir)
 
     def _populate_sysroot(self, pm, manifest):
         pkgs_to_install = manifest.parse_initial_manifest()
@@ -307,15 +313,21 @@ class DpkgSdk(Sdk):
         self.host_manifest = DpkgManifest(d, self.manifest_dir,
                                           Manifest.MANIFEST_TYPE_SDK_HOST)
 
+        deb_repo_workdir = "oe-sdk-repo"
+        if "sdk_ext" in d.getVar("BB_RUNTASK"):
+            deb_repo_workdir = "oe-sdk-ext-repo"
+
         self.target_pm = DpkgPM(d, self.sdk_target_sysroot,
                                 self.d.getVar("PACKAGE_ARCHS"),
                                 self.d.getVar("DPKG_ARCH"),
-                                self.target_conf_dir)
+                                self.target_conf_dir,
+                                deb_repo_workdir=deb_repo_workdir)
 
         self.host_pm = DpkgPM(d, self.sdk_host_sysroot,
                               self.d.getVar("SDK_PACKAGE_ARCHS"),
                               self.d.getVar("DEB_SDK_ARCH"),
-                              self.host_conf_dir)
+                              self.host_conf_dir,
+                              deb_repo_workdir=deb_repo_workdir)
 
     def _copy_apt_dir_to(self, dst_dir):
         staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE")
diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py
index afd5b8e75f6..1495f873210 100644
--- a/meta/lib/oeqa/utils/package_manager.py
+++ b/meta/lib/oeqa/utils/package_manager.py
@@ -22,13 +22,15 @@ def get_package_manager(d, root_path):
         pm = OpkgPM(d,
                     root_path,
                     d.getVar("IPKGCONF_TARGET"),
-                    d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"))
+                    d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"),
+                    filterbydependencies=False)
 
     elif pkg_class == "deb":
         pm = DpkgPM(d,
                     root_path,
                     d.getVar('PACKAGE_ARCHS'),
-                    d.getVar('DPKG_ARCH'))
+                    d.getVar('DPKG_ARCH'),
+                    filterbydependencies=False)
 
     pm.write_index()
     pm.update()
-- 
2.17.1

[PATCH 3/6] sstate: Ensure a given machine only removes things which it created

[Richard Purdie]

Currently if you build qemux86 and then generic86, the latter will
remove all of the former from deploy and workdir. This is because
qemux86 is i586, genericx86 is i686 and the architctures are compatible
therefore the sstate 'cleaup' code kicks in.

There was a valid reason for this to ensure i586 packages didn't get into
an i686 rootfs for example. With the rootfs creation being filtered now, this
is no longer necessary.

Instead, save out a list of stamps which a give machine has ever seen in
a given build and only clean up these things if they're no longer
"reachable".

In particular this means the autobuilder should no longer spend a load of time
deleting files when switching MACHINE, improving build times.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/sstate.bbclass | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 9927c76596e..a091c6e843e 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1045,6 +1045,15 @@ python sstate_eventhandler2() {
         with open(preservestampfile, 'r') as f:
             preservestamps = f.readlines()
     seen = []
+
+    # The machine index contains all the stamps this machine has ever seen in this build directory.
+    # We should only remove things which this machine once accessed but no longer does.
+    machineindex = set()
+    mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
+    if os.path.exists(mi):
+        with open(mi, "r") as f:
+            machineindex = set(f.readlines())
+
     for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
         toremove = []
         i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
@@ -1054,7 +1063,7 @@ python sstate_eventhandler2() {
             lines = f.readlines()
             for l in lines:
                 (stamp, manifest, workdir) = l.split()
-                if stamp not in stamps and stamp not in preservestamps:
+                if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
                     toremove.append(l)
                     if stamp not in seen:
                         bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
@@ -1083,6 +1092,11 @@ python sstate_eventhandler2() {
         with open(i, "w") as f:
             for l in lines:
                 f.write(l)
+    machineindex |= set(stamps)
+    with open(mi, "w") as f:
+        for l in machineindex:
+            f.write(l + "\n")
+
     if preservestamps:
         os.remove(preservestampfile)
 }
-- 
2.17.1

[PATCH 4/6] selftest: Replace bitbake -p with bitbake -e

[Richard Purdie]

Parsing all the recipes is annoying when trying to re-execute oe-selftest
and also unnecessary as its really just a sanity check. When the tests were
originally being developed the guard was useful but less so now.

Replace it with bitbake -e which is fast and checks the basic configuration
is valid.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/selftest/context.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index 3a70f9e77b0..c78947e200b 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -201,8 +201,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
 
         _add_layer_libs()
 
-        self.tc.logger.info("Running bitbake -p")
-        runCmd("bitbake -p")
+        self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable")
+        runCmd("bitbake -e")
 
     def _internal_run(self, logger, args):
         self.module_paths = self._get_cases_paths(
-- 
2.17.1

[PATCH 5/6] sstate/lib.oe.path: Ensure file sparseness is preserved

[Richard Purdie]

Files when restored from sstate were missing their sparseness. Fix up various
functions to preserve this and make things more deterministic.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/sstate.bbclass | 4 ++--
 meta/lib/oe/path.py         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index a091c6e843e..1c166ff6745 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -721,9 +721,9 @@ sstate_create_package () {
 	TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
 
         # Use pigz if available
-        OPT="-cz"
+        OPT="-czS"
         if [ -x "$(command -v pigz)" ]; then
-            OPT="-I pigz -c"
+            OPT="-I pigz -cS"
         fi
 
 	# Need to handle empty directories
diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index be02218c31d..1e24d0586b6 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -86,7 +86,7 @@ def copytree(src, dst):
     # This way we also preserve hardlinks between files in the tree.
 
     bb.utils.mkdirhier(dst)
-    cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
+    cmd = "tar --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
     subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 
 def copyhardlinktree(src, dst):
@@ -98,7 +98,7 @@ def copyhardlinktree(src, dst):
     if (os.stat(src).st_dev ==  os.stat(dst).st_dev):
         # Need to copy directories only with tar first since cp will error if two 
         # writers try and create a directory at the same time
-        cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst)
+        cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
         source = ''
         if os.path.isdir(src):
-- 
2.17.1

[PATCH 6/6] selftest/package: Improve test to cover sparseness and hardlinking from sstate

[Richard Purdie]

The sparseness test was sometimes working and sometimes failing depending
on whether sstate was valid. This adds an explict test of sstate
to the test for both hardlinking and sparseness. Tweak the test name to
cover the fact its tests sparseness too.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/selftest/cases/package.py | 27 ++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/package.py b/meta/lib/oeqa/selftest/cases/package.py
index 4ff9b08c96a..ee6430a1845 100644
--- a/meta/lib/oeqa/selftest/cases/package.py
+++ b/meta/lib/oeqa/selftest/cases/package.py
@@ -89,17 +89,26 @@ class VersionOrdering(OESelftestTestCase):
 class PackageTests(OESelftestTestCase):
     # Verify that a recipe which sets up hardlink files has those preserved into split packages
     # Also test file sparseness is preserved
-    def test_preserve_hardlinks(self):
-        result = bitbake("selftest-hardlink -c package")
+    def test_preserve_sparse_hardlinks(self):
+        bitbake("selftest-hardlink -c package")
 
         dest = get_bb_var('PKGDEST', 'selftest-hardlink')
         bindir = get_bb_var('bindir', 'selftest-hardlink')
 
-        # Recipe creates 4 hardlinked files, there is a copy in package/ and a copy in packages-split/
-        # so expect 8 in total.
-        self.assertEqual(os.stat(dest + "/selftest-hardlink" + bindir + "/hello").st_nlink, 8)
+        def checkfiles():
+            # Recipe creates 4 hardlinked files, there is a copy in package/ and a copy in packages-split/
+            # so expect 8 in total.
+            self.assertEqual(os.stat(dest + "/selftest-hardlink" + bindir + "/hello").st_nlink, 8)
 
-        # Test a sparse file remains sparse
-        sparsestat = os.stat(dest + "/selftest-hardlink" + bindir + "/sparsetest")
-        self.assertEqual(sparsestat.st_blocks, 0)
-        self.assertEqual(sparsestat.st_size, 1048576)
+            # Test a sparse file remains sparse
+            sparsestat = os.stat(dest + "/selftest-hardlink" + bindir + "/sparsetest")
+            self.assertEqual(sparsestat.st_blocks, 0)
+            self.assertEqual(sparsestat.st_size, 1048576)
+
+        checkfiles()
+
+        # Clean and reinstall so its now definitely from sstate, then retest.
+        bitbake("selftest-hardlink -c clean")
+        bitbake("selftest-hardlink -c package")
+
+        checkfiles()
-- 
2.17.1

✗ patchtest: failure for "package_manager: Remove rpm sp..." and 5 more

[Patchwork]

== Series Details ==

Series: "package_manager: Remove rpm sp..." and 5 more
Revision: 1
URL   : https://patchwork.openembedded.org/series/13512/
State : failure

== Summary ==


Thank you for submitting this patch series to OpenEmbedded Core. This is
an automated response. Several tests have been executed on the proposed
series by patchtest resulting in the following failures:



* Issue             Series does not apply on top of target branch [test_series_merge_on_head] 
  Suggested fix    Rebase your series on top of targeted branch
  Targeted branch  master (currently at e2b8a3d5a1)



If you believe any of these test results are incorrect, please reply to the
mailing list (openembedded-core@lists.openembedded.org) raising your concerns.
Otherwise we would appreciate you correcting the issues and submitting a new
version of the patchset if applicable. Please ensure you add/increment the
version number when sending the new version (i.e. [PATCH] -> [PATCH v2] ->
[PATCH v3] -> ...).

---
Guidelines:     https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines
Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest
Test suite:     http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe