[PATCH 1/2] package_manager: Filter to only rpms we depend upon

* [PATCH 1/2] package_manager: Filter to only rpms we depend upon
@ 2018-02-27 19:56 Richard Purdie
  2018-02-27 19:56 ` [PATCH 2/2] sstatesig/staging/package_manager: Create common sstate manifest code Richard Purdie
  2018-02-27 21:36 ` [PATCH 1/2] package_manager: Filter to only rpms we depend upon Denys Dmytriyenko
  0 siblings, 2 replies; 4+ messages in thread
From: Richard Purdie @ 2018-02-27 19:56 UTC (permalink / raw)
  To: openembedded-core

Currently do_rootfs gets to see all rpms in the deploy directory. This filters
that view to only rpms which the image recipe has actual depends upon which
potentially removes some sources of confusion in the image construction.

This makes builds more reproducibile and also fixes contamination issues
where dnf picks up packages it shouldn't be able to 'see'.

[YOCTO #12039]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oe/package_manager.py         | 114 ++++++++++++++++++++++++++++++++-
 meta/lib/oeqa/utils/package_manager.py |   3 +-
 2 files changed, 113 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index f7e0134..f59eaf7 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -454,6 +454,114 @@ class PackageManager(object, metaclass=ABCMeta):
             return res
         return _append(uris, base_paths)
 
+def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependencies):
+    """
+    Go through our do_package_write_X dependencies and hardlink the packages we depend
+    upon into the repo directory. This prevents us seeing other packages that may
+    have been built that we don't depend upon and also packages for architectures we don't
+    support.
+    """
+    import errno
+
+    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
+    mytaskname = d.getVar("BB_RUNTASK")
+    pn = d.getVar("PN")
+    seendirs = set()
+    multilibs = {}
+   
+    rpm_subrepo_dir = oe.path.join(rpm_repo_dir, "rpm")
+
+    bb.utils.remove(rpm_subrepo_dir, recurse=True)
+    bb.utils.mkdirhier(rpm_subrepo_dir)
+
+    # Detect bitbake -b usage
+    nodeps = d.getVar("BB_LIMITEDDEPS") or False
+    if nodeps or not filterbydependencies:
+        oe.path.symlink(deploydir, rpm_subrepo_dir, True)
+        return
+
+    start = None
+    for dep in taskdepdata:
+        data = taskdepdata[dep]
+        if data[1] == mytaskname and data[0] == pn:
+            start = dep
+            break
+    if start is None:
+        bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
+    rpmdeps = set()
+    start = [start]
+    seen = set(start)
+    # Support direct dependencies (do_rootfs -> rpms)
+    # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> rpms)
+    while start:
+        next = []
+        for dep2 in start:
+            for dep in taskdepdata[dep2][3]:
+                if taskdepdata[dep][0] != pn:
+                    if "do_" + taskname in dep:
+                        rpmdeps.add(dep)
+                elif dep not in seen:
+                    next.append(dep)
+                    seen.add(dep)
+        start = next
+
+    for dep in rpmdeps:
+        c = taskdepdata[dep][0]
+
+        d2 = d
+        variant = ''
+        if taskdepdata[dep][2].startswith("virtual:multilib"):
+            variant = taskdepdata[dep][2].split(":")[2]
+            if variant not in multilibs:
+                multilibs[variant] = oe.utils.get_multilib_datastore(variant, d)
+            d2 = multilibs[variant]
+
+        if c.endswith("-native"):
+            pkgarchs = ["${BUILD_ARCH}"]
+        elif c.startswith("nativesdk-"):
+            pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"]
+        elif "-cross-canadian" in c:
+            pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"]
+        elif "-cross-" in c:
+            pkgarchs = ["${BUILD_ARCH}_${TARGET_ARCH}"]
+        elif "-crosssdk" in c:
+            pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"]
+        else:
+            pkgarchs = ['${MACHINE_ARCH}']
+            pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split()))
+            pkgarchs.append('allarch')
+            pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}')
+
+        for pkgarch in pkgarchs:
+            manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, c, taskname))
+            if os.path.exists(manifest):
+                break
+        if not os.path.exists(manifest):
+            bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant))
+            continue
+        with open(manifest, "r") as f:
+            for l in f:
+                l = l.strip()
+                dest = l.replace(deploydir, "")
+                dest = rpm_subrepo_dir + dest
+                if l.endswith("/"):
+                    if dest not in seendirs:
+                        bb.utils.mkdirhier(dest)
+                        seendirs.add(dest)
+                    continue
+                # Try to hardlink the file, copy if that fails
+                destdir = os.path.dirname(dest)
+                if destdir not in seendirs:
+                    bb.utils.mkdirhier(destdir)
+                    seendirs.add(destdir)
+                try:
+                    os.link(l, dest)
+                except OSError as err:
+                    if err.errno == errno.EXDEV:
+                        bb.utils.copyfile(l, dest)
+                    else:
+                        raise
+
 class RpmPM(PackageManager):
     def __init__(self,
                  d,
@@ -462,7 +570,8 @@ class RpmPM(PackageManager):
                  task_name='target',
                  arch_var=None,
                  os_var=None,
-                 rpm_repo_workdir="oe-rootfs-repo"):
+                 rpm_repo_workdir="oe-rootfs-repo",
+                 filterbydependencies=True):
         super(RpmPM, self).__init__(d)
         self.target_rootfs = target_rootfs
         self.target_vendor = target_vendor
@@ -477,8 +586,7 @@ class RpmPM(PackageManager):
             self.primary_arch = self.d.getVar('MACHINE_ARCH')
 
         self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir)
-        bb.utils.mkdirhier(self.rpm_repo_dir)
-        oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True)
+        create_packages_dir(self.d, self.rpm_repo_dir, d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies)
 
         self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name)
         if not os.path.exists(self.d.expand('${T}/saved_packaging_data')):
diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py
index 724afb2..afd5b8e 100644
--- a/meta/lib/oeqa/utils/package_manager.py
+++ b/meta/lib/oeqa/utils/package_manager.py
@@ -14,7 +14,8 @@ def get_package_manager(d, root_path):
     if pkg_class == "rpm":
         pm = RpmPM(d,
                    root_path,
-                   d.getVar('TARGET_VENDOR'))
+                   d.getVar('TARGET_VENDOR'),
+                   filterbydependencies=False)
         pm.create_configs()
 
     elif pkg_class == "ipk":
-- 
2.7.4



^ permalink raw reply related	[flat|nested] 4+ messages in thread