[PATCH 1/2] cve-check: Add helper for symlink handling

[PATCH 1/2] cve-check: Add helper for symlink handling

[Ernst Sjöstrand]

Signed-off-by: Ernst Sjöstrand <ernstp@gmail.com>
---
 meta/classes/cve-check.bbclass | 34 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 3729d9cba8..0ab7ec7ae6 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -79,6 +79,12 @@ CVE_CHECK_LAYER_INCLUDELIST ??= ""
 # set to "alphabetical" for version using single alphabetical character as increment release
 CVE_VERSION_SUFFIX ??= ""
 
+def update_symlinks(target_path, link_path):
+    if link_path != target_path and os.path.exists(target_path):
+        if os.path.exists(os.path.realpath(link_path)):
+            os.remove(link_path)
+        os.symlink(os.path.basename(target_path), link_path)
+
 def generate_json_report(d, out_path, link_path):
     if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")):
         import json
@@ -98,10 +104,7 @@ def generate_json_report(d, out_path, link_path):
         with open(out_path, "w") as f:
             json.dump(summary, f, indent=2)
 
-        if link_path != out_path:
-            if os.path.exists(os.path.realpath(link_path)):
-                os.remove(link_path)
-            os.symlink(os.path.basename(out_path), link_path)
+        update_symlinks(out_path, link_path)
 
 python cve_save_summary_handler () {
     import shutil
@@ -118,14 +121,9 @@ python cve_save_summary_handler () {
 
     if os.path.exists(cve_tmp_file):
         shutil.copyfile(cve_tmp_file, cve_summary_file)
-
-        if cve_summary_file and os.path.exists(cve_summary_file):
-            cvefile_link = os.path.join(cvelogpath, cve_summary_name)
-            # if the paths are the same don't create the link
-            if cvefile_link != cve_summary_file:
-                if os.path.exists(os.path.realpath(cvefile_link)):
-                    os.remove(cvefile_link)
-                os.symlink(os.path.basename(cve_summary_file), cvefile_link)
+        cvefile_link = os.path.join(cvelogpath, cve_summary_name)
+        update_symlinks(cve_summary_file, cvefile_link)
+        bb.plain("Complete CVE report summary created at: %s" % cvefile_link)
 
     if d.getVar("CVE_CHECK_FORMAT_JSON") == "1":
         json_summary_link_name = os.path.join(cvelogpath, d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON"))
@@ -198,15 +196,9 @@ python cve_check_write_rootfs_manifest () {
         bb.utils.mkdirhier(os.path.dirname(manifest_name))
         shutil.copyfile(cve_tmp_file, manifest_name)
 
-        if manifest_name and os.path.exists(manifest_name):
-            manifest_link = os.path.join(deploy_dir, "%s.cve" % link_name)
-            # if they are the same don't create the link
-            if manifest_link != manifest_name:
-                # If we already have another manifest, update symlinks
-                if os.path.exists(os.path.realpath(manifest_link)):
-                    os.remove(manifest_link)
-                os.symlink(os.path.basename(manifest_name), manifest_link)
-            bb.plain("Image CVE report stored in: %s" % manifest_name)
+        manifest_link = os.path.join(deploy_dir, "%s.cve" % link_name)
+        update_symlinks(manifest_name, manifest_link)
+        bb.plain("Image CVE report stored in: %s" % manifest_name)
 
         if d.getVar("CVE_CHECK_FORMAT_JSON") == "1":
             link_path = os.path.join(deploy_dir, "%s.json" % link_name)
-- 
2.34.1

[PATCH 2/2] cve-check: Only include installed packages for rootfs manifest

[Ernst Sjöstrand]

Before this the rootfs manifest and the summary were identical.
We should separate the summary and rootfs manifest more clearly,
now the summary is for all CVEs and the rootfs manifest is only for
things in that image. This is even more useful if you build multiple
images.

Signed-off-by: Ernst Sjöstrand <ernstp@gmail.com>
---
 meta/classes/cve-check.bbclass | 69 ++++++++++++++++++++++++++--------
 1 file changed, 54 insertions(+), 15 deletions(-)

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 0ab7ec7ae6..3bb924ba34 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -176,6 +176,8 @@ python cve_check_write_rootfs_manifest () {
     """
 
     import shutil
+    import json
+    from oe.rootfs import image_list_installed_packages
     from oe.cve_check import cve_check_merge_jsons
 
     if d.getVar("CVE_CHECK_COPY_FILES") == "1":
@@ -186,26 +188,63 @@ python cve_check_write_rootfs_manifest () {
         if os.path.exists(deploy_file_json):
             bb.utils.remove(deploy_file_json)
 
-    if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")):
-        bb.note("Writing rootfs CVE manifest")
-        deploy_dir = d.getVar("DEPLOY_DIR_IMAGE")
-        link_name = d.getVar("IMAGE_LINK_NAME")
+    # Create a list of relevant recipies
+    recipies = set()
+    for pkg in list(image_list_installed_packages(d)):
+        pkg_info = os.path.join(d.getVar('PKGDATA_DIR'),
+                                'runtime-reverse', pkg)
+        pkg_data = oe.packagedata.read_pkgdatafile(pkg_info)
+        recipies.add(pkg_data["PN"])
+
+    bb.note("Writing rootfs CVE manifest")
+    deploy_dir = d.getVar("DEPLOY_DIR_IMAGE")
+    link_name = d.getVar("IMAGE_LINK_NAME")
+
+    json_data = {"version":"1", "package": []}
+    text_data = ""
+    enable_json = d.getVar("CVE_CHECK_FORMAT_JSON") == "1"
+    enable_text = d.getVar("CVE_CHECK_FORMAT_TEXT") == "1"
+
+    save_pn = d.getVar("PN")
+
+    for pkg in recipies:
+        # To be able to use the CVE_CHECK_RECIPE_FILE variable we have to evaluate
+        # it with the different PN names set each time.
+        d.setVar("PN", pkg)
+        if enable_text:
+            pkgfilepath = d.getVar("CVE_CHECK_RECIPE_FILE")
+            if os.path.exists(pkgfilepath):
+                with open(pkgfilepath) as pfile:
+                    text_data += pfile.read()
+
+        if enable_json:
+            pkgfilepath = d.getVar("CVE_CHECK_RECIPE_FILE_JSON")
+            if os.path.exists(pkgfilepath):
+                with open(pkgfilepath) as j:
+                    data = json.load(j)
+                    cve_check_merge_jsons(json_data, data)
+
+    d.setVar("PN", save_pn)
+
+    if enable_text:
+        link_path = os.path.join(deploy_dir, "%s.cve" % link_name)
         manifest_name = d.getVar("CVE_CHECK_MANIFEST")
-        cve_tmp_file = d.getVar("CVE_CHECK_TMP_FILE")
 
-        bb.utils.mkdirhier(os.path.dirname(manifest_name))
-        shutil.copyfile(cve_tmp_file, manifest_name)
+        with open(manifest_name, "w") as f:
+            f.write(text_data)
 
-        manifest_link = os.path.join(deploy_dir, "%s.cve" % link_name)
-        update_symlinks(manifest_name, manifest_link)
+        update_symlinks(manifest_name, link_path)
         bb.plain("Image CVE report stored in: %s" % manifest_name)
 
-        if d.getVar("CVE_CHECK_FORMAT_JSON") == "1":
-            link_path = os.path.join(deploy_dir, "%s.json" % link_name)
-            manifest_path = d.getVar("CVE_CHECK_MANIFEST_JSON")
-            bb.note("Generating JSON CVE manifest")
-            generate_json_report(d, manifest_path, link_path)
-            bb.plain("Image CVE JSON report stored in: %s" % link_path)
+    if enable_json:
+        link_path = os.path.join(deploy_dir, "%s.json" % link_name)
+        manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON")
+
+        with open(manifest_name, "w") as f:
+            json.dump(json_data, f, indent=2)
+
+        update_symlinks(manifest_name, link_path)
+        bb.plain("Image CVE JSON report stored in: %s" % manifest_name)
 }
 
 ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}"
-- 
2.34.1

Re: [OE-core] [PATCH 2/2] cve-check: Only include installed packages for rootfs manifest

[Marta Rybczynska]

On Tue, May 24, 2022 at 1:50 PM Ernst Sjöstrand <ernstp@gmail.com> wrote:
>
> Before this the rootfs manifest and the summary were identical.
> We should separate the summary and rootfs manifest more clearly,
> now the summary is for all CVEs and the rootfs manifest is only for
> things in that image. This is even more useful if you build multiple
> images.
>
> Signed-off-by: Ernst Sjöstrand <ernstp@gmail.com>

> +    for pkg in list(image_list_installed_packages(d)):

Thank you for submitting a new version of this patch. The references
posted on IRC suggested that using this way may cause us to skip
packages that should be included. Or I did misunderstand?

Kind regards,
Marta

Re: [OE-core] [PATCH 2/2] cve-check: Only include installed packages for rootfs manifest

[Ernst Sjöstrand]

[-- Attachment #1: Type: text/plain, Size: 1312 bytes --]

Hi,

it will not miss any installed package that Yocto knows about, i.e it will
be exact compared to the rootfs.manifest and so on.
There are of course ways to "fool" it like bundling a copy of a library in
another project and similar things.
And I think statically linked libraries are not really handled by Yocto
here either, they don't show up in the rootfs.manifest or the COPY_LIC_*
stuff either for example.

Regards
//Ernst

PS. My patch didn't show up on Patchwork?

Den ons 25 maj 2022 kl 11:42 skrev Marta Rybczynska <rybczynska@gmail.com>:

> On Tue, May 24, 2022 at 1:50 PM Ernst Sjöstrand <ernstp@gmail.com> wrote:
> >
> > Before this the rootfs manifest and the summary were identical.
> > We should separate the summary and rootfs manifest more clearly,
> > now the summary is for all CVEs and the rootfs manifest is only for
> > things in that image. This is even more useful if you build multiple
> > images.
> >
> > Signed-off-by: Ernst Sjöstrand <ernstp@gmail.com>
>
> > +    for pkg in list(image_list_installed_packages(d)):
>
> Thank you for submitting a new version of this patch. The references
> posted on IRC suggested that using this way may cause us to skip
> packages that should be included. Or I did misunderstand?
>
> Kind regards,
> Marta
>

[-- Attachment #2: Type: text/html, Size: 2680 bytes --]