[OE-core][PATCH v3 1/2] cve-check: add json format

[OE-core][PATCH v3 1/2] cve-check: add json format

[Marta Rybczynska]

Add an option to output the CVE check in a JSON-based format.
This format is easier to parse in software than the original
text-based one and allows post-processing by other tools.

Output formats are now handed by CVE_CHECK_FORMAT_TEXT and
CVE_CHECK_FORMAT_JSON. Both of them are enabled by default.

The JSON output format gets generated in a similar way to the
text format with the exception of the manifest: appending to
JSON arrays requires parsing the file. Because of that we
first write JSON fragments and then assemble them in one pass
at the end.

Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com>

---
Changes in v3:
- fix CVE_CHECK_REPORT_PATCHED==0 with the YAML output

Changes in v2:
- hardcoded the dictionary merge (cve_check_merge_jsons)
---
 meta/classes/cve-check.bbclass | 144 ++++++++++++++++++++++++++++++++-
 meta/lib/oe/cve_check.py       |  16 ++++
 2 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index dfad10c22b..f574f5daa4 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -34,15 +34,27 @@ CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
 CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve"
 CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary"
 CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}"
+CVE_CHECK_SUMMARY_FILE_NAME_JSON = "cve-summary.json"
+CVE_CHECK_SUMMARY_INDEX_PATH = "${CVE_CHECK_SUMMARY_DIR}/cve-summary-index.txt"
+
+CVE_CHECK_LOG_JSON ?= "${T}/cve.json"
 
 CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve"
 CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}"
+CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json"
 CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve"
+CVE_CHECK_MANIFEST_JSON ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json"
 CVE_CHECK_COPY_FILES ??= "1"
 CVE_CHECK_CREATE_MANIFEST ??= "1"
 
 CVE_CHECK_REPORT_PATCHED ??= "1"
 
+# Provide text output
+CVE_CHECK_FORMAT_TEXT ??= "1"
+
+# Provide JSON output
+CVE_CHECK_FORMAT_JSON ??= "1"
+
 # Skip CVE Check for packages (PN)
 CVE_CHECK_SKIP_RECIPE ?= ""
 
@@ -120,6 +132,7 @@ python cve_check_cleanup () {
     Delete the file used to gather all the CVE information.
     """
     bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE"))
+    bb.utils.remove(e.data.getVar("CVE_CHECK_SUMMARY_INDEX_PATH"))
 }
 
 addhandler cve_check_cleanup
@@ -131,11 +144,15 @@ python cve_check_write_rootfs_manifest () {
     """
 
     import shutil
+    from oe.cve_check import cve_check_merge_jsons
 
     if d.getVar("CVE_CHECK_COPY_FILES") == "1":
         deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE")
         if os.path.exists(deploy_file):
             bb.utils.remove(deploy_file)
+        deploy_file_json = d.getVar("CVE_CHECK_RECIPE_FILE_JSON")
+        if os.path.exists(deploy_file_json):
+            bb.utils.remove(deploy_file_json)
 
     if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")):
         bb.note("Writing rootfs CVE manifest")
@@ -154,6 +171,26 @@ python cve_check_write_rootfs_manifest () {
                 os.remove(manifest_link)
             os.symlink(os.path.basename(manifest_name), manifest_link)
             bb.plain("Image CVE report stored in: %s" % manifest_name)
+
+    if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")):
+        import json
+        bb.note("Generating JSON CVE manifest")
+        deploy_dir = d.getVar("DEPLOY_DIR_IMAGE")
+        link_name = d.getVar("IMAGE_LINK_NAME")
+        manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON")
+        index_file = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")
+        manifest = {"version":"1", "package": []}
+        with open(index_file) as f:
+            filename = f.readline()
+            while filename:
+                with open(filename.rstrip()) as j:
+                    data = json.load(j)
+                    cve_check_merge_jsons(manifest, data)
+                filename = f.readline()
+
+        with open(manifest_name, "w") as f:
+            json.dump(manifest, f, indent=2)
+        bb.plain("Image CVE report stored in: %s" % manifest_name)
 }
 
 ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}"
@@ -280,7 +317,7 @@ def get_cve_info(d, cves):
     conn.close()
     return cve_data
 
-def cve_write_data(d, patched, unpatched, ignored, cve_data):
+def cve_write_data_text(d, patched, unpatched, ignored, cve_data):
     """
     Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and
     CVE manifest if enabled.
@@ -346,3 +383,108 @@ def cve_write_data(d, patched, unpatched, ignored, cve_data):
 
             with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f:
                 f.write("%s" % write_string)
+
+def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file):
+    """
+    Write CVE information in the JSON format: to WORKDIR; and to
+    CVE_CHECK_DIR, if CVE manifest if enabled, write fragment
+    files that will be assembled at the end in cve_check_write_rootfs_manifest.
+    """
+
+    import json
+
+    write_string = json.dumps(output, indent=2)
+    with open(direct_file, "w") as f:
+        bb.note("Writing file %s with CVE information" % direct_file)
+        f.write(write_string)
+
+    if d.getVar("CVE_CHECK_COPY_FILES") == "1":
+        bb.utils.mkdirhier(os.path.dirname(deploy_file))
+        with open(deploy_file, "w") as f:
+            f.write(write_string)
+
+    if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1":
+        cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR")
+        index_path = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")
+        bb.utils.mkdirhier(cvelogpath)
+        fragment_file = os.path.basename(deploy_file)
+        fragment_path = os.path.join(cvelogpath, fragment_file)
+        with open(fragment_path, "w") as f:
+            f.write(write_string)
+        with open(index_path, "a+") as f:
+            f.write("%s\n" % fragment_path)
+
+def cve_write_data_json(d, patched, unpatched, ignored, cve_data):
+    """
+    Prepare CVE data for the JSON format, then write it.
+    """
+
+    output = {"version":"1", "package": []}
+    nvd_link = "https://nvd.nist.gov/vuln/detail/"
+
+    fdir_name  = d.getVar("FILE_DIRNAME")
+    layer = fdir_name.split("/")[-3]
+
+    include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split()
+    exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split()
+
+    if exclude_layers and layer in exclude_layers:
+        return
+
+    if include_layers and layer not in include_layers:
+        return
+
+    unpatched_cves = []
+
+    package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV"))
+    package_data = {
+        "name" : d.getVar("PN"),
+        "layer" : layer,
+        "version" : package_version
+    }
+    cve_list = []
+
+    for cve in sorted(cve_data):
+        is_patched = cve in patched
+        status = "Unpatched"
+        if is_patched and (d.getVar("CVE_CHECK_REPORT_PATCHED") != "1"):
+            continue
+        if cve in ignored:
+            status = "Ignored"
+        elif is_patched:
+            status = "Patched"
+        else:
+            # default value of status is Unpatched
+            unpatched_cves.append(cve)
+
+        issue_link = "%s%s" % (nvd_link, cve)
+
+        cve_item = {
+            "id" : cve,
+            "summary" : cve_data[cve]["summary"],
+            "scorev2" : cve_data[cve]["scorev2"],
+            "scorev3" : cve_data[cve]["scorev3"],
+            "vector" : cve_data[cve]["vector"],
+            "status" : status,
+            "link": issue_link
+        }
+        cve_list.append(cve_item)
+
+    package_data["issue"] = cve_list
+    output["package"].append(package_data)
+
+    direct_file = d.getVar("CVE_CHECK_LOG_JSON")
+    deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE_JSON")
+    manifest_file = d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON")
+
+    cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file)
+
+def cve_write_data(d, patched, unpatched, ignored, cve_data):
+    """
+    Write CVE data in each enabled format.
+    """
+
+    if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1":
+        cve_write_data_text(d, patched, unpatched, ignored, cve_data)
+    if d.getVar("CVE_CHECK_FORMAT_JSON") == "1":
+        cve_write_data_json(d, patched, unpatched, ignored, cve_data)
diff --git a/meta/lib/oe/cve_check.py b/meta/lib/oe/cve_check.py
index 0302beeb4a..e445b7a6ae 100644
--- a/meta/lib/oe/cve_check.py
+++ b/meta/lib/oe/cve_check.py
@@ -146,3 +146,19 @@ def get_cpe_ids(cve_product, version):
         cpe_ids.append(cpe_id)
 
     return cpe_ids
+
+def cve_check_merge_jsons(output, data):
+    """
+    Merge the data in the "package" property to the main data file
+    output
+    """
+    if output["version"] != data["version"]:
+        bb.error("Version mismatch when merging JSON outputs")
+        return
+
+    for product in output["package"]:
+        if product["name"] == data["package"][0]["name"]:
+            bb.error("Error adding the same package twice")
+            return
+
+    output["package"].append(data["package"][0])
-- 
2.33.0

[OE-core][PATCH v3 2/2] cve-check: add coverage statistics on recipes with/without CVEs

[Marta Rybczynska]

Until now the CVE checker was giving information about CVEs found for
a product (or more products) contained in a recipe. However, there was
no easy way to find out which products or recipes have no CVEs. Having
no reported CVEs might mean there are simply none, but can also mean
a product name (CPE) mismatch.

This patch adds CVE_CHECK_COVERAGE option enabling a new type of
statistics. Then we use the new JSON format to report the information.
The legacy text mode report does not contain it.

This option is expected to help with an identification of recipes with
mismatched CPEs, issues in the database and more.

This work is based on [1], but adding the JSON format makes it easier
to implement, without additional result files.

[1] https://lists.openembedded.org/g/openembedded-core/message/159873

Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com>
---
 meta/classes/cve-check.bbclass | 51 ++++++++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index f574f5daa4..78516d0bb6 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -55,6 +55,9 @@ CVE_CHECK_FORMAT_TEXT ??= "1"
 # Provide JSON output
 CVE_CHECK_FORMAT_JSON ??= "1"
 
+# Check for packages without CVEs (no issues or missing product name)
+CVE_CHECK_COVERAGE ??= "1"
+
 # Skip CVE Check for packages (PN)
 CVE_CHECK_SKIP_RECIPE ?= ""
 
@@ -114,10 +117,10 @@ python do_cve_check () {
             patched_cves = get_patched_cves(d)
         except FileNotFoundError:
             bb.fatal("Failure in searching patches")
-        ignored, patched, unpatched = check_cves(d, patched_cves)
-        if patched or unpatched:
+        ignored, patched, unpatched, status = check_cves(d, patched_cves)
+        if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status):
             cve_data = get_cve_info(d, patched + unpatched)
-            cve_write_data(d, patched, unpatched, ignored, cve_data)
+            cve_write_data(d, patched, unpatched, ignored, cve_data, status)
     else:
         bb.note("No CVE database found, skipping CVE check")
 
@@ -207,17 +210,19 @@ def check_cves(d, patched_cves):
     suffix = d.getVar("CVE_VERSION_SUFFIX")
 
     cves_unpatched = []
+    cves_status = []
+    cves_in_recipe = False
     # CVE_PRODUCT can contain more than one product (eg. curl/libcurl)
     products = d.getVar("CVE_PRODUCT").split()
     # If this has been unset then we're not scanning for CVEs here (for example, image recipes)
     if not products:
-        return ([], [], [])
+        return ([], [], [], {})
     pv = d.getVar("CVE_VERSION").split("+git")[0]
 
     # If the recipe has been skipped/ignored we return empty lists
     if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split():
         bb.note("Recipe has been skipped by cve-check")
-        return ([], [], [])
+        return ([], [], [], [])
 
     cve_ignore = d.getVar("CVE_CHECK_IGNORE").split()
 
@@ -227,6 +232,7 @@ def check_cves(d, patched_cves):
 
     # For each of the known product names (e.g. curl has CPEs using curl and libcurl)...
     for product in products:
+        cves_in_product = False
         if ":" in product:
             vendor, product = product.split(":", 1)
         else:
@@ -244,6 +250,11 @@ def check_cves(d, patched_cves):
             elif cve in patched_cves:
                 bb.note("%s has been patched" % (cve))
                 continue
+            # Write status once only for each product
+            if not cves_in_product:
+                cves_status.append([product, True])
+                cves_in_product = True
+                cves_in_recipe = True
 
             vulnerable = False
             for row in conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)):
@@ -290,9 +301,16 @@ def check_cves(d, patched_cves):
                 # TODO: not patched but not vulnerable
                 patched_cves.add(cve)
 
+        if not cves_in_product:
+            bb.note("No CVE records found for product %s, pn %s" % (product, pn))
+            cves_status.append([product, False])
+
     conn.close()
 
-    return (list(cve_ignore), list(patched_cves), cves_unpatched)
+    if not cves_in_recipe:
+        bb.note("No CVE records for products in recipe %s" % (pn))
+
+    return (list(cve_ignore), list(patched_cves), cves_unpatched, cves_status)
 
 def get_cve_info(d, cves):
     """
@@ -323,7 +341,6 @@ def cve_write_data_text(d, patched, unpatched, ignored, cve_data):
     CVE manifest if enabled.
     """
 
-
     cve_file = d.getVar("CVE_CHECK_LOG")
     fdir_name  = d.getVar("FILE_DIRNAME")
     layer = fdir_name.split("/")[-3]
@@ -337,6 +354,10 @@ def cve_write_data_text(d, patched, unpatched, ignored, cve_data):
     if include_layers and layer not in include_layers:
         return
 
+    # Early exit, the text format does not report packages without CVEs
+    if not patched+unpatched:
+        return
+
     nvd_link = "https://nvd.nist.gov/vuln/detail/"
     write_string = ""
     unpatched_cves = []
@@ -414,7 +435,7 @@ def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_fi
         with open(index_path, "a+") as f:
             f.write("%s\n" % fragment_path)
 
-def cve_write_data_json(d, patched, unpatched, ignored, cve_data):
+def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status):
     """
     Prepare CVE data for the JSON format, then write it.
     """
@@ -436,11 +457,19 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data):
 
     unpatched_cves = []
 
+    product_data = []
+    for s in cve_status:
+        p = {"product": s[0], "cvesInRecord": "Yes"}
+        if s[1] == False:
+            p["cvesInRecord"] = "No"
+        product_data.append(p)
+
     package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV"))
     package_data = {
         "name" : d.getVar("PN"),
         "layer" : layer,
-        "version" : package_version
+        "version" : package_version,
+        "products": product_data
     }
     cve_list = []
 
@@ -479,7 +508,7 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data):
 
     cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file)
 
-def cve_write_data(d, patched, unpatched, ignored, cve_data):
+def cve_write_data(d, patched, unpatched, ignored, cve_data, status):
     """
     Write CVE data in each enabled format.
     """
@@ -487,4 +516,4 @@ def cve_write_data(d, patched, unpatched, ignored, cve_data):
     if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1":
         cve_write_data_text(d, patched, unpatched, ignored, cve_data)
     if d.getVar("CVE_CHECK_FORMAT_JSON") == "1":
-        cve_write_data_json(d, patched, unpatched, ignored, cve_data)
+        cve_write_data_json(d, patched, unpatched, ignored, cve_data, status)
-- 
2.33.0

Re: [OE-core][PATCH v3 1/2] cve-check: add json format

[Sean Nyekjaer]

On 29/03/2022 14.54, Marta Rybczynska wrote:
> Add an option to output the CVE check in a JSON-based format.
> This format is easier to parse in software than the original
> text-based one and allows post-processing by other tools.
> 
> Output formats are now handed by CVE_CHECK_FORMAT_TEXT and
> CVE_CHECK_FORMAT_JSON. Both of them are enabled by default.
> 
> The JSON output format gets generated in a similar way to the
> text format with the exception of the manifest: appending to
> JSON arrays requires parsing the file. Because of that we
> first write JSON fragments and then assemble them in one pass
> at the end.
> 
> Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com>
> 
> ---

Hi Marta,

When upgrading from honister to kirkstone, this patch was included.

We have INHERIT += "cve" in our distro conf.
When doing a build from scratch it works fine with the cve json output, 
but when doing an incremental build it will throw:
	bb.error("Error adding the same package twice")

I have tried modifying that line to:
	bb.error("Error adding the same package twice. Offending package: %s" % 
product["name"])

It looks like it tries to add all packages again...

I have just disabled json output for now.
CVE_CHECK_FORMAT_JSON = "0"

Br,
/Sean

Re: [OE-core][PATCH v3 1/2] cve-check: add json format

[Marta Rybczynska]

[-- Attachment #1: Type: text/plain, Size: 1717 bytes --]

On Tue, Jun 21, 2022 at 12:45 PM Sean Nyekjaer <sean@geanix.com> wrote:

>
>
> On 29/03/2022 14.54, Marta Rybczynska wrote:
> > Add an option to output the CVE check in a JSON-based format.
> > This format is easier to parse in software than the original
> > text-based one and allows post-processing by other tools.
> >
> > Output formats are now handed by CVE_CHECK_FORMAT_TEXT and
> > CVE_CHECK_FORMAT_JSON. Both of them are enabled by default.
> >
> > The JSON output format gets generated in a similar way to the
> > text format with the exception of the manifest: appending to
> > JSON arrays requires parsing the file. Because of that we
> > first write JSON fragments and then assemble them in one pass
> > at the end.
> >
> > Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com>
> >
> > ---
>
> Hi Marta,
>
> When upgrading from honister to kirkstone, this patch was included.
>
> We have INHERIT += "cve" in our distro conf.
> When doing a build from scratch it works fine with the cve json output,
> but when doing an incremental build it will throw:
>         bb.error("Error adding the same package twice")
>
> I have tried modifying that line to:
>         bb.error("Error adding the same package twice. Offending package:
> %s" %
> product["name"])
>
> It looks like it tries to add all packages again...
>
> I have just disabled json output for now.
> CVE_CHECK_FORMAT_JSON = "0"
>
>
Dear Sean,
Thank you for the report. Could you please describe your configuration a
little bit, especially, which layers do you use? Do you do an image build
or a world build, or maybe a multiple image build?

I'm trying to imagine what is happening in your case, will try to reproduce.

Kind regards,
Marta

[-- Attachment #2: Type: text/html, Size: 2512 bytes --]

Re: [OE-core][PATCH v3 1/2] cve-check: add json format

[Sean Nyekjaer]

On 21/06/2022 15.06, Marta Rybczynska wrote:
> 
> 
> On Tue, Jun 21, 2022 at 12:45 PM Sean Nyekjaer <sean@geanix.com 
> <mailto:sean@geanix.com>> wrote:
> 
> 
> 
>     On 29/03/2022 14.54, Marta Rybczynska wrote:
>      > Add an option to output the CVE check in a JSON-based format.
>      > This format is easier to parse in software than the original
>      > text-based one and allows post-processing by other tools.
>      >
>      > Output formats are now handed by CVE_CHECK_FORMAT_TEXT and
>      > CVE_CHECK_FORMAT_JSON. Both of them are enabled by default.
>      >
>      > The JSON output format gets generated in a similar way to the
>      > text format with the exception of the manifest: appending to
>      > JSON arrays requires parsing the file. Because of that we
>      > first write JSON fragments and then assemble them in one pass
>      > at the end.
>      >
>      > Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com
>     <mailto:marta.rybczynska@huawei.com>>
>      >
>      > ---
> 
>     Hi Marta,
> 
>     When upgrading from honister to kirkstone, this patch was included.
> 
>     We have INHERIT += "cve" in our distro conf.
>     When doing a build from scratch it works fine with the cve json output,
>     but when doing an incremental build it will throw:
>              bb.error("Error adding the same package twice")
> 
>     I have tried modifying that line to:
>              bb.error("Error adding the same package twice. Offending
>     package: %s" %
>     product["name"])
> 
>     It looks like it tries to add all packages again...
> 
>     I have just disabled json output for now.
>     CVE_CHECK_FORMAT_JSON = "0"
> 
> 
> Dear Sean,
> Thank you for the report. Could you please describe your configuration a 
> little bit, especially, which layers do you use? Do you do an image 
> build or a world build, or maybe a multiple image build?
> 
> I'm trying to imagine what is happening in your case, will try to reproduce.
> 
> Kind regards,
> Marta

Hi Marta,

We are using:
meta-clang
meta-intel
meta-openembedded
meta-rauc
meta-security
meta-virtualization
poky

We are building an image :)

I can try to see if I can reproduce with a core-minimal-image, but it 
will have to wait until after summer...

/Sean