[OE-core][dunfell 4/6] cve-check: add json format

Steve Sakoman
2022-04-28T21:47:21Z

From: Marta Rybczynska <rybczynska@gmail.com>

Backport to dunfell from master df567de36ae5964bee433ebb97e8bf702034994a

Add an option to output the CVE check in a JSON-based format.
This format is easier to parse in software than the original
text-based one and allows post-processing by other tools.

Output formats are now handed by CVE_CHECK_FORMAT_TEXT and
CVE_CHECK_FORMAT_JSON. The text format is enabled by default
to maintain compatibility, while the JSON format is disabled
by default.

The JSON output format gets generated in a similar way to the
text format with the exception of the manifest: appending to
JSON arrays requires parsing the file. Because of that we
first write JSON fragments and then assemble them in one pass
at the end.

Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
meta/classes/cve-check.bbclass | 144 ++++++++++++++++++++++++++++++++-
meta/lib/oe/cve_check.py | 16 ++++
2 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 75c5b92b96..a7156cbdfb 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -34,15 +34,27 @@ CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve"
CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary"
CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}"
+CVE_CHECK_SUMMARY_FILE_NAME_JSON = "cve-summary.json"
+CVE_CHECK_SUMMARY_INDEX_PATH = "${CVE_CHECK_SUMMARY_DIR}/cve-summary-index.txt"
+
+CVE_CHECK_LOG_JSON ?= "${T}/cve.json"

CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve"
CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}"
+CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json"
CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve"
+CVE_CHECK_MANIFEST_JSON ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json"
CVE_CHECK_COPY_FILES ??= "1"
CVE_CHECK_CREATE_MANIFEST ??= "1"

CVE_CHECK_REPORT_PATCHED ??= "1"

+# Provide text output
+CVE_CHECK_FORMAT_TEXT ??= "1"
+
+# Provide JSON output - disabled by default for backward compatibility
+CVE_CHECK_FORMAT_JSON ??= "0"
+
# Whitelist for packages (PN)
CVE_CHECK_PN_WHITELIST ?= ""

@@ -118,6 +130,7 @@ python cve_check_cleanup () {
Delete the file used to gather all the CVE information.
"""
bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE"))
+ bb.utils.remove(e.data.getVar("CVE_CHECK_SUMMARY_INDEX_PATH"))
}

addhandler cve_check_cleanup
@@ -129,11 +142,15 @@ python cve_check_write_rootfs_manifest () {
"""

import shutil
+ from oe.cve_check import cve_check_merge_jsons

if d.getVar("CVE_CHECK_COPY_FILES") == "1":
deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE")
if os.path.exists(deploy_file):
bb.utils.remove(deploy_file)
+ deploy_file_json = d.getVar("CVE_CHECK_RECIPE_FILE_JSON")
+ if os.path.exists(deploy_file_json):
+ bb.utils.remove(deploy_file_json)

if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")):
bb.note("Writing rootfs CVE manifest")
@@ -152,6 +169,26 @@ python cve_check_write_rootfs_manifest () {
os.remove(manifest_link)
os.symlink(os.path.basename(manifest_name), manifest_link)
bb.plain("Image CVE report stored in: %s" % manifest_name)
+
+ if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")):
+ import json
+ bb.note("Generating JSON CVE manifest")
+ deploy_dir = d.getVar("DEPLOY_DIR_IMAGE")
+ link_name = d.getVar("IMAGE_LINK_NAME")
+ manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON")
+ index_file = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")
+ manifest = {"version":"1", "package": []}
+ with open(index_file) as f:
+ filename = f.readline()
+ while filename:
+ with open(filename.rstrip()) as j:
+ data = json.load(j)
+ cve_check_merge_jsons(manifest, data)
+ filename = f.readline()
+
+ with open(manifest_name, "w") as f:
+ json.dump(manifest, f, indent=2)
+ bb.plain("Image CVE report stored in: %s" % manifest_name)
}

ROOTFS_POSTPROCESS_COMMAND_prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}"
@@ -337,7 +374,7 @@ def get_cve_info(d, cves):
conn.close()
return cve_data

-def cve_write_data(d, patched, unpatched, whitelisted, cve_data):
+def cve_write_data_text(d, patched, unpatched, whitelisted, cve_data):
"""
Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and
CVE manifest if enabled.
@@ -403,3 +440,108 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data):

with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f:
f.write("%s" % write_string)
+
+def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file):
+ """
+ Write CVE information in the JSON format: to WORKDIR; and to
+ CVE_CHECK_DIR, if CVE manifest if enabled, write fragment
+ files that will be assembled at the end in cve_check_write_rootfs_manifest.
+ """
+
+ import json
+
+ write_string = json.dumps(output, indent=2)
+ with open(direct_file, "w") as f:
+ bb.note("Writing file %s with CVE information" % direct_file)
+ f.write(write_string)
+
+ if d.getVar("CVE_CHECK_COPY_FILES") == "1":
+ bb.utils.mkdirhier(os.path.dirname(deploy_file))
+ with open(deploy_file, "w") as f:
+ f.write(write_string)
+
+ if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1":
+ cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR")
+ index_path = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")
+ bb.utils.mkdirhier(cvelogpath)
+ fragment_file = os.path.basename(deploy_file)
+ fragment_path = os.path.join(cvelogpath, fragment_file)
+ with open(fragment_path, "w") as f:
+ f.write(write_string)
+ with open(index_path, "a+") as f:
+ f.write("%s\n" % fragment_path)
+
+def cve_write_data_json(d, patched, unpatched, ignored, cve_data):
+ """
+ Prepare CVE data for the JSON format, then write it.
+ """
+
+ output = {"version":"1", "package": []}
+ nvd_link = "https://nvd.nist.gov/vuln/detail/"
+
+ fdir_name = d.getVar("FILE_DIRNAME")
+ layer = fdir_name.split("/")[-3]
+
+ include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split()
+ exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split()
+
+ if exclude_layers and layer in exclude_layers:
+ return
+
+ if include_layers and layer not in include_layers:
+ return
+
+ unpatched_cves = []
+
+ package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV"))
+ package_data = {
+ "name" : d.getVar("PN"),
+ "layer" : layer,
+ "version" : package_version
+ }
+ cve_list = []
+
+ for cve in sorted(cve_data):
+ is_patched = cve in patched
+ status = "Unpatched"
+ if is_patched and (d.getVar("CVE_CHECK_REPORT_PATCHED") != "1"):
+ continue
+ if cve in ignored:
+ status = "Ignored"
+ elif is_patched:
+ status = "Patched"
+ else:
+ # default value of status is Unpatched
+ unpatched_cves.append(cve)
+
+ issue_link = "%s%s" % (nvd_link, cve)
+
+ cve_item = {
+ "id" : cve,
+ "summary" : cve_data[cve]["summary"],
+ "scorev2" : cve_data[cve]["scorev2"],
+ "scorev3" : cve_data[cve]["scorev3"],
+ "vector" : cve_data[cve]["vector"],
+ "status" : status,
+ "link": issue_link
+ }
+ cve_list.append(cve_item)
+
+ package_data["issue"] = cve_list
+ output["package"].append(package_data)
+
+ direct_file = d.getVar("CVE_CHECK_LOG_JSON")
+ deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE_JSON")
+ manifest_file = d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON")
+
+ cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file)
+
+def cve_write_data(d, patched, unpatched, ignored, cve_data):
+ """
+ Write CVE data in each enabled format.
+ """
+
+ if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1":
+ cve_write_data_text(d, patched, unpatched, ignored, cve_data)
+ if d.getVar("CVE_CHECK_FORMAT_JSON") == "1":
+ cve_write_data_json(d, patched, unpatched, ignored, cve_data)
diff --git a/meta/lib/oe/cve_check.py b/meta/lib/oe/cve_check.py
index a1d7c292af..1d3c775bbe 100644
--- a/meta/lib/oe/cve_check.py
+++ b/meta/lib/oe/cve_check.py
@@ -63,3 +63,19 @@ def _cmpkey(release, patch_l, pre_l, pre_v):
else:
_pre = float(pre_v) if pre_v else float('-inf')
return _release, _patch, _pre
+
+def cve_check_merge_jsons(output, data):
+ """
+ Merge the data in the "package" property to the main data file
+ output
+ """
+ if output["version"] != data["version"]:
+ bb.error("Version mismatch when merging JSON outputs")
+ return
+
+ for product in output["package"]:
+ if product["name"] == data["package"][0]["name"]:
+ bb.error("Error adding the same package twice")
+ return
+
+ output["package"].append(data["package"][0])
--
2.25.1