[Buildroot] [PATCH 0/3] Go: list vendored modules in show-info and the SBOM

[Buildroot] [PATCH 0/3] Go: list vendored modules in show-info and the SBOM

[Mattias Walström]

Go packages show up in the SBOM as one component, so CVE scanners never
see the modules vendored into them. This series records those modules
from the build info in the built binaries and adds them to the CycloneDX
output as nested components.

Patch 3 is a related fix: override and local sources skip the download
step that runs go mod vendor, so a tree without vendor/ failed to build.

Tested with amazon-ecr-credential-helper from an override tree without
vendor/: fails without patch 3, builds with it.

Mattias Walström (3):
  package/pkg-golang: record vendored Go modules in show-info
  utils/generate-cyclonedx: add vendored dependencies as nested
    components
  package/pkg-golang: vendor modules of local and overridden sources

 package/pkg-golang.mk                         | 24 ++++++
 package/pkg-utils.mk                          |  7 ++
 support/scripts/go-modules-info               | 67 ++++++++++++++++
 .../tests/utils/test_generate_cyclonedx.py    | 76 +++++++++++++++++++
 utils/generate-cyclonedx                      | 53 ++++++++++++-
 5 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100755 support/scripts/go-modules-info

-- 
2.43.0

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 1/3] package/pkg-golang: record vendored Go modules in show-info

[Mattias Walström]

The SBOM only listed Go packages as a whole, the modules vendored into
them were invisible to CVE scanners. Read them from the build info the
Go toolchain embeds in each binary, so the list matches what ships.

Signed-off-by: Mattias Walström <lazzer@gmail.com>
---
 package/pkg-golang.mk           |  8 ++++
 package/pkg-utils.mk            |  7 ++++
 support/scripts/go-modules-info | 67 +++++++++++++++++++++++++++++++++
 3 files changed, 82 insertions(+)
 create mode 100755 support/scripts/go-modules-info

diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
index cf12dd56b0..043efc94bd 100644
--- a/package/pkg-golang.mk
+++ b/package/pkg-golang.mk
@@ -206,6 +206,14 @@ define $(2)_INSTALL_CMDS
 endef
 endif
 
+# Record the Go modules linked into the built binaries, for show-info
+define $(2)_GO_MODULES_INFO
+	$$(Q)$$(TOPDIR)/support/scripts/go-modules-info --go $$(GO_BIN) \
+		-o $$(@D)/$$(PKG_VENDORED_DEPS_FILE) \
+		$$$$(find $$(@D)/bin -maxdepth 1 -type f 2>/dev/null)
+endef
+$(2)_POST_BUILD_HOOKS += $(2)_GO_MODULES_INFO
+
 # Call the generic package infrastructure to generate the necessary make
 # targets
 $(call inner-generic-package,$(1),$(2),$(3),$(4))
diff --git a/package/pkg-utils.mk b/package/pkg-utils.mk
index 17b0aa1760..21b434c6bd 100644
--- a/package/pkg-utils.mk
+++ b/package/pkg-utils.mk
@@ -111,6 +111,10 @@ define json-info
 	}
 endef
 
+# File in a package build directory listing the dependencies vendored into
+# it, as a JSON list of {name, version, purl}, added to show-info when present
+PKG_VENDORED_DEPS_FILE = .br_vendored_dependencies.json
+
 # _json-info-pkg, _json-info-pkg-details, _json-info-fs: private helpers
 # for json-info, above
 define _json-info-pkg
@@ -210,6 +214,9 @@ define _json-info-pkg-details
 			$(call mk-json-str,$(patsubst $(CONFIG_DIR)/%,%,$(patch)))$(comma) \
 		)
 	],
+	$(if $(wildcard $($(1)_DIR)/$(PKG_VENDORED_DEPS_FILE)), \
+		"vendored_dependencies": $(shell cat $($(1)_DIR)/$(PKG_VENDORED_DEPS_FILE))$(comma) \
+	)
 	$(if $($(1)_PROVIDES), \
 		"provides": [ \
 			$(foreach provide, \
diff --git a/support/scripts/go-modules-info b/support/scripts/go-modules-info
new file mode 100755
index 0000000000..476a4de333
--- /dev/null
+++ b/support/scripts/go-modules-info
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""List the Go modules linked into a set of Go binaries.
+
+Reads the build info that the Go toolchain embeds in every binary
+(go version -m -json) and writes the third-party modules, deduplicated
+across binaries, as a compact JSON list. Modules replaced by a local
+directory are part of the package source and are left out.
+
+Each entry has the name, version and purl of the module that was built.
+When a replace directive swapped in another module, the original is
+recorded under "replaces".
+"""
+
+import argparse
+import json
+import subprocess
+import sys
+import urllib.parse
+
+
+def purl(path, version):
+    segments = [urllib.parse.quote(s, safe="") for s in path.split("/")]
+    return f"pkg:golang/{'/'.join(segments)}@{urllib.parse.quote(version, safe='')}"
+
+
+def is_local(path):
+    return path.startswith(("./", "../", "/"))
+
+
+def module(path, version):
+    return {"name": path, "version": version, "purl": purl(path, version)}
+
+
+def binary_deps(go, binary):
+    out = subprocess.run([go, "version", "-m", "-json", binary],
+                         check=True, capture_output=True, text=True).stdout
+    return json.loads(out).get("Deps") or []
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--go", required=True, help="Path to the go tool")
+    parser.add_argument("-o", "--output", required=True, help="Output JSON file")
+    parser.add_argument("binaries", nargs="*", help="Go binaries to inspect")
+    args = parser.parse_args()
+
+    modules = {}
+    for binary in args.binaries:
+        for dep in binary_deps(args.go, binary):
+            repl = dep.get("Replace")
+            if repl:
+                if is_local(repl["Path"]):
+                    continue
+                entry = module(repl["Path"], repl["Version"])
+                entry["replaces"] = module(dep["Path"], dep["Version"])
+            else:
+                entry = module(dep["Path"], dep["Version"])
+            modules[entry["purl"]] = entry
+
+    with open(args.output, "w") as f:
+        json.dump([modules[k] for k in sorted(modules)], f, separators=(",", ":"))
+        f.write("\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())
-- 
2.43.0

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/3] package/pkg-golang: record vendored Go modules in show-info

[Thomas Perale via buildroot]

Hi Mattias,

In reply of:
> The SBOM only listed Go packages as a whole, the modules vendored into
> them were invisible to CVE scanners. Read them from the build info the
> Go toolchain embeds in each binary, so the list matches what ships.
> 
> Signed-off-by: Mattias Walström <lazzer@gmail.com>

I haven't totally check the series yet but during the ELCE2026 Buildroot dev
meeting we have discussed this issue. Go is not the only case this would be
required: Rust, the kernel, potentially NPM packages and other standalone
packages that support generating an SBOM (such as wolfssl, brotli, ...) would
also benefit from a proper SBOM generation support.

The idea was to add a "<pkg>_SBOM" step to the generic package infrastructure
so we can both have something shared per infra such as Go and individual package
implementing their own procedure.
There are still some missing links before we move forward this way:

- How do we support the variety of SBOM format (CycloneDX, SPDX, compilation
  database)
- Do we include in the CycloneDX SBOM or just reference it ?

Some question for you for the Go case:

- Would it make sense to use the utility CycloneDX maintain for Go instead [1] ?
  If we go with the "<pkg>_SBOM" step I described it's something we could add
  to the dependencies. And Rust packages could also use the CycloneDX utility
  for Rust.
- Can you compare the output of this utility to the one you propose so we can
  better assess the pros and cons ?

[1] https://github.com/CycloneDX/cyclonedx-gomod

Best Regards,
PERALE Thomas

> ---
>  package/pkg-golang.mk           |  8 ++++
>  package/pkg-utils.mk            |  7 ++++
>  support/scripts/go-modules-info | 67 +++++++++++++++++++++++++++++++++
>  3 files changed, 82 insertions(+)
>  create mode 100755 support/scripts/go-modules-info
> 
> diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
> index cf12dd56b0..043efc94bd 100644
> --- a/package/pkg-golang.mk
> +++ b/package/pkg-golang.mk
> @@ -206,6 +206,14 @@ define $(2)_INSTALL_CMDS
>  endef
>  endif
>  
> +# Record the Go modules linked into the built binaries, for show-info
> +define $(2)_GO_MODULES_INFO
> +	$$(Q)$$(TOPDIR)/support/scripts/go-modules-info --go $$(GO_BIN) \
> +		-o $$(@D)/$$(PKG_VENDORED_DEPS_FILE) \
> +		$$$$(find $$(@D)/bin -maxdepth 1 -type f 2>/dev/null)
> +endef
> +$(2)_POST_BUILD_HOOKS += $(2)_GO_MODULES_INFO
> +
>  # Call the generic package infrastructure to generate the necessary make
>  # targets
>  $(call inner-generic-package,$(1),$(2),$(3),$(4))
> diff --git a/package/pkg-utils.mk b/package/pkg-utils.mk
> index 17b0aa1760..21b434c6bd 100644
> --- a/package/pkg-utils.mk
> +++ b/package/pkg-utils.mk
> @@ -111,6 +111,10 @@ define json-info
>  	}
>  endef
>  
> +# File in a package build directory listing the dependencies vendored into
> +# it, as a JSON list of {name, version, purl}, added to show-info when present
> +PKG_VENDORED_DEPS_FILE = .br_vendored_dependencies.json
> +
>  # _json-info-pkg, _json-info-pkg-details, _json-info-fs: private helpers
>  # for json-info, above
>  define _json-info-pkg
> @@ -210,6 +214,9 @@ define _json-info-pkg-details
>  			$(call mk-json-str,$(patsubst $(CONFIG_DIR)/%,%,$(patch)))$(comma) \
>  		)
>  	],
> +	$(if $(wildcard $($(1)_DIR)/$(PKG_VENDORED_DEPS_FILE)), \
> +		"vendored_dependencies": $(shell cat $($(1)_DIR)/$(PKG_VENDORED_DEPS_FILE))$(comma) \
> +	)
>  	$(if $($(1)_PROVIDES), \
>  		"provides": [ \
>  			$(foreach provide, \
> diff --git a/support/scripts/go-modules-info b/support/scripts/go-modules-info
> new file mode 100755
> index 0000000000..476a4de333
> --- /dev/null
> +++ b/support/scripts/go-modules-info
> @@ -0,0 +1,67 @@
> +#!/usr/bin/env python3
> +"""List the Go modules linked into a set of Go binaries.
> +
> +Reads the build info that the Go toolchain embeds in every binary
> +(go version -m -json) and writes the third-party modules, deduplicated
> +across binaries, as a compact JSON list. Modules replaced by a local
> +directory are part of the package source and are left out.
> +
> +Each entry has the name, version and purl of the module that was built.
> +When a replace directive swapped in another module, the original is
> +recorded under "replaces".
> +"""
> +
> +import argparse
> +import json
> +import subprocess
> +import sys
> +import urllib.parse
> +
> +
> +def purl(path, version):
> +    segments = [urllib.parse.quote(s, safe="") for s in path.split("/")]
> +    return f"pkg:golang/{'/'.join(segments)}@{urllib.parse.quote(version, safe='')}"
> +
> +
> +def is_local(path):
> +    return path.startswith(("./", "../", "/"))
> +
> +
> +def module(path, version):
> +    return {"name": path, "version": version, "purl": purl(path, version)}
> +
> +
> +def binary_deps(go, binary):
> +    out = subprocess.run([go, "version", "-m", "-json", binary],
> +                         check=True, capture_output=True, text=True).stdout
> +    return json.loads(out).get("Deps") or []
> +
> +
> +def main():
> +    parser = argparse.ArgumentParser(description=__doc__,
> +                                     formatter_class=argparse.RawDescriptionHelpFormatter)
> +    parser.add_argument("--go", required=True, help="Path to the go tool")
> +    parser.add_argument("-o", "--output", required=True, help="Output JSON file")
> +    parser.add_argument("binaries", nargs="*", help="Go binaries to inspect")
> +    args = parser.parse_args()
> +
> +    modules = {}
> +    for binary in args.binaries:
> +        for dep in binary_deps(args.go, binary):
> +            repl = dep.get("Replace")
> +            if repl:
> +                if is_local(repl["Path"]):
> +                    continue
> +                entry = module(repl["Path"], repl["Version"])
> +                entry["replaces"] = module(dep["Path"], dep["Version"])
> +            else:
> +                entry = module(dep["Path"], dep["Version"])
> +            modules[entry["purl"]] = entry
> +
> +    with open(args.output, "w") as f:
> +        json.dump([modules[k] for k in sorted(modules)], f, separators=(",", ":"))
> +        f.write("\n")
> +
> +
> +if __name__ == "__main__":
> +    sys.exit(main())
> -- 
> 2.43.0
> 
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 2/3] utils/generate-cyclonedx: add vendored dependencies as nested components

[Mattias Walström]

A fork swapped in by a replace directive lists the original module as
its ancestor, so scanners can still match advisories for it.

Signed-off-by: Mattias Walström <lazzer@gmail.com>
---
 .../tests/utils/test_generate_cyclonedx.py    | 76 +++++++++++++++++++
 utils/generate-cyclonedx                      | 53 ++++++++++++-
 2 files changed, 127 insertions(+), 2 deletions(-)

diff --git a/support/testing/tests/utils/test_generate_cyclonedx.py b/support/testing/tests/utils/test_generate_cyclonedx.py
index 96b68beb40..37a2c1c673 100644
--- a/support/testing/tests/utils/test_generate_cyclonedx.py
+++ b/support/testing/tests/utils/test_generate_cyclonedx.py
@@ -150,6 +150,82 @@ class TestGenerateCycloneDX(unittest.TestCase):
         foo_deps = next(d for d in result["dependencies"] if d["ref"] == "package-foo")
         self.assertEqual(foo_deps["dependsOn"], ["package-bar", "skeleton-baz"])
 
+    def test_vendored_dependencies(self):
+        info = self._make_show_info()
+        info["package-foo"]["vendored_dependencies"] = [
+            {
+                "name": "example.org/lib",
+                "version": "v1.0.0",
+                "purl": "pkg:golang/example.org/lib@v1.0.0",
+            },
+            {
+                "name": "example.org/fork",
+                "version": "v1.1.0",
+                "purl": "pkg:golang/example.org/fork@v1.1.0",
+                "replaces": {
+                    "name": "example.org/upstream",
+                    "version": "v1.0.0",
+                    "purl": "pkg:golang/example.org/upstream@v1.0.0",
+                },
+            },
+        ]
+        info["package-bar"]["vendored_dependencies"] = [
+            {
+                "name": "example.org/lib",
+                "version": "v1.0.0",
+                "purl": "pkg:golang/example.org/lib@v1.0.0",
+            },
+        ]
+
+        result = self._run_script(show_info=info)
+
+        foo = self._find_component(result, "package-foo")
+        self.assertEqual(
+            foo["components"],
+            [
+                {
+                    "bom-ref": "package-foo:pkg:golang/example.org/lib@v1.0.0",
+                    "type": "library",
+                    "name": "example.org/lib",
+                    "version": "v1.0.0",
+                    "purl": "pkg:golang/example.org/lib@v1.0.0",
+                },
+                {
+                    "bom-ref": "package-foo:pkg:golang/example.org/fork@v1.1.0",
+                    "type": "library",
+                    "name": "example.org/fork",
+                    "version": "v1.1.0",
+                    "purl": "pkg:golang/example.org/fork@v1.1.0",
+                    "pedigree": {
+                        "ancestors": [
+                            {
+                                "type": "library",
+                                "name": "example.org/upstream",
+                                "version": "v1.0.0",
+                                "purl": "pkg:golang/example.org/upstream@v1.0.0",
+                            },
+                        ],
+                    },
+                },
+            ],
+        )
+
+        bar = self._find_component(result, "package-bar")
+        self.assertEqual(bar["components"][0]["bom-ref"], "package-bar:pkg:golang/example.org/lib@v1.0.0")
+
+        foo_deps = next(d for d in result["dependencies"] if d["ref"] == "package-foo")
+        self.assertEqual(
+            foo_deps["dependsOn"],
+            [
+                "package-bar",
+                "package-foo:pkg:golang/example.org/fork@v1.1.0",
+                "package-foo:pkg:golang/example.org/lib@v1.0.0",
+                "skeleton-baz",
+            ],
+        )
+
+        self.assertNotIn("components", self._find_component(result, "skeleton-baz"))
+
     def test_external_references(self):
         info = self._make_show_info()
         info["package-foo"]["downloads"] = [
diff --git a/utils/generate-cyclonedx b/utils/generate-cyclonedx
index a79ca41fe1..d85aec5973 100755
--- a/utils/generate-cyclonedx
+++ b/utils/generate-cyclonedx
@@ -405,6 +405,7 @@ def cyclonedx_component(name, comp):
         } if "cpe-id" in comp else {}),
         **cyclonedx_external_refs(comp),
         **(cyclonedx_patches(name, comp["patches"]) if comp.get("patches") else {}),
+        **(cyclonedx_vendored(name, comp["vendored_dependencies"]) if comp.get("vendored_dependencies") else {}),
         "properties": [{
             "name": "BR_TYPE",
             "value": comp["type"],
@@ -412,6 +413,53 @@ def cyclonedx_component(name, comp):
     }
 
 
+def vendored_ref(name, dep):
+    """Return the bom-ref of a dependency vendored into a package.
+
+    Args:
+        name (str): Key used for the package in the show-info output.
+        dep (dict): Vendored dependency from the show-info output.
+
+    Returns:
+        str: bom-ref, unique even when several packages vendor the same dependency.
+    """
+    return f"{name}:{dep['purl']}"
+
+
+def cyclonedx_vendored(name, deps):
+    """Translate the vendored dependencies of a package to nested CycloneDX components.
+
+    A dependency swapped in for another one, e.g. a fork through a Go
+    replace directive, lists the original as its ancestor.
+
+    Args:
+        name (str): Key used for the package in the show-info output.
+        deps (list): Vendored dependencies from the show-info output.
+
+    Returns:
+        dict: Nested components in CycloneDX format.
+    """
+    def component(dep):
+        return {
+            "type": "library",
+            "name": dep["name"],
+            "version": dep["version"],
+            "purl": dep["purl"],
+        }
+
+    return {
+        "components": [{
+            "bom-ref": vendored_ref(name, dep),
+            **component(dep),
+            **({
+                "pedigree": {
+                    "ancestors": [component(dep["replaces"])],
+                },
+            } if "replaces" in dep else {}),
+        } for dep in deps],
+    }
+
+
 def cyclonedx_dependency(ref, depends):
     """Create JSON for dependency relationships between components.
 
@@ -600,8 +648,9 @@ def main():
         ],
         "dependencies": [
             cyclonedx_dependency(args.project_name, br2_root_deps(filtered_show_info_dict, args.virtual)),
-            *[cyclonedx_dependency(ref, br2_parse_deps(ref, show_info_dict, args.virtual))
-              for ref in filtered_show_info_dict],
+            *[cyclonedx_dependency(ref, br2_parse_deps(ref, show_info_dict, args.virtual) +
+                                   [vendored_ref(ref, dep) for dep in comp.get("vendored_dependencies", [])])
+              for ref, comp in filtered_show_info_dict.items()],
         ],
         "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict),
     }
-- 
2.43.0

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 3/3] package/pkg-golang: vendor modules of local and overridden sources

[Mattias Walström]

These skip the download step that runs go mod vendor, so a tree without
a vendor/ directory failed to build with -mod=vendor.

Signed-off-by: Mattias Walström <lazzer@gmail.com>
---
 package/pkg-golang.mk | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
index 043efc94bd..2f2b5bdf0b 100644
--- a/package/pkg-golang.mk
+++ b/package/pkg-golang.mk
@@ -109,6 +109,22 @@ ifneq ($$($(2)_SUBDIR),)
 $(2)_DOWNLOAD_POST_PROCESS_OPTS += -s$$($(2)_SUBDIR)
 endif
 
+# Local and overridden sources skip the download step, so vendor their
+# modules here when the source tree does not carry a vendor/ directory.
+# The module cache lives in DL_DIR, go.sum verifies its contents. Run
+# from the source tree so relative replace directives resolve.
+define $(2)_GO_VENDOR_OVERRIDE
+	$$(Q)$$(if $$($(2)_OVERRIDE_SRCDIR), \
+		if [ -f $$($(2)_OVERRIDE_SRCDIR)/$$($(2)_SUBDIR)/go.mod ] && \
+		   [ ! -d $$($(2)_OVERRIDE_SRCDIR)/$$($(2)_SUBDIR)/vendor ]; then \
+			cd $$($(2)_OVERRIDE_SRCDIR)/$$($(2)_SUBDIR) && $$($(2)_DL_ENV) \
+				GOFLAGS=-mod=mod \
+				GOMODCACHE=$$(DL_DIR)/go-mod \
+				$$(GO_BIN) mod vendor -modcacherw -o $$(@D)/$$($(2)_SUBDIR)/vendor; \
+		fi)
+endef
+$(2)_PRE_BUILD_HOOKS += $(2)_GO_VENDOR_OVERRIDE
+
 # Because we append vendored info, we can't rely on the values being empty
 # once we eventually get into the generic-package infra. So, we duplicate
 # the heuristics here
-- 
2.43.0

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot