* [PATCH 1/2] fetch: Add package_identity() to the fetchers
2026-11-18 9:28 [PATCH 0/2] fetch: package identity of crate, gomod and npm URLs Stefano Tondo
@ 2026-11-18 9:28 ` Stefano Tondo
2026-11-18 9:28 ` [PATCH 2/2] fetch: Return the package identity of crate, gomod and npm URLs Stefano Tondo
1 sibling, 0 replies; 3+ messages in thread
From: Stefano Tondo @ 2026-11-18 9:28 UTC (permalink / raw)
To: bitbake-devel
Cc: Joshua Watt, Alexander Kanavin, Stefan Herbrechtsmeier,
Stefano Tondo, Stefano Tondo
From: Stefano Tondo <stefano.tondo.ext@siemens.com>
Fetchers of package registries know which package a URL downloads:
its name and its version are part of the URL. SBOM generation needs
this data, but it can only get it today by parsing fetcher internals.
Add FetchMethod.package_identity(). It returns the name, the version
and the Package URL (purl) of the downloaded package as a dict, or
None when the URL does not name a package. It uses only the URL and
the datastore, so it works before the fetch and without network
access. Add bb.fetch.package_url() to build the purl with the correct
encoding.
Signed-off-by: Stefano Tondo <stefano.tondo.ext@siemens.com>
---
lib/bb/fetch/__init__.py | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
index ce2bd62ad..a4b5860c0 100644
--- a/lib/bb/fetch/__init__.py
+++ b/lib/bb/fetch/__init__.py
@@ -1765,6 +1765,33 @@ class FetchMethod(object):
"""
return []
+ def package_identity(self, ud, d):
+ """
+ Return the identity of the package that the URL downloads, for
+ fetchers of package registries that know it, as a dict with the
+ keys "name", "version" and "purl" (Package URL). Return None when
+ the URL does not name a package.
+
+ The result depends only on the URL and the datastore. It is
+ available as soon as the FetchData exists, before the fetch and
+ without network access. It is meant for SBOM generation.
+ """
+ return None
+
+
+def package_url(purl_type, name, version, qualifiers=None):
+ """
+ Return a Package URL (https://github.com/package-url/purl-spec) for a
+ package. name can contain "/": the segments before the last one are
+ the namespace. Each segment, the version and the qualifier values are
+ percent-encoded. qualifiers is a dict, written sorted by key.
+ """
+ quote = lambda s: urllib.parse.quote(s, safe="")
+ purl = "pkg:%s/%s@%s" % (purl_type, "/".join(quote(s) for s in name.split("/")), quote(version))
+ if qualifiers:
+ purl += "?" + "&".join("%s=%s" % (k, quote(v)) for k, v in sorted(qualifiers.items()))
+ return purl
+
class DummyUnpackTracer(object):
"""
--
2.56.1
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH 2/2] fetch: Return the package identity of crate, gomod and npm URLs
2026-11-18 9:28 [PATCH 0/2] fetch: package identity of crate, gomod and npm URLs Stefano Tondo
2026-11-18 9:28 ` [PATCH 1/2] fetch: Add package_identity() to the fetchers Stefano Tondo
@ 2026-11-18 9:28 ` Stefano Tondo
1 sibling, 0 replies; 3+ messages in thread
From: Stefano Tondo @ 2026-11-18 9:28 UTC (permalink / raw)
To: bitbake-devel
Cc: Joshua Watt, Alexander Kanavin, Stefan Herbrechtsmeier,
Stefano Tondo, Stefano Tondo
From: Stefano Tondo <stefano.tondo.ext@siemens.com>
Implement package_identity() for the crate, gomod, gomodgit and npm
fetchers. Packages from a registry other than the default one get a
repository_url qualifier in the purl. A gomod URL with mod=1 downloads
only the go.mod file, which is not built into anything, so it has no
identity.
Signed-off-by: Stefano Tondo <stefano.tondo.ext@siemens.com>
---
lib/bb/fetch/crate.py | 7 +++++
lib/bb/fetch/gomod.py | 16 +++++++++++
lib/bb/fetch/npm.py | 7 +++++
lib/bb/tests/fetch.py | 63 +++++++++++++++++++++++++++++++++++++++++++
4 files changed, 93 insertions(+)
diff --git a/lib/bb/fetch/crate.py b/lib/bb/fetch/crate.py
index 16ea02d8a..bc0b032f2 100644
--- a/lib/bb/fetch/crate.py
+++ b/lib/bb/fetch/crate.py
@@ -103,11 +103,18 @@ class Crate(Wget):
ud.versionsurl = "%s://%s/%s/versions" % (proto, host, name)
ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
+ ud.crate = (name, version, None if host == 'crates.io' else "%s://%s" % (proto, host))
if 'name' not in ud.parm:
ud.parm['name'] = '%s-%s' % (name, version)
logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
+ def package_identity(self, ud, d):
+ name, version, registry = ud.crate
+ qualifiers = {"repository_url": registry} if registry else None
+ purl = bb.fetch.package_url("cargo", name, version, qualifiers)
+ return {"name": name, "version": version, "purl": purl}
+
def unpack(self, ud, rootdir, d):
"""
Uses the crate to build the necessary paths for cargo to utilize it
diff --git a/lib/bb/fetch/gomod.py b/lib/bb/fetch/gomod.py
index a62d643fb..1ec640096 100644
--- a/lib/bb/fetch/gomod.py
+++ b/lib/bb/fetch/gomod.py
@@ -80,6 +80,13 @@ def escape(path):
return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
+def go_package_identity(ud):
+ module = ud.parm['module']
+ version = ud.parm['version']
+ return {"name": module, "version": version,
+ "purl": bb.fetch.package_url("golang", module, version)}
+
+
class GoMod(Wget):
"""Class to fetch Go modules from a Go module proxy via wget"""
@@ -127,6 +134,12 @@ class GoMod(Wget):
super().urldata_init(ud, d)
+ def package_identity(self, ud, d):
+ # A go.mod file alone is not built into anything
+ if ud.parm.get('mod', '0') == '1':
+ return None
+ return go_package_identity(ud)
+
def unpack(self, ud, rootdir, d):
"""Unpack the module in the module cache."""
@@ -218,6 +231,9 @@ class GoModGit(Git):
super().urldata_init(ud, d)
+ def package_identity(self, ud, d):
+ return go_package_identity(ud)
+
def unpack(self, ud, rootdir, d):
"""Unpack the module in the module cache."""
diff --git a/lib/bb/fetch/npm.py b/lib/bb/fetch/npm.py
index c8995ba6b..356002381 100644
--- a/lib/bb/fetch/npm.py
+++ b/lib/bb/fetch/npm.py
@@ -225,6 +225,13 @@ class Npm(FetchMethod):
ud.needdonestamp = False
ud.resolvefile = self.localpath(ud, d) + ".resolved"
+ def package_identity(self, ud, d):
+ qualifiers = None
+ if ud.registry.rstrip("/") != "https://registry.npmjs.org":
+ qualifiers = {"repository_url": ud.registry}
+ purl = bb.fetch.package_url("npm", ud.package, ud.version, qualifiers)
+ return {"name": ud.package, "version": ud.version, "purl": purl}
+
def _resolve_proxy_url(self, ud, d):
"""Resolve the tarball URL from the registry and cache it without any checksum.
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 61dc06660..26066f566 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -4731,3 +4731,66 @@ class GitUnpackUpdateTest(FetcherTest):
with open(os.path.join(unpack_path, 'testfile.txt')) as f:
self.assertEqual(f.read(), 'updated content\n')
+
+
+class PackageIdentityTest(FetcherTest):
+ """package_identity() works from the URL alone, without network access."""
+
+ def identity(self, url):
+ fetcher = bb.fetch.Fetch([url], self.d)
+ ud = fetcher.ud[url]
+ return ud.method.package_identity(ud, self.d)
+
+ def test_crate(self):
+ self.assertEqual(self.identity("crate://crates.io/glob/0.2.11"),
+ {"name": "glob", "version": "0.2.11", "purl": "pkg:cargo/glob@0.2.11"})
+
+ def test_crate_alternate_registry(self):
+ self.assertEqual(self.identity("crate://crates.example.com/glob/0.2.11"),
+ {"name": "glob", "version": "0.2.11",
+ "purl": "pkg:cargo/glob@0.2.11?repository_url=https%3A%2F%2Fcrates.example.com"})
+
+ def test_gomod(self):
+ url = ('gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
+ 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad')
+ self.assertEqual(self.identity(url),
+ {"name": "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob", "version": "v1.0.0",
+ "purl": "pkg:golang/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0"})
+
+ def test_gomod_go_mod_only(self):
+ # A go.mod file alone is not built into anything
+ url = ('gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;'
+ 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873')
+ self.assertIsNone(self.identity(url))
+
+ def test_gomodgit(self):
+ url = ('gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;'
+ 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1')
+ self.assertEqual(self.identity(url),
+ {"name": "golang.org/x/net", "version": "v0.9.0",
+ "purl": "pkg:golang/golang.org/x/net@v0.9.0"})
+
+ def test_npm(self):
+ url = ('npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+ ';sha512sum=f2dd7d88cb9a129fbb97eb87a8b5103bab24f783420fd7587f8000a355a12bf7'
+ '83f1263230afc588123958b73e36bb241f63eaf08119aac5aa2a870bc4de9223')
+ self.assertEqual(self.identity(url),
+ {"name": "@savoirfairelinux/node-server-example", "version": "1.0.0",
+ "purl": "pkg:npm/%40savoirfairelinux/node-server-example@1.0.0"})
+
+ def test_npm_alternate_registry(self):
+ url = ('npm://npm.example.com/registry;package=foo;version=1.2.3'
+ ';sha512sum=f2dd7d88cb9a129fbb97eb87a8b5103bab24f783420fd7587f8000a355a12bf7'
+ '83f1263230afc588123958b73e36bb241f63eaf08119aac5aa2a870bc4de9223')
+ self.assertEqual(self.identity(url),
+ {"name": "foo", "version": "1.2.3",
+ "purl": "pkg:npm/foo@1.2.3?repository_url=https%3A%2F%2Fnpm.example.com%2Fregistry"})
+
+ def test_git_has_none(self):
+ url = ("git://git.yoctoproject.org/git-submodule-test;branch=master;protocol=https;"
+ "rev=f280847494763cdcf71197557a81ba7d8a6bce42")
+ self.assertIsNone(self.identity(url))
+
+ def test_package_url_encoding(self):
+ self.assertEqual(bb.fetch.package_url("golang", "example.com/m", "v1.0.0+incompatible"),
+ "pkg:golang/example.com/m@v1.0.0%2Bincompatible")
--
2.56.1
^ permalink raw reply related [flat|nested] 3+ messages in thread