* [PATCH v2] fetch2/crate: support configurable registry and index URLs
[not found] <20260514051143.67992-1-ms98.cho@gmail.com>
@ 2026-05-14 20:05 ` ms98.cho
2026-05-15 11:12 ` [bitbake-devel] " Mathieu Dubois-Briand
2026-05-21 16:30 ` [PATCH v3] " minsung.cho
0 siblings, 2 replies; 6+ messages in thread
From: ms98.cho @ 2026-05-14 20:05 UTC (permalink / raw)
To: bitbake-devel; +Cc: minsung.cho
The crate fetcher previously hardcoded the crates.io download URL and
only supported a fixed /crate/versions API shape for custom registries.
This prevented recipes from using private registries or mirrors with
different download paths or Cargo sparse indexes.
Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates
with {crate}, {version}, and {index_path} placeholders. Keep the
existing crates.io defaults, allow custom API-style version endpoints,
and mark sparse indexes explicitly so latest-version checks parse Cargo
index NDJSON instead of JSON API responses.
Add non-network tests for default crates.io URLs, custom registry
templates, sparse index paths, trailing slash handling, and both
latest-version parser paths. Also make the fetch test cleanup chmod
invocation portable by placing -R before the mode.
[YOCTO #16276]
Signed-off-by: minsung.cho <ms98.cho@gmail.com>
---
lib/bb/fetch2/crate.py | 39 +++++++++++--
lib/bb/tests/fetch.py | 122 ++++++++++++++++++++++++++++++++++++++++-
2 files changed, 156 insertions(+), 5 deletions(-)
diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py
index bb12f4e..a6dfc6b 100644
--- a/lib/bb/fetch2/crate.py
+++ b/lib/bb/fetch2/crate.py
@@ -77,12 +77,43 @@ class Crate(Wget):
# host (this is to allow custom crate registries to be specified
host = '/'.join(parts[2:-2])
- # If using crates.io use the CDN directly as per https://crates.io/data-access
+ # Allow overriding the registry and index URLs via bitbake variables.
+ # Example:
+ # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host.com/crates/{crate}/{version}/download"
+ # BB_CRATE_INDEX_URL[my-host] = "https://my-host.com/index/{index_path}"
+ dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host)
+ index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host)
+
if host == 'crates.io':
- ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version)
- ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name)
+ if not dl_url:
+ dl_url = "https://static.crates.io/crates/{crate}/{version}/download"
+ if not index_url:
+ index_url = "https://index.crates.io/"
+
+ if dl_url:
+ ud.url = dl_url.replace('{crate}', name).replace('{version}', version)
else:
ud.url = "https://%s/%s/%s/download" % (host, name, version)
+
+ ud.crate_index_format = 'api'
+ if index_url:
+ index_path = self._generate_index_path(name)
+ # Support {index_path} for sparse index registries. If not present
+ # and it's crates.io, append it for backward compatibility.
+ if '{index_path}' in index_url:
+ ud.versionsurl = index_url.replace('{index_path}', index_path)
+ ud.crate_index_format = 'sparse'
+ elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'):
+ if not index_url.endswith('/'):
+ index_url += '/'
+ ud.versionsurl = index_url + index_path
+ ud.crate_index_format = 'sparse'
+ else:
+ ud.versionsurl = index_url
+
+ # Apply other placeholders
+ ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version)
+ else:
ud.versionsurl = "https://%s/%s/versions" % (host, name)
ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -160,7 +191,7 @@ class Crate(Wget):
Return the latest upstream version, dispatching to the appropriate
parser based on the versionsurl format.
"""
- if ud.versionsurl.startswith('https://index.crates.io/'):
+ if getattr(ud, 'crate_index_format', None) == 'sparse':
return self._latest_versionstring_from_index(ud, d)
return self._latest_versionstring_from_api(ud, d)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 86dd929..0dff3fc 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -424,7 +424,7 @@ class FetcherTest(unittest.TestCase):
if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
print("Not cleaning up %s. Please remove manually." % self.tempdir)
else:
- bb.process.run('chmod u+rw -R %s' % self.tempdir)
+ bb.process.run('chmod -R u+rw %s' % self.tempdir)
bb.utils.prunedir(self.tempdir)
def git(self, cmd, cwd=None):
@@ -2590,6 +2590,126 @@ class FetchLocallyMissingTagFromRemote(FetcherTest):
class CrateTest(FetcherTest):
+ def test_crate_url_uses_crates_io_defaults(self):
+ ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.url,
+ "https://static.crates.io/crates/glob/0.2.11/download")
+ self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+ self.assertEqual(ud.crate_index_format, "sparse")
+ self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+ self.assertEqual(ud.parm["name"], "glob-0.2.11")
+
+ def test_crate_url_supports_custom_registry_templates(self):
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/{version}/download")
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/versions")
+
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.url,
+ "https://registry.example.com/api/v1/crates/glob/0.2.11/download")
+ self.assertEqual(ud.versionsurl,
+ "https://registry.example.com/api/v1/crates/glob/versions")
+ self.assertEqual(ud.crate_index_format, "api")
+
+ def test_crate_url_supports_custom_sparse_index_templates(self):
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+ "https://registry.example.com/crates/{crate}/{version}/download")
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/index/{index_path}")
+
+ ud = bb.fetch2.FetchData("crate://registry.example.com/aho-corasick/0.7.20", self.d)
+
+ self.assertEqual(ud.url,
+ "https://registry.example.com/crates/aho-corasick/0.7.20/download")
+ self.assertEqual(ud.versionsurl,
+ "https://registry.example.com/index/ah/o-/aho-corasick")
+ self.assertEqual(ud.crate_index_format, "sparse")
+
+ def test_crate_url_adds_sparse_index_path_with_missing_trailing_slash(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "crates.io", "https://index.crates.io")
+
+ ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+ self.assertEqual(ud.crate_index_format, "sparse")
+
+ def test_crate_latest_versionstring_supports_custom_sparse_index(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/index/{index_path}")
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+ index = '\n'.join([
+ '{"vers":"0.2.10","yanked":false}',
+ '{"vers":"0.2.11","yanked":true}',
+ '{"vers":"0.2.12","yanked":false}',
+ ])
+
+ with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+ def test_crate_latest_versionstring_supports_custom_api_index(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/versions")
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+ index = '{"versions":[{"num":"0.2.10"},{"num":"0.2.12"}]}'
+
+ with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+ def test_crate_fetches_from_local_sparse_registry(self):
+ registry = os.path.join(self.tempdir, "registry")
+ crate_name = "dummycrate"
+ crate_version = "1.0.0"
+ crate_basename = "%s-%s" % (crate_name, crate_version)
+ crate_path = os.path.join(registry, "crates", crate_name,
+ crate_version, "download")
+ index_path = os.path.join(registry, "index", "du", "mm", crate_name)
+ source_dir = os.path.join(self.tempdir, "crate-source", crate_basename)
+ bb.utils.mkdirhier(os.path.join(source_dir, "src"))
+ bb.utils.mkdirhier(os.path.dirname(crate_path))
+ bb.utils.mkdirhier(os.path.dirname(index_path))
+
+ with open(os.path.join(source_dir, "Cargo.toml"), "w") as f:
+ f.write('[package]\nname = "%s"\nversion = "%s"\n' %
+ (crate_name, crate_version))
+ with open(os.path.join(source_dir, "src", "lib.rs"), "w") as f:
+ f.write("pub fn answer() -> u32 { 42 }\n")
+ with tarfile.open(crate_path, "w:gz") as tar:
+ tar.add(source_dir, arcname=crate_basename)
+ with open(crate_path, "rb") as f:
+ crate_checksum = hashlib.sha256(f.read()).hexdigest()
+ with open(index_path, "w") as f:
+ f.write('{"name":"%s","vers":"%s","yanked":false}\n' %
+ (crate_name, crate_version))
+
+ server = HTTPService(registry, host="127.0.0.1")
+ server.start()
+ try:
+ host = "127.0.0.1:%s" % server.port
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", host,
+ "http://%s/crates/{crate}/{version}/download" % host)
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", host,
+ "http://%s/index/{index_path}" % host)
+ self.d.setVarFlag("SRC_URI", "%s.sha256sum" % crate_basename,
+ crate_checksum)
+ uri = "crate://%s/%s/%s" % (host, crate_name, crate_version)
+
+ fetcher = bb.fetch2.Fetch([uri], self.d)
+ ud = fetcher.ud[fetcher.urls[0]]
+ self.assertEqual(ud.crate_index_format, "sparse")
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d),
+ (crate_version, ""))
+
+ fetcher.download()
+ fetcher.unpack(self.tempdir)
+ unpacked_file = os.path.join(self.tempdir, "cargo_home", "bitbake",
+ crate_basename, "src", "lib.rs")
+ self.assertTrue(os.path.exists(unpacked_file))
+ finally:
+ server.stop()
+
@skipIfNoNetwork()
def test_crate_url(self):
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [bitbake-devel] [PATCH v2] fetch2/crate: support configurable registry and index URLs
2026-05-14 20:05 ` [PATCH v2] fetch2/crate: support configurable registry and index URLs ms98.cho
@ 2026-05-15 11:12 ` Mathieu Dubois-Briand
2026-05-15 16:03 ` Minsung Cho
2026-05-21 16:30 ` [PATCH v3] " minsung.cho
1 sibling, 1 reply; 6+ messages in thread
From: Mathieu Dubois-Briand @ 2026-05-15 11:12 UTC (permalink / raw)
To: ms98.cho, bitbake-devel
On Thu May 14, 2026 at 10:05 PM CEST, minsung.cho via lists.openembedded.org wrote:
> The crate fetcher previously hardcoded the crates.io download URL and
> only supported a fixed /crate/versions API shape for custom registries.
> This prevented recipes from using private registries or mirrors with
> different download paths or Cargo sparse indexes.
>
> Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates
> with {crate}, {version}, and {index_path} placeholders. Keep the
> existing crates.io defaults, allow custom API-style version endpoints,
> and mark sparse indexes explicitly so latest-version checks parse Cargo
> index NDJSON instead of JSON API responses.
>
> Add non-network tests for default crates.io URLs, custom registry
> templates, sparse index paths, trailing slash handling, and both
> latest-version parser paths. Also make the fetch test cleanup chmod
> invocation portable by placing -R before the mode.
>
> [YOCTO #16276]
>
> Signed-off-by: minsung.cho <ms98.cho@gmail.com>
> ---
Hi,
Thanks for your patch.
I note it does conflict with some other patches currently in testing,
and once these conflicts solved, I get some bitbake-selftest errors:
ERROR: test_crate_fetches_from_local_sparse_registry (bb.tests.fetch.CrateTest.test_crate_fetches_from_local_sparse_registry)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/tests/fetch.py", line 2829, in test_crate_fetches_from_local_sparse_registry
self.assertEqual(ud.method.latest_versionstring(ud, self.d),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py", line 197, in latest_versionstring
return self._latest_versionstring_from_api(ud, d)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py", line 204, in _latest_versionstring_from_api
versions = [(0, i["num"], "") for i in json_data["versions"]]
~~~~~~~~~^^^^^^^^^^^^
KeyError: 'versions'
https://autobuilder.yoctoproject.org/valkyrie/#/builders/35/builds/3864
https://autobuilder.yoctoproject.org/valkyrie/#/builders/48/builds/3727
https://autobuilder.yoctoproject.org/valkyrie/#/builders/23/builds/3960
Maybe the easiest solution is to wait a few days until said patches are
either merged or rejected, then send a refreshed version.
If you want to have a look at the current situation, here is the branch
that was used for above builds:
https://git.yoctoproject.org/poky-ci-archive/log/?h=bitbake/autobuilder.yoctoproject.org/valkyrie/a-full-3832
Thanks,
Mathieu
--
Mathieu Dubois-Briand, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [bitbake-devel] [PATCH v2] fetch2/crate: support configurable registry and index URLs
2026-05-15 11:12 ` [bitbake-devel] " Mathieu Dubois-Briand
@ 2026-05-15 16:03 ` Minsung Cho
0 siblings, 0 replies; 6+ messages in thread
From: Minsung Cho @ 2026-05-15 16:03 UTC (permalink / raw)
To: mathieu.dubois-briand; +Cc: bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 4527 bytes --]
Hi Mathieu,
Thanks for testing this and digging into the failure.
I reproduced it locally with master + Chen Qi's filter_regex series + this
patch. The KeyError comes from the conflict resolution in
latest_versionstring(), not from either patch on its own. The merged tree
kept the old prefix check (ud.versionsurl.startswith('
https://index.crates.io/')), but this patch intentionally drops that in
favour of an explicit ud.crate_index_format == 'sparse' check, since the
whole point is to support custom sparse registries whose index URL isn't on
index.crates.io. The local sparse registry test serves its index from
127.0.0.1, so the prefix check sends NDJSON to the JSON API parser.
The right merge keeps the format check and threads filter_regex through
both helpers:
def latest_versionstring(self, ud, d, filter_regex=None):
if getattr(ud, 'crate_index_format', None) == 'sparse':
return self._latest_versionstring_from_index(ud, d, filter_regex)
return self._latest_versionstring_from_api(ud, d, filter_regex)
With that, CrateTest passes here (12 tests, 5 skipped), including
test_crate_fetches_from_local_sparse_registry.
I've got a v3 ready with exactly that resolution, rebased on the
filter_regex series. Agreed it's cleaner to wait until that series lands or
gets dropped, then I'll send the refreshed v3. I'll watch the thread.
Thanks,
Minsung
On Fri, May 15, 2026 04:12 AM, Mathieu Dubois-Briand <
mathieu.dubois-briand@bootlin.com> wrote:
> On Thu May 14, 2026 at 10:05 PM CEST, minsung.cho via
> lists.openembedded.org wrote:
> > The crate fetcher previously hardcoded the crates.io download URL and
> > only supported a fixed /crate/versions API shape for custom registries.
> > This prevented recipes from using private registries or mirrors with
> > different download paths or Cargo sparse indexes.
> >
> > Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates
> > with {crate}, {version}, and {index_path} placeholders. Keep the
> > existing crates.io defaults, allow custom API-style version endpoints,
> > and mark sparse indexes explicitly so latest-version checks parse Cargo
> > index NDJSON instead of JSON API responses.
> >
> > Add non-network tests for default crates.io URLs, custom registry
> > templates, sparse index paths, trailing slash handling, and both
> > latest-version parser paths. Also make the fetch test cleanup chmod
> > invocation portable by placing -R before the mode.
> >
> > [YOCTO #16276]
> >
> > Signed-off-by: minsung.cho <ms98.cho@gmail.com>
> > ---
>
> Hi,
>
> Thanks for your patch.
>
> I note it does conflict with some other patches currently in testing,
> and once these conflicts solved, I get some bitbake-selftest errors:
>
> ERROR: test_crate_fetches_from_local_sparse_registry
> (bb.tests.fetch.CrateTest.test_crate_fetches_from_local_sparse_registry)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File
> "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/tests/fetch.py",
> line 2829, in test_crate_fetches_from_local_sparse_registry
> self.assertEqual(ud.method.latest_versionstring(ud, self.d),
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File
> "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py",
> line 197, in latest_versionstring
> return self._latest_versionstring_from_api(ud, d)
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File
> "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py",
> line 204, in _latest_versionstring_from_api
> versions = [(0, i["num"], "") for i in json_data["versions"]]
> ~~~~~~~~~^^^^^^^^^^^^
> KeyError: 'versions'
>
> https://autobuilder.yoctoproject.org/valkyrie/#/builders/35/builds/3864
> https://autobuilder.yoctoproject.org/valkyrie/#/builders/48/builds/3727
> https://autobuilder.yoctoproject.org/valkyrie/#/builders/23/builds/3960
>
> Maybe the easiest solution is to wait a few days until said patches are
> either merged or rejected, then send a refreshed version.
>
> If you want to have a look at the current situation, here is the branch
> that was used for above builds:
>
> https://git.yoctoproject.org/poky-ci-archive/log/?h=bitbake/autobuilder.yoctoproject.org/valkyrie/a-full-3832
>
> Thanks,
> Mathieu
>
> --
> Mathieu Dubois-Briand, Bootlin
> Embedded Linux and Kernel engineering
> https://bootlin.com
>
>
[-- Attachment #2: Type: text/html, Size: 6363 bytes --]
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v3] fetch2/crate: support configurable registry and index URLs
2026-05-14 20:05 ` [PATCH v2] fetch2/crate: support configurable registry and index URLs ms98.cho
2026-05-15 11:12 ` [bitbake-devel] " Mathieu Dubois-Briand
@ 2026-05-21 16:30 ` minsung.cho
2026-06-03 11:30 ` [bitbake-devel] " Ross Burton
2026-06-04 21:19 ` [PATCH v4] " minsung.cho
1 sibling, 2 replies; 6+ messages in thread
From: minsung.cho @ 2026-05-21 16:30 UTC (permalink / raw)
To: bitbake-devel; +Cc: mathieu.dubois-briand, minsung.cho
The crate fetcher previously hardcoded the crates.io download URL and
only supported a fixed /crate/versions API shape for custom registries.
This prevented recipes from using private registries or mirrors with
different download paths or Cargo sparse indexes.
Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates
with {crate}, {version}, and {index_path} placeholders. Keep the
existing crates.io defaults, allow custom API-style version endpoints,
and mark sparse indexes explicitly so latest-version checks parse Cargo
index NDJSON instead of JSON API responses.
Add non-network tests for default crates.io URLs, custom registry
templates, sparse index paths, trailing slash handling, and both
latest-version parser paths. Also make the fetch test cleanup chmod
invocation portable by placing -R before the mode.
[YOCTO #16276]
Signed-off-by: minsung.cho <ms98.cho@gmail.com>
---
lib/bb/fetch2/crate.py | 39 +++++++++++--
lib/bb/tests/fetch.py | 122 ++++++++++++++++++++++++++++++++++++++++-
2 files changed, 156 insertions(+), 5 deletions(-)
diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py
index 8f928ea..3989c31 100644
--- a/lib/bb/fetch2/crate.py
+++ b/lib/bb/fetch2/crate.py
@@ -78,12 +78,43 @@ class Crate(Wget):
# host (this is to allow custom crate registries to be specified
host = '/'.join(parts[2:-2])
- # If using crates.io use the CDN directly as per https://crates.io/data-access
+ # Allow overriding the registry and index URLs via bitbake variables.
+ # Example:
+ # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host.com/crates/{crate}/{version}/download"
+ # BB_CRATE_INDEX_URL[my-host] = "https://my-host.com/index/{index_path}"
+ dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host)
+ index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host)
+
if host == 'crates.io':
- ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version)
- ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name)
+ if not dl_url:
+ dl_url = "https://static.crates.io/crates/{crate}/{version}/download"
+ if not index_url:
+ index_url = "https://index.crates.io/"
+
+ if dl_url:
+ ud.url = dl_url.replace('{crate}', name).replace('{version}', version)
else:
ud.url = "https://%s/%s/%s/download" % (host, name, version)
+
+ ud.crate_index_format = 'api'
+ if index_url:
+ index_path = self._generate_index_path(name)
+ # Support {index_path} for sparse index registries. If not present
+ # and it's crates.io, append it for backward compatibility.
+ if '{index_path}' in index_url:
+ ud.versionsurl = index_url.replace('{index_path}', index_path)
+ ud.crate_index_format = 'sparse'
+ elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'):
+ if not index_url.endswith('/'):
+ index_url += '/'
+ ud.versionsurl = index_url + index_path
+ ud.crate_index_format = 'sparse'
+ else:
+ ud.versionsurl = index_url
+
+ # Apply other placeholders
+ ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version)
+ else:
ud.versionsurl = "https://%s/%s/versions" % (host, name)
ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -161,7 +192,7 @@ class Crate(Wget):
Return the latest upstream version, dispatching to the appropriate
parser based on the versionsurl format.
"""
- if ud.versionsurl.startswith('https://index.crates.io/'):
+ if getattr(ud, 'crate_index_format', None) == 'sparse':
return self._latest_versionstring_from_index(ud, d, filter_regex)
return self._latest_versionstring_from_api(ud, d, filter_regex)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 7026645..7855422 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -424,7 +424,7 @@ class FetcherTest(unittest.TestCase):
if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
print("Not cleaning up %s. Please remove manually." % self.tempdir)
else:
- bb.process.run('chmod u+rw -R %s' % self.tempdir)
+ bb.process.run('chmod -R u+rw %s' % self.tempdir)
bb.utils.prunedir(self.tempdir)
def git(self, cmd, cwd=None):
@@ -2666,6 +2666,126 @@ class FetchLocallyMissingTagFromRemote(FetcherTest):
class CrateTest(FetcherTest):
+ def test_crate_url_uses_crates_io_defaults(self):
+ ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.url,
+ "https://static.crates.io/crates/glob/0.2.11/download")
+ self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+ self.assertEqual(ud.crate_index_format, "sparse")
+ self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+ self.assertEqual(ud.parm["name"], "glob-0.2.11")
+
+ def test_crate_url_supports_custom_registry_templates(self):
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/{version}/download")
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/versions")
+
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.url,
+ "https://registry.example.com/api/v1/crates/glob/0.2.11/download")
+ self.assertEqual(ud.versionsurl,
+ "https://registry.example.com/api/v1/crates/glob/versions")
+ self.assertEqual(ud.crate_index_format, "api")
+
+ def test_crate_url_supports_custom_sparse_index_templates(self):
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+ "https://registry.example.com/crates/{crate}/{version}/download")
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/index/{index_path}")
+
+ ud = bb.fetch2.FetchData("crate://registry.example.com/aho-corasick/0.7.20", self.d)
+
+ self.assertEqual(ud.url,
+ "https://registry.example.com/crates/aho-corasick/0.7.20/download")
+ self.assertEqual(ud.versionsurl,
+ "https://registry.example.com/index/ah/o-/aho-corasick")
+ self.assertEqual(ud.crate_index_format, "sparse")
+
+ def test_crate_url_adds_sparse_index_path_with_missing_trailing_slash(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "crates.io", "https://index.crates.io")
+
+ ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+ self.assertEqual(ud.crate_index_format, "sparse")
+
+ def test_crate_latest_versionstring_supports_custom_sparse_index(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/index/{index_path}")
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+ index = '\n'.join([
+ '{"vers":"0.2.10","yanked":false}',
+ '{"vers":"0.2.11","yanked":true}',
+ '{"vers":"0.2.12","yanked":false}',
+ ])
+
+ with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+ def test_crate_latest_versionstring_supports_custom_api_index(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/versions")
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+ index = '{"versions":[{"num":"0.2.10"},{"num":"0.2.12"}]}'
+
+ with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+ def test_crate_fetches_from_local_sparse_registry(self):
+ registry = os.path.join(self.tempdir, "registry")
+ crate_name = "dummycrate"
+ crate_version = "1.0.0"
+ crate_basename = "%s-%s" % (crate_name, crate_version)
+ crate_path = os.path.join(registry, "crates", crate_name,
+ crate_version, "download")
+ index_path = os.path.join(registry, "index", "du", "mm", crate_name)
+ source_dir = os.path.join(self.tempdir, "crate-source", crate_basename)
+ bb.utils.mkdirhier(os.path.join(source_dir, "src"))
+ bb.utils.mkdirhier(os.path.dirname(crate_path))
+ bb.utils.mkdirhier(os.path.dirname(index_path))
+
+ with open(os.path.join(source_dir, "Cargo.toml"), "w") as f:
+ f.write('[package]\nname = "%s"\nversion = "%s"\n' %
+ (crate_name, crate_version))
+ with open(os.path.join(source_dir, "src", "lib.rs"), "w") as f:
+ f.write("pub fn answer() -> u32 { 42 }\n")
+ with tarfile.open(crate_path, "w:gz") as tar:
+ tar.add(source_dir, arcname=crate_basename)
+ with open(crate_path, "rb") as f:
+ crate_checksum = hashlib.sha256(f.read()).hexdigest()
+ with open(index_path, "w") as f:
+ f.write('{"name":"%s","vers":"%s","yanked":false}\n' %
+ (crate_name, crate_version))
+
+ server = HTTPService(registry, host="127.0.0.1")
+ server.start()
+ try:
+ host = "127.0.0.1:%s" % server.port
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", host,
+ "http://%s/crates/{crate}/{version}/download" % host)
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", host,
+ "http://%s/index/{index_path}" % host)
+ self.d.setVarFlag("SRC_URI", "%s.sha256sum" % crate_basename,
+ crate_checksum)
+ uri = "crate://%s/%s/%s" % (host, crate_name, crate_version)
+
+ fetcher = bb.fetch2.Fetch([uri], self.d)
+ ud = fetcher.ud[fetcher.urls[0]]
+ self.assertEqual(ud.crate_index_format, "sparse")
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d),
+ (crate_version, ""))
+
+ fetcher.download()
+ fetcher.unpack(self.tempdir)
+ unpacked_file = os.path.join(self.tempdir, "cargo_home", "bitbake",
+ crate_basename, "src", "lib.rs")
+ self.assertTrue(os.path.exists(unpacked_file))
+ finally:
+ server.stop()
+
@skipIfNoNetwork()
def test_crate_url(self):
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [bitbake-devel] [PATCH v3] fetch2/crate: support configurable registry and index URLs
2026-05-21 16:30 ` [PATCH v3] " minsung.cho
@ 2026-06-03 11:30 ` Ross Burton
2026-06-04 21:19 ` [PATCH v4] " minsung.cho
1 sibling, 0 replies; 6+ messages in thread
From: Ross Burton @ 2026-06-03 11:30 UTC (permalink / raw)
To: ms98.cho@gmail.com; +Cc: bitbake-devel@lists.openembedded.org
Hi,
On 21 May 2026, at 17:30, minsung.cho via lists.openembedded.org <ms98.cho=gmail.com@lists.openembedded.org> wrote:
> + if dl_url:
> + ud.url = dl_url.replace('{crate}', name).replace('{version}', version)
> else:
> ud.url = "https://%s/%s/%s/download" % (host, name, version)
Instead of using replace() calls, we can treat the string that looks exactly like a f-string as an f-string, and use format(). eg:
ud.url = dl_url.format(version=version, crate=name)
> + elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'):
> + if not index_url.endswith('/'):
> + index_url += '/'
> + ud.versionsurl = index_url + index_path
> + ud.crate_index_format = 'sparse'
It looks like setting index_url to include {index_path} in the crates.io <http://crates.io/> special-casing would mean this entire block can be deleted, as the block above would handle it?
> + ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version)
Again, use format().
> - if ud.versionsurl.startswith('https://index.crates.io/'):
> + if getattr(ud, 'crate_index_format', None) == 'sparse':
When is crate_index_format not set? If it’s not been set in the codepaths above, the other variables haven’t been set either. I think this can be a normal attribute lookup.
> - bb.process.run('chmod u+rw -R %s' % self.tempdir)
> + bb.process.run('chmod -R u+rw %s' % self.tempdir)
Separate commit please, as this is unrelated to crate registry fetching. Also, as you’re changing this command please switch it to use a list instead of a string, that is [‘chmod’, ‘-R’, ‘u+rw’, self.tempdir].
Many thanks for writing an actual test case!
Thanks,
Ross
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v4] fetch2/crate: support configurable registry and index URLs
2026-05-21 16:30 ` [PATCH v3] " minsung.cho
2026-06-03 11:30 ` [bitbake-devel] " Ross Burton
@ 2026-06-04 21:19 ` minsung.cho
1 sibling, 0 replies; 6+ messages in thread
From: minsung.cho @ 2026-06-04 21:19 UTC (permalink / raw)
To: bitbake-devel; +Cc: minsung.cho, Ross.Burton, mathieu.dubois-briand
The crate fetcher hardcoded the crates.io download URL and assumed every
other host served a fixed /name/versions JSON API. Recipes pointing at a
private registry or a Cargo sparse index mirror just did not work.
Two new per-host varflags fix this: BB_CRATE_REGISTRY_URL[host] and
BB_CRATE_INDEX_URL[host]. Both take {crate} and {version} placeholders;
the index template additionally takes {index_path}, and its presence is
how the fetcher knows to treat the response as a Cargo sparse index
(NDJSON) rather than as a JSON versions API. The crates.io defaults are
expressed with the same templates, so there is no separate code path for
that host.
Tests cover the crates.io defaults, custom API and sparse templates, both
latest-version parser paths, and an end-to-end fetch+unpack against a
local sparse registry served over HTTP.
[YOCTO #16276]
Signed-off-by: minsung.cho <ms98.cho@gmail.com>
---
v4: address Ross Burton's review of v3
- build the download/index URLs with str.format() instead of chained
.replace() calls, as the templates are already f-string shaped
- express the crates.io index default with the {index_path} template
too, so the generic path handles it and the crates.io-specific
index branch (and its trailing-slash handling) is gone
- latest_versionstring() dispatches on ud.crate_index_format directly
instead of a getattr() guard; it is always set in _crate_urldata_init
- dropped the separate test-cleanup chmod tweak: master already
converted that call to an argument list
- dropped the test for the removed trailing-slash branch
Rebased onto master (resolved against the filter_regex series so
latest_versionstring keeps the filter_regex parameter).
lib/bb/fetch2/crate.py | 29 +++++++++--
lib/bb/tests/fetch.py | 112 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 137 insertions(+), 4 deletions(-)
diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py
index 8f928ea..fe8e70a 100644
--- a/lib/bb/fetch2/crate.py
+++ b/lib/bb/fetch2/crate.py
@@ -78,12 +78,33 @@ class Crate(Wget):
# host (this is to allow custom crate registries to be specified
host = '/'.join(parts[2:-2])
- # If using crates.io use the CDN directly as per https://crates.io/data-access
+ # Allow overriding the registry and index URLs per host via varflags.
+ # The templates accept {crate}, {version} and {index_path} placeholders:
+ # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host/crates/{crate}/{version}/download"
+ # BB_CRATE_INDEX_URL[my-host] = "https://my-host/index/{index_path}"
+ # A {index_path} placeholder marks a Cargo sparse index; without it the
+ # index URL is treated as a JSON versions API endpoint.
+ dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host)
+ index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host)
+
+ # crates.io uses the CDN directly as per https://crates.io/data-access
if host == 'crates.io':
- ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version)
- ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name)
+ if not dl_url:
+ dl_url = "https://static.crates.io/crates/{crate}/{version}/download"
+ if not index_url:
+ index_url = "https://index.crates.io/{index_path}"
+
+ if dl_url:
+ ud.url = dl_url.format(crate=name, version=version)
else:
ud.url = "https://%s/%s/%s/download" % (host, name, version)
+
+ if index_url:
+ ud.crate_index_format = 'sparse' if '{index_path}' in index_url else 'api'
+ ud.versionsurl = index_url.format(crate=name, version=version,
+ index_path=self._generate_index_path(name))
+ else:
+ ud.crate_index_format = 'api'
ud.versionsurl = "https://%s/%s/versions" % (host, name)
ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -161,7 +182,7 @@ class Crate(Wget):
Return the latest upstream version, dispatching to the appropriate
parser based on the versionsurl format.
"""
- if ud.versionsurl.startswith('https://index.crates.io/'):
+ if ud.crate_index_format == 'sparse':
return self._latest_versionstring_from_index(ud, d, filter_regex)
return self._latest_versionstring_from_api(ud, d, filter_regex)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 95cf6c4..c4a1f33 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -2756,6 +2756,118 @@ class FetchLocallyMissingTagFromRemote(FetcherTest):
class CrateTest(FetcherTest):
+ def test_crate_url_uses_crates_io_defaults(self):
+ ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.url,
+ "https://static.crates.io/crates/glob/0.2.11/download")
+ self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+ self.assertEqual(ud.crate_index_format, "sparse")
+ self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+ self.assertEqual(ud.parm["name"], "glob-0.2.11")
+
+ def test_crate_url_supports_custom_registry_templates(self):
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/{version}/download")
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/versions")
+
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+
+ self.assertEqual(ud.url,
+ "https://registry.example.com/api/v1/crates/glob/0.2.11/download")
+ self.assertEqual(ud.versionsurl,
+ "https://registry.example.com/api/v1/crates/glob/versions")
+ self.assertEqual(ud.crate_index_format, "api")
+
+ def test_crate_url_supports_custom_sparse_index_templates(self):
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+ "https://registry.example.com/crates/{crate}/{version}/download")
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/index/{index_path}")
+
+ ud = bb.fetch2.FetchData("crate://registry.example.com/aho-corasick/0.7.20", self.d)
+
+ self.assertEqual(ud.url,
+ "https://registry.example.com/crates/aho-corasick/0.7.20/download")
+ self.assertEqual(ud.versionsurl,
+ "https://registry.example.com/index/ah/o-/aho-corasick")
+ self.assertEqual(ud.crate_index_format, "sparse")
+
+ def test_crate_latest_versionstring_supports_custom_sparse_index(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/index/{index_path}")
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+ index = '\n'.join([
+ '{"vers":"0.2.10","yanked":false}',
+ '{"vers":"0.2.11","yanked":true}',
+ '{"vers":"0.2.12","yanked":false}',
+ ])
+
+ with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+ def test_crate_latest_versionstring_supports_custom_api_index(self):
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+ "https://registry.example.com/api/v1/crates/{crate}/versions")
+ ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+ index = '{"versions":[{"num":"0.2.10"},{"num":"0.2.12"}]}'
+
+ with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+ def test_crate_fetches_from_local_sparse_registry(self):
+ registry = os.path.join(self.tempdir, "registry")
+ crate_name = "dummycrate"
+ crate_version = "1.0.0"
+ crate_basename = "%s-%s" % (crate_name, crate_version)
+ crate_path = os.path.join(registry, "crates", crate_name,
+ crate_version, "download")
+ index_path = os.path.join(registry, "index", "du", "mm", crate_name)
+ source_dir = os.path.join(self.tempdir, "crate-source", crate_basename)
+ bb.utils.mkdirhier(os.path.join(source_dir, "src"))
+ bb.utils.mkdirhier(os.path.dirname(crate_path))
+ bb.utils.mkdirhier(os.path.dirname(index_path))
+
+ with open(os.path.join(source_dir, "Cargo.toml"), "w") as f:
+ f.write('[package]\nname = "%s"\nversion = "%s"\n' %
+ (crate_name, crate_version))
+ with open(os.path.join(source_dir, "src", "lib.rs"), "w") as f:
+ f.write("pub fn answer() -> u32 { 42 }\n")
+ with tarfile.open(crate_path, "w:gz") as tar:
+ tar.add(source_dir, arcname=crate_basename)
+ with open(crate_path, "rb") as f:
+ crate_checksum = hashlib.sha256(f.read()).hexdigest()
+ with open(index_path, "w") as f:
+ f.write('{"name":"%s","vers":"%s","yanked":false}\n' %
+ (crate_name, crate_version))
+
+ server = HTTPService(registry, host="127.0.0.1")
+ server.start()
+ try:
+ host = "127.0.0.1:%s" % server.port
+ self.d.setVarFlag("BB_CRATE_REGISTRY_URL", host,
+ "http://%s/crates/{crate}/{version}/download" % host)
+ self.d.setVarFlag("BB_CRATE_INDEX_URL", host,
+ "http://%s/index/{index_path}" % host)
+ self.d.setVarFlag("SRC_URI", "%s.sha256sum" % crate_basename,
+ crate_checksum)
+ uri = "crate://%s/%s/%s" % (host, crate_name, crate_version)
+
+ fetcher = bb.fetch2.Fetch([uri], self.d)
+ ud = fetcher.ud[fetcher.urls[0]]
+ self.assertEqual(ud.crate_index_format, "sparse")
+ self.assertEqual(ud.method.latest_versionstring(ud, self.d),
+ (crate_version, ""))
+
+ fetcher.download()
+ fetcher.unpack(self.tempdir)
+ unpacked_file = os.path.join(self.tempdir, "cargo_home", "bitbake",
+ crate_basename, "src", "lib.rs")
+ self.assertTrue(os.path.exists(unpacked_file))
+ finally:
+ server.stop()
+
@skipIfNoNetwork()
def test_crate_url(self):
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 6+ messages in thread
end of thread, other threads:[~2026-06-04 21:22 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
[not found] <20260514051143.67992-1-ms98.cho@gmail.com>
2026-05-14 20:05 ` [PATCH v2] fetch2/crate: support configurable registry and index URLs ms98.cho
2026-05-15 11:12 ` [bitbake-devel] " Mathieu Dubois-Briand
2026-05-15 16:03 ` Minsung Cho
2026-05-21 16:30 ` [PATCH v3] " minsung.cho
2026-06-03 11:30 ` [bitbake-devel] " Ross Burton
2026-06-04 21:19 ` [PATCH v4] " minsung.cho
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox