[PATCH v2] fetch2/crate: support configurable registry and index URLs

[ms98.cho@gmail.com]

The crate fetcher previously hardcoded the crates.io download URL and
only supported a fixed /crate/versions API shape for custom registries.
This prevented recipes from using private registries or mirrors with
different download paths or Cargo sparse indexes.

Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates
with {crate}, {version}, and {index_path} placeholders. Keep the
existing crates.io defaults, allow custom API-style version endpoints,
and mark sparse indexes explicitly so latest-version checks parse Cargo
index NDJSON instead of JSON API responses.

Add non-network tests for default crates.io URLs, custom registry
templates, sparse index paths, trailing slash handling, and both
latest-version parser paths. Also make the fetch test cleanup chmod
invocation portable by placing -R before the mode.

[YOCTO #16276]

Signed-off-by: minsung.cho <ms98.cho@gmail.com>
---
 lib/bb/fetch2/crate.py |  39 +++++++++++--
 lib/bb/tests/fetch.py  | 122 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 156 insertions(+), 5 deletions(-)

diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py
index bb12f4e..a6dfc6b 100644
--- a/lib/bb/fetch2/crate.py
+++ b/lib/bb/fetch2/crate.py
@@ -77,12 +77,43 @@ class Crate(Wget):
         # host (this is to allow custom crate registries to be specified
         host = '/'.join(parts[2:-2])
 
-        # If using crates.io use the CDN directly as per https://crates.io/data-access
+        # Allow overriding the registry and index URLs via bitbake variables.
+        # Example:
+        # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host.com/crates/{crate}/{version}/download"
+        #          BB_CRATE_INDEX_URL[my-host] = "https://my-host.com/index/{index_path}"
+        dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host)
+        index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host)
+
         if host == 'crates.io':
-            ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version)
-            ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name)
+            if not dl_url:
+                dl_url = "https://static.crates.io/crates/{crate}/{version}/download"
+            if not index_url:
+                index_url = "https://index.crates.io/"
+
+        if dl_url:
+            ud.url = dl_url.replace('{crate}', name).replace('{version}', version)
         else:
             ud.url = "https://%s/%s/%s/download" % (host, name, version)
+
+        ud.crate_index_format = 'api'
+        if index_url:
+            index_path = self._generate_index_path(name)
+            # Support {index_path} for sparse index registries. If not present
+            # and it's crates.io, append it for backward compatibility.
+            if '{index_path}' in index_url:
+                ud.versionsurl = index_url.replace('{index_path}', index_path)
+                ud.crate_index_format = 'sparse'
+            elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'):
+                if not index_url.endswith('/'):
+                    index_url += '/'
+                ud.versionsurl = index_url + index_path
+                ud.crate_index_format = 'sparse'
+            else:
+                ud.versionsurl = index_url
+
+            # Apply other placeholders
+            ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version)
+        else:
             ud.versionsurl = "https://%s/%s/versions" % (host, name)
 
         ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -160,7 +191,7 @@ class Crate(Wget):
         Return the latest upstream version, dispatching to the appropriate
         parser based on the versionsurl format.
         """
-        if ud.versionsurl.startswith('https://index.crates.io/'):
+        if getattr(ud, 'crate_index_format', None) == 'sparse':
             return self._latest_versionstring_from_index(ud, d)
         return self._latest_versionstring_from_api(ud, d)
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 86dd929..0dff3fc 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -424,7 +424,7 @@ class FetcherTest(unittest.TestCase):
         if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
             print("Not cleaning up %s. Please remove manually." % self.tempdir)
         else:
-            bb.process.run('chmod u+rw -R %s' % self.tempdir)
+            bb.process.run('chmod -R u+rw %s' % self.tempdir)
             bb.utils.prunedir(self.tempdir)
 
     def git(self, cmd, cwd=None):
@@ -2590,6 +2590,126 @@ class FetchLocallyMissingTagFromRemote(FetcherTest):
 
 
 class CrateTest(FetcherTest):
+    def test_crate_url_uses_crates_io_defaults(self):
+        ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+        self.assertEqual(ud.url,
+                         "https://static.crates.io/crates/glob/0.2.11/download")
+        self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+        self.assertEqual(ud.crate_index_format, "sparse")
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+
+    def test_crate_url_supports_custom_registry_templates(self):
+        self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+                          "https://registry.example.com/api/v1/crates/{crate}/{version}/download")
+        self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+                          "https://registry.example.com/api/v1/crates/{crate}/versions")
+
+        ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+
+        self.assertEqual(ud.url,
+                         "https://registry.example.com/api/v1/crates/glob/0.2.11/download")
+        self.assertEqual(ud.versionsurl,
+                         "https://registry.example.com/api/v1/crates/glob/versions")
+        self.assertEqual(ud.crate_index_format, "api")
+
+    def test_crate_url_supports_custom_sparse_index_templates(self):
+        self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com",
+                          "https://registry.example.com/crates/{crate}/{version}/download")
+        self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+                          "https://registry.example.com/index/{index_path}")
+
+        ud = bb.fetch2.FetchData("crate://registry.example.com/aho-corasick/0.7.20", self.d)
+
+        self.assertEqual(ud.url,
+                         "https://registry.example.com/crates/aho-corasick/0.7.20/download")
+        self.assertEqual(ud.versionsurl,
+                         "https://registry.example.com/index/ah/o-/aho-corasick")
+        self.assertEqual(ud.crate_index_format, "sparse")
+
+    def test_crate_url_adds_sparse_index_path_with_missing_trailing_slash(self):
+        self.d.setVarFlag("BB_CRATE_INDEX_URL", "crates.io", "https://index.crates.io")
+
+        ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d)
+
+        self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob")
+        self.assertEqual(ud.crate_index_format, "sparse")
+
+    def test_crate_latest_versionstring_supports_custom_sparse_index(self):
+        self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+                          "https://registry.example.com/index/{index_path}")
+        ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+        index = '\n'.join([
+            '{"vers":"0.2.10","yanked":false}',
+            '{"vers":"0.2.11","yanked":true}',
+            '{"vers":"0.2.12","yanked":false}',
+        ])
+
+        with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+            self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+    def test_crate_latest_versionstring_supports_custom_api_index(self):
+        self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com",
+                          "https://registry.example.com/api/v1/crates/{crate}/versions")
+        ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d)
+        index = '{"versions":[{"num":"0.2.10"},{"num":"0.2.12"}]}'
+
+        with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index):
+            self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", ""))
+
+    def test_crate_fetches_from_local_sparse_registry(self):
+        registry = os.path.join(self.tempdir, "registry")
+        crate_name = "dummycrate"
+        crate_version = "1.0.0"
+        crate_basename = "%s-%s" % (crate_name, crate_version)
+        crate_path = os.path.join(registry, "crates", crate_name,
+                                  crate_version, "download")
+        index_path = os.path.join(registry, "index", "du", "mm", crate_name)
+        source_dir = os.path.join(self.tempdir, "crate-source", crate_basename)
+        bb.utils.mkdirhier(os.path.join(source_dir, "src"))
+        bb.utils.mkdirhier(os.path.dirname(crate_path))
+        bb.utils.mkdirhier(os.path.dirname(index_path))
+
+        with open(os.path.join(source_dir, "Cargo.toml"), "w") as f:
+            f.write('[package]\nname = "%s"\nversion = "%s"\n' %
+                    (crate_name, crate_version))
+        with open(os.path.join(source_dir, "src", "lib.rs"), "w") as f:
+            f.write("pub fn answer() -> u32 { 42 }\n")
+        with tarfile.open(crate_path, "w:gz") as tar:
+            tar.add(source_dir, arcname=crate_basename)
+        with open(crate_path, "rb") as f:
+            crate_checksum = hashlib.sha256(f.read()).hexdigest()
+        with open(index_path, "w") as f:
+            f.write('{"name":"%s","vers":"%s","yanked":false}\n' %
+                    (crate_name, crate_version))
+
+        server = HTTPService(registry, host="127.0.0.1")
+        server.start()
+        try:
+            host = "127.0.0.1:%s" % server.port
+            self.d.setVarFlag("BB_CRATE_REGISTRY_URL", host,
+                              "http://%s/crates/{crate}/{version}/download" % host)
+            self.d.setVarFlag("BB_CRATE_INDEX_URL", host,
+                              "http://%s/index/{index_path}" % host)
+            self.d.setVarFlag("SRC_URI", "%s.sha256sum" % crate_basename,
+                              crate_checksum)
+            uri = "crate://%s/%s/%s" % (host, crate_name, crate_version)
+
+            fetcher = bb.fetch2.Fetch([uri], self.d)
+            ud = fetcher.ud[fetcher.urls[0]]
+            self.assertEqual(ud.crate_index_format, "sparse")
+            self.assertEqual(ud.method.latest_versionstring(ud, self.d),
+                             (crate_version, ""))
+
+            fetcher.download()
+            fetcher.unpack(self.tempdir)
+            unpacked_file = os.path.join(self.tempdir, "cargo_home", "bitbake",
+                                         crate_basename, "src", "lib.rs")
+            self.assertTrue(os.path.exists(unpacked_file))
+        finally:
+            server.stop()
+
     @skipIfNoNetwork()
     def test_crate_url(self):
 
-- 
2.50.1 (Apple Git-155)