[PATCH 1/2] fetch2: kill parameters in uri_find_decoded

[PATCH 1/2] fetch2: kill parameters in uri_find_decoded

[Pavel Zhukov]

Fixes:
  File "/bitbake/./lib/bb/fetch2/__init__.py", line 446, in uri_replace
    if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
KeyError: 'module'

Signed-off-by: Pavel Zhukov <pavel@zhukoff.net>
---
 bitbake/lib/bb/fetch2/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 893ec6f6bd..b1cd6b25c2 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -469,6 +469,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                     basename = os.path.basename(mirrortarball)
                     # Kill parameters, they make no sense for mirror tarballs
                     uri_decoded[5] = {}
+                    uri_find_decoded[5] = {}
                 elif ud.localpath and ud.method.supports_checksum(ud):
                     basename = os.path.basename(ud.localpath)
                 if basename:
-- 
2.39.0

[PATCH 2/2] fetch2/tests: Add test for Mercurial

[Pavel Zhukov]

Basic test for mercurial functionality including PREMIRRORONLY with
disabled network

Signed-off-by: Pavel Zhukov <pavel@zhukoff.net>
---
 bitbake/lib/bb/tests/fetch.py | 44 +++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index ad3d4dea7d..2e5cd14205 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -2936,6 +2936,50 @@ class FetchPremirroronlyNetworkTest(FetcherTest):
         with self.assertRaises(bb.fetch2.NetworkAccess):
             fetcher.download()
 
+class FetchPremirroronlyMercurialTest(FetcherTest):
+    """ Test for premirrors with mercurial repos
+        the test covers also basic hg:// clone (see fetch_and_create_tarball
+    """
+    def skipIfNoHg():
+        import shutil
+        if not shutil.which('hg'):
+            return unittest.skip('Mercurial not installed')
+        return lambda f: f
+
+    def setUp(self):
+        super(FetchPremirroronlyMercurialTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "libgnt"
+        self.clonedir = os.path.join(self.tempdir, "hg")
+        self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt"
+        self.d.setVar("SRCREV", "53e8b422faaf")
+        self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz"
+
+    def fetch_and_create_tarball(self):
+        """
+        Ask bitbake to download repo and prepare mirror tarball for us
+        """
+        self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname)
+        self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile))
+        ## moving tarball to mirror directory
+        os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname))
+        self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0")
+
+
+    @skipIfNoNetwork()
+    @skipIfNoHg()
+    def test_premirror_mercurial(self):
+        self.fetch_and_create_tarball()
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+
 class FetchPremirroronlyBrokenTarball(FetcherTest):
 
     def setUp(self):
-- 
2.39.0

Re: [bitbake-devel] [PATCH 1/2] fetch2: kill parameters in uri_find_decoded

[Richard Purdie]

On Tue, 2023-01-17 at 16:36 +0100, Pavel Zhukov wrote:
> Fixes:
>   File "/bitbake/./lib/bb/fetch2/__init__.py", line 446, in uri_replace
>     if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
> KeyError: 'module'
> 
> Signed-off-by: Pavel Zhukov <pavel@zhukoff.net>
> ---
>  bitbake/lib/bb/fetch2/__init__.py | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
> index 893ec6f6bd..b1cd6b25c2 100644
> --- a/bitbake/lib/bb/fetch2/__init__.py
> +++ b/bitbake/lib/bb/fetch2/__init__.py
> @@ -469,6 +469,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
>                      basename = os.path.basename(mirrortarball)
>                      # Kill parameters, they make no sense for mirror tarballs
>                      uri_decoded[5] = {}
> +                    uri_find_decoded[5] = {}
>                  elif ud.localpath and ud.method.supports_checksum(ud):
>                      basename = os.path.basename(ud.localpath)
>                  if basename:

The fix looks reasonable but is there a test we should be adding to
cover this scenario?

Cheers,

Richard

Re: [bitbake-devel] [PATCH 1/2] fetch2: kill parameters in uri_find_decoded

[Pavel Zhukov]

Richard Purdie <richard.purdie@linuxfoundation.org> writes:

> On Tue, 2023-01-17 at 16:36 +0100, Pavel Zhukov wrote:
>> Fixes:
>>   File "/bitbake/./lib/bb/fetch2/__init__.py", line 446, in uri_replace
>>     if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
>> KeyError: 'module'
>> 
>> Signed-off-by: Pavel Zhukov <pavel@zhukoff.net>
>> ---
>>  bitbake/lib/bb/fetch2/__init__.py | 1 +
>>  1 file changed, 1 insertion(+)
>> 
>> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
>> index 893ec6f6bd..b1cd6b25c2 100644
>> --- a/bitbake/lib/bb/fetch2/__init__.py
>> +++ b/bitbake/lib/bb/fetch2/__init__.py
>> @@ -469,6 +469,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
>>                      basename = os.path.basename(mirrortarball)
>>                      # Kill parameters, they make no sense for mirror tarballs
>>                      uri_decoded[5] = {}
>> +                    uri_find_decoded[5] = {}
>>                  elif ud.localpath and ud.method.supports_checksum(ud):
>>                      basename = os.path.basename(ud.localpath)
>>                  if basename:
>
> The fix looks reasonable but is there a test we should be adding to
> cover this scenario?
Mercurial test (second patch in the series) covers this but it will be
executed only if network enabled and mercurial installed so I've modified
existing git premirror local test to cover this change as well:

Subject: [PATCH] fetch2/tests: Add parameter to recipe_uri

>
> Cheers,
>
> Richard